+
    ƒ°ñjC  ã                   ój  € ^ RI t ^ RIHt ^ RIt^ RIHtHt ^ RIt^ RIt^ RI	H
t
Ht ^RI HtHt Rt ! R R]P                  4      tR!R R	 llt ! R
 R]P                  4      t ! R R]P                  4      tR"R lt ! R R]P
                  P                  4      t ! R R]P                  4      t] ! R R4      4       t ! R R]P                  4      tR#R ltR$R lt ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R ]P                  4      tR# )%é    N)Ú	dataclass)ÚTensorÚnn)ÚAttentionTensorContainerÚComfyAttention)Ú	attentionÚropec                   óP   a a€ ] tR t^t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚEmbedNDc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )é   ÚdimÚthetaÚaxes_dim)ÚintÚlist)ÚformatÚ__classdict__s   "€Ú9C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\flux\layers.pyÚ__annotate__ÚEmbedND.__annotate__   s"   ø€ ÷ !ñ !™Cð !©ð !±tñ !ó    c                óH   <€ \         SV `  4        Wn        W n        W0n        R # ©N)ÚsuperÚ__init__r   r   r   )Úselfr   r   r   Ú	__class__s   &&&&€r   r   ÚEmbedND.__init__   s   ø€ Ü‰ÑÔØŒØŒ
Ø Žr   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   ÚidsÚreturn©r   )r   r   s   "€r   r   r      s   ø€ ÷  ñ  ™6ð  ¡fñ  r   c           
     ó  € VP                   R,          p\        P                  ! \        V4       Uu. uF4  p\	        VRV3,          V P
                  V,          V P                  4      NK6  	  upRR7      pVP                  ^4      # u upi )é   .©r   éÿÿÿÿéýÿÿÿ)ÚshapeÚtorchÚcatÚranger	   r   r   Ú	unsqueeze)r   r!   Ún_axesÚiÚembs   &&   r   ÚforwardÚEmbedND.forward   sk   € Ø—‘˜2•ˆÜ�iŠiÜFKÈFÄmÓTÁmÀŒT�#�c˜1�f•+˜tŸ}™}¨QÕ/°·±Ö<ÁmÑTØô
ˆð
 �}‰}˜QÓÐùò	 Us   ±:B)r   r   r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r1   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   r   s   @@r   r   r      s   ù‡ € ÷!ó !÷ ÷  ð  r   r   c                ó0   € V ^8„  d   QhR\         R\        /# )r   ÚtÚtime_factor)r   Úfloat)r   s   "r   r   r       s   € ÷ ñ œ&ð Äeñ r   c           
     óª  € W0,          p V^,          p\         P                  ! \        P                  ! V4      ) \         P                  ! ^ V\         P
                  V P                  R7      ,          V,          4      pV R,          P                  4       VR,          ,          p\         P                  ! \         P                  ! V4      \         P                  ! V4      .RR7      pV^,          '       d6   \         P                  ! V\         P                  ! VR,          4      .RR7      p\         P                  ! V 4      '       d   VP                  V 4      pV# )a1  
Create sinusoidal timestep embeddings.
:param t: a 1-D Tensor of N indices, one per batch element.
                  These may be fractional.
:param dim: the dimension of the output.
:param max_period: controls the minimum frequency of the embeddings.
:return: an (N, D) Tensor of positional embeddings.
)ÚstartÚendÚdtypeÚdeviceNr&   )ºNNNNr'   )rE   :Nr%   N)r*   ÚexpÚmathÚlogÚarangeÚfloat32rD   r?   r+   ÚcosÚsinÚ
zeros_likeÚis_floating_pointÚto)r=   r   Ú
max_periodr>   ÚhalfÚfreqsÚargsÚ	embeddings   &&&&    r   Útimestep_embeddingrU       sé   € ð 	�€AØ�!�8€DÜ�IŠI”t—x’x 
Ó+Ð+¬e¯lªlÀÈÔTY×TaÑTaÐjk×jrÑjrÔ.sÕsÐvzÕzÓ{€EàˆW�:×ÑÓ  d¥Õ+€DÜ—	’	œ5Ÿ9š9 T›?¬E¯IªI°d«OÐ<À"ÔE€IØ
ˆQ‡w„wÜ—I’I˜y¬%×*:Ò*:¸9ÀUÕ;KÓ*LÐMÐSUÔVˆ	Ü×Ò˜q×!Ò!Ø—L‘L “Oˆ	ØÐr   c                   óT   a a€ ] tR t^5t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMLPEmbedderc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Úin_dimÚ
hidden_dim©r   )r   r   s   "€r   r   ÚMLPEmbedder.__annotate__6   s   ø€ ÷ jñ j™sð j±ñ jr   c                ó¼   <€ \         SV `  4        VP                  WW4VR 7      V n        \        P
                  ! 4       V n        VP                  W"W4VR 7      V n        R# )©ÚbiasrC   rD   N)r   r   ÚLinearÚin_layerr   ÚSiLUÚsiluÚ	out_layer)r   rY   rZ   r_   rC   rD   Ú
operationsr   s   &&&&&&&€r   r   ÚMLPEmbedder.__init__6   sO   ø€ Ü‰ÑÔØ"×)Ñ)¨&À4Ð]cÐ)ÓdˆŒÜ—G’G“IˆŒ	Ø#×*Ñ*¨:ÈÐbhÐ*ÓiˆŽr   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# ©r   Úxr"   r#   )r   r   s   "€r   r   r\   <   s   ø€ ÷ ;ñ ;™ð ;¡Fñ ;r   c                ó`   € V P                  V P                  V P                  V4      4      4      # r   )rd   rc   ra   )r   ri   s   &&r   r1   ÚMLPEmbedder.forward<   s#   € Ø�~‰~˜dŸi™i¨¯©°aÓ(8Ó9Ó:Ð:r   )ra   rd   rc   ©TNNNr3   r;   s   @@r   rW   rW   5   s    ù‡ € ÷jõ j÷;÷ ;ð ;r   rW   c                   óT   a a€ ] tR t^?t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚYakMLPc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Úhidden_sizeÚintermediate_sizer[   )r   r   s   "€r   r   ÚYakMLP.__annotate__@   s   ø€ ÷  ñ  ¡Cð  ¹Cñ  r   c                ó„  <€ \         SV `  4        Wn        W n        VP	                  V P                  V P                  R W4R7      V n        VP	                  V P                  V P                  R W4R7      V n        VP	                  V P                  V P                  R W4R7      V n        \        P                  ! 4       V n
        R# )Tr^   N)r   r   rp   rq   r`   Ú	gate_projÚup_projÚ	down_projr   rb   Úact_fn)r   rp   rq   rC   rD   re   r   s   &&&&&&€r   r   ÚYakMLP.__init__@   s¢   ø€ Ü‰ÑÔØ&ÔØ!2ÔØ#×*Ñ*¨4×+;Ñ+;¸T×=SÑ=SÐZ^ÐfkÐ*Ó{ˆŒØ!×(Ñ(¨×)9Ñ)9¸4×;QÑ;QÐX\ÐdiÐ(ÓyˆŒØ#×*Ñ*¨4×+AÑ+AÀ4×CSÑCSÐZ^ÐfkÐ*Ó{ˆŒÜ—g’g“iˆŽr   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# rh   r#   )r   r   s   "€r   r   rr   I   s   ø€ ÷ ñ ™ð ¡Fñ r   c                ó�   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pV# r   )rv   rw   rt   ru   )r   ri   rv   s   && r   r1   ÚYakMLP.forwardI   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÕ#SÓTˆ	ØÐr   )rw   rv   rt   rp   rq   ru   ©NNNr3   r;   s   @@r   rn   rn   ?   s   ù‡ € ÷ õ  ÷÷ ð r   rn   c                 óv  € V'       d   \        WWEVR 7      # V'       dL   \        P                  ! VP                  W^,          RWER7      \	        4       VP                  WRWER7      4      # \        P                  ! VP                  WRWER7      \        P
                  ! RR7      VP                  WRWER7      4      # )©rC   rD   re   Fr^   TÚtanh©Úapproximate)rn   r   Ú
Sequentialr`   ÚSiLUActivationÚGELU)rp   Úmlp_hidden_dimÚmlp_silu_actÚyak_mlprC   rD   re   s   &&&&&&&r   Ú	build_mlprˆ   M   s¨   € ßÜ�k¸ÐZdÔeÐeßÜ�}Š}Ø×Ñ˜k¸AÕ+=ÀEÐQVÐÓfÜÓØ×Ñ˜nÀÈUÐÓbó
ð 	
ô �}Š}Ø×Ñ˜kÀÈEÐÓaÜ�GŠG Ô'Ø×Ñ˜nÀÈEÐÓaó
ð 	
r   c                   óT   a a€ ] tR t^^t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚQKNormc                ó    <€ V ^8„  d   QhRS[ /# )r   r   r[   )r   r   s   "€r   r   ÚQKNorm.__annotate___   s   ø€ ÷ Lñ L™Cñ Lr   c                ó„   <€ \         SV `  4        VP                  WVR 7      V n        VP                  WVR 7      V n        R# ))rC   rD   N)r   r   ÚRMSNormÚ
query_normÚkey_norm)r   r   rC   rD   re   r   s   &&&&&€r   r   ÚQKNorm.__init___   s<   ø€ Ü‰ÑÔØ$×,Ñ,¨SÀfÐ,ÓMˆŒØ"×*Ñ*¨3ÀFÐ*ÓKˆŽr   c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r   ÚqÚkÚvr"   ©r   Útuple)r   r   s   "€r   r   rŒ   d   s)   ø€ ÷  ñ  ™ð  ¡Fð  ©vð  ¹%ñ  r   c                óŠ   € V P                  V4      pV P                  V4      pVP                  V4      VP                  V4      3# r   )r�   r�   rO   )r   r“   r”   r•   s   &&&&r   r1   ÚQKNorm.forwardd   s9   € Ø�O‰O˜AÓˆØ�M‰M˜!ÓˆØ�t‰t�A‹w˜Ÿ™˜Q›ÐÐr   )r�   r�   r|   r3   r;   s   @@r   rŠ   rŠ   ^   s    ù‡ € ÷Lõ L÷
 ÷  ð  r   rŠ   c                   óB   a a€ ] tR t^jt oRV3R lV 3R llltRtVtV ;t# )ÚSelfAttentionc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[RS[/# )r   r   Ú	num_headsÚqkv_biasÚ	proj_bias©r   Úbool)r   r   s   "€r   r   ÚSelfAttention.__annotate__k   s0   ø€ ÷ \ñ \™Cð \©Cð \¹tð \ÑX\ñ \r   c                óÖ   <€ \         S	V `  4        W n        W,          pVP                  W^,          W5VR7      V n        \        W…WgR7      V n        VP                  WWEVR7      V n        R# )é   r^   r~   N)r   r   r�   r`   ÚqkvrŠ   ÚnormÚproj)
r   r   r�   rž   rŸ   rC   rD   re   Úhead_dimr   s
   &&&&&&&& €r   r   ÚSelfAttention.__init__k   sa   ø€ Ü‰ÑÔØ"ŒØÕ#ˆà×$Ñ$ S°­'¸ÐV\Ð$Ó]ˆŒÜ˜8¸ÔWˆŒ	Ø×%Ñ% c°YÐTZÐ%Ó[ˆŽ	r   )r¦   r�   r§   r¥   )é   FTNNN)r4   r5   r6   r7   r   r8   r9   r:   r;   s   @@r   r›   r›   j   s   ù‡ € ÷\÷ \õ \r   r›   c                   ó,   a € ] tR t^ut o V 3R ltRtV tR# )ÚModulationOutc                ó>   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   # )r   ÚshiftÚscaleÚgater#   )r   r   s   "€r   r   ÚModulationOut.__annotate__u   s'   ø‡ ‚ á�Mñ ñ �Mñ ñ �Lò	 r   © N)r4   r5   r6   r7   Ú__annotate_func__r8   r9   )r   s   @r   r¬   r¬   u   s   ø‡ ‡ „ r   r¬   c                   óT   a a€ ] tR t^|t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )Ú
Modulationc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r   Údoubler    )r   r   s   "€r   r   ÚModulation.__annotate__}   s   ø€ ÷ hñ h™Cð h©ñ hr   c                ó¦   <€ \         SV `  4        W n        V'       d   ^M^V n        VP	                  WP                  V,          W4VR7      V n        R# )é   r^   N)r   r   Ú	is_doubleÚ
multiplierr`   Úlin)r   r   r·   r_   rC   rD   re   r   s   &&&&&&&€r   r   ÚModulation.__init__}   sA   ø€ Ü‰ÑÔØŒß%™!¨1ˆŒØ×$Ñ$ S¯/©/¸CÕ*?ÀdÐ`fÐ$ÓgˆŽr   c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   Úvecr"   r–   )r   r   s   "€r   r   r¸   ƒ   s   ø€ ÷ 
ñ 
™6ð 
¡eñ 
r   c                ó2  € VP                   ^8X  d
   VR,          pV P                  \        P                  P	                  V4      4      P                  V P                  RR7      p\        VR,          !  V P                  '       d   \        VR,          !  3# R3# )r   Nr&   :Nr¤   N:r¤   NN©rE   NrE   r'   )	Úndimr½   r   Ú
functionalrc   Úchunkr¼   r¬   r»   )r   rÀ   Úouts   && r   r1   ÚModulation.forwardƒ   s€   € Ø�8‰8�qŒ=Ø�j•/ˆCØ�h‰h”r—}‘}×)Ñ)¨#Ó.Ó/×5Ñ5°d·o±oÈ2Ð5ÓNˆô ˜3˜r�7Ñ#Ø'+§~§~ ~ŒM˜3˜r�7Ñ#ð
ð 	
à;?ð
ð 	
r   )r»   r½   r¼   rl   r3   r;   s   @@r   rµ   rµ   |   s    ù‡ € ÷hõ h÷
÷ 
ð 
r   rµ   c           	      óŽ  € Vf$   Ve   \         P                  ! W V4      # W,          # V F—  pV RV^ ,          V^,          13;;,          VRV^,          V^,          ^,           13,          ,          uu&   Vf   KP  V RV^ ,          V^,          13;;,          VRV^,          V^,          ^,           13,          ,          uu&   K™  	  V # )NrE   )r*   Úaddcmul)ÚtensorÚm_multÚm_addÚmodulation_dimsÚds   &&&& r   Ú	apply_modrÏ   Ž   s²   € ØÒØÒÜ—=’= °Ó7Ð7à•?Ð"ã ˆAØ�1�a˜•d˜1˜Q�4�i�<×  F¨1¨a°­d°1°Qµ4¸!µ8¨mÐ+;Õ$<Õ<Ó ØÔ Ø�q˜!˜A�$˜q �t˜)�|×$¨¨a°°1µ°a¸µd¸Qµh°Ð.>Õ(?Õ?Õ$ñ !ð ˆr   c                 óæ   € Vf!   \         P                  P                  '       d   \        V! V 4      ^V,           W44      # \         P                  P
                  P                  WW1P                  4      # r   )ÚcomfyÚmodel_managementÚin_trainingrÏ   Ú	quant_opsÚckÚadalnÚeps)ri   r¦   r¯   r®   rÍ   s   &&&&&r   Úmodulated_normrØ   œ   sQ   € ØÒ"¤e×&<Ñ&<×&H×&HÐ&HÜ™˜a› ! e¥)¨UÓDÐDÜ�?‰?×Ñ×#Ñ# A¨e·X±XÓ>Ð>r   c                   óD   a a€ ] tR t^¢t oV 3R ltV3R lR ltRtVtV ;t# )rƒ   c                óX   <€ \         SV `  4        \        P                  ! 4       V n        R # r   )r   r   r   rb   Úgate_fn)r   r   s   &€r   r   ÚSiLUActivation.__init__£   s   ø€ Ü‰ÑÔÜ—w’w“yˆŽr   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# rh   r#   )r   r   s   "€r   r   ÚSiLUActivation.__annotate__§   s   ø€ ÷ %ñ %™ð %¡Fñ %r   c                ó\   € VP                  ^RR7      w  r#V P                  V4      V,          # )r   r&   r'   )rÅ   rÛ   )r   ri   Úx1Úx2s   &&  r   r1   ÚSiLUActivation.forward§   s*   € Ø—‘˜ �Ó#‰ˆØ�|‰|˜BÓ "Õ$Ð$r   )rÛ   r3   r;   s   @@r   rƒ   rƒ   ¢   s   ù‡ € õ!÷%÷ %ð %r   rƒ   c                   ó`   a a€ ] tR t^¬t oRV3R lV 3R llltRRR/ 3V3R lR lltRtVtV ;t# )ÚDoubleStreamBlockNc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[RS[/# )r   rp   r�   Ú	mlp_ratiorž   )r   r?   r¡   )r   r   s   "€r   r   ÚDoubleStreamBlock.__annotate__­   s0   ø€ ÷ ]ñ ]¡Cð ]±Cð ]ÁEð ]ÑUYñ ]r   c           
     ó�  <€ \         SV `  4        \        4       V n        \	        W,          4      pW n        Wn        WPn        V P                  '       d   \        VR WšVR7      V n	        VP                  VRRWšR7      V n        \        WWGWšVR7      V n        VP                  VRRWšR7      V n        \        WWhWšVR7      V n        V P                  '       d   \        VR WšVR7      V n        VP                  VRRWšR7      V n        \        WWGWšVR7      V n        VP                  VRRWšR7      V n        \        WWhWšVR7      V n        R# )T©r·   rC   rD   re   Fç�íµ ÷Æ°>©Úelementwise_affiner×   rC   rD   )r   r�   rž   rŸ   rC   rD   re   )r†   r‡   rC   rD   re   N)r   r   r   Úcomfy_attentionr   r�   rp   Ú
modulationrµ   Úimg_modÚ	LayerNormÚ	img_norm1r›   Úimg_attnÚ	img_norm2rˆ   Úimg_mlpÚtxt_modÚ	txt_norm1Útxt_attnÚ	txt_norm2Útxt_mlp)r   rp   r�   ræ   rž   rî   r†   rŸ   r‡   rC   rD   re   r…   r   s   &&&&&&&&&&&& €r   r   ÚDoubleStreamBlock.__init__­   sP  ø€ Ü‰ÑÔÜ-Ó/ˆÔä˜[Õ4Ó5ˆØ"ŒØ&ÔØ$Œà�?�?ˆ?Ü% k¸$ÀeÐgqÔrˆDŒLà#×-Ñ-¨kÈeÐY]ÐejÐ-ÓzˆŒÜ%¨+ÐU]Ðzð  \fô  gˆŒà#×-Ñ-¨kÈeÐY]ÐejÐ-ÓzˆŒä  È<Ðpuð  R\ô  ]ˆŒà�?�?ˆ?Ü% k¸$ÀeÐgqÔrˆDŒLà#×-Ñ-¨kÈeÐY]ÐejÐ-ÓzˆŒÜ%¨+ÐU]Ðzð  \fô  gˆŒà#×-Ñ-¨kÈeÐY]ÐejÐ-ÓzˆŒä  È<Ðpuð  R\ô  ]ˆŽr   c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )r   ÚimgÚtxtrÀ   Úper#   )r   r   s   "€r   r   rç   Ê   s/   ø€ ÷ Añ A™6ð A©ð A±Vð AÁñ Ar   c	                óÀ	  € V P                   '       d(   V P                  V4      w  ršV P                  V4      w  r¼M	Vw  w  ršw  r¼VP                  R / 4      pVP	                  4       p\        WP                  V	P                  V	P                  V4      pV P                  P                  V4      p?VP                  VP                  ^ ,          VP                  ^,          ^V P                  R4      P                  ^^ ^^^4      w  ppp?V P                  P                  VVV4      w  pp\        W P                   VP                  VP                  V4      pV P"                  P                  V4      p?VP                  VP                  ^ ,          VP                  ^,          ^V P                  R4      P                  ^^ ^^^4      w  ppp?V P"                  P                  VVV4      w  pp\$        P&                  ! VV3^R7      p??\$        P&                  ! VV3^R7      p??\$        P&                  ! VV3^R7      p??VP                  ^,          VP                  ^,          .VR&   RV9   d|   VR,          pV Fl  pV! VVVWEVR7      pVP                  RV4      VP                  RV4      VP                  RV4      VP                  RV4      VP                  R	V4      3w  ppprEKn  	  \)        V4      \)        V4      \)        V4      ppp\+        VVVWEW€P,                  R
7      p???RV9   d   VR,          pV F  pV! VV4      pK  	  VRRVP                  ^,          13,          VRVP                  ^,          R13,          p!p V\/        V P                  P1                  V!4      V	P2                  RV4      ,          p?!V\/        V P5                  \        WP6                  V
P                  V
P                  V4      4      V
P2                  RV4      ,          pV\/        V P"                  P1                  V 4      VP2                  RV4      ,          p? V\/        V P9                  \        W P:                  VP                  VP                  V4      4      VP2                  RV4      ,          pVP<                  \$        P>                  8X  d   \$        P@                  ! VRRRR7      pW3# )Úpatchesr&   Ú	img_sliceÚattn1_patch©rþ   Ú	attn_maskÚextra_optionsr“   r”   r•   rþ   r  ©rþ   ÚmaskÚtransformer_optionsÚpreferred_attentionÚattn1_output_patchrE   Nç        éàÿ  ©ÚnanÚposinfÚneginfr'   é  ÿÿ)!rî   rï   rõ   ÚgetÚcopyrØ   rñ   r¯   r®   rò   r¥   Úviewr)   r�   Úpermuter¦   rö   r÷   r*   r+   r   r   rí   rÏ   r§   r°   rô   ró   rù   rø   rC   Úfloat16Ú
nan_to_num)"r   rü   rý   rÀ   rþ   r  Úmodulation_dims_imgÚmodulation_dims_txtr  Úimg_mod1Úimg_mod2Útxt_mod1Útxt_mod2Útransformer_patchesr  Úimg_modulatedÚimg_qkvÚimg_qÚimg_kÚimg_vÚtxt_modulatedÚtxt_qkvÚtxt_qÚtxt_kÚtxt_vr“   r”   r•   ÚpatchÚprÆ   Úattnr÷   rò   s"   &&&&&&&&&                         r   r1   ÚDoubleStreamBlock.forwardÊ   s—  € Ø�?�?ˆ?Ø!%§¡¨cÓ!2ÑˆHØ!%§¡¨cÓ!2ÑˆH�hà9<Ñ6Ñ ˆXÑ"6 8à1×5Ñ5°iÀÓDÐØ+×0Ñ0Ó2ˆô ' s¯N©N¸H¿N¹NÈHÏNÉNÐ\oÓpˆØ—-‘-×#Ñ# MÓ2ˆØØ%Ÿl™l¨7¯=©=¸Õ+;¸W¿]¹]È1Õ=MÈqÐRV×R`ÑR`ÐbdÓe×mÑmÐnoÐqrÐtuÐwxÐz{Ó|Ñˆˆu�eØØ—}‘}×)Ñ)¨%°¸Ó>‰ˆˆuô ' s¯N©N¸H¿N¹NÈHÏNÉNÐ\oÓpˆØ—-‘-×#Ñ# MÓ2ˆØØ%Ÿl™l¨7¯=©=¸Õ+;¸W¿]¹]È1Õ=MÈqÐRV×R`ÑR`ÐbdÓe×mÑmÐnoÐqrÐtuÐwxÐz{Ó|Ñˆˆu�eØØ—}‘}×)Ñ)¨%°¸Ó>‰ˆˆuä�IŠI�u˜e�n¨!Ô,ˆØ�5Ü�IŠI�u˜e�n¨!Ô,ˆØ�5Ü�IŠI�u˜e�n¨!Ô,ˆØ�5à&)§i¡i°¥l°A·G±G¸AµJÐ%?ˆ�kÑ"ØÐ/Ô/Ø'¨Õ6ˆEÛ�Ù˜˜1˜a BÈ=ÔY�Ø),¯©°°a«¸#¿'¹'À#Àq»/È3Ï7É7ÐSVÐXYË?Ð\_×\cÑ\cÐdhÐjlÓ\mÐor×ovÑovð  xCð  ENó  pOð  *OÑ&��1�a˜™Yñ ô
 +¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆ1ˆÜ˜˜A˜q RÐM`÷  wKñ  wKô  LˆØˆq�!àÐ#6Ô6Ø'Ð(<Õ=ˆEÛ�Ù˜˜}Ó-’ñ ð " ! ^ s§y¡y°¥| ^Ð"3Õ4°d¸1¸c¿i¹iÈ½l¹mÐ;KÕ6L�(ˆð 	Œy˜Ÿ™×+Ñ+¨HÓ5°x·}±}ÀdÐL_Ó`Õ`ˆØØŒy˜Ÿ™¤n°S¿.¹.È(Ï.É.ÐZb×ZhÑZhÐj}Ó&~Óð  BJ÷  BOñ  BOð  QUð  Wjó  kõ  	kˆð 	Œy˜Ÿ™×+Ñ+¨HÓ5°x·}±}ÀdÐL_Ó`Õ`ˆØØŒy˜Ÿ™¤n°S¿.¹.È(Ï.É.ÐZb×ZhÑZhÐj}Ó&~Óð  BJ÷  BOñ  BOð  QUð  Wjó  kõ  	kˆà�9‰9œŸ™Ô%Ü×"Ò" 3¨C¸ÀfÔMˆCàˆxˆr   )rí   rp   rò   rô   rï   rñ   ró   rî   r�   r÷   rù   rõ   rö   rø   )FTFTFNNNr3   r;   s   @@r   rä   rä   ¬   s;   ù‡ € ÷]õ ]ð: TXÐmqð  HLð  bd÷ A÷ Aò Ar   rä   c                   ób   a a€ ] tR tRt oRtR	V3R lV 3R llltRR/ 3V3R lR lltRtVtV ;t	# )
ÚSingleStreamBlocki  z|
A DiT block with parallel linear layers as described in
https://arxiv.org/abs/2302.05442 and adapted modulation interface.
Nc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[RS[/# )r   rp   r�   ræ   Úqk_scale)r   r?   )r   r   s   "€r   r   ÚSingleStreamBlock.__annotate__  s3   ø€ ÷ 0#ñ 0#áð0#ñ ð0#ñ ð	0#ñ
 ñ0#r   c                ó‚  <€ \         SV `  4        \        4       V n        Wn        W n        W,          pT;'       g
    VR
,          V n        \        W,          4      V n        V P                  V n	        W€n
        V'       d.   \        W,          ^,          4      V n	        \        4       V n        M\        P                  ! RR7      V n        V P                  '       d4   V ;P                  ^,          un	        \        P                  ! 4       V n        VP!                  W^,          V P                  ,           WyV
R7      V n        VP!                  WP                  ,           WWšR7      V n        \'        WÉW«R7      V n        Wn        VP-                  VRRWšR7      V n        V'       d   \1        VRWšVR7      V n        R	# R	V n        R	# )g      à?r   r€   r^   r~   Frê   rë   ré   Ng      à¿)r   r   r   rí   rZ   r�   r¯   r   r…   Úmlp_hidden_dim_firstr‡   rƒ   Úmlp_actr   r„   rb   r`   Úlinear1Úlinear2rŠ   r¦   rp   rð   Úpre_normrµ   rî   )r   rp   r�   ræ   r0  rî   r†   r_   r‡   rC   rD   re   r¨   r   s   &&&&&&&&&&&& €r   r   ÚSingleStreamBlock.__init__  s`  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØ%ŒØ"ŒØÕ+ˆØ×/Ð/ ¨4¥ˆŒ
ä! +Õ"9Ó:ˆÔà$(×$7Ñ$7ˆÔ!ØŒßÜ(+¨KÕ,CÀaÕ,GÓ(HˆDÔ%Ü)Ó+ˆD�LäŸ7š7¨vÔ6ˆDŒLà�<�<ˆ<Ø×%Ò%¨Õ*Õ%ÜŸ7š7›9ˆDŒLð "×(Ñ(¨ÀAµoÈ×HaÑHaÕ6aÐhlð  CIÐ(ó  JˆŒà!×(Ñ(¨×7JÑ7JÕ)JÈKÐjoÐ(ÓˆŒä˜8¸ÔWˆŒ	à&ÔØ"×,Ñ,¨[ÈUÐX\ÐdiÐ,ÓyˆŒçÜ(¨¸UÈ%ÐkuÔvˆDŽOà"ˆDŽOr   c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )r   ri   rÀ   rþ   r"   r#   )r   r   s   "€r   r   r1  F  s.   ø€ ÷ (ñ (™ð (¡fð (±&ð (ñ  |Bñ (r   c                óö  € V P                   '       d   V P                  V4      w  rxMTpVP                  R / 4      p	VP                  4       p
\        P                  ! V P                  \        WP                  VP                  VP                  V4      4      ^V P                  ,          V P                  .RR7      w  r¼VP                  VP                  ^ ,          VP                  ^,          ^V P                  R4      P                  ^^ ^^^4      w  rÞp?V P!                  WÞV4      w  rÞRV	9   dy   V	R,          pV Fi  pV! WÞWóWJR7      pVP                  RV4      VP                  RV4      VP                  RV4      VP                  RV4      VP                  RV4      3w  rÞrópKk  	  \#        V4      \#        V4      \#        V4      rþp\%        WÞWóWFV P&                  R	7      p???R
V	9   d   V	R
,          pV F  pV! VV
4      pK  	  V P(                  '       dR   V P+                  VRV P                  ^,          R13,          4      VRRV P                  ^,          13,          ,          pMV P+                  V4      pV P-                  \        P.                  ! VV3^4      4      pV\1        VVP2                  RV4      ,          pVP4                  \        P6                  8X  d   \        P8                  ! VRRRR7      pV# )r   r&   r  r  r“   r”   r•   rþ   r  r  r
  .Nr  r  r  r'   r  )rî   r  r  r*   Úsplitr5  rØ   r7  r¯   r®   rp   r3  r  r)   r�   r  r¦   r   r   rí   r‡   r4  r6  r+   rÏ   r°   rC   r  r  )r   ri   rÀ   rþ   r  rÍ   r  ÚmodÚ_r  r  r¥   Úmlpr“   r”   r•   r)  r*  rÆ   r+  Úoutputs   &&&&&&&              r   r1   ÚSingleStreamBlock.forwardF  sÙ  € Ø�?�?ˆ?Ø—_‘_ SÓ)‰FˆC�àˆCà1×5Ñ5°iÀÓDÐØ+×0Ñ0Ó2ˆä—;’;˜tŸ|™|¬N¸1¿m¹mÈSÏYÉYÐX[×XaÑXaÐcrÓ,sÓtÐwxÐ{÷  |Lñ  |Lõ  xLð  NR÷  Ngñ  Ngð  whð  npô  q‰ˆà—(‘(˜3Ÿ9™9 Q�<¨¯©°1­°q¸$¿.¹.È"ÓM×UÑUÐVWÐYZÐ\]Ð_`ÐbcÓd‰ˆˆaØØ�y‰y˜˜qÓ!‰ˆàÐ/Ô/Ø'¨Õ6ˆEÛ�Ù˜˜a°)ÔY�Ø),¯©°°a«¸#¿'¹'À#Àq»/È3Ï7É7ÐSVÐXYË?Ð\_×\cÑ\cÐdhÐjlÓ\mÐor×ovÑovð  xCð  ENó  pOð  *OÑ&��ašYñ ô
 +¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆÜ˜˜q¨iÐvz÷  wKñ  wKô  LˆØˆq�!àÐ#6Ô6Ø'Ð(<Õ=ˆEÛ�Ù˜˜}Ó-’ñ ð �<�<ˆ<Ø—,‘,˜s 3¨×(AÑ(AÀQÕ(FÑ(GÐ#GÕHÓIÈCÐPSÐUtÐVZ×VoÑVoÐstÕVtÐUtÐPtÕLuÕu‰Cà—,‘,˜sÓ#ˆCØ—‘œeŸiši¨¨s¨°QÓ7Ó8ˆØ	ŒY�v˜sŸx™x¨¨Ó?Õ?ˆØ�7‰7”e—m‘mÔ#Ü× Ò  ¨°EÀ&ÔIˆAØˆr   )rí   rZ   rp   r5  r6  r4  r…   r3  rî   r¦   r�   r7  r¯   r‡   )	g      @NTFTFNNN)
r4   r5   r6   r7   Ú__doc__r   r1   r8   r9   r:   r;   s   @@r   r.  r.    s1   ù‡ € ñ÷
0#õ 0#ðd EIÐZ^Ðtv÷ (÷ (ò (r   r.  c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	Ú	LastLayeriq  c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   rp   Ú
patch_sizeÚout_channelsr[   )r   r   s   "€r   r   ÚLastLayer.__annotate__r  s'   ø€ ÷ Qñ Q¡Cð Q±Sð QÉñ Qr   c                ó8  <€ \         SV `  4        VP                  VR RWVR7      V n        VP	                  WV,          V,          WEVR7      V n        \        P                  ! \        P                  ! 4       VP	                  V^V,          WEVR7      4      V n	        R# )Frê   rë   r^   N)
r   r   rð   Ú
norm_finalr`   Úlinearr   r‚   rb   ÚadaLN_modulation)	r   rp   rE  rF  r_   rC   rD   re   r   s	   &&&&&&&&€r   r   ÚLastLayer.__init__r  s—   ø€ Ü‰ÑÔØ$×.Ñ.¨{ÈuÐZ^ÐfkÐ.Ó{ˆŒØ ×'Ñ'¨À*Õ5LÈ|Õ5[Ðbfð  }CÐ'ó  DˆŒÜ "§¢¬b¯gªg«i¸×9JÑ9JÈ;ÐXYÐ\gÕXgÐnrð  IOÐ9Jó  :Pó  !QˆÖr   c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   ri   rÀ   r"   r#   )r   r   s   "€r   r   rG  x  s"   ø€ ÷ ñ ™ð ¡fð Ávñ r   c                óÒ   € VP                   ^8X  d
   VR,          pV P                  V4      P                  ^RR7      w  rE\        WP                  WTV4      pV P                  V4      pV# )r   r&   rÂ   r'   )rÃ   rK  rÅ   rØ   rI  rJ  )r   ri   rÀ   rÍ   r®   r¯   s   &&&&  r   r1   ÚLastLayer.forwardx  s\   € Ø�8‰8�qŒ=Ø�j•/ˆCà×,Ñ,¨SÓ1×7Ñ7¸¸rÐ7ÓB‰ˆÜ˜1Ÿo™o¨u¸_ÓMˆØ�K‰K˜‹NˆØˆr   )rK  rJ  rI  rl   r   r3   r;   s   @@r   rC  rC  q  s    ù‡ € ÷Qõ Q÷÷ ò r   rC  )i'  g     @�@)FFNNN)NNr   )rG   Údataclassesr   r*   r   r   Úcomfy.model_managementrÑ   Úcomfy.quant_opsÚcomfy.ldm.modules.attentionr   r   r   r	   rŽ   ÚModuler   rU   rW   rn   rˆ   rŠ   r›   r¬   rµ   rÏ   rØ   rƒ   rä   r.  rC  r²   r   r   Ú<module>rU     sý   ðÛ Ý !ã ß ã Û ß Pß !ð €ô ˆb�i‰iô  ÷"ô*;�"—)‘)ô ;ôˆR�Y‰Yô ô
ô"	 ˆU�X‰X�_‰_ô 	 ô\�B—I‘Iô \ð ÷ð ó ðô
�—‘ô 
ô$ô?ô%�R—Y‘Yô %ô_˜Ÿ	™	ô _ôD`˜Ÿ	™	ô `ôF�—	‘	ö r   