+
    ƒ°ñj’œ  ã                   óˆ  € ^ RI Ht ^ RIHtHtHt ^ RIt^ RIt^ RI	H
t
 ^RIHtHtHt ^ RIHtHt ^RIHt ^ RIt^ RItR t ! R R	]
P0                  4      t ! R
 R]
P0                  4      tR tR3R ltR tR tR]P>                  3R lt RRR]P>                  3R lt! ! R R]
P0                  4      t" ! R R]
P0                  4      t#R t$ ! R R]
P0                  4      t% ! R R]P                  P0                  4      t& ! R R]
P0                  4      t' ! R R]
P0                  4      t(R  t)R!R"/R# lt*/ 3R$ lt+ ! R% R&]
P0                  4      t, ! R' R(]
P0                  4      t- ! R) R*]
P0                  4      t. ! R+ R,]
P0                  4      t/ ! R- R.]
P0                  4      t0 ! R/ R0]
P0                  4      t1 ! R1 R2]14      t2R# )4é    )Úpartial)ÚDictÚOptionalÚListN)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention)Ú	rearrangeÚrepeat)Útimestep_embeddingc                 ó   € V e   V # V# ©N© )ÚxÚys   &&ÚLC:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\modules\diffusionmodules\mmdit.pyÚdefaultr      s   € Ø‚}ØˆØ€Hó    c            
       óh   a a€ ] tR t^t oRtRR]P                  RRRRRRR3
V 3R lltR tRt	Vt
V ;t# )	ÚMlpzFMLP as used in Vision Transformer, MLP-Mixer and related networks
    NTç        Fc                ó¾  <€ \         SV `  4        T;'       g    TpT;'       g    TpTpV'       d   \        VP                  ^R7      MVP                  pV! WWiV
R7      V n        V! 4       V n        \        P                  ! V4      V n	        Ve	   V! V4      M\        P                  ! 4       V n        V! W#WiV
R7      V n        \        P                  ! V4      V n        R# )é   )Úkernel_size©ÚbiasÚdtypeÚdeviceN)ÚsuperÚ__init__r   ÚConv2dÚLinearÚfc1ÚactÚnnÚDropoutÚdrop1ÚIdentityÚnormÚfc2Údrop2)ÚselfÚin_featuresÚhidden_featuresÚout_featuresÚ	act_layerÚ
norm_layerr   ÚdropÚuse_convr   r   Ú
operationsÚ
drop_probsÚlinear_layerÚ	__class__s   &&&&&&&&&&&&  €r   r    ÚMlp.__init__   s°   ø€ ô 	‰ÑÔØ#×2Ð2 {ˆØ)×8Ð8¨[ˆØˆ
ßDL”w˜z×0Ñ0¸aÕ@ÐR\×RcÑRcˆá À4Ð]cÔdˆŒÙ“;ˆŒÜ—Z’Z 
Ó+ˆŒ
Ø3=Ò3I‘J˜Ô/ÌrÏ{Ê{Ë}ˆŒ	Ù ÀDÐ^dÔeˆŒÜ—Z’Z 
Ó+ˆŽ
r   c                óÒ   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P	                  V4      pV P                  V4      pV# r   )r#   r$   r'   r)   r*   r+   ©r,   r   s   &&r   ÚforwardÚMlp.forward0   sU   € Ø�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ�J‰J�q‹MˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØ�J‰J�q‹MˆØˆr   )r$   r'   r+   r#   r*   r)   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r%   ÚGELUr    r;   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r7   Ú__classdict__s   @@r   r   r      s=   ù‡ € ñð
 !ØØ—g‘gØØØØØØØ÷,÷6ò r   r   c                   óX   a a€ ] tR t^9t oRtRV3R lV 3R llltR tV3R ltRtVt	V ;t
# )Ú
PatchEmbedz 2D Image to Patch Embedding
    c                óZ   <€ V ^8„  d   QhRS[ S[,          RS[RS[RS[RS[RS[RS[RS[/# )	é   Úimg_sizeÚ
patch_sizeÚin_chansÚ	embed_dimÚflattenr   Ústrict_img_sizeÚdynamic_img_pad)r   ÚintÚbool)ÚformatrG   s   "€r   Ú__annotate__ÚPatchEmbed.__annotate__>   si   ø€ ÷ $Kñ $Ká™s•mð$Kñ ð$Kñ ð	$Kñ
 ð$Kñ ð$Kñ ð$Kñ "ð$Kñ "ñ$Kr   c           
     ó˜  <€ \         SV `  4         \        V4       W n        Y n        Y`n        Y€n        Y�n        T'       d   TP                  Y4Y"Y|TR 7      T n	        MTP                  Y4Y"Y|TR 7      T n	        T'       d   T! T4      T n        R# \        P                  ! 4       T n        R#    T'       d   Y"T3T n         L�Y"3T n         L§; i))r   Ústrider   r   r   N)r   r    ÚlenrM   Úpadding_moderP   rQ   rR   ÚConv3dÚprojr!   r%   r(   r)   )r,   rL   rM   rN   rO   r1   rP   r   rQ   rR   r[   Úconv3dr   r   r4   r7   s   &&&&&&&&&&&&&&&€r   r    ÚPatchEmbed.__init__>   sÁ   ø€ ô" 	‰ÑÔð	;Ü�
ŒOØ(ŒOð )Ôð ŒØ.ÔØ.ÔßØ"×)Ñ)¨(È:Ðosð  JPÐ)ó  QˆD�Ià"×)Ñ)¨(È:Ðosð  JPÐ)ó  QˆDŒIß-7‘J˜yÓ)ˆŽ	¼R¿[º[»]ˆŽ	øð	;ßØ#-¸:Ð"F�–à#-Ð":�–ús   ‘B* Â*C	Â?C	c                óR  € V P                   '       d@   \        P                  P                  P	                  WP
                  V P                  R 7      pV P                  V4      pV P                  '       d"   VP                  ^4      P                  ^^4      pV P                  V4      pV# ))r[   )rR   ÚcomfyÚldmÚ
common_ditÚpad_to_patch_sizerM   r[   r]   rP   Ú	transposer)   r:   s   &&r   r;   ÚPatchEmbed.forwardd   sz   € Ø××ÐÜ—	‘	×$Ñ$×6Ñ6°q¿/¹/ÐX\×XiÑXiÐ6ÓjˆAØ�I‰I�a‹LˆØ�<�<ˆ<Ø—	‘	˜!“×&Ñ& q¨!Ó,ˆAØ�I‰I�a‹LˆØˆr   c                ó^   <€ V ^8„  d   Qh/ S[ P                  P                  S[,          ;R&   # )rK   rR   )ÚtorchÚjitÚFinalrT   )rU   rG   s   "€r   rV   rW   9   s   ø‡ ‚ ñ —Y‘Y—_‘_¡TÕ*Ñ*ò r   )rR   rP   r)   r[   rM   r]   rQ   )éà   é   é   i   NTTTTÚcircularFNNN)r=   r>   r?   r@   rA   r    r;   Ú__annotate_func__rC   rD   rE   rF   s   @@r   rI   rI   9   s$   ù‡ € ñ÷$Kõ $KòL÷W … r   rI   c                 ó°   € Vf   \         P                  ! V4      p\         P                  ! VP                  ^4      V ^VP                  ^4      ,           4      # r   )rh   Ú
zeros_likeÚaddcmulÚ	unsqueeze)r   ÚshiftÚscales   &&&r   Úmodulaterv   m   sA   € Ø‚}Ü× Ò  Ó'ˆÜ�=Š=˜Ÿ™¨Ó+¨Q°°5·?±?À1Ó3EÕ0EÓFÐFr   c                óä  € \         P                  ! V\         P                  R7      p\         P                  ! V\         P                  R7      p\         P                  ! Wv4      p\         P                  ! V^ R7      pVe	   W„,          pVe	   W…,
          pVP                  ^^W.4      p\        W4      p	V'       d7   V^ 8”  d0   \         P                  ! \         P                  ! W0.4      V	.^ R7      p	V	# )z�
grid_size: int of the grid height and width
return:
pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
©r   ©Úaxis)	ÚnpÚarangeÚfloat32ÚmeshgridÚstackÚreshapeÚ!get_2d_sincos_pos_embed_from_gridÚconcatenateÚzeros)
rO   Ú	grid_sizeÚ	cls_tokenÚextra_tokensÚscaling_factorÚoffsetÚgrid_hÚgrid_wÚgridÚ	pos_embeds
   &&&&&&    r   Úget_2d_sincos_pos_embedr�   x   sº   € ô �YŠY�y¬¯
©
Ô3€FÜ�YŠY�y¬¯
©
Ô3€FÜ�;Š;�vÓ&€DÜ�8Š8�D˜qÔ!€DØÒ!ØÕ$ˆØÒØ�}ˆà�<‰<˜˜A˜yÐ4Ó5€DÜ1°)ÓB€Iß�\ AÔ%Ü—N’NÜ�XŠX�|Ð/Ó0°)Ð<À1ô
ˆ	ð Ðr   c                 óÀ   € V ^,          ^ 8X  g   Q h\        V ^,          V^ ,          4      p\        V ^,          V^,          4      p\        P                  ! W#.^R7      pV# )rK   ry   )Ú!get_1d_sincos_pos_embed_from_gridr{   r‚   )rO   r‹   Úemb_hÚemb_wÚembs   &&   r   r�   r�   —   sU   € Ø�q�=˜AÔÐÐô .¨i¸1­n¸dÀ1½gÓF€EÜ-¨i¸1­n¸dÀ1½gÓF€Eä
�.Š.˜%˜¨aÔ
0€CØ€Jr   c                ó˜  € V ^,          ^ 8X  g   Q h\         P                  ! V ^,          \         P                  R7      pW R,          ,          pRRV,          ,          pVP                  R4      p\         P                  ! RW4      p\         P
                  ! V4      p\         P                  ! V4      p\         P                  ! WE.^R7      pV# )zm
embed_dim: output dimension for each position
pos: a list of positions to be encoded: size (M,)
out: (M, D)
rx   ç       @ç      ð?é'  úm,d->mdry   éÿÿÿÿ)r{   r|   Úfloat64r€   ÚeinsumÚsinÚcosr‚   )rO   ÚposÚomegaÚoutÚemb_sinÚemb_cosr’   s   &&     r   r�   r�   ¢   s—   € ð �q�=˜AÔÐÐÜ�IŠI�i 1•n¬B¯J©JÔ7€EØ	˜�_Õ€EØ�%˜•,Õ€Eà
�+‰+�b‹/€CÜ
�)Š)�I˜sÓ
*€Cä�fŠf�S‹k€GÜ�fŠf�S‹k€Gä
�.Š.˜'Ð+°!Ô
4€CØ€Jr   c                 ó\  € \         P                  ! V ^,          W#R7      pW@R,          ,          pRRV,          ,          pVP                  R4      p\         P                  ! RW4      p\         P                  ! V4      p\         P
                  ! V4      p\         P                  ! Wg.^R7      pV# )rK   ©r   r   r”   r•   r–   r—   ©Údimr˜   )rh   r|   r€   rš   r›   rœ   Úcat)	rO   r�   r   r   rž   rŸ   r    r¡   r’   s	   &&&&     r   Ú'get_1d_sincos_pos_embed_from_grid_torchr§   ¶   s   € Ü�LŠL˜ a�°ÔD€EØ	˜�_Õ€EØ�%˜•,Õ€EØ
�+‰+�b‹/€CÜ
�,Š,�y #Ó
-€CÜ�iŠi˜‹n€GÜ�iŠi˜‹n€GÜ
�)Š)�WÐ&¨AÔ
.€CØ€Jr   g      @c                 ó¬  € \        W!4      pW',          V,          pW,          V,          p	\        P                  ! \        P                  ! V) V,           Wƒ,           W%VR 7      \        P                  ! V	) V,           W“,           WVR 7      RR7      w  r«\	        V ^,          W¥VR 7      p\	        V ^,          WµVR 7      p\        P
                  ! WÜ.^R7      pV# )r£   Úij)Úindexingr¤   )Úminrh   r~   Úlinspacer§   r¦   )rO   ÚwÚhÚ
val_centerÚval_magnituder   r   ÚsmallÚval_hÚval_wr‰   rŠ   r�   r‘   r’   s   &&&&&&&        r   Úget_2d_sincos_pos_embed_torchr´   Á   sé   € Ü�‹I€EØ�Y˜-Õ'€EØ�Y˜-Õ'€EÜ—^’^¤E§N¢N°E°6¸JÕ3FÈÕHZÐ\]ÐtyÔ$zô  }B÷  }Kò  }Kð  MRð  LRð  U_õ  L_ð  afõ  asð  uvð  MRô  }Sð  ^bô  c�N€FÜ3°IÀµNÀFÐafÔg€EÜ3°IÀµNÀFÐafÔg€EÜ
�)Š)�U�N¨Ô
*€CØ€Jr   c                   ó@   a a€ ] tR t^Ñt oRtRV 3R lltR tRtVtV ;t	# )ÚTimestepEmbedderz6
Embeds scalar timesteps into vector representations.
c                óð   <€ \         SV `  4        Vf   Tp\        P                  ! VP	                  W!RWER7      \        P
                  ! 4       VP	                  WRWER7      4      V n        W n        Wpn        R # )NTr   )	r   r    r%   Ú
Sequentialr"   ÚSiLUÚmlpÚfrequency_embedding_sizeÚ
max_period)	r,   Úhidden_sizer»   Úoutput_sizer   r   r4   r¼   r7   s	   &&&&&&&&€r   r    ÚTimestepEmbedder.__init__Ö   sl   ø€ Ü‰ÑÔØÒØ%ˆKÜ—=’=Ø×ÑÐ6È$ÐV[ÐÓkÜ�GŠG‹IØ×Ñ˜k¸TÈÐÓ^ó
ˆŒð
 )AÔ%Ø$Žr   c                óˆ   € \        WP                  V P                  R 7      P                  V4      pV P	                  V4      pV# ))r¼   )r   r»   r¼   Útorº   )r,   Útr   ÚkwargsÚt_freqÚt_embs   &&&,  r   r;   ÚTimestepEmbedder.forwardâ   s:   € Ü# A×'DÑ'DÐQU×Q`ÑQ`Ôa×dÑdÐejÓkˆØ—‘˜Ó ˆØˆr   )r»   r¼   rº   )é   NNNNr–   ©
r=   r>   r?   r@   rA   r    r;   rC   rD   rE   rF   s   @@r   r¶   r¶   Ñ   s   ù‡ € ñ÷
%÷ò r   r¶   c                   óX   a a€ ] tR t^èt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚVectorEmbedderz-
Embeds a flat vector of dimension input_dim
c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )rK   Ú	input_dimr½   )rS   )rU   rG   s   "€r   rV   ÚVectorEmbedder.__annotate__í   s   ø€ ÷ 
ñ 
¡#ð 
±Cñ 
r   c                óÌ   <€ \         SV `  4        \        P                  ! VP	                  WR W4R7      \        P
                  ! 4       VP	                  W"R W4R7      4      V n        R# )Tr   N)r   r    r%   r¸   r"   r¹   rº   )r,   rÌ   r½   r   r   r4   r7   s   &&&&&&€r   r    ÚVectorEmbedder.__init__í   sQ   ø€ Ü‰ÑÔÜ—=’=Ø×Ñ˜i¸4ÀuÐÓ\Ü�GŠG‹IØ×Ñ˜k¸TÈÐÓ^ó
ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rK   r   Úreturn©rh   ÚTensor)rU   rG   s   "€r   rV   rÍ   õ   s#   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ñ r   c                ó(   € V P                  V4      pV# r   ©rº   )r,   r   r’   s   && r   r;   ÚVectorEmbedder.forwardõ   s   € Ø�h‰h�q‹kˆØˆ
r   rÖ   ©NNNrÈ   rF   s   @@r   rÊ   rÊ   è   s#   ù‡ € ñ÷
õ 
÷÷ ð r   rÊ   c                 óÄ   € V P                  V P                  ^ ,          V P                  ^,          ^RV4      P                  ^^ 4      p V ^ ,          V ^,          V ^,          3# ©r   r˜   )r€   ÚshapeÚmovedim)ÚqkvÚhead_dims   &&r   Ú	split_qkvrß   ÿ   sN   € Ø
�+‰+�c—i‘i •l C§I¡I¨a¥L°!°R¸Ó
B×
JÑ
JÈ1ÈaÓ
P€CØˆq�6�3�q•6˜3˜q�6Ð!Ð!r   c                   ó|   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltV3R lR ltV3R lR	 ltR
t	Vt
V ;t# )ÚSelfAttentioni  c                óp   <€ V ^8„  d   QhRS[ RS[ RS[RS[S[,          RS[RS[RS[RS[S[,          R	S[/	# )
rK   r¥   Ú	num_headsÚqkv_biasÚqk_scaleÚ	proj_dropÚ	attn_modeÚpre_onlyÚqk_normÚrmsnorm)rS   rT   r   ÚfloatÚstr)rU   rG   s   "€r   rV   ÚSelfAttention.__annotate__  sm   ø€ ÷ &&ñ &&áð&&ñ ð&&ñ ð	&&ñ
 ™5•/ð&&ñ ð&&ñ ð&&ñ ð&&ñ ™#•ð&&ñ ñ&&r   c                ó  <€ \         SV `  4        \        4       V n        W n        W,          V n        VP                  W^,          W:VR7      V n        V'       g4   VP                  WW«R7      V n        \        P                  ! V4      V n        W`P                  9   g   Q hW`n        Wpn        VR8X  d?   \        V P
                  RRW«R7      V n        \        V P
                  RRW«R7      V n        R# VR8X  dK   VP%                  V P
                  RRW«R7      V n        VP%                  V P
                  RRW«R7      V n        R# Vf7   \        P&                  ! 4       V n        \        P&                  ! 4       V n        R# \)        V4      h)	rm   r   ©r   r   ÚrmsTç�íµ ÷Æ°>©Úelementwise_affineÚepsr   r   ÚlnN)r   r    r   Úcomfy_attentionrã   rÞ   r"   rÝ   r]   r%   r&   ræ   ÚATTENTION_MODESrç   rè   ÚRMSNormÚln_qÚln_kÚ	LayerNormr(   Ú
ValueError)r,   r¥   rã   rä   rå   ræ   rç   rè   ré   rê   r   r   r4   r7   s   &&&&&&&&&&&&&€r   r    ÚSelfAttention.__init__  s7  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØ"ŒØÕ(ˆŒà×$Ñ$ S°­'¸ÐV\Ð$Ó]ˆŒßØ"×)Ñ)¨#¸%Ð)ÓOˆDŒIÜŸZšZ¨	Ó2ˆDŒNØ×0Ñ0Ô0Ð0Ð0Ø"ŒØ Œà�eÔÜ §¡À$ÈFÐZ_ÔoˆDŒIÜ §¡À$ÈFÐZ_ÔoˆDŽIØ˜Œ_Ø"×,Ñ,¨T¯]©]ÈtÐY_ÐglÐ,Ó|ˆDŒIØ"×,Ñ,¨T¯]©]ÈtÐY_ÐglÐ,Ó|ˆDŽIØŠ_ÜŸš›ˆDŒIÜŸš›ˆDŽIä˜WÓ%Ð%r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÑ   rÓ   )rU   rG   s   "€r   rV   rí   /  s#   ø€ ÷ ñ ™uŸ|™|ð ±·±ñ r   c                óŠ  € VP                   w  r#pV P                  V4      p\        WPP                  4      w  rgpV P	                  V4      P                  VP                   ^ ,          VP                   ^,          R4      pV P                  V4      P                  VP                   ^ ,          VP                   ^,          R4      pWgV3# rÚ   )rÛ   rÝ   rß   rÞ   rù   r€   rú   )	r,   r   ÚBÚLÚCrÝ   ÚqÚkÚvs	   &&       r   Úpre_attentionÚSelfAttention.pre_attention/  s�   € Ø—'‘'‰ˆˆaØ�h‰h�q‹kˆÜ˜C§¡Ó/‰ˆˆaØ�I‰I�a‹L× Ñ  §¡¨¥¨Q¯W©W°Q­Z¸Ó<ˆØ�I‰I�a‹L× Ñ  §¡¨¥¨Q¯W©W°Q­Z¸Ó<ˆØ�aˆyÐr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÑ   rÓ   )rU   rG   s   "€r   rV   rí   7  s#   ø€ ÷ ñ ¡§¡ð ±·±ñ r   c                ór   € V P                   '       d   Q hV P                  V4      pV P                  V4      pV# r   )rè   r]   ræ   r:   s   &&r   Úpost_attentionÚSelfAttention.post_attention7  s1   € Ø—=—=�=Ð Ð Ø�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rÑ   rÓ   )rU   rG   s   "€r   rV   rí   =  s#   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ñ r   c                óÔ   € V P                  V4      w  r#p\        V4      \        V4      \        V4      rCp\        W#W@P                  V P                  R 7      pV P                  V4      pV# )©ÚheadsÚpreferred_attention)r  r   r	   rã   rö   r
  )r,   r   r  r  r  s   &&   r   r;   ÚSelfAttention.forward=  se   € Ø×$Ñ$ QÓ'‰ˆˆaÜ*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆÜØ�!Ÿ>™>¸t×?SÑ?Sô
ˆð ×Ñ Ó"ˆØˆr   )
rç   rö   rÞ   rú   rù   rã   rè   r]   ræ   rÝ   ©Úxformersrh   ztorch-hbÚmathÚdebug)é   FNr   r  FNFNNN)r=   r>   r?   r@   r÷   r    r  r
  r;   rC   rD   rE   rF   s   @@r   rá   rá     s8   ù‡ € ØH€O÷&&õ &&÷Pð ÷ð ÷÷ ð r   rá   c                   óH   a a€ ] tR tRt oRV3R lV 3R llltR tRtVtV ;t# )rø   iG  c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )rK   r¥   ró   rô   )rS   rT   rë   )rU   rG   s   "€r   rV   ÚRMSNorm.__annotate__H  s%   ø€ ÷ 4ñ 4Ùð4Ù,0ð4Ù?Dñ4r   c                óê   <€ \         SV `  4        W0n        W n        V P                  '       d4   \        P
                  ! \        P                  ! WVR7      4      V n        R# V P                  RR4       R# )af  
Initialize the RMSNorm normalization layer.
Args:
    dim (int): The dimension of the input tensor.
    eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6.
Attributes:
    eps (float): A small value added to the denominator for numerical stability.
    weight (nn.Parameter): Learnable scaling parameter.
r£   ÚweightN)
r   r    rô   Úlearnable_scaler%   Ú	Parameterrh   Úemptyr  Úregister_parameter)r,   r¥   ró   rô   r   r   rÃ   r7   s   &&&&&&,€r   r    ÚRMSNorm.__init__H  sR   ø€ ô 	‰ÑÔØŒØ1ÔØ××ÐÜŸ,š,¤u§{¢{°3ÈUÔ'SÓTˆDŽKà×#Ñ# H¨dÖ3r   c                ó~   € \         P                  P                  P                  WP                  V P
                  4      # r   )ra   rb   rc   Úrms_normr  rô   r:   s   &&r   r;   ÚRMSNorm.forward\  s'   € Ü�y‰y×#Ñ#×,Ñ,¨Q·±¸T¿X¹XÓFÐFr   )rô   r  r  )Frñ   NN©	r=   r>   r?   r@   r    r;   rC   rD   rE   rF   s   @@r   rø   rø   G  s   ù‡ € ÷4õ 4÷(Gò Gr   rø   c                   óH   a a€ ] tR tRt oRV3R lV 3R llltR tRtVtV ;t# )ÚSwiGLUFeedForwardia  c          	      óB   <€ V ^8„  d   QhRS[ RS[ RS[ RS[S[,          /# )rK   r¥   Ú
hidden_dimÚmultiple_ofÚffn_dim_multiplier)rS   r   rë   )rU   rG   s   "€r   rV   ÚSwiGLUFeedForward.__annotate__b  s7   ø€ ÷ 9ñ 9áð9ñ ð9ñ ð	9ñ
 %¡U�Oñ9r   c                óh  <€ \         SV `  4        \        ^V,          ^,          4      pVe   \        WB,          4      pW2V,           ^,
          V,          ,          p\        P                  ! WRR7      V n        \        P                  ! W!RR7      V n        \        P                  ! WRR7      V n        R# )a/  
Initialize the FeedForward module.

Args:
    dim (int): Input dimension.
    hidden_dim (int): Hidden dimension of the feedforward layer.
    multiple_of (int): Value to ensure hidden dimension is a multiple of this value.
    ffn_dim_multiplier (float, optional): Custom multiplier for hidden dimension. Defaults to None.

Attributes:
    w1 (ColumnParallelLinear): Linear transformation for the first layer.
    w2 (RowParallelLinear): Linear transformation for the second layer.
    w3 (ColumnParallelLinear): Linear transformation for the third layer.

NF)r   )r   r    rS   r%   r"   Úw1Úw2Úw3)r,   r¥   r(  r)  r*  r7   s   &&&&&€r   r    ÚSwiGLUFeedForward.__init__b  s„   ø€ ô, 	‰ÑÔÜ˜˜Z�¨!Õ+Ó,ˆ
àÒ)ÜÐ/Õ<Ó=ˆJØ °+Õ%=ÀÕ%AÀkÕ$QÕRˆ
ä—)’)˜C°%Ô8ˆŒÜ—)’)˜J°%Ô8ˆŒÜ—)’)˜C°%Ô8ˆŽr   c                ó¨   € V P                  \        P                  P                  V P	                  V4      4      V P                  V4      ,          4      # r   )r.  r%   Ú
functionalÚsilur-  r/  r:   s   &&r   r;   ÚSwiGLUFeedForward.forwardƒ  s5   € Ø�w‰w”r—}‘}×)Ñ)¨$¯'©'°!«*Ó5¸¿¹À»
ÕBÓCÐCr   )r-  r.  r/  r   r$  rF   s   @@r   r&  r&  a  s   ù‡ € ÷9õ 9÷BDò Dr   r&  c                   óŒ   a a€ ] tR tRt oRtRtRV3R lV 3R llltV3R lR ltR tV3R lR	 lt	R
 t
V3R lR ltRtVtV ;t# )ÚDismantledBlocki‡  zB
A DiT block with gated adaptive layer norm (adaLN) conditioning.
c                ól   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[RS[RS[RS[R	S[R
S[S[,          RS[/# )rK   r½   rã   Ú	mlp_ratiorç   rä   rè   rê   Úscale_mod_onlyÚswigluré   Úx_block_self_attn)rS   rë   rì   rT   r   )rU   rG   s   "€r   rV   ÚDismantledBlock.__annotate__Ž  sŠ   ø€ ÷ [!ñ [!áð[!ñ ð[!ñ ð	[!ñ
 ð[!ñ ð[!ñ ð[!ñ ð[!ñ ð[!ñ ð[!ñ ™#•ð[!ñ  ñ[!r   c                ó²  <€ \         SV `  4        W@P                  9   g   Q hV'       g   VP                  VR RWÍR7      V n        M\        VR RR7      V n        \        VVVVVV
VVVVR7
      V n        V'       d7   V'       d   Q hV'       d   Q hRV n        \        VVVVR V
VVVVR7
      V n	        MR V n        V'       g7   V'       g   VP                  VR RWÍR7      V n
        M\        VR RR7      V n
        \        W,          4      pV'       g5   V	'       g   \        VVR ^ VVVR7      V n        M\        VVRR	7      V n        W€n        V'       d   V'       d   Q hV'       d   Q h^	pM!V'       g   V'       g   ^M^pMV'       g   ^M^p\         P"                  ! \         P$                  ! 4       VP'                  VVV,          RWÍR
7      4      V n        W`n        R# )Frñ   rò   )ró   rô   )
r¥   rã   rä   rç   rè   ré   rê   r   r   r4   Tc                  ó0   € \         P                  ! R R7      # ©Útanh)Úapproximate©r%   rB   r   r   r   Ú<lambda>Ú*DismantledBlock.__init__.<locals>.<lambda>Ñ  s   € ¤b§g¢g¸&Õ&Ar   ©r-   r.   r0   r2   r   r   r4   rÇ   )r¥   r(  r)  r   N)r   r    r÷   rû   Únorm1rø   rá   Úattnr;  Úattn2Únorm2rS   r   rº   r&  r9  r%   r¸   r¹   r"   ÚadaLN_modulationrè   )r,   r½   rã   r8  rç   rä   rè   rê   r9  r:  ré   r;  r   r   r4   Úblock_kwargsÚmlp_hidden_dimÚn_modsr7   s   &&&&&&&&&&&&&&&,  €r   r    ÚDismantledBlock.__init__Ž  sÆ  ø€ ô$ 	‰ÑÔØ×0Ñ0Ô0Ð0Ð0ßØ#×-Ñ-¨kÈeÐY]ÐejÐ-ÓzˆD�Jä  ÀÈDÔQˆDŒJÜ!ØØØØØØØØØØ!ô
ˆŒ	÷ ßÐ�<ß%Ð%Ð%Ø%)ˆDÔ"Ü&ØØ#Ø!Ø#ØØØØØØ%ôˆD�Jð &+ˆDÔ"ßßØ'×1Ñ1Ø°E¸tÈ5ð 2ó �•
ô % [ÀUÐPTÔU�”
Ü˜[Õ4Ó5ˆßßÜØ +Ø$2ÙAØØØ!Ø)ô�•ô -Ø#Ø-Ø #ô�”ð
 -ÔßßÐ�<ß%Ð%Ð%Ø‰Fßß&‘Q¨A‰Fç&‘Q¨AˆFÜ "§¢Ü�GŠG‹I�z×(Ñ(¨°f¸{Õ6JÐQUÐ]bÐ(Óró!
ˆÔð !Žr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©rK   r   ÚcrÒ   rÓ   )rU   rG   s   "€r   rV   r<  ë  s.   ø€ ÷ *ñ *™uŸ|™|ð *±·±ð *ÁÇÁñ *r   c                ót  € V P                   '       g¥   V P                  '       g+   V P                  V4      P                  ^^R7      w  ppppppM+RpRpV P                  V4      P                  ^^R7      w  ppppV P                  P                  \        V P                  V4      W44      4      p	V	VVVVV33# V P                  '       g'   V P                  V4      P                  ^^R7      w  ppMRpV P                  V4      pV P                  P                  \        V P                  V4      W44      4      p	V	R3# )é   r¤   N)rè   r9  rJ  ÚchunkrG  r  rv   rF  )
r,   r   rQ  Ú	shift_msaÚ	scale_msaÚgate_msaÚ	shift_mlpÚ	scale_mlpÚgate_mlprÝ   s
   &&&       r   r  ÚDismantledBlock.pre_attentionë  sN  € Ø�}�}ˆ}Ø×&×&Ð&ð ×)Ñ)¨!Ó,×2Ñ2°1¸!Ð2Ó<ñØØØØØÙð !�	Ø �	ð ×)Ñ)Øóç‘%˜˜q�%“/ñØØØØð —)‘)×)Ñ)¬(°4·:±:¸a³=À)Ó*WÓXˆCØØØØØØðð ð ð ×&×&Ð&ð ×)Ñ)Øóç‘%˜˜q�%“/ñØÙð
 !�	Ø ×1Ñ1°!Ó4�	Ø—)‘)×)Ñ)¬(°4·:±:¸a³=À)Ó*WÓXˆCØ˜�9Ðr   c           	     ó(  € V P                   '       d   Q hW#P                  ^4      V P                  P                  V4      ,          ,           pW&P                  ^4      V P	                  \        V P                  V4      WE4      4      ,          ,           pV# ©r   )rè   rs   rG  r
  rº   rv   rI  )r,   rG  r   rW  rX  rY  rZ  s   &&&&&&&r   r
  ÚDismantledBlock.post_attention  sv   € Ø—=—=�=Ð Ð Ø×"Ñ" 1Ó%¨¯	©	×(@Ñ(@ÀÓ(FÕFÕFˆØ×"Ñ" 1Ó%¨¯©Ü�T—Z‘Z “] IÓ9ó)
õ 
õ 
ˆð ˆr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rP  rÓ   )rU   rG   s   "€r   rV   r<    s.   ø€ ÷ 
ñ 
¡§¡ð 
±%·,±,ð 
Á5Ç<Á<ñ 
r   c           	     óL  € V P                   '       g   Q hV P                  V4      P                  ^	^R7      w	  ppppppp	p
pV P                  V4      pV P                  P                  \        WÃV4      4      pV P                  P                  \        WÉV
4      4      pWÞVVVVVV33# )é	   r¤   )r;  rJ  rT  rF  rG  r  rv   rH  )r,   r   rQ  rU  rV  rW  rX  rY  rZ  Ú
shift_msa2Ú
scale_msa2Ú	gate_msa2Úx_normrÝ   Úqkv2s   &&&            r   Úpre_attention_xÚDismantledBlock.pre_attention_x  s»   € Ø×%×%Ð%Ð%Ð%ð ×!Ñ! !Ó$×*Ñ*¨1°!Ð*Ó4ñ
	
ØØØØØØØØØà—‘˜A“ˆØ�i‰i×%Ñ%¤h¨vÀ)Ó&LÓMˆØ�z‰z×'Ñ'¬°ÀZÓ(PÓQˆØØØØØØØð
ð 
ð 	
r   c	           	     ó@  € V P                   '       d   Q hV P                  P                  V4      p	V P                  P                  V4      p\	        W4W‰V4      pW7P                  ^4      V P                  \        V P                  V4      WV4      4      ,          ,           pV# r]  )	rè   rG  r
  rH  Úgate_catrs   rº   rv   rI  )
r,   rG  rH  r   rW  rX  rY  rZ  rd  Úattn1s
   &&&&&&&&& r   Úpost_attention_xÚ DismantledBlock.post_attention_x8  s…   € Ø—=—=�=Ð Ð Ø—	‘	×(Ñ(¨Ó.ˆØ—
‘
×)Ñ)¨%Ó0ˆÜ�Q )°EÓ:ˆØ×"Ñ" 1Ó%¨¯©Ü�T—Z‘Z “] IÓ9ó)
õ 
õ 
ˆð ˆr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rP  rÓ   )rU   rG   s   "€r   rV   r<  B  s.   ø€ ÷ =ñ =™Ÿ™ð =©%¯,©,ð =¹5¿<¹<ñ =r   c                óÄ  € V P                   '       d   Q hV P                  '       Ed   V P                  W4      w  r4p\        ;QJ d    . R  V 4       F  NK  	  5M! R  V 4       4      p\        ;QJ d    . R V 4       F  NK  	  5M! R V 4       4      p\	        V^ ,          V^,          V^,          V P
                  P                  V P
                  P                  R7      p\	        V^ ,          V^,          V^,          V P                  P                  V P                  P                  R7      pV P                  ! Wg.VO5!  # V P                  W4      w  r5\        ;QJ d    . R V 4       F  NK  	  5M! R V 4       4      p\	        V^ ,          V^,          V^,          V P
                  P                  V P
                  P                  R7      pV P                  ! V.VO5!  # )c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5ir   ©r   ©Ú.0rÂ   s   & r   Ú	<genexpr>Ú*DismantledBlock.forward.<locals>.<genexpr>F  ó   é € ÐA¹S¸Ô0°×3Ð3»Sùó   ‚c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5ir   rq  rr  s   & r   rt  ru  G  s   é € ÐC¹d¸Ô1°!×4Ð4»dùrw  r  c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5ir   rq  rr  s   & r   rt  ru  S  rv  rw  )rè   r;  rg  Útupler	   rG  rã   rö   rH  rl  r  r
  )r,   r   rQ  rÝ   rf  ÚintermediatesrG  rH  s   &&&     r   r;   ÚDismantledBlock.forwardB  ss  € Ø—=—=�=Ð Ð Ø×!×!Ñ!Ø'+×';Ñ';¸AÓ'AÑ$ˆC�}ß”%ÑA¹SÓA—%‘%ÑA¹SÓAÓAˆCß”5ÑC¹dÓC—5‘5ÑC¹dÓCÓCˆDÜ&Ø�A•˜˜A�  A¥Ø—i‘i×)Ñ)¸t¿y¹y×?XÑ?XôˆDô (Ø�Q•˜˜a� $ q¥'Ø—j‘j×*Ñ*ÀÇ
Á
×@ZÑ@ZôˆEð ×(Ò(¨ÐE°}ÓEÐEà!%×!3Ñ!3°AÓ!9ÑˆCß”%ÑA¹SÓA—%‘%ÑA¹SÓAÓAˆCÜ&Ø�A•˜˜A�  A¥Ø—i‘i×)Ñ)¸t¿y¹y×?XÑ?XôˆDð ×&Ò& tÐ<¨mÓ<Ð<r   )	rJ  rG  rH  rº   rF  rI  rè   r9  r;  r  )ç      @r  FFFFFNFNNN)r=   r>   r?   r@   rA   r÷   r    r  r
  rg  rl  r;   rC   rD   rE   rF   s   @@r   r6  r6  ‡  sM   ù‡ € ñð I€O÷[!õ [!÷z*ð *òX÷
ð 
ò2÷=÷ =ð =r   r6  c                 óº   € VP                  ^4      V,          pVP                  ^4      V,          p\        P                  ! WV.^ R7      P                  ^ R7      p V # )r   r¤   )rs   rh   r   Úsum)r   rW  rd  rk  rH  Úout1Úout2s   &&&&&  r   rj  rj  Z  sS   € Ø×Ñ˜aÓ  5Õ(€DØ×Ñ˜qÓ! EÕ)€DÜ�Š�Q˜d�O¨Ô+×/Ñ/°AÐ/Ó6€AØ€Hr   Úuse_checkpointTc                 ó�   € V '       d5   \         P                  P                  P                  ! \        .VO5R R/VB # \        V/ VB # )Úuse_reentrantF)rh   ÚutilsÚ
checkpointÚ_block_mixing)r‚  ÚargsrÃ   s   $*,r   Úblock_mixingr‰  `  sO   € ßÜ�{‰{×%Ñ%×0Ò0Üð
Ø ò
Ø05ð
Ø9?ñ
ð 	
ô ˜dÐ- fÑ-Ð-r   c           	      óü  aa€ VP                  W4      w  opVP                  '       d   VP                  W4      w  orxMVP                  W4      w  opS^ ,          P                  ^,          p	\        ;QJ d!    . VV3R l\        ^4       4       F  NK  	  5M! VV3R l\        ^4       4       4      p
>>\        V
^ ,          V
^,          V
^,          VP                  P                  VP                  P                  VR7      pVRRV	13,          VRV	R13,          rÜVP                  '       g   VP                  ! V.VO5!  p MRp VP                  '       d’   \        ;QJ d    . R X 4       F  NK  	  5M! R X 4       4      p\        V^ ,          V^,          V^,          VP                  P                  VP                  P                  VR7      pVP                  ! WÞ.VO5!  pW3# VP                  ! V.VO5!  pW3# )r   c              3   ó†   <"  € T F6  p\        \        P                  ! SV,          SV,          3^R7      4      x € K8  	  R# 5i)r   r¤   N)r   rh   r¦   )rs  rÂ   Úcontext_qkvÚx_qkvs   & €€r   rt  Ú _block_mixing.<locals>.<genexpr>r  s6   øé € ÐiÑ`hÐ[\Ô(¬¯ª°KÀµNÀEÈ!ÅHÐ3MÐSTÔ)U×VÐVÓ`hùs   ƒ>A)r  r  Útransformer_optionsºNNNNc              3   ó8   "  € T F  p\        V4      x € K  	  R # 5ir   rq  rr  s   & r   rt  rŽ  †  s   é € ÐC¹F°qÔ/°×2Ð2»Fùrw  )r  r;  rg  rÛ   rz  Úranger	   rG  rã   rö   rè   r
  rH  rl  )Úcontextr   Úcontext_blockÚx_blockrQ  r�  Úcontext_intermediatesÚx_qkv2Úx_intermediatesÚcontext_lenrÝ   rG  Úcontext_attnÚx_attnrH  rŒ  r�  s   &&&&&&         @@r   r‡  r‡  i  sÄ  ù€ Ø)6×)DÑ)DÀWÓ)PÑ&€KÐ&à× × Ð Ø)0×)@Ñ)@ÀÓ)FÑ&ˆˆv�à!(×!6Ñ!6°qÓ!<Ñˆˆà˜a•.×&Ñ& qÕ)€Kß
Œ%ÕiÔ`eÐfgÔ`hÓi�%‰%ÕiÔ`eÐfgÔ`hÓiÓ
i€CØ�UäØˆA���A•˜˜A�Ø�l‰l×$Ñ$Ø#ŸL™L×8Ñ8Ø/ô	€Dð 	ˆQ���ˆ_ÕØˆQ�‘ˆ_Õð ð
 ×!×!Ð!Ø×.Ò.¨|ÐTÐ>SÓT‰ð ˆØ× × Ð ß”ÑC¹FÓC—‘ÑC¹FÓCÓCˆÜ#Ø�q•	˜6 !�9 f¨Q¥iØ—m‘m×-Ñ-Ø$+§M¡M×$AÑ$AØ$7ô	ˆð ×$Ò$ VÐE°_ÓEˆð ˆ:Ðð ×"Ò" 6Ð<¨OÓ<ˆØˆ:Ðr   c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )Ú
JointBlocki“  z,just a small wrapper to serve as a fsdp unitc           	     óâ   <€ \         SV `  4        VP                  R 4      pVP                  RR4      pVP                  RR4      p\        VR VRV/VB V n        \        VR RRVRV/VB V n        R# )rè   ré   Nr;  F)r   r    Úpopr6  r”  r•  )r,   rˆ  rÃ   rè   ré   r;  r7   s   &*,   €r   r    ÚJointBlock.__init__–  s�   ø€ ô
 	‰ÑÔØ—:‘:˜jÓ)ˆØ—*‘*˜Y¨Ó-ˆØ"ŸJ™JÐ':¸EÓBÐÜ,¨dÐa¸XÐaÈwÐaÐZ`ÑaˆÔÜ&¨ð 1Ø05ð1à/6ð1ð :Kð1ð *0ñ	1ˆŽr   c                óH   € \        VR V P                  RV P                  /VB # ©r”  r•  )r‰  r”  r•  )r,   rˆ  rÃ   s   &*,r   r;   ÚJointBlock.forward¦  s4   € ÜØð
Ø!%×!3Ñ!3ð
Ø=A¿\¹\ð
ØMSñ
ð 	
r   r¢  rÈ   rF   s   @@r   r�  r�  “  s   ù‡ € Ù6õ1÷ 
ò 
r   r�  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	Ú
FinalLayeri¬  z
The final layer of DiT.
c          	      óB   <€ V ^8„  d   QhRS[ RS[ RS[ RS[S[ ,          /# )rK   r½   rM   Úout_channelsÚtotal_out_channels)rS   r   )rU   rG   s   "€r   rV   ÚFinalLayer.__annotate__±  s7   ø€ ÷ 
ñ 
áð
ñ ð
ñ ð	
ñ
 %¡S�Mñ
r   c                óh  <€ \         SV `  4        VP                  VR RWVR7      V n        Vf#   VP	                  WV,          V,          RWVR7      MVP	                  WRWVR7      V n        \        P                  ! \        P                  ! 4       VP	                  V^V,          RWVR7      4      V n	        R# )Frñ   rò   NTr   )
r   r    rû   Ú
norm_finalr"   Úlinearr%   r¸   r¹   rJ  )	r,   r½   rM   r§  r¨  r   r   r4   r7   s	   &&&&&&&&€r   r    ÚFinalLayer.__init__±  s¬   ø€ ô 	‰ÑÔØ$×.Ñ.¨{ÈuÐZ^ÐfkÐ.Ó{ˆŒð #Ò*ð ×Ñ˜k¸
Õ+BÀ\Õ+QÐX\ÐdiÐÔyà×"Ñ" ;ÈÐUZÐ"Ójð 	Œô
 !#§¢Ü�GŠG‹I�z×(Ñ(¨°a¸+µoÈDÐX]Ð(Ómó!
ˆÖr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# rP  rÓ   )rU   rG   s   "€r   rV   r©  Æ  s.   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ð ¹5¿<¹<ñ r   c                ó¦   € V P                  V4      P                  ^^R7      w  r4\        V P                  V4      W44      pV P	                  V4      pV# )rK   r¤   )rJ  rT  rv   r«  r¬  )r,   r   rQ  rt   ru   s   &&&  r   r;   ÚFinalLayer.forwardÆ  sK   € Ø×,Ñ,¨QÓ/×5Ñ5°a¸QÐ5Ó?‰ˆÜ�T—_‘_ QÓ'¨Ó6ˆØ�K‰K˜‹NˆØˆr   )rJ  r¬  r«  )NNNNrÈ   rF   s   @@r   r¥  r¥  ¬  s#   ù‡ € ñ÷
õ 
÷*÷ ð r   r¥  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚSelfAttentionContextiÌ  c                óÞ   <€ \         SV `  4        \        4       V n        W,          pTpW n        W0n        VP                  W^,          RWER7      V n        VP                  WqWER7      V n        R# )rm   Tr   rï   N)	r   r    r   rö   r  Údim_headr"   rÝ   r]   )	r,   r¥   r  r´  r   r   r4   Ú	inner_dimr7   s	   &&&&&&& €r   r    ÚSelfAttentionContext.__init__Í  sc   ø€ Ü‰ÑÔÜ-Ó/ˆÔØ•<ˆØˆ	àŒ
Ø Œà×$Ñ$ S°­'¸ÀEÐ$ÓYˆŒà×%Ñ% i¸EÐ%ÓQˆŽ	r   c                ób  € V P                  V4      p\        W P                  4      w  r4p\        VP	                  VP
                  ^ ,          VP
                  ^,          R4      4      p\        V4      \        V4      rT?\        W4WPP                  V P                  R7      pV P                  V4      # )r   r  r˜   )
rÝ   rß   r´  r   r€   rÛ   r	   r  rö   r]   )r,   r   rÝ   r  r  r  s   &&    r   r;   ÚSelfAttentionContext.forwardÚ  sƒ   € Ø�h‰h�q‹kˆÜ˜C§¡Ó/‰ˆˆaÜ$ Q§Y¡Y¨q¯w©w°q­z¸1¿7¹7À1½:ÀrÓ%JÓKˆÜ'¨Ó*Ô,DÀQÓ,Gˆ1ØÜ  a¯z©zÈt×OcÑOcÔdˆØ�y‰y˜‹|Ðr   )rö   r´  r  r]   rÝ   )r  é@   NNNr$  rF   s   @@r   r²  r²  Ì  s   ù‡ € ÷R÷ò r   r²  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚContextProcessorBlockiã  c           
     óê   <€ \         SV `  4        VP                  VR RW#R7      V n        \	        WW4R7      V n        VP                  VR RW#R7      V n        \        W^,          R ^ W#VR7      V n        R# )Frñ   rò   ©r   r   r4   c                  ó0   € \         P                  ! R R7      # r?  rB  r   r   r   rC  Ú0ContextProcessorBlock.__init__.<locals>.<lambda>é  s   € Ôgi×gnÒgnð  |Bõ  hCr   rE  N)	r   r    rû   rF  r²  rG  rI  r   rº   )r,   Úcontext_sizer   r   r4   r7   s   &&&&&€r   r    ÚContextProcessorBlock.__init__ä  s‡   ø€ Ü‰ÑÔØ×)Ñ)¨,È5ÐVZÐbgÐ)ÓwˆŒ
Ü(¨È6ÔiˆŒ	Ø×)Ñ)¨,È5ÐVZÐbgÐ)ÓwˆŒ
Ü <ÐQRÕBRñ  `Cð  JKð  SXð  t~ô  ˆŽr   c                óž   € WP                  V P                  V4      4      ,          pWP                  V P                  V4      4      ,          pV# r   )rG  rF  rº   rI  r:   s   &&r   r;   ÚContextProcessorBlock.forwardë  s9   € Ø	�Y‰Y�t—z‘z !“}Ó%Õ%ˆØ	�X‰X�d—j‘j “mÓ$Õ$ˆØˆr   )rG  rº   rF  rI  rØ   r$  rF   s   @@r   r»  r»  ã  s   ù‡ € ÷÷ò r   r»  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚContextProcessorið  c                óò   <€ \         SV `  4        \        P                  P	                  \        V4       Uu. uF  p\        WWER 7      NK  	  up4      V n        VP                  VRRW4R7      V n	        R# u upi )r½  Frñ   rò   N)
r   r    rh   r%   Ú
ModuleListr’  r»  Úlayersrû   r)   )r,   rÀ  Ú
num_layersr   r   r4   Úir7   s   &&&&&& €r   r    ÚContextProcessor.__init__ñ  s�   ø€ Ü‰ÑÔÜ—h‘h×)Ñ)ô  LQð  R\ô  L]ó  +^ñ  L]ð  GHÔ+@ÀÐci÷  ,Bñ  L]ñ  +^ó  _ˆŒØ×(Ñ(¨È%ÐUYÐafÐ(ÓvˆŽ	ùò +^s   ¶A4c                óp   € \        V P                  4       F  w  r#V! V4      pK  	  V P                  V4      # r   )Ú	enumeraterÈ  r)   )r,   r   rÊ  Úls   &&  r   r;   ÚContextProcessor.forwardö  s.   € Ü˜dŸk™kÖ*‰DˆAÙ�!“ŠAñ +à�y‰y˜‹|Ðr   )rÈ  r)   rØ   r$  rF   s   @@r   rÅ  rÅ  ð  s   ù‡ € ÷w÷
ò r   rÅ  c                    óÔ   a a€ ] tR tRt oRt^ ^^^RRRRRR^ RRRRRRRRRRRRR. RRRRRRR3 V3R	 lV 3R
 llltRR ltRR ltRR/ 3V3R lR lltRRR/ 3V3R lR llt	Rt
VtV ;t# )ÚMMDiTiû  z.
Diffusion model with a Transformer backbone.
r}  FNrh   Ti   c          0      ó4  <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[RS[RS[S[ ,          RS[S[,          R	S[R
S[RS[ RS[RS[RS[RS[RS[S[ ,          RS[S[,          RS[S[,          RS[S[ ,          RS[S[,          RS[RS[RS[S[S[ ,          ,          /# )rK   Ú
input_sizerM   Úin_channelsÚdepthr8  Úlearn_sigmaÚadm_in_channelsÚcontext_embedder_configÚcompile_corer‚  Úregister_lengthrç   rê   r9  r:  r§  Úpos_embed_scaling_factorÚpos_embed_offsetÚpos_embed_max_sizeré   rä   r;  Úx_block_self_attn_layers)rS   rë   rT   r   r   rì   r   )rU   rG   s   "€r   rV   ÚMMDiT.__annotate__   sH  ø€ ÷ EYñ EYáðEYñ ðEYñ ð	EYñ
 ðEYñ ðEYñ ðEYñ "¡#�ðEYñ "*©$¥ðEYñ ðEYñ ðEYñ ðEYñ ðEYñ  ð!EYñ" ð#EYñ$ ð%EYñ& ™s•mð'EYñ( #+©5¥/ð)EYñ* #¡5�/ð+EYñ, %¡S�Mð-EYñ0 ™#•ð1EYñ2 ð3EYñ6  ð7EYñ8 #+©4±­9Õ"5ñ9EYr   c!                óà  <€ \         S$V `  4        VV n        W`n        W0n        V'       d
   V^,          MTp!\        VV!4      V n        W n        VV n        VV n	        VV n
        VV n        ^@V,          V n        Tp"Vf   TpW@n        V"V n        \        VVVV P                  RV P                  RJ VVV R7	      V n        \#        V P                  VVV R7      V n        RV n        Ve7   \)        V\*        4      '       g   Q h\-        WpP                  VVV R7      V n        Ve   \/        VVVVV R7      V n        MRV n        \2        P4                  ! 4       V n        Ve3   VR,          R8X  d%   V P8                  ! R/ VR,          BRVR	V/B V n        W°n        V P:                  ^ 8”  d>   \2        P<                  ! \>        P@                  ! ^W°P                  VVR
7      4      V n!        Ve7   V PE                  R\>        PF                  ! ^VV P                  VVR
7      4       MRV n$        W n%        V'       g�   \2        PL                  ! \O        V4       U#u. uFQ  p#\Q        V P                  T"TTTV#V^,
          8H  ;'       d    TTTTTV#V P                  9   ;'       g    TVVV R7      NKS  	  up#4      V n)        V'       d*   \U        V P                  W P                  VVV R7      V n+        V	'       d   Q hR# u up#i )rK   NT)r   rQ   r   r   r4   r½  Útargetztorch.nn.LinearÚparamsr   r   rï   rŒ   )r8  rä   rç   rè   rê   r9  r:  ré   r;  r   r   r4   r   ).r   r    r   rÖ  rÔ  r   r§  rM   rÛ  rÜ  rÝ  rÞ  r½   rÕ  rã   rI   Ú
x_embedderr¶   Ú
t_embedderÚ
y_embedderÚ
isinstancerS   rÊ   rÅ  Úcontext_processorr%   r(   Úcontext_embedderr"   rÚ  r  rh   ÚrandnÚregisterÚregister_bufferr  rŒ   r‚  rÇ  r’  r�  Újoint_blocksr¥  Úfinal_layerÚcompileÚforward_core_with_concat)%r,   rÓ  rM   rÔ  rÕ  r8  rÖ  r×  rØ  rÙ  r‚  rÚ  rç   rê   r9  r:  r§  rÛ  rÜ  rÝ  Únum_patchesré   rä   Úcontext_processor_layersr;  rÞ  rÀ  Ú
num_blocksrí  Úskip_blocksr   r   r4   Údefault_out_channelsrã   rÊ  r7   s%   &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&   €r   r    ÚMMDiT.__init__   sô  ø€ ôJ 	‰ÑÔØˆŒ
Ø&ÔØ&Ôß2=˜{¨QžÀ;ÐÜ# LÐ2FÓGˆÔØ$ŒØ(@ˆÔ%Ø 0ˆÔØ"4ˆÔØ(@ˆÔ%ð  �:ˆÔØˆ	ØÒØˆJàŒ
Ø"ˆŒä$ØØØØ×ÑØØ ×3Ñ3°tÐ;ØØØ!ô

ˆŒô +¨4×+;Ñ+;À5ÐQWÐdnÔoˆŒàˆŒØÒ&Ü˜o¬s×3Ò3Ð3Ð3Ü,¨_×>NÑ>NÐV[Ðdjð  xBô  CˆDŒOà#Ò/Ü%5°lÐD\ÐdiÐrxð  FPô  &QˆDÕ"à%)ˆDÔ"ä "§¢£ˆÔØ"Ò.Ø& xÕ0Ð4EÔEØ(2×(9Ò(9Ñ(zÐ<SÐT\Õ<]Ñ(zÐejÐ(zÐsyÒ(z�Ô%à.ÔØ×Ñ !Ô#ÜŸLšL¬¯ª°Q¸×IYÑIYÐafÐouÔ)vÓwˆDŒMð
 Ò"Ø× Ñ ØÜ—’˜A˜{¨D×,<Ñ,<ÀEÐRXÔYõð
 "ˆDŒNà,ÔßÜ "§¢ô$ # :Ô.ó#ñ" /˜ô! Ø×(Ñ(Ø!Ø"+Ø!)Ø"+Ø"# z°A¥~Ñ"5×!FÐ!F¸;Ø 'Ø'5Ø%Ø 'Ø+,°×0MÑ0MÑ+M×*cÐ*cÐRcØ#Ø%Ø#-÷ñ  /ñ#ó!ˆDÔ÷, Ü)¨$×*:Ñ*:¸J×HYÑHYÐafÐouð  CMô   NˆDÔçØ�5ñ ùò1s   É+K+É7K+ÊK+c                ó�  € V P                   P                  ^ ,          pVw  rEV^,           V,          pV^,           V,          pV P                  f   \        V P                  WTVR7      # V P
                  f   Q hW@P
                  8:  g   Q W@P
                  34       hWPP
                  8:  g   Q WPP
                  34       hV P
                  V,
          ^,          pV P
                  V,
          ^,          p\        V P                  RV P
                  V P
                  R7      pVRWfV,           1WwV,           1R3,          p\        VR4      pV# )r   ©r   z1 (h w) c -> 1 h w c)r®   r­   r�  z1 h w c -> 1 (h w) c)rã  rM   rŒ   r´   r½   rÝ  r
   )	r,   Úhwr   Úpr®   r­   ÚtopÚleftÚspatial_pos_embeds	   &&&      r   Úcropped_pos_embedÚMMDiT.cropped_pos_embed‡  s4  € Ø�O‰O×&Ñ& qÕ)ˆØ‰ˆà��U�q�LˆØ��U�q�LˆØ�>‰>Ò!Ü0°×1AÑ1AÀ1ÐPVÔWÐWØ×&Ñ&Ò2Ð2Ð2Ø×+Ñ+Ô+ÐI¨a×1HÑ1HÐ-IÓIÐ+Ø×+Ñ+Ô+ÐI¨a×1HÑ1HÐ-IÓIÐ+Ø×&Ñ&¨Õ*¨qÕ0ˆØ×'Ñ'¨!Õ+°Õ1ˆÜ%Ø�N‰NØ"Ø×%Ñ%Ø×%Ñ%ô	
Ðð .¨a°¸Qµw°ÀÈaÅxÀÐQRÐ.RÕSÐÜ%Ð&7Ð9OÓPÐð !Ð r   c                ó  € V P                   pV P                  P                  ^ ,          pVf&   \        VP                  ^,          R,          4      ;rVM$Vw  rVV^,           V,          pV^,           V,          pWV,          VP                  ^,          8X  g   Q hVP                  VP                  ^ ,          WVWDV3R7      p\        P                  ! RV4      pVP                  VP                  ^ ,          W5V,          Wd,          3R7      pV# )z1
x: (N, T, patch_size**2 * C)
imgs: (N, H, W, C)
g      à?)rÛ   znhwpqc->nchpwq)r§  rã  rM   rS   rÛ   r€   rh   rš   )r,   r   rø  rQ  rù  r®   r­   Úimgss   &&&     r   Ú
unpatchifyÚMMDiT.unpatchify£  sÙ   € ð
 ×ÑˆØ�O‰O×&Ñ& qÕ)ˆØŠ:Ü˜Ÿ™ �
 cÕ)Ó*Ð*ˆA�à‰DˆAØ�Q•˜1•ˆAØ�Q•˜1•ˆAØ�u˜Ÿ™ �
Ô"Ð"Ð"à�I‰I˜QŸW™W Q�Z¨¨q°QÐ7ˆIÓ8ˆÜ�LŠLÐ)¨1Ó-ˆØ�y‰y §¡¨¥
¨A°1­u°aµeÐ<ˆyÓ=ˆØˆr   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[ P                  /# )rK   r   Úc_modr“  rÒ   ©rh   rÔ   r   )rU   rG   s   "€r   rV   rß  ·  sH   ø€ ÷ 0ñ 0á�<‰<ð0ñ �|‰|ð0ñ ™%Ÿ,™,Õ'ð	0ñ 
�‰ñ0r   c                ó  a a€ VP                  R / 4      pS P                  ^ 8”  dn   \        P                  ! \	        S P
                  RVP                  ^ ,          R7      \        V\        P                  ! . 4      P                  V4      4      3^4      pVP                  R/ 4      p\        S P                  4      p\        V4       F¬  oRS3V9   d7   VV 3R lp	VRS3,          ! RVRVRVR	V/R
V	/4      p
V
R,          pV
R,          pM*S P                  S,          ! VVVS P                  VR7      w  r1Vf   Kr  VP                  R4      pS\        V4      8  g   K•  VS,          pVf   K¤  W,          pK®  	  S P                  W4      pV# )Úpatches_replacez1 ... -> b ...)ÚbÚditÚdouble_blockc                 ó�   <€ / pSP                   S,          ! V R ,          V R,          V R,          V R,          R7      w  VR &   VR&   V# )ÚtxtÚimgÚvecr�  )rQ  r�  )rì  )rˆ  rŸ   rÊ  r,   s   & €€r   Ú
block_wrapÚ2MMDiT.forward_core_with_concat.<locals>.block_wrapÏ  s[   ø€ Ø�CØ-1×->Ñ->¸qÖ-AÀ$ÀuÅ+ÈtÐTYÍ{Ð^bÐchÕ^ið  @Dð  EZõ  @[ô  .\Ñ*�C˜‘J  E¡
Ø�Jr   r  r  r  r�  Úoriginal_block)rQ  r‚  r�  Úoutput)ÚgetrÚ  rh   r¦   r   rê  rÛ   r   rÔ   Útype_asrZ   rì  r’  r‚  rí  )r,   r   r  r“  Úcontrolr�  r  Úblocks_replaceÚblocksr  rŸ   Ú	control_oÚaddrÊ  s   f&&&&&       @r   rï  ÚMMDiT.forward_core_with_concat·  s�  ù€ ð .×1Ñ1Ð2CÀRÓHˆØ×Ñ !Ô#Ü—i’iä˜4Ÿ=™=Ð*:¸a¿g¹gÀa½jÔIÜ˜G¤U§\¢\°"Ó%5×%=Ñ%=¸aÓ%@ÓAðð óˆGð )×,Ñ,¨U°BÓ7ˆÜ�T×&Ñ&Ó'ˆÜ�v–ˆAØ Ð" nÔ4öð
 % n°aÐ%8Ö9¸5À!ÀUÈGÐUZÐ\aÐcxð  {Nð  ;Oð  Rbð  dnð  Qoó  p�Ø˜e�*�Ø˜•J‘à!×.Ñ.¨qÖ1ØØØØ#'×#6Ñ#6Ø(;ô‘
�ð Ô"Ø#ŸK™K¨Ó1�	Ø”s˜9“~Ö%Ø# A�,�CØ”Ø�šñ/ ð2 ×Ñ˜QÓ&ˆØˆr   c          
      ó¼   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  /# )rK   r   rÂ   r   r“  rÒ   r  )rU   rG   s   "€r   rV   rß  é  s[   ø€ ÷  &ñ  &á�<‰<ð &ñ �<‰<ð &ñ ‘E—L‘LÕ!ð	 &ñ
 ™%Ÿ,™,Õ'ð &ñ 
�‰ñ &r   c                ó>  € V P                   e   V P                  V4      pVP                  RR pV P                  V4      \        P                  P                  V P                  WqP                  R7      V4      ,           pV P                  W!P                  R7      pVe(   V P                  e   V P                  V4      pWƒ,           pVe   V P                  V4      pV P                  WWEV4      pV P                  WR7      pVRRRVR,          1RVR,          13,          # )z±
Forward pass of DiT.
x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
t: (N,) tensor of diffusion timesteps
y: (N,) tensor of class labels
Nr÷  rx   )rø  r�  éþÿÿÿr˜   )rç  rÛ   rã  ra   ÚopsÚcast_to_inputrý  r   rä  r   rå  rè  rï  r  )	r,   r   rÂ   r   r“  r  r�  rø  rQ  s	   &&&&&&&  r   r;   ÚMMDiT.forwardé  sÿ   € ð  ×!Ñ!Ò-Ø×,Ñ,¨WÓ5ˆGà�W‰W�R�Sˆ\ˆØ�O‰O˜AÓ¤§¡×!8Ñ!8¸×9OÑ9OÐPR×[cÑ[cÐ9OÓ9dÐfgÓ!hÕhˆØ�O‰O˜A§W¡WˆOÓ-ˆØŠ=˜TŸ_™_Ò8Ø—‘ Ó"ˆAØ•ˆAàÒØ×+Ñ+¨GÓ4ˆGà×)Ñ)¨!°ÐBUÓVˆà�O‰O˜AˆOÓ%ˆØ��1�W�b˜•f�W˜W˜b �f˜WÐ$Õ%Ð%r   )rè  rç  rÕ  r   rí  rï  r½   rÔ  rì  rÖ  rã   r§  rM   rŒ   rÝ  rÜ  rÛ  rê  rÚ  rä  r‚  rÞ  rã  rå  r   )r=   r>   r?   r@   rA   r    rý  r  rï  r;   rC   rD   rE   rF   s   @@r   rÑ  rÑ  û  s»   ù‡ € ñð ØØØð Ø!Ø)-Ø26Ø"Ø$Ø Ø ØØ$ØØ&*Ø48Ø,0Ø,0ØØ!%ØØ#'Ø"'Ø8:ØØØØØØØ÷GEYõ EYôN!ô8ð0 +/ØØ ÷0ò 0ðl %)Ø*.ØØ ÷ &÷  &ò  &r   rÑ  c                   óJ   a a€ ] tR tRt oRRR/ 3V3R lV 3R llltRtVtV ;t# )ÚOpenAISignatureMMDITWrapperi  Nc          
      ó¼   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  /# )rK   r   Ú	timestepsr“  r   rÒ   r  )rU   rG   s   "€r   rV   Ú(OpenAISignatureMMDITWrapper.__annotate__  sb   ø€ ÷ 
}ñ 
}á�<‰<ð
}ñ —<‘<ð
}ñ ™%Ÿ,™,Õ'ð	
}ñ
 ‘E—L‘LÕ!ð
}ñ 
�‰ñ
}r   c           	     ó(   <€ \         SV `  WW4WVR 7      # ))r“  r   r  r�  )r   r;   )	r,   r   r$  r“  r   r  r�  rÃ   r7   s	   &&&&&&&,€r   r;   Ú#OpenAISignatureMMDITWrapper.forward  s   ø€ ô ‰w‰˜q°WÈ7ˆÓ|Ð|r   r   )r=   r>   r?   r@   r;   rC   rD   rE   rF   s   @@r   r"  r"    s%   ù‡ € ð
 +/Ø$(ØØ ÷
}÷ 
}õ 
}r   r"  )Fr   NN)3Ú	functoolsr   Útypingr   r   r   Únumpyr{   rh   Útorch.nnr%   Ú	attentionr   r   r	   Úeinopsr
   r   Úutilr   Ú	comfy.opsra   Úcomfy.ldm.common_ditr   ÚModuler   rI   rv   r�   r�   r�   r}   r§   r´   r¶   rÊ   rß   rá   rø   r&  r6  rj  r‰  r‡  r�  r¥  r²  r»  rÅ  rÑ  r"  r   r   r   Ú<module>r2     s•  ðÝ ß 'Ñ 'ã Û Ý ß UÑ Uß $Ý $Û Û òô
%ˆ"�)‰)ô %ôN2�—‘ô 2òhGôò>òð( DHÈuÏ}É}ô 	ð ?BÐQTÐ]aÐin×ivÑivô ô �r—y‘yô ô.�R—Y‘Yô ò."ô
@�B—I‘Iô @ôFGˆe�h‰h�o‰oô Gô4#D˜Ÿ	™	ô #DôLQ=�b—i‘iô Q=òfð. tô .ð NPô 'ôT
�—‘ô 
ô2�—‘ô ô@˜2Ÿ9™9ô ô.˜BŸI™Iô ô	�r—y‘yô 	ôN&ˆB�I‰Iô N&ôb} %ö }r   