+
    ƒ°ñj¾S  ã                   óÂ  € ^ RI t ^ RIt^ RIHt ^ RIHu Ht ^ RIHtH	t	H
t
 ^ RIt^ RIt^ RItR tR R lt ! R R]P"                  4      t ! R R	]P"                  4      t ! R
 R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      tR# )é    N)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc                 ól   € V ^VP                  ^4      ,           ,          VP                  ^4      ,           # ©é   )Ú	unsqueeze)ÚxÚshiftÚscales   &&&Ú8C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\aura\mmdit.pyÚmodulater      s)   € Ø��E—O‘O AÓ&Õ&Õ'¨%¯/©/¸!Ó*<Õ<Ð<ó    c                ó<   € V ^8„  d   QhR\         R\         R\         /# )é   ÚnÚkÚreturn)Úint)Úformats   "r   Ú__annotate__r      s!   € ÷ ñ ”Sð œSð ¤Sñ r   c                 óJ   € W,          ^ 8X  d   V # W,           W,          ,
          # )r   © )r   r   s   &&r   Úfind_multipler      s   € Ø…u�„zØˆØ�5�A•E�?Ðr   c                   óT   a a€ ] tR t^t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMLPc                ó   <€ V ^8„  d   QhRR/# )r   r   Nr   )r   Ú__classdict__s   "€r   r   ÚMLP.__annotate__   s   ø€ ÷ 
_ñ 
_ÐZ^ñ 
_r   c                ó  <€ \         SV `  4        Vf
   ^V,          p\        ^V,          ^,          4      p\        VR4      pVP	                  WRW4R7      V n        VP	                  WRW4R7      V n        VP	                  WaRW4R7      V n        R # )Né   F©ÚbiasÚdtypeÚdevice)ÚsuperÚ__init__r   r   ÚLinearÚc_fc1Úc_fc2Úc_proj)ÚselfÚdimÚ
hidden_dimr$   r%   Ú
operationsÚn_hiddenÚ	__class__s   &&&&&& €r   r'   ÚMLP.__init__   s‡   ø€ Ü‰ÑÔØÒØ˜S�ˆJä�q˜:•~¨Õ)Ó*ˆÜ  ¨3Ó/ˆà×&Ñ& s¸5ÈÐ&Ó]ˆŒ
Ø×&Ñ& s¸5ÈÐ&Ó]ˆŒ
Ø ×'Ñ'¨¸EÈÐ'Ó^ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r
   r   )ÚtorchÚTensor)r   r   s   "€r   r   r   &   s#   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ñ r   c                óž   € \         P                  ! V P                  V4      4      V P                  V4      ,          pV P	                  V4      pV# ©N)ÚFÚsilur)   r*   r+   )r,   r
   s   &&r   ÚforwardÚMLP.forward&   s7   € Ü�FŠF�4—:‘:˜a“=Ó! D§J¡J¨q£MÕ1ˆØ�K‰K˜‹NˆØˆr   )r)   r*   r+   )NNNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r'   r:   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r1   r   s   @@r   r   r      s    ù‡ € ÷
_õ 
_÷÷ ð r   r   c                   ó<   a a€ ] tR t^,t oRV 3R lltR tRtVtV ;t# )ÚMultiHeadLayerNormc                ó’   <€ \         SV `  4        \        P                  ! \        P
                  ! WVR 7      4      V n        W n        R# ©©r$   r%   N)r&   r'   ÚnnÚ	Parameterr4   ÚemptyÚweightÚvariance_epsilon)r,   Úhidden_sizeÚepsr$   r%   r1   s   &&&&&€r   r'   ÚMultiHeadLayerNorm.__init__-   s2   ø€ ô 	‰ÑÔÜ—l’l¤5§;¢;¨{ÐPVÔ#WÓXˆŒØ #Ör   c                ó¶  € VP                   pVP                  \        P                  4      pVP	                  RRR7      pW,
          P                  ^4      P	                  RRR7      pW,
          \        P                  ! W@P                  ,           4      ,          pV P                  P                  \        P                  4      V,          pVP                  V4      # )r   T)Úkeepdiméÿÿÿÿ)	r$   Útor4   Úfloat32ÚmeanÚpowÚrsqrtrN   rM   )r,   Úhidden_statesÚinput_dtyperW   Úvariances   &&   r   r:   ÚMultiHeadLayerNorm.forward4   s«   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ×!Ñ! "¨dÐ!Ó3ˆØ!Õ(×-Ñ-¨aÓ0×5Ñ5°bÀ$Ð5ÓGˆØ&Õ-´·²Ø×,Ñ,Õ,ó2
õ 
ˆð Ÿ™Ÿ™¤u§}¡}Ó5¸ÕEˆØ×Ñ Ó,Ð,r   )rN   rM   )Ngñhãˆµøä>NNr<   rD   s   @@r   rF   rF   ,   s   ù‡ € ÷$÷	-ò 	-r   rF   c                   óB   a a€ ] tR t^?t oRV 3R llt/ 3R ltRtVtV ;t# )ÚSingleAttentionc                óv  <€ \         SV `  4        \        4       V n        W n        W,          V n        VP                  WR WER7      V n        VP                  WR WER7      V n        VP                  WR WER7      V n	        VP                  WR WER7      V n
        V'       d$   \        V P                  V P
                  3WER7      MVP                  V P
                  R WER7      V n        V'       d+   \        V P                  V P
                  3WER7      V n        R# VP                  V P
                  R WER7      V n        R# ©Fr"   rI   ©Úelementwise_affiner$   r%   N)r&   r'   r   Úcomfy_attentionÚn_headsÚhead_dimr(   Úw1qÚw1kÚw1vÚw1orF   Ú	LayerNormÚq_norm1Úk_norm1©r,   r-   re   Ú	mh_qknormr$   r%   r/   r1   s   &&&&&&&€r   r'   ÚSingleAttention.__init__@   s  ø€ Ü‰ÑÔÜ-Ó/ˆÔàŒØ�ˆŒð ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒ÷ ô  §¡¨d¯m©mÐ<ÀEÕYà×%Ñ% d§m¡mÈÐUZÐ%Ójð 	Œ÷ ô  §¡¨d¯m©mÐ<ÀEÔYð 	Žð ×%Ñ% d§m¡mÈÐUZÐ%Ójð 	Žr   c           
     óÊ  € VP                   w  r4pV P                  V4      V P                  V4      V P                  V4      r‡pVP	                  W4V P
                  V P                  4      pVP	                  W4V P
                  V P                  4      pVP	                  W4V P
                  V P                  4      pV P                  V4      V P                  V4      rv\        VP                  ^ ^^^4      4      \        VP                  ^ ^^^4      4      \        VP                  ^ ^^^4      4      r‡p\        WgW€P
                  RV P                  VR7      p	V P                  V	4      pV# )r   T©Úskip_reshapeÚpreferred_attentionÚtransformer_options)Úshaperg   rh   ri   Úviewre   rf   rl   rm   r   Úpermuter   rd   rj   )
r,   Úcru   ÚbszÚseqlen1Ú_Úqr   ÚvÚoutputs
   &&&       r   r:   ÚSingleAttention.forwardY   sN  € àŸ'™'‰ˆ�aà—(‘(˜1“+˜tŸx™x¨›{¨D¯H©H°Q«KˆaˆØ�F‰F�3 §¡¨t¯}©}Ó=ˆØ�F‰F�3 §¡¨t¯}©}Ó=ˆØ�F‰F�3 §¡¨t¯}©}Ó=ˆØ�|‰|˜A‹ §¡¨Q£ˆ1ä*¨1¯9©9°Q¸¸1¸aÓ+@ÓAÔC[Ð\]×\eÑ\eÐfgÐijÐlmÐopÓ\qÓCrô  uMð  NO÷  NWñ  NWð  XYð  [\ð  ^_ð  abó  Ncó  udˆaˆÜ$ Q¨1¯l©lÈÐcg×cwÑcwð  Naô  bˆØ�H‰H�VÓˆØˆr   )	rd   rf   rm   re   rl   rh   rj   rg   ri   ©FNNNr<   rD   s   @@r   r_   r_   ?   s   ù‡ € ÷
ð2 .0÷ ô r   r_   c                   óB   a a€ ] tR t^jt oRV 3R llt/ 3R ltRtVtV ;t# )ÚDoubleAttentionc                óv  <€ \         SV `  4        \        4       V n        W n        W,          V n        VP                  WR WER7      V n        VP                  WR WER7      V n        VP                  WR WER7      V n	        VP                  WR WER7      V n
        VP                  WR WER7      V n        VP                  WR WER7      V n        VP                  WR WER7      V n        VP                  WR WER7      V n        V'       d$   \        V P                  V P
                  3WER7      MVP!                  V P
                  R WER7      V n        V'       d$   \        V P                  V P
                  3WER7      MVP!                  V P
                  R WER7      V n        V'       d$   \        V P                  V P
                  3WER7      MVP!                  V P
                  R WER7      V n        V'       d+   \        V P                  V P
                  3WER7      V n        R# VP!                  V P
                  R WER7      V n        R# ra   )r&   r'   r   rd   re   rf   r(   rg   rh   ri   rj   Úw2qÚw2kÚw2vÚw2orF   rk   rl   rm   Úq_norm2Úk_norm2rn   s   &&&&&&&€r   r'   ÚDoubleAttention.__init__k   sø  ø€ Ü‰ÑÔÜ-Ó/ˆÔàŒØ�ˆŒð ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒð ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒØ×$Ñ$ S°EÀÐ$ÓVˆŒ÷ ô  §¡¨d¯m©mÐ<ÀEÕYà×%Ñ% d§m¡mÈÐUZÐ%Ójð 	Œ÷ ô  §¡¨d¯m©mÐ<ÀEÕYà×%Ñ% d§m¡mÈÐUZÐ%Ójð 	Œ÷ ô  §¡¨d¯m©mÐ<ÀEÕYà×%Ñ% d§m¡mÈÐUZÐ%Ójð 	Œ÷ ô  §¡¨d¯m©mÐ<ÀEÔYð 	Žð ×%Ñ% d§m¡mÈÐUZÐ%Ójð 	Žr   c           
     ój  € VP                   w  rEpVP                   w  rGpV P                  V4      V P                  V4      V P                  V4      r©pVP	                  WEV P
                  V P                  4      pV	P	                  WEV P
                  V P                  4      p	V
P	                  WEV P
                  V P                  4      p
V P                  V4      V P                  V	4      r˜V P                  V4      V P                  V4      V P                  V4      rÜpVP	                  WGV P
                  V P                  4      pVP	                  WGV P
                  V P                  4      pVP	                  WGV P
                  V P                  4      pV P                  V4      V P                  V4      rË\        P                  ! W‹.^R7      \        P                  ! Wœ.^R7      \        P                  ! W­.^R7      prþ??	?
???\!        VP#                  ^ ^^^4      4      \!        VP#                  ^ ^^^4      4      \!        VP#                  ^ ^^^4      4      prþ\%        WïVV P
                  RV P&                  VR7      pVP)                  WW.^R7      w  rV P+                  V4      pV P-                  V4      pW3# )r   ©r-   Trr   )rv   rg   rh   ri   rw   re   rf   rl   rm   r…   r†   r‡   r‰   rŠ   r4   Úcatr   rx   r   rd   Úsplitrj   rˆ   )r,   ry   r
   ru   rz   r{   r|   Úseqlen2ÚcqÚckÚcvÚxqÚxkÚxvr}   r   r~   r   s   &&&&              r   r:   ÚDoubleAttention.forward–   sV  € àŸ'™'‰ˆ�aØŸ'™'‰ˆ�aà—X‘X˜a“[ $§(¡(¨1£+¨t¯x©x¸«{�ˆØ�W‰W�S 4§<¡<°·±Ó?ˆØ�W‰W�S 4§<¡<°·±Ó?ˆØ�W‰W�S 4§<¡<°·±Ó?ˆØ—‘˜bÓ! 4§<¡<°Ó#3ˆBà—X‘X˜a“[ $§(¡(¨1£+¨t¯x©x¸«{�ˆØ�W‰W�S 4§<¡<°·±Ó?ˆØ�W‰W�S 4§<¡<°·±Ó?ˆØ�W‰W�S 4§<¡<°·±Ó?ˆØ—‘˜bÓ! 4§<¡<°Ó#3ˆBô �IŠI�r�h AÔ&Ü�IŠI�r�h AÔ&Ü�IŠI�r�h AÔ&ð ˆ1ð ��B˜˜B Ü*¨1¯9©9°Q¸¸1¸aÓ+@ÓAÔC[Ð\]×\eÑ\eÐfgÐijÐlmÐopÓ\qÓCrô  uMð  NO÷  NWñ  NWð  XYð  [\ð  ^_ð  abó  Ncó  udˆaˆ1Ü$ Q¨1¨d¯l©lÈÐcg×cwÑcwð  Naô  bˆà�|‰|˜WÐ.°Aˆ|Ó6‰ˆØ�H‰H�Q‹KˆØ�H‰H�Q‹Kˆàˆtˆr   )rd   rf   rm   rŠ   re   rl   r‰   rh   rj   rg   ri   r†   rˆ   r…   r‡   r�   r<   rD   s   @@r   rƒ   rƒ   j   s   ù‡ € ÷'
ðV 13÷  ô  r   rƒ   c                   óB   a a€ ] tR t^¹t oRV 3R llt/ 3R ltRtVtV ;t# )Ú
MMDiTBlockc                óN  <€ \         SV `  4        VP                  VR WVR7      V n        VP                  VR WVR7      V n        V'       ge   \        W^,          WVVR7      V n        \        P                  ! \        P                  ! 4       VP                  V^V,          R WVR7      4      V n        MI\        P                  ! \        P                  ! 4       VP                  V^V,          R WVR7      4      V n        VP                  VR WVR7      V n        VP                  VR WVR7      V n        \        W^,          WVVR7      V n        \        P                  ! \        P                  ! 4       VP                  V^V,          R WVR7      4      V n        \!        WWVVR7      V n        W@n        R# )Frb   ©r.   r$   r%   r/   r"   ©r$   r%   r/   N)r&   r'   rk   ÚnormC1ÚnormC2r   ÚmlpCrJ   Ú
SequentialÚSiLUr(   ÚmodCÚnormX1ÚnormX2ÚmlpXÚmodXrƒ   ÚattnÚis_last)	r,   r-   ÚheadsÚglobal_conddimr¨   r$   r%   r/   r1   s	   &&&&&&&&€r   r'   ÚMMDiTBlock.__init__º   s[  ø€ Ü‰ÑÔà ×*Ñ*¨3À5ÐPUÐ*ÓeˆŒØ ×*Ñ*¨3À5ÐPUÐ*ÓeˆŒßÜ˜C°!­G¸5Ð\fÔgˆDŒIÜŸšÜ—’“	Ø×!Ñ! .°!°cµ'ÀÈUÐ!ÓbóˆD�Iô
 ŸšÜ—’“	Ø×!Ñ! .°!°cµ'ÀÈUÐ!ÓbóˆDŒIð
 !×*Ñ*¨3À5ÐPUÐ*ÓeˆŒØ ×*Ñ*¨3À5ÐPUÐ*ÓeˆŒÜ˜¨a­°uÐXbÔcˆŒ	Ü—M’MÜ�GŠG‹IØ×Ñ˜n¨a°#­g¸EÈÐÓ^ó
ˆŒ	ô
 $ C°eÐWaÔbˆŒ	ØŽr   c                óÚ  € YrvV P                  V4      P                  ^^R7      w  r‰r«rÍ\        V P                  V4      W‰4      pV P	                  V4      P                  ^^R7      w  rïpppp\        V P                  V4      Wï4      pV P                  WVR7      w  rV P                  WjP                  ^4      V,          ,           4      pVP                  ^4      V P                  \        WV4      4      ,          pWa,           pV P                  VVP                  ^4      V,          ,           4      pVP                  ^4      V P                  \        VVV4      4      ,          pWr,           pW3# ©é   r�   ©ru   )r¢   Úchunkr   r�   r¦   r£   r§   rž   r	   rŸ   r¤   r¥   )r,   ry   r
   Úglobal_condru   ÚkwargsÚcresÚxresÚ
cshift_msaÚ
cscale_msaÚ	cgate_msaÚ
cshift_mlpÚ
cscale_mlpÚ	cgate_mlpÚ
xshift_msaÚ
xscale_msaÚ	xgate_msaÚ
xshift_mlpÚ
xscale_mlpÚ	xgate_mlps   &&&&&,              r   r:   ÚMMDiTBlock.forward×   sI  € àˆdð �I‰I�kÓ"×(Ñ(¨°Ð(Ó2ñ 	Mˆ
 	°zô �T—[‘[ “^ ZÓ<ˆð �I‰I�kÓ"×(Ñ(¨°Ð(Ó2ñ 	Mˆ
 	¨:°zÀ9ô �T—[‘[ “^ ZÓ<ˆð �y‰y˜Ð3FˆyÓG‰ˆð �K‰K˜×2Ñ2°1Ó5¸Õ9Õ9Ó:ˆØ×Ñ Ó" T§Y¡Y¬x¸ÀzÓ/RÓ%SÕSˆØ�Hˆà�K‰K˜˜y×2Ñ2°1Ó5¸Õ9Õ9Ó:ˆØ×Ñ Ó" T§Y¡Y¬x¸¸:ÀzÓ/RÓ%SÕSˆØ�Hˆàˆtˆr   )
r§   r¨   rŸ   r¥   r¢   r¦   r�   rž   r£   r¤   )é   é   FNNNr<   rD   s   @@r   r™   r™   ¹   s   ù‡ € ÷ð: >@÷ ô r   r™   c                   óB   a a€ ] tR t^öt oRV 3R llt/ 3R ltRtVtV ;t# )ÚDiTBlockc                ót  <€ \         SV `  4        VP                  VR WER7      V n        VP                  VR WER7      V n        \
        P                  ! \
        P                  ! 4       VP                  V^V,          R WER7      4      V n	        \        WWEVR7      V n        \        W^,          WEVR7      V n        R# )Frb   r"   rœ   r›   N)r&   r'   rk   Únorm1Únorm2rJ   r    r¡   r(   ÚmodCXr_   r§   r   Úmlp)r,   r-   r©   rª   r$   r%   r/   r1   s   &&&&&&&€r   r'   ÚDiTBlock.__init__ø   sœ   ø€ Ü‰ÑÔà×)Ñ)¨#À%ÈuÐ)ÓdˆŒ
Ø×)Ñ)¨#À%ÈuÐ)ÓdˆŒ
ä—]’]Ü�GŠG‹IØ×Ñ˜n¨a°#­g¸EÈÐÓ^ó
ˆŒ
ô
 $ C°eÐWaÔbˆŒ	Ü�s¨Q¥w°eÐWaÔbˆŽr   c                ó€  € TpV P                  V4      P                  ^^R7      w  rgr‰r«\        V P                  V4      Wg4      pV P	                  WR7      pV P                  WXP                  ^4      V,          ,           4      pV P                  \        WV
4      4      pVP                  ^4      V,          pWQ,           pV# r­   )rÉ   r°   r   rÇ   r§   rÈ   r	   rÊ   )r,   Úcxr±   ru   r²   ÚcxresÚ	shift_msaÚ	scale_msaÚgate_msaÚ	shift_mlpÚ	scale_mlpÚgate_mlpÚmlpouts   &&&&,        r   r:   ÚDiTBlock.forward  s­   € ØˆØIMÏÉØóJ
ç
‰%��qˆ%‹/ñ 	Gˆ	˜h°9ô �d—j‘j “n iÓ;ˆØ�Y‰Y�rˆYÓCˆØ�Z‰Z˜× 2Ñ 2°1Ó 5¸Õ :Õ:Ó;ˆØ—‘œ( 2°)Ó<Ó=ˆØ×Ñ Ó" VÕ+ˆà�Zˆàˆ	r   )r§   rÊ   rÉ   rÇ   rÈ   )rÂ   rÃ   NNNr<   rD   s   @@r   rÅ   rÅ   ö   s   ù‡ € ÷cð <>÷ ô r   rÅ   c                   óP   a a€ ] tR tRt oRV 3R llt]RR l4       tR tRtVt	V ;t
# )ÚTimestepEmbedderi  c                óÔ   <€ \         SV `  4        \        P                  ! VP	                  W!W4R 7      \        P
                  ! 4       VP	                  WW4R 7      4      V n        W n        R# rH   )r&   r'   rJ   r    r(   r¡   rÊ   Úfrequency_embedding_size)r,   rO   rÚ   r$   r%   r/   r1   s   &&&&&&€r   r'   ÚTimestepEmbedder.__init__  sW   ø€ Ü‰ÑÔÜ—=’=Ø×ÑÐ6È5ÐÓ`Ü�GŠG‹IØ×Ñ˜k¸eÐÓSó
ˆŒð
 )AÖ%r   c           	     ó0  € V^,          pR\         P                  ! \        P                  ! V4      ) \         P                  ! ^ VR7      ,          V,          4      P                  V P                  4      ,          pV R,          VR,          ,          p\         P                  ! \         P                  ! V4      \         P                  ! V4      .RR7      pV^,          '       d6   \         P                  ! V\         P                  ! VR,          4      .RR7      pV# )r   iè  )ÚstartÚendNr�   )ºNNNNrT   )rß   :Nr   N)r4   ÚexpÚmathÚlogÚarangerU   r%   rŽ   ÚcosÚsinÚ
zeros_like)Útr-   Ú
max_periodÚhalfÚfreqsÚargsÚ	embeddings   &&&    r   Útimestep_embeddingÚ#TimestepEmbedder.timestep_embedding"  sÃ   € à�a�xˆØ”u—y’yÜ�XŠX�jÓ!Ð!¤E§L¢L°q¸dÔ$CÕCÀdÕJó
ç
‰"ˆQ�X‰X‹,õˆð ��z˜E $�KÕ'ˆÜ—I’IœuŸyšy¨›´·	²	¸$³Ð@ÀbÔIˆ	Ø��7Œ7ÜŸ	š	ØœE×,Ò,¨Y°uÕ-=Ó>Ð?ÀRôˆIð Ðr   c                ó|   € V P                  WP                  4      P                  V4      pV P                  V4      pV# r7   )rí   rÚ   rU   rÊ   )r,   rç   r$   Út_freqÚt_embs   &&&  r   r:   ÚTimestepEmbedder.forward1  s6   € Ø×(Ñ(¨×,IÑ,IÓJ×MÑMÈeÓTˆØ—‘˜Ó ˆØˆr   )rÚ   rÊ   )r!   NNN)i'  )r=   r>   r?   r@   r'   Ústaticmethodrí   r:   rA   rB   rC   rD   s   @@r   rØ   rØ     s)   ù‡ € ÷Að óó ð÷ò r   rØ   c                   ó˜   a a€ ] tR tRt oRV 3R llt]P                  ! 4       RR l4       tR tR t	R t
R t/ 3R lt/ 3R	 ltR
tVtV ;t# )ÚMMDiTi7  c                ón  <€ \         SV `  4        WÀn        \        WŒW½R 7      V n        VP                  W”RWËR7      V n        VP                  W3,          V,          WLVR7      V n        \        P                  ! \        P                  ! ^W¤WËR7      4      V n        \        P                  ! \        P                  ! ^^WLVR7      4      V n        \        P                  ! . 4      V n        \        P                  ! . 4      V n        \#        V4       F5  pV P                  P%                  \'        WGWŽV^,
          8H  WËVR7      4       K7  	  \#        We4       F*  pV P                   P%                  \)        WGWŒW½R 7      4       K,  	  VP                  WCV,          V,          RWËR7      V n        \        P,                  ! \        P.                  ! 4       VP                  V^V,          RWËR7      4      V n        W n        W0n        W`n        WPn        \;        V
R,          4      V n        \;        V
R,          4      V n        R# )rœ   Fr"   rI   )r¨   r$   r%   r/   g      à?N) r&   r'   r$   rØ   Ú
t_embedderr(   Úcond_seq_linearÚinit_x_linearrJ   rK   r4   rL   Úpositional_encodingÚregister_tokensÚ
ModuleListÚdouble_layersÚsingle_layersÚrangeÚappendr™   rÅ   Úfinal_linearr    r¡   ÚmodFÚout_channelsÚ
patch_sizeÚn_double_layersÚn_layersÚroundÚh_maxÚw_max)r,   Úin_channelsr  r  r-   r  r  re   rª   Úcond_seq_dimÚmax_seqr%   r$   r/   Úidxr1   s   &&&&&&&&&&&&&& €r   r'   ÚMMDiT.__init__8  sß  ø€ ô  	‰ÑÔØŒ
ä*¨>ÈvÔmˆŒà)×0Ñ0Ø E°ð  1ó  
ˆÔð (×.Ñ.ØÕ# kÕ1°3ÈFð /ó 
ˆÔô $&§<¢<´·²¸A¸wÐSXÔ0hÓ#iˆÔ Ü!Ÿ|š|¬E¯KªK¸¸1¸cÐW]Ô,^Ó_ˆÔäŸ]š]¨2Ó.ˆÔÜŸ]š]¨2Ó.ˆÔô ˜Ö)ˆCØ×Ñ×%Ñ%Ü˜3¨ÈÐTUÍÑBUÐ^cð  @Jô  Köñ *ô
 ˜Ö3ˆCØ×Ñ×%Ñ%Ü˜ ~È6Ôiöñ 4ð '×-Ñ-Ø˜jÕ(¨<Õ7¸eÈ5ð .ó 
ˆÔô —M’MÜ�GŠG‹IØ×Ñ˜n¨a°#­g¸EÈÐÓ^ó
ˆŒ	ð
 )ÔØ$ŒØ.ÔØ Œä˜7 C�<Ó(ˆŒ
Ü˜7 C�<Ó(ˆŽ
r   c                ó  € V P                   P                  P                  ^ 4      RV^ ,          V^,          ,           pVP                  V^ ,          V^,          R4      P	                  ^^ ^4      p\
        P                  ! VP                  ^ 4      VRR7      pVP                  ^ 4      P	                  ^^^ 4      P                  ^ ^4      pVP                  ^ 4      P                  4       V P                   n        Vw  V n
        V n        R# )r   NÚbilinear)ÚsizeÚmoderT   )rú   ÚdataÚsqueezerw   rx   r8   Úinterpolater	   ÚflattenÚ
contiguousr  r	  )r,   Úinit_dimÚ
target_dimÚpe_dataÚpe_as_2dÚpe_news   &&&   r   Ú	extend_peÚMMDiT.extend_pew  sß   € ð ×*Ñ*×/Ñ/×7Ñ7¸Ó:Ð;V¸XÀa½[È8ÐTUÍ;Õ=VÐWˆà—<‘< ¨¥¨X°a­[¸"Ó=×EÑEÀaÈÈAÓNˆô —=’=Ø×Ñ˜qÓ!¨
¸ô
ˆð ×!Ñ! !Ó$×,Ñ,¨Q°°1Ó5×=Ñ=¸aÀÓCˆØ(.×(8Ñ(8¸Ó(;×(FÑ(FÓ(Hˆ× Ñ Ô%Ø!+ÑˆŒ
�D–Jr   c                óÔ  € WP                   ,          W P                   ,          rC\        P                  ! V P                  P                  ^,          4      pVP                  V P                  V P                  4      pV P                  ^,          V^,          ,
          pWc,           pV P                  ^,          V^,          ,
          pW„,           p	VWg1W‰13,          pVP                  4       # r   )	r  r4   rã   rú   rv   rw   r  r	  r  )
r,   ÚhÚwÚh_pÚw_pÚoriginal_pe_indexesÚstarthÚendhÚstartwÚendws
   &&&       r   Úpe_selection_index_based_on_dimÚ%MMDiT.pe_selection_index_based_on_dim‡  s´   € ØŸ™Õ'¨¯o©oÕ)=ˆSÜ#Ÿlšl¨4×+CÑ+C×+IÑ+IÈ!Õ+LÓMÐØ1×6Ñ6°t·z±zÀ4Ç:Á:ÓNÐØ—*‘* •/ C¨1¥HÕ,ˆØ�lˆØ—‘˜q• 3¨!¥8Õ+ˆØ�|ˆØ1ØˆK˜˜Ð$õ
Ðð #×*Ñ*Ó,Ð,r   c                ó  € V P                   pV P                  pVP                  VP                  ^ ,          W#WUV3R7      p\        P
                  ! RV4      pVP                  VP                  ^ ,          WBV,          W5,          3R7      pV# )r   )rv   znhwpqc->nchpwq)r  r  Úreshaperv   r4   Úeinsum)r,   r
   r   r!  ry   ÚpÚimgss   &&&&   r   Ú
unpatchifyÚMMDiT.unpatchify”  sr   € Ø×ÑˆØ�O‰Oˆà�I‰I˜QŸW™W Q�Z¨¨q°QÐ7ˆIÓ8ˆÜ�LŠLÐ)¨1Ó-ˆØ�y‰y §¡¨¥
¨A°1­u°aµeÐ<ˆyÓ=ˆØˆr   c                óÈ  € VP                  4       w  r#rE\        P                  P                  P	                  WP
                  V P
                  34      pVP                  VVV^,           V P
                  ,          V P
                  V^,           V P
                  ,          V P
                  4      pVP                  ^ ^^^^^4      P                  R4      P                  ^^4      pV# )r   éýÿÿÿ)	r  ÚcomfyÚldmÚ
common_ditÚpad_to_patch_sizer  rw   rx   r  )r,   r
   ÚBÚCÚHÚWs   &&    r   ÚpatchifyÚMMDiT.patchify�  s±   € Ø—V‘V“X‰
ˆˆaÜ�I‰I× Ñ ×2Ñ2°1·±ÈÏÉÐ6XÓYˆØ�F‰FØØØ��U�t—‘Õ&Ø�O‰OØ��U�t—‘Õ&Ø�O‰Oó
ˆð �I‰I�a˜˜A˜q ! QÓ'×/Ñ/°Ó3×;Ñ;¸A¸qÓAˆØˆr   c                ón  € V^,           V P                   ,          pV^,           V P                   ,          p\        W#4      pV P                  p\        P                  P                  V P                  P                  ^WUR4      V4      pWE8”  d=   \        P                  ! VP                  R^4      WD3RR7      P                  ^R4      pTpWR,
          ^,          pWS,
          ^,          pVRWwV,           1WˆV,           13,          pWP                  ^RV P                  P                  R,          4      ,           # )r   r  )r  rß   rT   )r  Úmaxr  r4  ÚopsÚcast_to_inputrú   r,  r8   r  Úmovedimrv   )	r,   r
   r   r!  Úmax_dimÚcur_dimÚpos_encodingÚfrom_hÚfrom_ws	   &&&&     r   Úapply_pos_embedsÚMMDiT.apply_pos_embeds«  s  € Ø��U�t—‘Õ&ˆØ��U�t—‘Õ&ˆÜ�a“)ˆà—*‘*ˆÜ—y‘y×.Ñ.¨t×/GÑ/G×/OÑ/OÐPQÐSZÐegÓ/hÐjkÓlˆàÔÜŸ=š=¨×)=Ñ)=¸bÀ!Ó)DÀwÐFXÐ_iÔj×rÑrÐstÐvxÓyˆLØˆGà•+ !Õ#ˆØ•+ !Õ#ˆØ# A f°A­X o°fÀA½X°oÐ$EÕFˆØ×'Ñ'¨¨2¨t×/GÑ/G×/MÑ/MÈbÕ/QÓRÕRÐRr   c                ó  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  ! WW43/ VB # r7   )	r4  Úpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)r,   r
   ÚtimestepÚcontextru   r²   s   &&&&&,r   r:   ÚMMDiT.forward¼  s{   € Ü×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4´U×5LÑ5L×5WÑ5W×5gÑ5gÐi|Ó}ó
÷ ‰'ð	ð ˜wñ	Gð @Fñ	Gð 	Gr   c                óŠ  a€ VP                  R / 4      pVP                  w  rxršV P                  V P                  V4      4      pV P	                  WV
4      pTpTpV P                  V4      p\        P                  ! \        P                  P                  V P                  V4      P                  VP                  ^ 4      ^^4      V.^R7      pV P                  WÁP                  4      pVP                  R/ 4      p\!        V P"                  4      ^ 8”  dm   \%        V P"                  4       FS  w  poRV3V9   d7   V3R lpVRV3,          ! RVRVRVRV/R	V/4      pVR,          pVR,          pKE  S! W�V3RV/VB w  r�KU  	  \!        V P&                  4      ^ 8”  d—   VP                  ^4      p\        P                  ! W�.^R7      p\%        V P&                  4       FF  w  poR
V3V9   d,   V3R lpVR
V3,          ! RVRVRV/R	V/4      pVR,          pK:  S! VV3RV/VB pKH  	  VRVR13,          pV P)                  V4      P+                  ^^R7      w  pp\-        VVV4      pV P/                  V4      pV P1                  W^,           V P2                  ,          V
^,           V P2                  ,          4      RRRV	1RV
13,          pV# )Úpatches_replacer�   ÚditÚdouble_blockc                 ón   <€ / pS! V R ,          V R,          V R,          V R,          R7      w  VR &   VR&   V# )ÚtxtÚimgÚvecru   r¯   r   ©rë   ÚoutÚlayers   & €r   Ú
block_wrapÚ"MMDiT._forward.<locals>.block_wrapÝ  sD   ø€ Ø ˜Ù16°t¸Eµ{Ø7;¸Eµ{Ø7;¸Eµ{ØKOÐPeÕKfô2hÑ.˜˜E™
 C¨¡Jð  #˜
r   r\  r[  r]  ru   Úoriginal_blockÚsingle_blockc                 óR   <€ / pS! V R ,          V R,          V R,          R7      VR &   V# )r\  r]  ru   r¯   r   r^  s   & €r   ra  rb  ï  s.   ø€ Ø ˜Ù%*¨4°­;¸¸U½ÐY]Ð^sÕYtÔ%u˜˜E™
Ø"˜
r   rß   N)Úgetrv   rù   r<  rH  rø   r4   rŽ   r4  r@  rA  rû   Úrepeatr  r÷   r$   Úlenrý   Ú	enumeraterþ   r  r°   r   r  r0  r  )r,   r
   rS  rT  ru   r²   rW  Úbry   r   r!  Úc_seqrç   r±   Úblocks_replaceÚira  r_  Úc_lenrÍ   ÚfshiftÚfscaler`  s   &&&&&,                @r   rN  ÚMMDiT._forwardÃ  sò  ø€ Ø-×1Ñ1Ð2CÀRÓHˆà—W‘W‰
ˆˆað
 ×Ñ˜tŸ}™}¨QÓ/Ó0ˆØ×!Ñ! !¨Ó*ˆð
 ˆØˆà× Ñ  Ó'ˆÜ�IŠI”u—y‘y×.Ñ.¨t×/CÑ/CÀQÓG×NÑNÈqÏvÉvÐVWËyÐZ[Ð]^Ó_ÐabÐcÐijÔkˆà—o‘o a¯©Ó1ˆà(×,Ñ,¨U°BÓ7ˆÜˆt×!Ñ!Ó" QÔ&Ü% d×&8Ñ&8Ö9‘��5Ø" AÐ&¨.Ô8õ#ð )¨.¸!Ð)<Ö=¸uÀaÈÐPQÐSXÐZeÐg|ð  Rð  ?Sð  Vfð  hrð  Usó  t�CØ˜E�
�AØ˜E�
’Aá   {ÑfÐH[ÐfÐ_eÑf‘D�A‘qñ :ô ˆt×!Ñ!Ó" QÔ&Ø—F‘F˜1“IˆEÜ—’˜A˜6 qÔ)ˆBÜ% d×&8Ñ&8Ö9‘��5Ø" AÐ&¨.Ô8õ#ð
 )¨.¸!Ð)<Ö=¸uÀbÈ%ÐQ\Ð^sð  vIð  ?Jð  M]ð  _ið  Ljó  k�CØ˜U�’Bá˜r ;ÑbÐDWÐbÐ[aÑb’Bñ :ð �1�e‘f�9•ˆAàŸ™ ;Ó/×5Ñ5°a¸QÐ5Ó?‰ˆ�ä�Q˜ Ó'ˆØ×Ñ˜aÓ ˆØ�O‰O˜A A¥¨$¯/©/Õ9¸AÀ½EÀdÇoÁoÕ;UÓVÐWXÐYZÐ[]Ð\]Ð[]Ð^`Ð_`Ð^`ÐW`ÕaˆØˆr   )rø   rý   r$   r  r  rù   r  r  r  r  r  rú   rû   rþ   r÷   r	  )é   rr  r   é   é$   rr  é   rs  i   rÃ   NNN))é   rv  )é@   rw  )r=   r>   r?   r@   r'   r4   Úno_gradr  r)  r0  r<  rH  r:   rN  rA   rB   rC   rD   s   @@r   rõ   rõ   7  sX   ù‡ € ÷=)ð~ ‡]‚]ƒ_ó,ó ð,ò-òòòSð" ACô Gð BD÷ =ô =r   rõ   )rá   r4   Útorch.nnrJ   Útorch.nn.functionalÚ
functionalr8   Úcomfy.ldm.modules.attentionr   r   r   Ú	comfy.opsr4  Úcomfy.patcher_extensionÚcomfy.ldm.common_ditr   r   ÚModuler   rF   r_   rƒ   r™   rÅ   rØ   rõ   r   r   r   Ú<module>r�     s·   ðó ã Ý ß Ð ç eÑ eÛ Û Û ò=õôˆ"�)‰)ô ô&-˜Ÿ™ô -ô&'�b—i‘iô 'ôVL�b—i‘iô Lô^;�—‘ô ;ôzˆr�y‰yô ôD�r—y‘yô ô>IˆB�I‰Iö Ir   