+
    ƒ°ñjOY  ã                   ó:  € ^ RI t ^ RI Ht ^ RIt^ RIt^ RIHtHtHt ^ RI	H
t
 ^ RIHt / R3R ltR tR tR	 tR!R
 lt ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R ]P$                  4      tR# )"é    N)Únn)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention)Úapply_rope1)ÚEmbedNDc           
      óP  € \        V \        4      '       d0   V P                  4       VP                  4       VP                  4       r!p \        V P                  ^^4      4      p \        VP                  ^^4      4      p\        VP                  ^^4      4      p\	        WVVVRVR7      # )é   T)ÚheadsÚpreferred_attentionÚskip_reshapeÚtransformer_options)Ú
isinstancer   ÚtakeÚ	transposer   )ÚqÚkÚvr   r   r   s   &&&&&&Ú>C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\kandinsky5\model.pyÚ	attentionr   
   sŠ   € Ü�!Ô-×.Ò.Ø—&‘&“(˜AŸF™F›H a§f¡f£hˆaˆÜ  §¡¨Q°Ó!2Ó3€AÜ  §¡¨Q°Ó!2Ó3€AÜ  §¡¨Q°Ó!2Ó3€AÜØ	ˆaØØ/ØØ/ôð ó    c                 óJ   € \         P                  ! W0! V4      VR ,           4      # )ç      ð?©ÚtorchÚaddcmul)ÚnormÚxÚscaleÚshifts   &&&&r   Úapply_scale_shift_normr!      s   € Ü�=Š=˜  Q£¨°­Ó5Ð5r   c                 ó0   € \         P                  ! WV4      # ©Nr   )r   ÚoutÚgates   &&&r   Úapply_gate_sumr&      s   € Ü�=Š=˜ #Ó&Ð&r   c                 óš   € \         P                  ! V ^RR7      w  rp\        ;QJ d    . R WV3 4       F  NK  	  5# ! R WV3 4       4      # )é   ©Údimc              3   óB   "  € T F  qP                  ^4      x € K  	  R# 5i)r
   N)Ú	unsqueeze)Ú.0r   s   & r   Ú	<genexpr>Ú'get_shift_scale_gate.<locals>.<genexpr>    s   é € Ð>Ñ)= A—‘˜Q—�Ó)=ùs   ‚éÿÿÿÿ)r   ÚchunkÚtuple)Úparamsr    r   r%   s   &   r   Úget_shift_scale_gater4      sE   € ÜŸš V¨Q°BÔ7Ñ€E�$ßŒ5Ñ>¨%¸Ñ)=Ó>�5Ð>ˆ5Ñ>¨%¸Ñ)=Ó>Ó>Ð>r   c           	      ó¾   € \         P                  ! \        P                  ! V4      ) \         P                  ! ^ V \         P
                  R7      ,          V ,          4      # )r   )ÚstartÚendÚdtype)r   ÚexpÚmathÚlogÚarangeÚfloat32)r*   Ú
max_periods   &&r   Ú	get_freqsr?   "   s<   € Ü�9Š9”d—h’h˜zÓ*Ð*¬U¯\ª\ÀÀsÔRW×R_ÑR_Ô-`Õ`ÐcfÕfÓgÐgr   c                   ó<   a a€ ] tR t^&t oRV 3R lltR tRtVtV ;t# )ÚTimeEmbeddingsc           	     óÜ  <€ \         SV `  4        V^,          ^ 8X  g   Q hWn        W0n        V P	                  R\        V^,          V4      RR7       VP                  R4      pVP                  WRVP                  R4      VP                  R4      R7      V n        \        P                  ! 4       V n        VP                  W"RVP                  R4      VP                  R4      R7      V n        R	# )
é   ÚfreqsF)Ú
persistentÚ
operationsTÚdevicer8   ©ÚbiasrG   r8   N)ÚsuperÚ__init__Ú	model_dimr>   Úregister_bufferr?   ÚgetÚLinearÚin_layerr   ÚSiLUÚ
activationÚ	out_layer)ÚselfrL   Útime_dimr>   Úoperation_settingsrF   Ú	__class__s   &&&&& €r   rK   ÚTimeEmbeddings.__init__'   s   ø€ Ü‰ÑÔØ˜1�} Ô!Ð!Ð!Ø"ŒØ$ŒØ×Ñ˜W¤i°	¸QµÀ
Ó&KÐX]ÐÔ^Ø'×+Ñ+¨LÓ9ˆ
Ø"×)Ñ)¨)ÀDÐQc×QgÑQgÐhpÓQqð  zL÷  zPñ  zPð  QXó  zYÐ)ó  ZˆŒÜŸ'š'›)ˆŒØ#×*Ñ*¨8ÀDÐQc×QgÑQgÐhpÓQqð  zL÷  zPñ  zPð  QXó  zYÐ*ó  ZˆŽr   c                óz  € \         P                  ! WP                  P                  VP                  R 7      4      p\         P
                  ! \         P                  ! V4      \         P                  ! V4      .RR7      P                  V4      pV P                  V P                  V P                  V4      4      4      pV# ))rG   r)   r0   )r   ÚouterrD   ÚtorG   ÚcatÚcosÚsinrS   rR   rP   )rT   Útimestepr8   ÚargsÚ
time_embeds   &&&  r   ÚforwardÚTimeEmbeddings.forward2   sy   € Ü�{Š{˜8§Z¡Z§]¡]¸(¿/¹/ ]Ó%JÓKˆÜ—Y’Y¤§	¢	¨$£´·²¸4³ÐAÀrÔJ×MÑMÈeÓTˆ
Ø—^‘^ D§O¡O°D·M±MÀ*Ó4MÓ$NÓOˆ
ØÐr   )rR   rP   r>   rL   rS   )ç     ˆÃ@N©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rK   rb   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rW   Ú__classdict__s   @@r   rA   rA   &   s   ù‡ € ÷	Z÷ò r   rA   c                   ó<   a a€ ] tR t^9t oRV 3R lltR tRtVtV ;t# )ÚTextEmbeddingsc           	     ó&  <€ \         SV `  4        VP                  R 4      pVP                  WRVP                  R4      VP                  R4      R7      V n        VP                  VRVP                  R4      VP                  R4      R7      V n        R# )rF   TrG   r8   rH   ©Úelementwise_affinerG   r8   N)rJ   rK   rN   rO   rP   Ú	LayerNormr   )rT   Útext_dimrL   rV   rF   rW   s   &&&& €r   rK   ÚTextEmbeddings.__init__:   s¶   ø€ Ü‰ÑÔØ'×+Ñ+¨LÓ9ˆ
Ø"×)Ñ)¨(ÀDÐQc×QgÑQgÐhpÓQqð  zL÷  zPñ  zPð  QXó  zYÐ)ó  ZˆŒØ×(Ñ(¨ÀtÐTf×TjÑTjÐksÓTtð  }O÷  }Sñ  }Sð  T[ó  }\Ð(ó  ]ˆŽ	r   c                ód   € V P                  V4      pV P                  V4      P                  V4      # r#   )rP   r   Útype_as)rT   Ú
text_embeds   &&r   rb   ÚTextEmbeddings.forward@   s*   € Ø—]‘] :Ó.ˆ
Ø�y‰y˜Ó$×,Ñ,¨ZÓ8Ð8r   )rP   r   r#   re   rm   s   @@r   rp   rp   9   s   ù‡ € ÷]÷9ò 9r   rp   c                   ó<   a a€ ] tR t^Et oRV 3R lltR tRtVtV ;t# )ÚVisualEmbeddingsc                óÀ   <€ \         SV `  4        W0n        VP                  R 4      pVP	                  WVP                  R4      VP                  R4      R7      V n        R# ©rF   rG   r8   ©rG   r8   N)rJ   rK   Ú
patch_sizerN   rO   rP   )rT   Ú
visual_dimrL   r€   rV   rF   rW   s   &&&&& €r   rK   ÚVisualEmbeddings.__init__F   sm   ø€ Ü‰ÑÔØ$ŒØ'×+Ñ+¨LÓ9ˆ
Ø"×)Ñ)¨*ÐHZ×H^ÑH^Ð_gÓHhð  qC÷  qGñ  qGð  HOó  qPÐ)ó  QˆŽr   c           
     ó&  € VP                  ^R4      pVP                  w  r#rEpV P                  w  rxp	VP                  VW7,          VWH,          VWY,          V	V4      P	                  ^ ^^^^^^^4      P                  ^^4      pV P                  V4      # )r
   r0   )ÚmovedimÚshaper€   ÚviewÚpermuteÚflattenrP   )
rT   r   ÚBÚTÚHÚWr*   ÚptÚphÚpws
   &&        r   rb   ÚVisualEmbeddings.forwardL   s�   € Ø�I‰I�a˜ÓˆØŸ'™'‰ˆˆa�CØ—_‘_‰
ˆ�à�F‰FØØ�G�RØ�G�RØ�G�RØó
÷ ‰'�!�Q˜˜1˜a  A qÓ
)¯'©'°!°Q«-ð 	
ð �}‰}˜QÓÐr   )rP   r€   r#   re   rm   s   @@r   r|   r|   E   s   ù‡ € ÷Q÷ ò  r   r|   c                   ó<   a a€ ] tR t^\t oRV 3R lltR tRtVtV ;t# )Ú
Modulationc                óò   <€ \         SV `  4        \        P                  ! 4       V n        VP                  R 4      P                  WV,          VP                  R4      VP                  R4      R7      V n        R# r~   )rJ   rK   r   rQ   rR   rN   rO   rS   )rT   rU   rL   Ú
num_paramsrV   rW   s   &&&&&€r   rK   ÚModulation.__init__]   sŠ   ø€ Ü‰ÑÔÜŸ'š'›)ˆŒØ+×/Ñ/°Ó=×DÑDÀXÐ\eÕOeð  oA÷  oEñ  oEð  FNó  oOð  Wi÷  Wmñ  Wmð  nuó  WvÐDó  wˆŽr   c                óB   € V P                  V P                  V4      4      # r#   )rS   rR   ©rT   r   s   &&r   rb   ÚModulation.forwardb   s   € Ø�~‰~˜dŸo™o¨aÓ0Ó1Ð1r   )rR   rS   r#   re   rm   s   @@r   r’   r’   \   s   ù‡ € ÷w÷
2ò 2r   r’   c                   ó`   a a€ ] tR t^ft oRV 3R lltR t/ 3R lt/ 3R lt/ 3R ltRt	Vt
V ;t# )ÚSelfAttentionc           	     óN  <€ \         SV `  4        \        4       V n        W,          ^ 8X  g   Q hW,          V n        W n        VP                  R4      pVP                  WRVP                  R4      VP                  R4      R7      V n        VP                  WRVP                  R4      VP                  R4      R7      V n	        VP                  WRVP                  R4      VP                  R4      R7      V n
        VP                  W#P                  R4      VP                  R4      R7      V n        VP                  W#P                  R4      VP                  R4      R7      V n        VP                  WRVP                  R4      VP                  R4      R7      V n        ^V n        R# )r   rF   TrG   r8   rH   r   N)rJ   rK   r   Úcomfy_attentionÚ	num_headsÚhead_dimrN   rO   Úto_queryÚto_keyÚto_valueÚRMSNormÚ
query_normÚkey_normrS   Ú
num_chunks)rT   Únum_channelsrž   rV   rF   rW   s   &&&& €r   rK   ÚSelfAttention.__init__g   sö  ø€ Ü‰ÑÔÜ-Ó/ˆÔØÕ&¨!Ô+Ð+Ð+Ø%Õ1ˆŒØ Œà'×+Ñ+¨LÓ9ˆ
Ø"×)Ñ)¨,È4ÐXj×XnÑXnÐowÓXxð  AS÷  AWñ  AWð  X_ó  A`Ð)ó  aˆŒØ ×'Ñ'¨ÈÐVh×VlÑVlÐmuÓVvð  Q÷  Uñ  Uð  V]ó  ^Ð'ó  _ˆŒØ"×)Ñ)¨,È4ÐXj×XnÑXnÐowÓXxð  AS÷  AWñ  AWð  X_ó  A`Ð)ó  aˆŒØ$×,Ñ,¨X×>TÑ>TÐU]Ó>^Ðfx×f|Ñf|ð  ~Eó  gFÐ,ó  GˆŒØ"×*Ñ*¨8×<RÑ<RÐS[Ó<\Ðdv×dzÑdzð  |Có  eDÐ*ó  EˆŒà#×*Ñ*¨<ÈDÐYk×YoÑYoÐpxÓYyð  BT÷  BXñ  BXð  Y`ó  BaÐ*ó  bˆŒØˆŽr   c                óŽ   € V! V4      P                   ! . VP                  R R OV P                  NRN5!  p\        V! V4      V4      # ©Nr0   )r†   r…   r�   r   )rT   r   rD   Úproj_fnÚnorm_fnÚresults   &&&&& r   Ú_compute_qkÚSelfAttention._compute_qkx   s@   € Ù˜“—’ÐC !§'¡'¨#¨2 ,ÐC°·±ÐCÀÓCˆÜ™7 6›?¨EÓ2Ð2r   c           	     óÀ  € V P                  WV P                  V P                  4      pV P                  WV P                  V P                  4      pV P                  V4      P                  ! . VP                  R R OV P                  NRN5!  p\        V4      \        V4      \        V4      rep\        WEW`P                  V P                  VR7      pV P                  V4      # )N©r   r   r0   )r­   rŸ   r£   r    r¤   r¡   r†   r…   r�   r   r   rœ   rS   )rT   r   rD   r   r   r   r   r$   s   &&&&    r   Ú_forwardÚSelfAttention._forward|   s¼   € Ø×Ñ˜Q t§}¡}°d·o±oÓFˆØ×Ñ˜Q t§{¡{°D·M±MÓBˆØ�M‰M˜!Ó×!Ò!ÐD 1§7¡7¨3¨B <ÐD°·±ÐDÀÓDˆÜ*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆÜ˜˜a§¡ÀT×EYÑEYð  pCô  DˆØ�~‰~˜cÓ"Ð"r   c           	     ó°  a aa€ VV V3R  lpV! S P                   S P                  4      pV! S P                  S P                  4      pS P	                  S4      P
                  ! . SP                  RR OS P                  NRN5!  p\        V4      \        V4      \        V4      rvp\        WVVS P                  S P                  VR7      pS P                  V4      # )c           	      ó.  <€ \         P                  ! S	SP                  ^R7      p\         P                  ! SSP                  ^R7      p. p\        W#4       F&  w  rVVP	                  SP                  WVW4      4       K(  	  \         P                  ! V^R7      # ©r
   r)   )r   r1   r¥   ÚzipÚappendr­   r\   )
rª   r«   Úx_chunksÚfreqs_chunksÚchunksÚx_chunkÚfreqs_chunkrD   rT   r   s
   &&     €€€r   Úprocess_chunksÚ6SelfAttention._forward_chunked.<locals>.process_chunks…   sq   ø€ Ü—{’{ 1 d§o¡o¸1Ô=ˆHÜ Ÿ;š; u¨d¯o©oÀ1ÔEˆLØˆFÜ(+¨HÖ(CÑ$�Ø—‘˜d×.Ñ.¨wÀWÓVÖWñ )Dä—9’9˜V¨Ô+Ð+r   Nr°   r0   )rŸ   r£   r    r¤   r¡   r†   r…   r�   r   r   rœ   rS   )	rT   r   rD   r   r½   r   r   r   r$   s	   fff&     r   Ú_forward_chunkedÚSelfAttention._forward_chunked„   s¶   ú€ ÷	,ñ ˜4Ÿ=™=¨$¯/©/Ó:ˆÙ˜4Ÿ;™;¨¯©Ó6ˆØ�M‰M˜!Ó×!Ò!ÐD 1§7¡7¨3¨B <ÐD°·±ÐDÀÓDˆÜ*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆÜ˜˜a §¡ÀT×EYÑEYð  pCô  DˆØ�~‰~˜cÓ"Ð"r   c                ó~   € VP                   ^,          R8”  d   V P                  WVR7      # V P                  WVR7      # )r
   é    ©r   ©r…   r¿   r±   )rT   r   rD   r   s   &&&&r   rb   ÚSelfAttention.forward”   s=   € Ø�7‰7�1�:˜ÔØ×(Ñ(¨ÐGZÐ(Ó[Ð[à—=‘= Ð?R�=ÓSÐSr   )
rœ   rž   r¤   r¥   r�   rS   r£   r    rŸ   r¡   r#   )rf   rg   rh   ri   rK   r­   r±   r¿   rb   rj   rk   rl   rm   s   @@r   rš   rš   f   s5   ù‡ € ÷ò"3ð 68ô #ð >@ô #ð  57÷ Tô Tr   rš   c                   ó2   a € ] tR t^›t o R t/ 3R ltRtV tR# )ÚCrossAttentionc                óx  € V P                  V4      P                  ! . VP                  R R OV P                  NRN5!  pV P	                  V4      P                  ! . VP                  R R OV P                  NRN5!  pV P                  V4      P                  ! . VP                  R R OV P                  NRN5!  pW4V3# r©   )rŸ   r†   r…   r�   r    r¡   )rT   r   Úcontextr   r   r   s   &&&   r   Úget_qkvÚCrossAttention.get_qkvœ   s©   € Ø�M‰M˜!Ó×!Ò!ÐD 1§7¡7¨3¨B <ÐD°·±ÐDÀÓDˆØ�K‰K˜Ó ×%Ò%ÐN w§}¡}°S°bÐ'9ÐN¸4¿>¹>ÐNÈ2ÓNˆØ�M‰M˜'Ó"×'Ò'ÐP¨¯©°s¸Ð);ÐP¸T¿^¹^ÐPÈRÓPˆØ�Qˆwˆr   c           	     ó  € V P                  W4      w  rEp\        V P                  V4      4      \        V P                  V4      4      \        V4      rep\	        WEW`P
                  V P                  VR 7      pV P                  V4      # )r°   )rÊ   r   r£   r¤   r   r�   rœ   rS   )rT   r   rÉ   r   r   r   r   r$   s   &&&&    r   rb   ÚCrossAttention.forward¢   s„   € Ø—,‘,˜qÓ*‰ˆˆaÜ*¨4¯?©?¸1Ó+=Ó>Ô@XÐY]×YfÑYfÐghÓYiÓ@jô  mEð  FGó  mHˆaˆÜ˜˜a§¡ÀT×EYÑEYð  pCô  DˆØ�~‰~˜cÓ"Ð"r   © N)rf   rg   rh   ri   rÊ   rb   rj   rk   )rn   s   @r   rÇ   rÇ   ›   s   ø‡ € òð 79÷ #ò #r   rÇ   c                   óH   a a€ ] tR t^©t oRV 3R lltR tR tR tRtVt	V ;t
# )ÚFeedForwardc           	     óh  <€ \         SV `  4        VP                  R 4      pVP                  WRVP                  R4      VP                  R4      R7      V n        \
        P                  ! 4       V n        VP                  W!RVP                  R4      VP                  R4      R7      V n        ^V n	        R# )rF   FrG   r8   rH   N)
rJ   rK   rN   rO   rP   r   ÚGELUrR   rS   r¥   )rT   r*   Úff_dimrV   rF   rW   s   &&&& €r   rK   ÚFeedForward.__init__ª   sÊ   ø€ Ü‰ÑÔØ'×+Ñ+¨LÓ9ˆ
Ø"×)Ñ)¨#¸EÐJ\×J`ÑJ`ÐaiÓJjð  sE÷  sIñ  sIð  JQó  sRÐ)ó  SˆŒÜŸ'š'›)ˆŒØ#×*Ñ*¨6¸UÐK]×KaÑKaÐbjÓKkð  tF÷  tJñ  tJð  KRó  tSÐ*ó  TˆŒØˆŽr   c                ó`   € V P                  V P                  V P                  V4      4      4      # r#   )rS   rR   rP   r—   s   &&r   r±   ÚFeedForward._forward²   s#   € Ø�~‰~˜dŸo™o¨d¯m©m¸AÓ.>Ó?Ó@Ð@r   c                óÌ   € \         P                  ! WP                  ^R7      p. pV F#  pVP                  V P	                  V4      4       K%  	  \         P
                  ! V^R7      # rµ   )r   r1   r¥   r·   r±   r\   )rT   r   rº   Úoutput_chunksr1   s   &&   r   r¿   ÚFeedForward._forward_chunkedµ   sM   € Ü—’˜Q§¡°QÔ7ˆØˆÛˆEØ× Ñ  §¡¨uÓ!5Ö6ñ ä�yŠy˜¨AÔ.Ð.r   c                óv   € VP                   ^,          R8”  d   V P                  V4      # V P                  V4      # )r
   rÂ   rÄ   r—   s   &&r   rb   ÚFeedForward.forward¼   s2   € Ø�7‰7�1�:˜ÔØ×(Ñ(¨Ó+Ð+à—=‘= Ó#Ð#r   )rR   rP   r¥   rS   r#   )rf   rg   rh   ri   rK   r±   r¿   rb   rj   rk   rl   rm   s   @@r   rÐ   rÐ   ©   s   ù‡ € ÷òAò/÷$ò $r   rÐ   c                   ó<   a a€ ] tR t^Ãt oRV 3R lltR tRtVtV ;t# )ÚOutLayerc           	     ó�  <€ \         SV `  4        W@n        \        W!^VR7      V n        VP                  R4      pVP                  VRVP                  R4      VP                  R4      R7      V n        VP                  V\        P                  ! V4      V,          RVP                  R4      VP                  R4      R7      V n        R	# )
rC   ©rV   rF   FrG   r8   rr   TrH   N)rJ   rK   r€   r’   Ú
modulationrN   rt   r   rO   r:   ÚprodrS   )rT   rL   rU   r�   r€   rV   rF   rW   s   &&&&&& €r   rK   ÚOutLayer.__init__Ä   sí   ø€ Ü‰ÑÔØ$ŒÜ$ X¸!ÐPbÔcˆŒØ'×+Ñ+¨LÓ9ˆ
Ø×(Ñ(¨ÀuÐUg×UkÑUkÐltÓUuð  ~P÷  ~Tñ  ~Tð  U\ó  ~]Ð(ó  ^ˆŒ	Ø#×*Ñ*¨9´d·i²iÀ
Ó6KÈjÕ6XÐ_cÐl~÷  mCñ  mCð  DLó  mMð  Ug÷  Ukñ  Ukð  lsó  UtÐ*ó  uˆŽr   c                óÂ  € VP                   w  r4rVp\        P                  ! V P                  V4      ^RR7      w  r‰V	R,          p	VR,          p\	        V P
                  WV4      pV P                  V4      p
V
P                   R,          V P                  ^ ,          V P                  ^,          ,          V P                  ^,          ,          ,          pV
P                  W4WVVV P                  ^ ,          V P                  ^,          V P                  ^,          4      p
V
P                  ^ ^^^^^^^4      P                  ^^4      P                  ^^4      P                  ^^4      # )rC   r)   r0   )ºNNNNNNrä   )r…   r   r1   rà   r!   r   rS   r€   r†   r‡   rˆ   )rT   Úvisual_embedra   r‰   rŠ   r‹   rŒ   Ú_r    r   r   Úout_dims   &&&         r   rb   ÚOutLayer.forwardÌ   s  € Ø$×*Ñ*‰ˆˆa�AÜ—{’{ 4§?¡?°:Ó#>ÀÀrÔJ‰ˆØÐ,Õ-ˆØÐ,Õ-ˆÜ-¨d¯i©i¸ÈeÓTˆØ�N‰N˜<Ó(ˆà—'‘'˜"•+ $§/¡/°!Õ"4°t·±ÀqÕ7IÕ"IÈDÏOÉOÐ\]ÕL^Õ"^Õ_ˆØ�F‰FØ�!ØØ�O‰O˜AÕ §¡°Õ 2°D·O±OÀAÕ4Fó
ˆð
 �y‰y˜˜A˜q ! Q¨¨1¨aÓ0×8Ñ8¸¸AÓ>×FÑFÀqÈ!ÓL×TÑTÐUVÐXYÓZÐZr   )rà   r   rS   r€   r#   re   rm   s   @@r   rÝ   rÝ   Ã   s   ù‡ € ÷u÷[ò [r   rÝ   c                   óB   a a€ ] tR t^Ýt oRV 3R llt/ 3R ltRtVtV ;t# )ÚTransformerEncoderBlockc                ó”  <€ \         SV `  4        \        W!^VR7      V n        VP	                  R4      pVP                  VRVP	                  R4      VP	                  R4      R7      V n        \        WVR7      V n        VP                  VRVP	                  R4      VP	                  R4      R7      V n	        \        WVR7      V n        R# )é   rß   rF   FrG   r8   rr   N)rJ   rK   r’   Útext_modulationrN   rt   Úself_attention_normrš   Úself_attentionÚfeed_forward_normrÐ   Úfeed_forward©rT   rL   rU   rÓ   rž   rV   rF   rW   s   &&&&&& €r   rK   Ú TransformerEncoderBlock.__init__Þ   sü   ø€ Ü‰ÑÔÜ)¨(¸qÐUgÔhˆÔØ'×+Ñ+¨LÓ9ˆ
à#-×#7Ñ#7¸	ÐV[Ðdv×dzÑdzð  |Dó  eEð  M_÷  Mcñ  Mcð  dkó  MlÐ#7ó  $mˆÔ Ü+¨IÐTfÔgˆÔà!+×!5Ñ!5°iÐTYÐbt×bxÑbxð  zBó  cCð  K]÷  Kañ  Kað  bió  KjÐ!5ó  "kˆÔÜ'¨	ÐN`ÔaˆÖr   c                óf  € \         P                  ! V P                  V4      ^RR7      w  rV\        V4      w  rxp	\	        V P
                  WV4      p
V P                  W£VR7      p
\        WV	4      p\        V4      w  rxp	\	        V P                  WV4      p
V P                  V
4      p
\        WV	4      pV# )rC   r)   rÃ   r0   )
r   r1   rí   r4   r!   rî   rï   r&   rð   rñ   )rT   r   ra   rD   r   Úself_attn_paramsÚ	ff_paramsr    r   r%   r$   s   &&&&&      r   rb   ÚTransformerEncoderBlock.forwardé   s¯   € Ü&+§k¢k°$×2FÑ2FÀzÓ2RÐTUÐ[]Ô&^Ñ#ÐÜ1Ð2BÓCÑˆ�dÜ$ T×%=Ñ%=¸qÈÓOˆØ×!Ñ! #ÐBUÐ!ÓVˆÜ˜1 4Ó(ˆä1°)Ó<Ñˆ�dÜ$ T×%;Ñ%;¸QÀuÓMˆØ×Ñ Ó$ˆÜ˜1 4Ó(ˆØˆr   )rñ   rð   rï   rî   rí   r#   re   rm   s   @@r   rê   rê   Ý   s   ù‡ € ÷	bð AC÷ ô r   rê   c                   óB   a a€ ] tR t^÷t oRV 3R llt/ 3R ltRtVtV ;t# )ÚTransformerDecoderBlockc                ó(  <€ \         SV `  4        \        W!^	VR7      V n        VP	                  R4      pVP                  VRVP	                  R4      VP	                  R4      R7      V n        \        WVR7      V n        VP                  VRVP	                  R4      VP	                  R4      R7      V n	        \        WVR7      V n        VP                  VRVP	                  R4      VP	                  R4      R7      V n        \        WVR7      V n        R# )é	   rß   rF   FrG   r8   rr   N)rJ   rK   r’   Úvisual_modulationrN   rt   rî   rš   rï   Úcross_attention_normrÇ   Úcross_attentionrð   rÐ   rñ   rò   s   &&&&&& €r   rK   Ú TransformerDecoderBlock.__init__ø   sb  ø€ Ü‰ÑÔÜ!+¨HÀÐWiÔ!jˆÔà'×+Ñ+¨LÓ9ˆ
Ø#-×#7Ñ#7¸	ÐV[Ðdv×dzÑdzð  |Dó  eEð  M_÷  Mcñ  Mcð  dkó  MlÐ#7ó  $mˆÔ Ü+¨IÐTfÔgˆÔà$.×$8Ñ$8¸ÐW\Ðew×e{Ñe{ð  }Eó  fFð  N`÷  Ndñ  Ndð  eló  NmÐ$8ó  %nˆÔ!Ü-¨iÐVhÔiˆÔà!+×!5Ñ!5°iÐTYÐbt×bxÑbxð  zBó  cCð  K]÷  Kañ  Kað  bió  KjÐ!5ó  "kˆÔÜ'¨	ÐN`ÔaˆÖr   c                óð  € \         P                  ! V P                  V4      ^RR7      w  rgp\        V4      w  ršp\	        V P
                  WV	4      pV P                  WÄVR7      p\        WV4      p\        V4      w  ršp\	        V P                  WV	4      pV P                  WÂVR7      p\        WV4      p\        V4      w  ršp\	        V P                  WV	4      pV P                  V4      p\        WV4      pV# )r(   r)   rÃ   r0   )r   r1   rü   r4   r!   rî   rï   r&   rý   rþ   rð   rñ   )rT   rå   ry   ra   rD   r   rõ   Úcross_attn_paramsrö   r    r   r%   Ú
visual_outs   &&&&&&       r   rb   ÚTransformerDecoderBlock.forward  sü   € Ü9>¿ºÀT×E[ÑE[Ð\fÓEgÐijÐprÔ9sÑ6Ð¨Yä1Ð2BÓCÑˆ�dÜ+¨D×,DÑ,DÀlÐ[`Óaˆ
Ø×(Ñ(¨ÐPcÐ(Ódˆ
Ü% lÀÓEˆä1Ð2CÓDÑˆ�dÜ+¨D×,EÑ,EÀ|Ð\aÓbˆ
Ø×)Ñ)¨*ÐViÐ)Ójˆ
Ü% lÀÓEˆä1°)Ó<Ñˆ�dÜ+¨D×,BÑ,BÀLÐY^Ó_ˆ
Ø×&Ñ& zÓ2ˆ
Ü% lÀÓEˆØÐr   )rþ   rý   rñ   rð   rï   rî   rü   r#   re   rm   s   @@r   rù   rù   ÷   s   ù‡ € ÷bð XZ÷ ô r   rù   c                   óŠ   a a€ ] tR tRt oR
V 3R llt^ RRR/ 3R lt^ RRRRR/ 3R lt/ 3R ltR/ 3R ltR/ 3R lt	R	t
VtV ;t# )Ú
Kandinsky5i  Nc                ó¼  <€ \         SV `  4        \        V4      pWÐn        Wn        W`n        W�n        W€n        Wàn        Wðn	        R VRVRV/p\        WeVR7      V n        \        W6VR7      V n        \        WEVR7      V n        \        W†V	VR7      V n        \"        P$                  ! \'        V
4       Uu. uF  p\)        WeVVVR7      NK  	  up4      V n        \"        P$                  ! \'        V4       Uu. uF  p\-        WeVVVR7      NK  	  up4      V n        \1        WeW)VR7      V n        \5        VRVR7      V n        \5        VRV.R7      V n        R# u upi u upi )rF   rG   r8   rß   rd   )r*   ÚthetaÚaxes_dimN)rJ   rK   ÚsumÚrope_scale_factorÚin_visual_dimrL   r€   Úvisual_embed_dimr8   rG   rA   Útime_embeddingsrp   Útext_embeddingsÚpooled_text_embeddingsr|   Úvisual_embeddingsr   Ú
ModuleListÚrangerê   Útext_transformer_blocksrù   Úvisual_transformer_blocksrÝ   rS   r   Úrope_embedder_3dÚrope_embedder_1d)rT   r  Úout_visual_dimÚin_text_dimÚin_text_dim2rU   rL   rÓ   r  r€   Únum_text_blocksÚnum_visual_blocksÚ	axes_dimsr
  r8   rG   rF   Úkwargsrž   rV   ræ   rW   s   &&&&&&&&&&&&&&&&&,   €r   rK   ÚKandinsky5.__init__  sˆ  ø€ ô 	‰ÑÔÜ�y“>ˆØ!2ÔØ*ÔØ"ŒØ$ŒØ 0ÔØŒ
ØŒØ*¨J¸À&È'ÐSXÐYÐä-¨iÐVhÔiˆÔÜ-¨kÐYkÔlˆÔÜ&4°\Ð`rÔ&sˆÔ#Ü!1Ð2BÈzð  oAô  "BˆÔä')§}¢}ô }Bð  CRô  }Só  Tñ  }SÐwxÔ$ Y¸&À(Ð_q×rñ  }Sñ  Tó(
ˆÔ$ô *,¯ªô }Bð  CTô  }Uó  Vñ  }UÐwxÔ$ Y¸&À(Ð_q×rñ  }Uñ  Vó*
ˆÔ&ô " )°~ÐfxÔyˆŒä '¨H¸GÈiÔ XˆÔÜ '¨H¸GÈxÈjÔ YˆÖùò Tùò Vs   Â3EÃ4Ec                óè   € Vf   TMTp\         P                  ! W"V^,
          ,           W4VR7      pVP                  R^4      P                  ^ 4      pV P	                  V4      P                  ^^4      pV# )N©ÚstepsrG   r8   r0   )r   ÚlinspaceÚreshaper,   r  r„   )	rT   Úseq_lenÚ	seq_startr!  rG   r8   r   Úseq_idsrD   s	   &&&&&&&  r   Úrope_encode_1dÚKandinsky5.rope_encode_1d?  sf   € Ø š=‘¨eˆÜ—.’. ¸À1½Õ,EÈUÐinÔoˆØ—/‘/ " aÓ(×2Ñ2°1Ó5ˆØ×%Ñ% gÓ.×6Ñ6°q¸!Ó<ˆØˆr   c           	     óT  € V P                   pW^ ,          ^,          ,           V^ ,          ,          pW+^,          ^,          ,           V^,          ,          pW;^,          ^,          ,           V^,          ,          pVf   TpVf   TpVf   Tp^ p^ pV
P                  RR4      pVeÂ   VR,
          VP                  RR4      ,          R,           pVR,
          VP                  RR4      ,          R,           pVR,
          VP                  RR4      ,          R,           pVVP                  RR4      ,          pVVP                  R	R4      ,          pVVP                  R
R4      ,          pM†V P                  pV P                  R8X  d   W#,          R8¼  d   RpVR,
          V^ ,          ,          R,           pVR,
          V^,          ,          R,           pVR,
          V^,          ,          R,           p\        P
                  ! WVV^3W‰R7      pVR,          \        P                  ! WDV^,
          ,           WXV	R7      P                  R^^4      ,           VR&   VR,          \        P                  ! WÿV^,
          ,           WhV	R7      P                  ^R^4      ,           VR&   VR,          \        P                  ! VVV^,
          ,           WxV	R7      P                  ^^R4      ,           VR&   VP                  ^RVP                  R,          4      pV P                  V4      P                  ^^4      pV# )r   NÚrope_optionsr   Úscale_tÚscale_yÚscale_xÚshift_tg        Úshift_yÚshift_xi   i 7  r   r   )r   çHáz®G	@r1  )rä   rä   rä   r   r0   )rä   rä   rä   r
   )rä   rä   rä   rC   )r€   rN   r
  rL   r   Úzerosr"  r#  r…   r  r„   )rT   ÚtÚhÚwÚt_startÚsteps_tÚsteps_hÚsteps_wrG   r8   r   r€   Út_lenÚh_lenÚw_lenÚh_startÚw_startr*  r
  Úimg_idsrD   s   &&&&&&&&&&&          r   Úrope_encode_3dÚKandinsky5.rope_encode_3dF  s&  € à—_‘_ˆ
Ø !•}¨Õ)Õ*¨z¸!­}Õ<ˆØ !•}¨Õ)Õ*¨z¸!­}Õ<ˆØ !•}¨Õ)Õ*¨z¸!­}Õ<ˆàŠ?ØˆGØŠ?ØˆGØŠ?ØˆGàˆØˆØ*×.Ñ.¨~¸tÓDˆØÒ#Ø˜S•[ L×$4Ñ$4°YÀÓ$DÕDÀsÕJˆEØ˜S•[ L×$4Ñ$4°YÀÓ$DÕDÀsÕJˆEØ˜S•[ L×$4Ñ$4°YÀÓ$DÕDÀsÕJˆEà�|×'Ñ'¨	°3Ó7Õ7ˆGØ�|×'Ñ'¨	°3Ó7Õ7ˆGØ�|×'Ñ'¨	°3Ó7Õ7‰Gà $× 6Ñ 6ÐØ�~‰~ Ô%Ø•5˜E”>Ø(9Ð%à˜S•[Ð$5°aÕ$8Õ8¸3Õ>ˆEØ˜S•[Ð$5°aÕ$8Õ8¸3Õ>ˆEØ˜S•[Ð$5°aÕ$8Õ8¸3Õ>ˆEä—+’+˜w°¸!Ð<ÀVÔYˆØ% jÕ1´E·N²NÀ7ÐW\Ð_`ÕW`ÕLaÐipð  HMô  5N÷  5Vñ  5Vð  WYð  [\ð  ^_ó  5`õ  `ˆ�
ÑØ% jÕ1´E·N²NÀ7ÐW\Ð_`ÕW`ÕLaÐipð  HMô  5N÷  5Vñ  5Vð  WXð  Z\ð  ^_ó  5`õ  `ˆ�
ÑØ% jÕ1´E·N²NÀ7ÈGÐW\Ð_`ÕW`ÕLaÐipð  HMô  5N÷  5Vñ  5Vð  WXð  Z[ð  ]_ó  5`õ  `ˆ�
ÑØ—/‘/ ! R¨¯©°rÕ):Ó;ˆà×%Ñ% gÓ.×6Ñ6°q¸!Ó<ˆØˆr   c                ó¼  a€ VP                  R / 4      p	V P                  V4      pV P                  W!P                  4      V P	                  V4      ,           p
V P
                   F  oS! W:WgR7      pK  	  V P                  V4      pVP                  RR pVP                  ^R4      pV	P                  R/ 4      p\        V P                  4      VR&   RVR&   \        V P                  4       FK  w  poWçR&   RV3V9   d.   V3R	 lpVRV3,          ! R
VRVRV
RVRV/RV/4      R
,          pK@  S! W³W¥VR7      pKM  	  VP                  ! . VORN5!  pV P                  Wº4      # )Úpatches_replacerÃ   NÚditÚtotal_blocksÚdoubleÚ
block_typeÚblock_indexÚdouble_blockc           	      ót   <€ S! V R ,          V R,          V R,          V R,          V P                  R4      R7      # )r   rÉ   ra   rD   r   )r   rÉ   ra   rD   r   )rN   )r`   Úblocks   &€r   Ú
block_wrapÚ+Kandinsky5.forward_orig.<locals>.block_wrap„  sS   ø€ Ù  4¨¥9°d¸9µoÐRVÐWcÕRdÐlpÐqxÕlyð  PT÷  PXñ  PXð  Ynó  Poô  pð  pr   r   rÉ   ra   rD   r   Úoriginal_block)rD   r   r0   éþÿÿÿ)rN   r  r  r8   r  r  r  r…   rˆ   Úlenr  Ú	enumerater#  rS   )rT   r   r_   rÉ   ÚyrD   Ú
freqs_textr   r  rC  ra   rå   Úvisual_shapeÚblocks_replaceÚirL  rK  s   &&&&&&&&,       @r   Úforward_origÚKandinsky5.forward_origr  s«  ø€ Ø-×1Ñ1Ð2CÀRÓHˆØ×&Ñ& wÓ/ˆØ×)Ñ)¨(·G±GÓ<¸t×?ZÑ?ZÐ[\Ó?]Õ]ˆ
à×1Ô1ˆEÙ˜G°ÔeŠGñ 2ð ×-Ñ-¨aÓ0ˆØ#×)Ñ)¨#¨2Ð.ˆØ#×+Ñ+¨A¨rÓ2ˆà(×,Ñ,¨U°BÓ7ˆÜ.1°$×2PÑ2PÓ.QÐ˜NÑ+Ø,4Ð˜LÑ)Ü! $×"@Ñ"@ÖA‰HˆAˆuØ12 Ñ.Ø Ð" nÔ4õpà-¨~¸qÐ.AÖBÀCÈÐW`ÐbiÐkwð  zDð  FMð  OTð  Vkð  m@ð  DAð  DTð  V`ð  Caó   bð  cfõ   g’á$ \¸JÐi|Ô}’ñ Bð $×+Ò+Ð>¨\Ð>¸2Ó>ˆØ�~‰~˜lÓ7Ð7r   c           	     óÀ  € VP                   pV^8X  d   VP                  ^4      pVP                  w  ršr¼p\        P                  P
                  P                  WP                  4      pVea   \        P                  P
                  P                  WPP                  4      pWQRRVP                  ^,          1RVP                  ^,          13&   V P                  W¼WÑP                  VP                  VR7      pV P                  VP                  ^,          VP                  VP                  VR7      pV P                  ! WW4Wï3RV/VB pV^8X  d   VP                  ^4      pV# )é   Nrä   )rG   r8   r   r   )Úndimr,   r…   ÚcomfyÚldmÚ
common_ditÚpad_to_patch_sizer€   r@  rG   r8   r'  rW  Úsqueeze)rT   r   r_   rÉ   rR  Útime_dim_replacer   r  Úoriginal_dimsÚbsÚcr:  r4  r5  rD   rS  r$   s   &&&&&&&,         r   r±   ÚKandinsky5._forward�  s:  € ØŸ™ˆØ˜AÔØ—‘˜A“ˆAØŸW™WÑˆˆu˜Ü�I‰I× Ñ ×2Ñ2°1·o±oÓFˆàÒ'Ü$Ÿy™y×3Ñ3×EÑEÐFV×XgÑXgÓhÐØK[ˆaÐ+Ð"×(Ñ(¨Õ+Ð+Ð-GÐ.>×.DÑ.DÀQÕ.GÐ-GÐGÑHà×#Ñ# E¨a¿¹ÈÏÉÐexÐ#ÓyˆØ×(Ñ(¨¯©°qÕ)9À!Ç(Á(ÐRS×RYÑRYð  pCÐ(ó  Dˆ
à×Ò ¨W¸Ñ~Ð`sÐ~Ðw}Ñ~ˆØ˜AÔØ—+‘+˜a“.ˆCØˆ
r   c                ó  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  ! WW43R VRV/VB # )ra  r   )	r\  Úpatcher_extensionÚWrapperExecutorÚnew_class_executorr±   Úget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)rT   r   r_   rÉ   rR  ra  r   r  s   &&&&&&&,r   rb   ÚKandinsky5.forward   s”   € Ü×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4´U×5LÑ5L×5WÑ5W×5gÑ5gÐi|Ó}ó
÷ ‰'ð	ð ˜wñ	Að =Mð	Að cvð	Að z@ñ	Að 	Ar   )rG   r8   r  rL   rS   r€   r  r  r  r
  r  r  r  r  r  r  )é   ro  i   i   i   i   i   é„   )r
   rC   rC   rC   é    )ro  é   rr  )r   ç       @rs  NNN)rf   rg   rh   ri   rK   r'  r@  rW  r±   rb   rj   rk   rl   rm   s   @@r   r  r    sp   ù‡ € ÷"ZðH 12¸ÀdÐRVÐlnô ð /0¸ÀtÐUYÐbfÐnrð  IKô *ðX \^ô 8ð6 BFÐ[]ô ð& AEÐZ\÷ Aô Ar   r  )rd   )r   r   r:   Úcomfy.ldm.common_ditr\  Úcomfy.ldm.modules.attentionr   r   r   Úcomfy.ldm.flux.mathr   Úcomfy.ldm.flux.layersr   r   r!   r&   r4   r?   ÚModulerA   rp   r|   r’   rš   rÇ   rÐ   rÝ   rê   rù   r  rÎ   r   r   Ú<module>ry     sò   ðÛ Ý Û ã ß eÑ eÝ +Ý )à24È$ô ò6ò'ò?ôhô�R—Y‘Yô ô&	9�R—Y‘Yô 	9ô �r—y‘yô  ô.2�—‘ô 2ô2T�B—I‘Iô 2Tôj#�]ô #ô$�"—)‘)ô $ô4[ˆr�y‰yô [ô4˜bŸi™iô ô4 ˜bŸi™iô  ôFKA�—‘ö KAr   