+
    ƒ°ñj=k  ã                   óÀ  € ^ RI t ^ RIHt ^ RIHu Ht ^ RIHtHt ^ RI	H
t
 ^ RIHtHt ^ RIHtHtHt ^ RIHt ^ RIt^ RIt^ RIHt  ! R R	]P2                  4      t ! R
 R]P2                  4      tR t ! R R]P2                  4      t ! R R]P2                  4      t ! R R]P2                  4      t ! R R]P2                  4      t  ! R R]P2                  4      t!R# )é    N)ÚOptionalÚTuple)Úrepeat)ÚTimestepEmbeddingÚ	Timesteps)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention_masked)ÚEmbedND)Úapply_rope1c                   óH   a a€ ] tR t^t oRV3R lV 3R llltR tRtVtV ;t# )ÚGELUc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[RS[/# )é   Údim_inÚdim_outÚapproximateÚbias)ÚintÚstrÚbool)ÚformatÚ__classdict__s   "€Ú>C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\qwen_image\model.pyÚ__annotate__ÚGELU.__annotate__   s*   ø€ ÷ 'ñ '™sð '©Sð '¹sð 'ÑSWñ 'ó    c                ób   <€ \         SV `  4        VP                  WWEVR 7      V n        W0n        R# )©r   ÚdtypeÚdeviceN)ÚsuperÚ__init__ÚLinearÚprojr   )	Úselfr   r   r   r   r    r!   Ú
operationsÚ	__class__s	   &&&&&&&&€r   r#   ÚGELU.__init__   s.   ø€ Ü‰ÑÔØ×%Ñ% f¸DÐV\Ð%Ó]ˆŒ	Ø&Ör   c                ój   € V P                  V4      p\        P                  ! WP                  R 7      pV# ))r   )r%   ÚFÚgelur   )r&   Úhidden_statess   &&r   ÚforwardÚGELU.forward   s*   € ØŸ	™	 -Ó0ˆÜŸš˜}×:JÑ:JÔKˆØÐr   )r   r%   )ÚnoneTNNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r#   r.   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r(   r   s   @@r   r   r      s   ù‡ € ÷'õ '÷
ò r   r   c                   óT   a a€ ] tR t^t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚFeedForwardc          
      óH   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[ RS[RS[/# )r   Údimr   ÚmultÚdropoutr   )r   r   Úfloatr   )r   r   s   "€r   r   ÚFeedForward.__annotate__   sH   ø€ ÷ fñ fáðfñ ™#•ðfñ ð	fñ
 ðfñ ñfr   c
                óœ  <€ \         S
V `  4        Vf   \        W,          4      pVe   TMTp\        P                  ! . 4      V n        V P
                  P                  \        WRWgW‰R7      4       V P
                  P                  \        P                  ! V4      4       V P
                  P                  V	P                  WRWgVR7      4       R # )NÚtanh)r   r   r    r!   r'   r   )
r"   r#   r   ÚnnÚ
ModuleListÚnetÚappendr   ÚDropoutr$   )r&   r=   r   r>   r?   Ú	inner_dimr   r    r!   r'   r(   s   &&&&&&&&&&€r   r#   ÚFeedForward.__init__   s•   ø€ ô 	‰ÑÔØÒÜ˜C�J›ˆIØ$Ò0‘'°cˆä—=’= Ó$ˆŒØ�‰�‰œ˜S¸ÀdÐ`fÔ~ÔØ�‰�‰œŸ
š
 7Ó+Ô,Ø�‰�‰˜
×)Ñ)¨)À4Ð]cÐ)ÓdÖer   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r-   Úreturn©ÚtorchÚTensor)r   r   s   "€r   r   rA   0   s#   ø€ ÷ ñ ¡U§\¡\ð ÁuÇ|Á|ñ r   c                ó<   € V P                    F  pV! V4      pK  	  V# ©N©rF   )r&   r-   ÚargsÚkwargsÚmodules   &&*, r   r.   ÚFeedForward.forward0   s    € Ø—h”hˆFÙ" =Ó1ŠMñ àÐr   rR   )Né   ç        NTNNNr1   r9   s   @@r   r;   r;      s    ù‡ € ÷fõ f÷(÷ ð r   r;   c                 ó  € V P                   ^,          ^ 8X  d   V # V P                  ! . V P                   RR ORN^N^N5!  pVR,          VR,          ,          VR,          VR,          ,          ,           pVP                  ! V P                   !  # )é   Néÿÿÿÿ).r   ).rZ   )ÚshapeÚreshape)ÚxÚ	freqs_cisÚt_Út_outs   &&  r   Úapply_rotary_embrb   9   sy   € Ø‡w�wˆq…z�Q„Øˆà	
�ŠÐ	+�A—G‘G˜C˜R�LÐ	+ "Ð	+ aÐ	+¨Ó	+€BØ�fÕ  6¥
Õ*¨Y°vÕ->ÀÀFÅÕ-KÕK€EØ�=Š=˜!Ÿ'™'Ñ"Ð"r   c                   ó@   a a€ ] tR t^Bt oRV 3R lltRR ltRtVtV ;t# )ÚQwenTimestepProjEmbeddingsc                óÜ   <€ \         SV `  4        \        R R^ RR7      V n        \	        R VVVVR7      V n        W0n        V P                  '       d   VP                  ^WVR7      V n        R# R# )é   Tiè  )Únum_channelsÚflip_sin_to_cosÚdownscale_freq_shiftÚscale)Úin_channelsÚtime_embed_dimr    r!   r'   ©r!   r    N)	r"   r#   r   Ú	time_projr   Útimestep_embedderÚuse_additional_t_condÚ	EmbeddingÚaddition_t_embedding)r&   Úembedding_dimÚpooled_projection_dimrp   r    r!   r'   r(   s   &&&&&&&€r   r#   Ú#QwenTimestepProjEmbeddings.__init__C   sr   ø€ Ü‰ÑÔÜ"°ÀTÐ`aÐimÔnˆŒÜ!2ØØ(ØØØ!ô"
ˆÔð &;Ô"Ø×%×%Ð%Ø(2×(<Ñ(<¸QÀÐdiÐ(<Ó(jˆDÖ%ñ &r   c                ór  € V P                  V4      pV P                  VP                  VP                  R 7      4      pV P                  '       di   VfC   \
        P                  ! VP                  ^ ,          VP                  \
        P                  R7      pWPP                  W5P                  R7      ,          pV# ))r    rm   )Ú	out_dtype)rn   ro   Útor    rp   rN   Úzerosr\   r!   Úlongrr   )r&   Útimestepr-   Úaddition_t_condÚtimesteps_projÚtimesteps_embs   &&&&  r   r.   Ú"QwenTimestepProjEmbeddings.forwardR   s’   € ØŸ™¨Ó1ˆØ×.Ñ.¨~×/@Ñ/@À}×GZÑGZÐ/@Ó/[Ó\ˆà×%×%Ð%ØÒ&Ü"'§+¢+¨}×/BÑ/BÀ1Õ/EÈ}×OcÑOcÔkp×kuÑkuÔ"v�Ø×6Ñ6°×ReÑReÐ6ÓfÕfˆMàÐr   )rr   rn   ro   rp   )FNNNrQ   r1   r9   s   @@r   rd   rd   B   s   ù‡ € ÷k÷	ô 	r   rd   c                   ób   a a€ ] tR t^^t oRV3R lV 3R llltRRRR/ 3V3R lR lltRtVtV ;t# )Ú	AttentionNc                óP   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[RS[RS[RS[ R	S[ /	# )
r   Ú	query_dimÚdim_headÚheadsr?   r   ÚepsÚout_biasÚout_dimÚout_context_dim)r   r@   r   )r   r   s   "€r   r   ÚAttention.__annotate___   sp   ø€ ÷ .}ñ .}áð.}ñ ð.}ñ ð	.}ñ
 ð.}ñ ð.}ñ ð.}ñ ð.}ñ ð.}ñ ñ.}r   c           
     óV  <€ \         SV `  4        \        4       V n        Ve   TMW#,          V n        V P                  V n        W0n        W n        Ve   TMTV n        V	e   T	MTV n	        W@n
        VP                  W&RW«R7      V n        VP                  W&RW«R7      V n        VP                  W&W«R7      V n        VP                  W&W«R7      V n        VP!                  WP                  WZVR7      V n        VP!                  WP
                  WZVR7      V n        VP!                  WP
                  WZVR7      V n        VP!                  WP                  WZVR7      V n        VP!                  WP
                  WZVR7      V n        VP!                  WP
                  WZVR7      V n        \.        P0                  ! VP!                  V P                  V P                  WzVR7      \.        P2                  ! V4      .4      V n        VP!                  V P                  V P                  WzVR7      V n        R # )NT)r†   Úelementwise_affiner    r!   ©r†   r    r!   r   )r"   r#   r	   Úcomfy_attentionrI   Úinner_kv_dimr…   r„   rˆ   r‰   r?   ÚRMSNormÚnorm_qÚnorm_kÚnorm_added_qÚnorm_added_kr$   Úto_qÚto_kÚto_vÚ
add_q_projÚ
add_k_projÚ
add_v_projrD   rE   rH   Úto_outÚ
to_add_out)r&   rƒ   r„   r…   r?   r   r†   r‡   rˆ   r‰   r    r!   r'   r(   s   &&&&&&&&&&&&&€r   r#   ÚAttention.__init___   s÷  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØ$+Ò$7™¸XÕ=MˆŒØ ŸN™NˆÔØŒ
Ø ŒØ")Ò"5‘w¸9ˆŒØ2AÒ2M™ÐS\ˆÔØŒð !×(Ñ(¨ÈtÐ[`Ð(ÓpˆŒØ ×(Ñ(¨ÈtÐ[`Ð(ÓpˆŒØ&×.Ñ.¨xÈÐ.Ó]ˆÔØ&×.Ñ.¨xÈÐ.Ó]ˆÔð ×%Ñ% i·±ÀdÐ`fÐ%ÓgˆŒ	Ø×%Ñ% i×1BÑ1BÈÐciÐ%ÓjˆŒ	Ø×%Ñ% i×1BÑ1BÈÐciÐ%ÓjˆŒ	ð %×+Ñ+¨I·~±~ÈDÐflÐ+ÓmˆŒØ$×+Ñ+¨I×7HÑ7HÈtÐioÐ+ÓpˆŒØ$×+Ñ+¨I×7HÑ7HÈtÐioÐ+ÓpˆŒô —m’mØ×Ñ˜dŸn™n¨d¯l©lÀÐ_eÐÓfÜ�JŠJ�wÓð%
ó ˆŒð %×+Ñ+¨D¯N©N¸D×<PÑ<PÐW_Ðu{Ð+Ó|ˆŽr   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[ P                  S[ P                  3,          /# )r   r-   Úencoder_hidden_statesÚencoder_hidden_states_maskÚattention_maskÚimage_rotary_embrL   )rN   ÚFloatTensorr   rO   r   )r   r   s   "€r   r   rŠ   �   s‡   ø€ ÷ A0ñ A0á×(Ñ(ðA0ñ  %×0Ñ0ðA0ñ %*×$5Ñ$5ð	A0ñ
 !¡×!2Ñ!2Õ3ðA0ñ #¡5§<¡<Õ0ðA0ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñA0r   c                ón  € VP                   ^ ,          pVP                   ^,          pVP                   ^,          p	VP                  R/ 4      p
VP                  4       pV P                  V4      P	                  WxV P
                  R4      P                  ^^4      P                  4       pV P                  V4      P	                  WxV P
                  R4      P                  ^^4      P                  4       pV P                  V4      P	                  WxV P
                  R4      P                  ^^4      pV P                  V4      P	                  WyV P
                  R4      P                  ^^4      P                  4       pV P                  V4      P	                  WyV P
                  R4      P                  ^^4      P                  4       pV P                  V4      P	                  WyV P
                  R4      P                  ^^4      pV P                  V4      pV P                  V4      pV P                  V4      pV P!                  V4      p\"        P$                  ! Wü.^R7      p\"        P$                  ! VV.^R7      p\"        P$                  ! VV.^R7      p??????VeB   \"        P&                  ! V^W˜,           3VP(                  VP*                  R7      pVVR^ RV	13&   MRpV	VP                   ^,          .VR&   RV
9   d|   V
R,          pV Fl  pV! VVVWSVR7      pVP                  R	V4      VP                  R
V4      VP                  RV4      VP                  RV4      VP                  RV4      3w  ppprSKn  	  \-        VV4      p\-        VV4      p\/        V4      p\/        V4      p\/        V4      p\1        VVVV P
                  VVRV P2                  R7      pVRRV	1R3,          pVRV	R1R3,          pV P4                  ^ ,          ! V4      pV P4                  ^,          ! V4      pV P7                  V4      pVV3# )r   Úpatches©r=   N©r    r!   ºNNNÚ	img_sliceÚattn1_patch)ÚpeÚ	attn_maskÚextra_optionsÚqÚkÚvr«   r¬   T)Útransformer_optionsÚskip_reshapeÚpreferred_attentionr[   )r\   ÚgetÚcopyr•   Úviewr…   Ú	transposeÚ
contiguousr–   r—   r˜   r™   rš   r‘   r’   r“   r”   rN   Úcatry   r    r!   r   r   r
   rŽ   r›   rœ   )r&   r-   rŸ   r    r¡   r¢   r±   Ú
batch_sizeÚseq_imgÚseq_txtÚtransformer_patchesr­   Ú	img_queryÚimg_keyÚ	img_valueÚ	txt_queryÚtxt_keyÚ	txt_valueÚjoint_queryÚ	joint_keyÚjoint_valuer¬   ÚpatchÚpÚoutÚjoint_hidden_statesÚtxt_attn_outputÚimg_attn_outputs   &&&&&&&                     r   r.   ÚAttention.forward�   sm  € ð #×(Ñ(¨Õ+ˆ
Ø×%Ñ% aÕ(ˆØ'×-Ñ-¨aÕ0ˆà1×5Ñ5°iÀÓDÐØ+×0Ñ0Ó2ˆð —I‘I˜mÓ,×1Ñ1°*ÀtÇzÁzÐSUÓV×`Ñ`ÐabÐdeÓf×qÑqÓsˆ	Ø—)‘)˜MÓ*×/Ñ/°
ÀTÇZÁZÐQSÓT×^Ñ^Ð_`ÐbcÓd×oÑoÓqˆØ—I‘I˜mÓ,×1Ñ1°*ÀtÇzÁzÐSUÓV×`Ñ`ÐabÐdeÓfˆ	à—O‘OÐ$9Ó:×?Ñ?À
ÐUY×U_ÑU_ÐacÓd×nÑnÐopÐrsÓt×Ñó  Bˆ	Ø—/‘/Ð"7Ó8×=Ñ=¸jÐSW×S]ÑS]Ð_aÓb×lÑlÐmnÐpqÓr×}Ñ}ÓˆØ—O‘OÐ$9Ó:×?Ñ?À
ÐUY×U_ÑU_ÐacÓd×nÑnÐopÐrsÓtˆ	à—K‘K 	Ó*ˆ	Ø—+‘+˜gÓ&ˆØ×%Ñ% iÓ0ˆ	Ø×#Ñ# GÓ,ˆä—i’i Ð 6¸AÔ>ˆÜ—I’I˜w¨Ð0°aÔ8ˆ	Ü—i’i ¨IÐ 6¸AÔ>ˆØ�w 	¨9°g¸yà%Ò1ÜŸš Z°°GÕ4EÐ$FÈm×NaÑNaÐjw×j~Ñj~ÔˆIØ(BˆI�a˜˜H˜W˜H�nÒ%àˆIà&-¨{×/@Ñ/@ÀÕ/CÐ%Dˆ�kÑ"ØÐ/Ô/Ø'¨Õ6ˆEÛ�Ù˜ Y°Ð@Pð  GTô  U�Ødg×dkÑdkÐloÐq|Ód}ð  @C÷  @Gñ  @Gð  HKð  MVó  @Wð  Y\÷  Y`ñ  Y`ð  adð  fqó  Yrð  tw÷  t{ñ  t{ð  |@ð  BRó  tSð  UX÷  U\ñ  U\ð  ]hð  jDó  UEð  eEÑa�˜Y¨Ð5EÑGañ ô " +Ð/?Ó@ˆÜ 	Ð+;Ó<ˆ	Ü.¨{Ó;ˆÜ,¨YÓ7ˆ	Ü.¨{Ó;ˆä8¸ÀiÐQ\Ð^b×^hÑ^hØ9BÐXkØFJÐ`d×`tÑ`tôvÐð .¨a°°'°¸1¨nÕ=ˆØ-¨a°±¸1¨nÕ=ˆàŸ+™+ až.¨Ó9ˆØŸ+™+ až.¨Ó9ˆØŸ/™/¨/Ó:ˆà Ð/Ð/r   )r™   r˜   rš   rŽ   r„   r?   r…   rI   r�   r”   r“   r’   r‘   r‰   rˆ   rœ   r–   r›   r•   r—   )é@   é   rX   Fgñhãˆµøä>TNNNNNr1   r9   s   @@r   r�   r�   ^   s5   ù‡ € ÷.}õ .}ðf 48Ø8<Ø6:Ø37Ø÷A0÷ A0ò A0r   r�   c                   ó~   a a€ ] tR t^Ót oR
V3R lV 3R llltRR ltRV3R lR lltRR/ 3V3R lR lltR	tVt	V ;t
# )ÚQwenImageTransformerBlockNc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r   r=   Únum_attention_headsÚattention_head_dimr†   )r   r@   )r   r   s   "€r   r   Ú&QwenImageTransformerBlock.__annotate__Ô   s3   ø€ ÷ )
ñ )
áð)
ñ !ð)
ñ  ð	)
ñ
 ñ)
r   c                óº  <€ \         SV `  4        Wn        W n        W0n        \
        P                  ! \
        P                  ! 4       VP                  V^V,          RWVR7      4      V n	        VP                  VRWEVR7      V n        VP                  VRWEVR7      V n        \        WWVVR7      V n        \
        P                  ! \
        P                  ! 4       VP                  V^V,          RWVR7      4      V n        VP                  VRWEVR7      V n        VP                  VRWEVR7      V n        \        WWVVR7      V n        \'        VVVVRVVVVR7	      V n        R# )é   Tr   F)rŒ   r†   r    r!   )r=   r   r    r!   r'   )	rƒ   r„   r…   rˆ   r   r†   r    r!   r'   N)r"   r#   r=   rÓ   rÔ   rD   Ú
SequentialÚSiLUr$   Úimg_modÚ	LayerNormÚ	img_norm1Ú	img_norm2r;   Úimg_mlpÚtxt_modÚ	txt_norm1Ú	txt_norm2Útxt_mlpr�   Úattn)	r&   r=   rÓ   rÔ   r†   r    r!   r'   r(   s	   &&&&&&&&€r   r#   Ú"QwenImageTransformerBlock.__init__Ô   sJ  ø€ ô 	‰ÑÔØŒØ#6Ô Ø"4Ôä—}’}Ü�GŠG‹IØ×Ñ˜c 1 s¥7°¸UÐÓRó
ˆŒð $×-Ñ-¨cÀeÐQTÐjpÐ-ÓqˆŒØ#×-Ñ-¨cÀeÐQTÐjpÐ-ÓqˆŒÜ" s¸uÐ`jÔkˆŒä—}’}Ü�GŠG‹IØ×Ñ˜c 1 s¥7°¸UÐÓRó
ˆŒð $×-Ñ-¨cÀeÐQTÐjpÐ-ÓqˆŒØ#×-Ñ-¨cÀeÐQTÐjpÐ-ÓqˆŒÜ" s¸uÐ`jÔkˆŒäØØ'Ø%ØØØØØØ!ô

ˆŽ	r   c                óÞ   € VeT   V\         P                  ! VRR V13,          V^ ,          ,          VRVR 13,          V^,          ,          3^R7      ,           # \         P                  ! W#V4      # )Nr¨   r¦   )rN   r¹   Úaddcmul)r&   r^   ÚyÚgateÚtimestep_zero_indexs   &&&&&r   Ú_apply_gateÚ%QwenImageTransformerBlock._apply_gateÿ   si   € ØÒ*Ø”u—y’y ! AÐ';Ð(;Ð';Ð$;Õ"<¸tÀA½wÕ"FÈÈ!ÐM`ÑMaÐJaÕHbÐeiÐjkÕelÕHlÐ!mÐstÔuÕuÐuä—=’= ¨!Ó,Ð,r   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# )r   r^   Ú
mod_paramsrL   )rN   rO   r   )r   r   s   "€r   r   rÕ     sX   ø€ ÷ cñ c™5Ÿ<™<ð c±U·\±\ð cÑ`eÑfk×frÑfrÑty÷  uAñ  uAð  gAõ  aBñ cr   c                óÔ  € \         P                  ! V^RR7      w  rEpVeü   VP                  ^ 4      ^,          pVRV WGR r„VRV WWR r•VRV WgR r¦\         P                  ! VP	                  ^4      VRRV13,          ^VP	                  ^4      ,           4      p\         P                  ! VP	                  ^4      VRVR13,          ^V	P	                  ^4      ,           4      p\         P
                  ! W¼3^R7      VP	                  ^4      V
P	                  ^4      33# \         P                  ! VP	                  ^4      V^VP	                  ^4      ,           4      VP	                  ^4      3# )é   r¦   Nr¨   r[   )rN   ÚchunkÚsizeræ   Ú	unsqueezer¹   )r&   r^   rí   ré   Úshiftrj   rè   Úactual_batchÚshift_0Úscale_0Úgate_0ÚregÚzeros   &&&&         r   Ú	_modulateÚ#QwenImageTransformerBlock._modulate  sU  € Ü"Ÿ[š[¨°Q¸BÔ?Ñˆ�dØÒ*Ø Ÿ:™: a›=¨AÕ-ˆLØ" = LÐ1°5¸Ð3G�7Ø" = LÐ1°5¸Ð3G�7Ø  Ð.°°]Ð0C�&Ü—-’- §¡°Ó 2°A°aÐ9MÐ:MÐ9MÐ6MÕ4NÐPQÐTY×TcÑTcÐdeÓTfÕPfÓgˆCÜ—=’= ×!2Ñ!2°1Ó!5°q¸Ð<OÑ<PÐ9PÕ7QÐSTÐW^×WhÑWhÐijÓWkÕSkÓlˆDÜ—9’9˜c˜[¨aÔ0°4·>±>À!Ó3DÀf×FVÑFVÐWXÓFYÐ2ZÐZÐZä—=’= §¡°Ó!3°Q¸¸E¿O¹OÈAÓ<NÕ8NÓOÐQU×Q_ÑQ_Ð`aÓQbÐbÐbr   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[S[S[ P                  S[ P                  3,          ,          RS[S[ P                  S[ P                  3,          /# )r   r-   rŸ   r    Útembr¢   rL   )rN   rO   r   r   )r   r   s   "€r   r   rÕ     s€   ø€ ÷ /4ñ /4á—|‘|ð/4ñ  %Ÿ|™|ð/4ñ %*§L¡Lð	/4ñ
 �l‰lð/4ñ #¡5©¯©±u·|±|Ð)CÕ#DÕEð/4ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ/4r   c                ó  € V P                  V4      pVe   VP                  ^^ R7      ^ ,          pV P                  V4      p	VP                  ^RR7      w  r«V	P                  ^RR7      w  rÍV P                  V P	                  V4      W¦4      w  rï?
V P                  V P                  V4      V4      w  pp?V P                  VVVVVR7      w  pp??V P                  VWV4      pVVV,          ,           p????V P                  V P                  V4      W¶4      w  ppV P                  V P                  V4      VVV4      pV P                  V P                  V4      V4      w  pp\        P                  ! VVV P                  V4      4      pW!3# )Nr¦   )r-   rŸ   r    r¢   r±   r[   )rÚ   rð   rß   rú   rÜ   rà   rã   rê   rÝ   rÞ   rá   rN   ræ   râ   )r&   r-   rŸ   r    rý   r¢   ré   r±   Úimg_mod_paramsÚtxt_mod_paramsÚimg_mod1Úimg_mod2Útxt_mod1Útxt_mod2Úimg_modulatedÚ	img_gate1Útxt_modulatedÚ	txt_gate1rÌ   rË   Úimg_modulated2Ú	img_gate2Útxt_modulated2Ú	txt_gate2s   &&&&&&&&                r   r.   Ú!QwenImageTransformerBlock.forward  sš  € ð Ÿ™ dÓ+ˆàÒ*Ø—:‘:˜a Q�:Ó'¨Õ*ˆDàŸ™ dÓ+ˆØ+×1Ñ1°!¸Ð1Ó<ÑˆØ+×1Ñ1°!¸Ð1Ó<Ñˆà#'§>¡>°$·.±.ÀÓ2OÐQYÓ#oÑ ˆØØ#'§>¡>°$·.±.ÐAVÓ2WÐYaÓ#bÑ ˆ�yØà+/¯9©9Ø'Ø"/Ø'AØ-Ø 3ð ,5ó ,
Ñ(ˆ˜ð Øà×(Ñ(¨¸-ÐTgÓhˆØ 5¸	ÀOÕ8SÕ SÐØØØØà$(§N¡N°4·>±>À-Ó3PÐRZÓ$pÑ!ˆ˜	Ø×(Ñ(¨¯©°nÓ)EÀ}ÐV_ÐatÓuˆà$(§N¡N°4·>±>ÐBWÓ3XÐZbÓ$cÑ!ˆ˜	Ü %§¢Ð.CÀYÐPT×P\ÑP\Ð]kÓPlÓ mÐà$Ð3Ð3r   )rÔ   rã   r=   rÞ   rÚ   rÜ   rÝ   rÓ   râ   rß   rà   rá   )ç�íµ ÷Æ°>NNNrQ   )r2   r3   r4   r5   r#   rê   rú   r.   r6   r7   r8   r9   s   @@r   rÑ   rÑ   Ó   s=   ù‡ € ÷)
õ )
ôV-÷cò cð& IMØ Ø÷/4÷ /4ò /4r   rÑ   c                   óT   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltRtVtV ;t# )Ú	LastLayeriD  c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   rs   Úconditioning_embedding_dim)r   )r   r   s   "€r   r   ÚLastLayer.__annotate__E  s#   ø€ ÷ ~ñ ~áð~ñ %(ñ~r   c	           	     óÌ   <€ \         S	V `  4        \        P                  ! 4       V n        VP                  W!^,          WVVR7      V n        VP                  WRWVVR7      V n        R# )r   r   F)rŒ   r   r    r!   N)	r"   r#   rD   rÙ   Úsilur$   ÚlinearrÛ   Únorm)
r&   rs   r  rŒ   r†   r   r    r!   r'   r(   s
   &&&&&&&&&€r   r#   ÚLastLayer.__init__E  s]   ø€ ô 	‰ÑÔÜ—G’G“IˆŒ	Ø ×'Ñ'Ð(BÐTUÕDUÐ\`Ðv|Ð'Ó}ˆŒØ×(Ñ(¨ÐPUÐ\`Ðv|Ð(Ó}ˆŽ	r   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   r^   Úconditioning_embeddingrL   rM   )r   r   s   "€r   r   r  S  s1   ø€ ÷ ñ ™Ÿ™ð ¹u¿|¹|ð ÑPU×P\ÑP\ñ r   c                óô   € V P                  V P                  V4      4      p\        P                  ! V^^R7      w  rE\        P                  ! VR,          V P                  V4      ^V,           R,          4      pV# )r   r¦   )r¨   Nr¨   )r  r  rN   rð   ræ   r  )r&   r^   r  Úembrj   ró   s   &&&   r   r.   ÚLastLayer.forwardS  s[   € Ø�k‰k˜$Ÿ)™)Ð$:Ó;Ó<ˆÜ—{’{ 3¨¨qÔ1‰ˆÜ�MŠM˜% 
Õ+¨T¯Y©Y°q«\¸AÀ½IÀzÕ;RÓSˆØˆr   )r  r  r  )Fr  TNNNr1   r9   s   @@r   r  r  D  s    ù‡ € ÷~õ ~÷÷ ð r   r  c                   ór   a a€ ] tR tRt oR	V3R lV 3R llltR
R ltRRR/ 3R ltRRR/ R3R ltRtVt	V ;t
# )ÚQwenImageTransformer2DModeliZ  Nc                óz   <€ V ^8„  d   QhRS[ RS[ RS[S[ ,          RS[ RS[ RS[ RS[ RS[ R	S[S[ S[ S[ 3,          /	# )
r   Ú
patch_sizerk   Úout_channelsÚ
num_layersrÔ   rÓ   Újoint_attention_dimrt   Úaxes_dims_rope)r   r   r   )r   r   s   "€r   r   Ú(QwenImageTransformer2DModel.__annotate__[  s   ø€ ÷ ;Rñ ;Ráð;Rñ ð;Rñ ™s•mð	;Rñ
 ð;Rñ  ð;Rñ !ð;Rñ !ð;Rñ  #ð;Rñ ™c¡3©˜mÕ,ñ;Rr   c                ó®  <€ \         SV `  4        Wàn        Wn        W n        T;'       g    TV n        We,          V n        W n        \        VR \        V	4      R7      V n
        \        V P                  VVVVVR7      V n        VP                  VRWïR7      V n        VP                  W P                  WïR7      V n        VP                  WpP                  WïR7      V n        \$        P&                  ! \)        V4       Uu. uF  p\+        V P                  VVVVVR7      NK   	  up4      V n        V P                  R8X  d'   V P/                  R\0        P2                  ! . 4      4       V'       df   \5        V P                  V P                  WïVR	7      V n        VP                  V P                  W,          V P
                  ,          R
WïR7      V n        R# R# u upi )i'  )r=   ÚthetaÚaxes_dim)rs   rt   rp   r    r!   r'   r  r�   r§   )r=   rÓ   rÔ   r    r!   r'   Úindex_timestep_zeroÚ__index_timestep_zero__)r    r!   r'   Tr   N)r"   r#   r    r!  rk   r"  rI   Údefault_ref_methodr   ÚlistÚpe_embedderrd   Útime_text_embedr�   Útxt_normr$   Úimg_inÚtxt_inrD   rE   ÚrangerÑ   Útransformer_blocksÚregister_bufferrN   Útensorr  Únorm_outÚproj_out)r&   r!  rk   r"  r#  rÔ   rÓ   r$  rt   r%  r,  Úimage_modelÚfinal_layerrp   r    r!   r'   Ú_r(   s   &&&&&&&&&&&&&&&&& €r   r#   Ú$QwenImageTransformer2DModel.__init__[  s­  ø€ ô& 	‰ÑÔØŒ
Ø$ŒØ&ÔØ(×7Ð7¨KˆÔØ,ÕAˆŒØ"4Ôä"Ð'9ÀÔQUÐVdÓQeÔfˆÔä9ØŸ.™.Ø"7Ø"7ØØØ!ô 
ˆÔð #×*Ñ*Ð+>ÀDÐPUÐ*ÓeˆŒØ ×'Ñ'¨·^±^È5Ð'Ó`ˆŒØ ×'Ñ'Ð(;¿^¹^ÐSXÐ'ÓhˆŒä"$§-¢-ô ˜:Ô&ó
1
ñ '�ô &Ø—N‘NØ$7Ø#5ØØØ%÷ñ 'ñ
1
ó 
#ˆÔð ×"Ñ"Ð&;Ô;Ø× Ñ Ð!:¼E¿LºLÈÓ<LÔMçÜ% d§n¡n°d·n±nÈEÐmwÔxˆDŒMØ&×-Ñ-¨d¯n©n¸jÕ>UÐX\×XiÑXiÕ>iÐptð  }BÐ-ó  RˆDŽMñ ùò
1
s   Ã<$Gc           	     óB  € VP                   w  rVrxp	V P                  p
\        P                  P                  P                  V^V P                  V P                  34      pVP                   pVP                  V^ ,          V^,          VR,          VR,          ^,          ^VR,          ^,          ^4      pVP                  ^ ^^^^^^4      pVP                  V^ ,          VR,          VR,          ^,          ,          VR,          ^,          ,          V^,          ^,          4      pTpWŠ^,          ,           V
,          pWš^,          ,           V
,          pW:^,          ,           V
,          pWJ^,          ,           V
,          p\        P                  ! WÞV^3VP                  R7      pV^8”  df   VR,          \        P                  ! ^ V^,
          WÑP                  VP                  R7      P                  ^4      P                  ^4      ,           VR&   MVR,          V,           VR&   VR	,          \        P                  ! W>^,
          V,           WáP                  VP                  R7      P                  ^4      P                  ^ 4      ,           V^,          ,
          VR	&   VR
,          \        P                  ! WO^,
          V,           WñP                  VP                  R7      P                  ^ 4      P                  ^ 4      ,           V^,          ,
          VR
&   V\        VRVR7      V3# )rZ   ©r!   )Ústepsr!   r    zt h w c -> b (t h w) c)Úbéýÿÿÿéþÿÿÿr[   )r¨   r¨   r¨   r   )r¨   r¨   r¨   rZ   )r¨   r¨   r¨   r   )r\   r!  ÚcomfyÚldmÚ
common_ditÚpad_to_patch_sizer¶   Úpermuter]   rN   ry   r!   Úlinspacer    rò   r   )r&   r^   ÚindexÚh_offsetÚw_offsetÚbsÚcÚtÚhÚwr!  r-   Ú
orig_shapeÚt_lenÚh_lenÚw_lenÚimg_idss   &&&&&            r   Úprocess_imgÚ'QwenImageTransformer2DModel.process_img˜  sc  € ØŸ™‰ˆˆq�QØ—_‘_ˆ
ÜŸ	™	×,Ñ,×>Ñ>¸qÀ1ÀdÇoÁoÐW[×WfÑWfÐBgÓhˆØ"×(Ñ(ˆ
Ø%×*Ñ*¨:°a­=¸*ÀQ½-ÈÐTVÍÐYcÐdfÕYgÐklÕYlÐnoÐq{Ð|~Õqð  DEõ  rEð  GHó  IˆØ%×-Ñ-¨a°°A°q¸!¸QÀÓBˆØ%×-Ñ-¨j¸­m¸ZÈ½^ÈzÐZ\Í~ÐabÕObÕ=cÐgqÐrtÕguÐyzÕgzÕ={ð  ~Hð  IJõ  ~Kð  NOõ  ~Oó  PˆØˆØ Q•Õ'¨JÕ6ˆØ Q•Õ'¨JÕ6ˆà¨q¥Õ1°jÕ@ˆØ¨q¥Õ1°jÕ@ˆä—+’+˜u¨U°AÐ6¸q¿x¹xÔHˆà�1Œ9Ø")¨*Õ"5¼¿ºÀqÈ%ÐRSÍ)Ð[`×iqÑiqÐyz÷  zAñ  zAô  9B÷  9Lñ  9Lð  MNó  9O÷  9Yñ  9Yð  Z[ó  9\õ  #\ˆG�JÒà")¨*Õ"5¸Õ"=ˆG�JÑà% jÕ1´E·N²NÀ8ÐUVÍYÐYaÕMaÐin×wÑwð  HI÷  HOñ  HOô  5P÷  5Zñ  5Zð  [\ó  5]÷  5gñ  5gð  hió  5jõ  jð  nsð  wxõ  nxõ  yˆ�
ÑØ% jÕ1´E·N²NÀ8ÐUVÍYÐYaÕMaÐin×wÑwð  HI÷  HOñ  HOô  5P÷  5Zñ  5Zð  [\ó  5]÷  5gñ  5gð  hió  5jõ  jð  nsð  wxõ  nxõ  yˆ�
ÑØœf WÐ.FÈ"ÔMÈzÐYÐYr   c           	     ó  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  ! WW4WVV3/ VB # rQ   )	rC  Úpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)	r&   r^   r{   Úcontextr¡   Úref_latentsÚadditional_t_condr±   rT   s	   &&&&&&&&,r   r.   Ú#QwenImageTransformer2DModel.forward²  s€   € Ü×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4´U×5LÑ5L×5WÑ5W×5gÑ5gÐi|Ó}ó
÷ ‰'ð	ð ˜w¸ÐXkñ	wð pvñ	wð 	wr   c	                ó@  a,a-a.€ Tp
TpVo-S-en   \         P                  ! S-4      '       gR   S-^,
          P                  VP                  4      \         P                  ! VP                  4      P
                  ,          o-V P                  V4      w  rÍpVP                  ^,          pR o.VEe·   . p^ p^ p^ pV	P                  RV P                  4      pVR8H  ;'       g    VR8H  pVR8H  pVR8H  pV EF)  pV'       d   V^,          p^ p^ pM V'       d   V^,          p^ p^ pMŠ^p^ p^ pVP                  R,          V,           VP                  R ,          V,           8”  d   TpMTp\        VVP                  R,          V,           4      p\        VVP                  R ,          V,           4      pV P                  VVVVR7      w  ppp\         P                  ! VV.^R7      p\         P                  ! VV.^R7      pVP                  VP                  ^,          4       EK,  	  V'       d*   V^ 8”  d#   \         P                  ! Wª^ ,          .^ R7      p
Vo.VP                  4       pVVR&   \        \        VP                  R ,          V P                  ^,          ,           V P                  ,          ^,          VP                  R,          V P                  ^,          ,           V P                  ,          ^,          4      4      p\         P                  ! VVVP                  ^,          ,           VP                   R7      P#                  ^R ^4      P%                  VP                  ^ ,          ^^4      pV P'                  V4      pV P)                  V4      pV P+                  V4      pV P-                  W¬V4      p VP                  R	/ 4      p!VP                  R
/ 4      p"V!P                  R/ 4      p#RV"9   dG   V"R,           F9  p$V$! RVRVRVRVRV/4      p%V%R,          pV%R,          pV%R,          pV%R,          pK;  	  \         P                  ! VV3^R7      p&V P/                  V&4      P                  VP                  4      P1                  4       p'?&??\3        V P4                  4      VR&   RVR&   \7        V P4                  4       Fû  w  p(o,V(VR&   RV(3V#9   d:   V,V-V.3R lp)V#RV(3,          ! RVRVRV RV'RV/RV)/4      p%V%R,          pV%R,          pMS,! VVS-V V'S.VR7      w  r¼RV"9   d5   V"R,           F'  p$V$! RVRVRVRV(RV/4      p%V%R,          pV%R,          pK)  	  Vf   KŸ  VP                  R4      p*V(\3        V*4      8  g   KÂ  V*V(,          p+V+f   KÑ  VRR V+P                  ^,          13;;,          V+,          uu&   Ký  	  S.e   V P9                  ^^ R7      ^ ,          p V P;                  VV 4      pV P=                  V4      pVRR V13,          P?                  V^ ,          VR!,          VR,          ^,          VR ,          ^,          V^,          ^^4      pVPA                  ^ ^^^^^^4      pVP#                  V4      RRRR VP                  R,          1R VP                  R ,          13,          # )"NÚref_latents_methodrI  r*  Únegative_index)rI  rJ  rK  r¦   Úreference_image_num_tokensr>  Úpatches_replacer¥   ÚditÚ
post_inputÚimgÚtxtrU  Útxt_idsr±   Útotal_blocksÚdoubleÚ
block_typeÚblock_indexÚdouble_blockc           
      ó‚   <€ / pS! V R ,          V R,          SV R,          V R,          SV R,          R7      w  VR&   VR &   V# )rl  rm  Úvecr«   r±   ©r-   rŸ   r    rý   r¢   ré   r±   © )rS   rÉ   Úblockr    ré   s   & €€€r   Ú
block_wrapÚ8QwenImageTransformer2DModel._forward.<locals>.block_wrap  s�   ø€ Ø�CÙ-2ÀÀeÅÐdhÐinÕdoð  Mgð  nrð  sxõ  nyð  LPð  QUõ  LVð  lð  UYð  Zoõ  Upô  .qÑ*�C˜‘J  E¡
Ø�Jr   ru  r«   Úoriginal_blockrv  r^   Úinputr¨   rB  r[   rA  )!rN   Úis_floating_pointrx   r    ÚfinfoÚmaxrV  r\   r´   r,  r¹   rG   rµ   Úroundr!  Úaranger!   r]   r   r1  r0  r2  r/  r.  r¸   Úlenr4  Ú	enumeraterð   r7  r8  r¶   rG  )/r&   r^   Ú	timestepsra  r¡   rb  rc  r±   ÚcontrolrT   r{   rŸ   r-   rU  rQ  Ú
num_embedsÚref_num_tokensrO  rP  rI  Ú
ref_methodÚindex_ref_methodÚnegative_ref_methodÚtimestep_zeroÚrefrJ  rK  ÚkontextÚkontext_idsr;  Ú	txt_startrn  rý   ri  r¥   Úblocks_replacerÈ   rÉ   Úidsr¢   Úiry  Ú	control_iÚaddrx  r    ré   s/   &&&&&&&&&,                                  @@@r   r\  Ú$QwenImageTransformer2DModel._forward¹  s  ú€ ð ˆØ 'ÐØ%3Ð"à%Ò1¼%×:QÒ:QÐRl×:mÒ:mØ*DÀqÕ*H×)LÑ)LÈQÏWÉWÓ)UÔX]×XcÒXcÐde×dkÑdkÓXl×XpÑXpÕ)pÐ&à-1×-=Ñ-=¸aÓ-@Ñ*ˆ 
Ø"×(Ñ(¨Õ+ˆ
à"ÐØÓ"ØˆNØˆAØˆAØˆEØŸ™Ð$8¸$×:QÑ:QÓRˆJØ *¨gÑ 5×_Ð_¸:ÐI^Ñ;^ÐØ",Ð0@Ñ"@ÐØ&Ð*?Ñ?ˆMÜ"�ß#Ø˜Q•J�EØ �HØ ‘Hß(Ø˜Q•J�EØ �HØ ‘Hà�EØ �HØ �HØ—y‘y •} qÕ(¨3¯9©9°R­=¸1Õ+<Ô<Ø#$™à#$˜Ü˜A˜sŸy™y¨�}¨xÕ7Ó8�AÜ˜A˜sŸy™y¨�}¨xÕ7Ó8�Aà*.×*:Ñ*:¸3ÀeÐV^ÐiqÐ*:Ó*rÑ'�˜ aÜ %§	¢	¨=¸'Ð*BÈÔ J�ÜŸ)š) W¨kÐ$:ÀÔB�Ø×%Ñ% g§m¡m°AÕ&6×7ñ/ #÷0 Ø˜1”9Ü$Ÿyšy¨(¸qµLÐ)AÀqÔI�HØ*4Ð'Ø"5×":Ñ":Ó"<ÐØ@NÐÐ <Ñ=äœ# §¡¨¥¨t¯©À!Õ/CÕ DÈÏÉÕXÐ]^Õ^Ðbc×biÑbiÐjlÕbmÐqu÷  rAñ  rAð  EFõ  rFõ  cGð  LP÷  L[ñ  L[õ  b[ð  `aõ  aaó  bó  cˆ	Ü—,’,˜y¨)°g·m±mÀAÕ6FÕ*FÈqÏxÉxÔX×`Ñ`ÐabÐdfÐhiÓj×qÑqÐrs×ryÑryÐz{Õr|Ð~ð  BCó  DˆàŸ™ MÓ2ˆØ $§¡Ð.CÓ DÐØ $§¡Ð,AÓ BÐà×#Ñ# HÐ=NÓOˆà-×1Ñ1Ð2CÀRÓHˆØ%×)Ñ)¨)°RÓ8ˆØ(×,Ñ,¨U°BÓ7ˆà˜7Ô"Ø˜\×*Ð*�Ù˜ ¨uÐ6KÈYÐX_ÐajÐlsð  vKð  M`ð  aó  b�Ø # E¥
�Ø(+¨E­
Ð%Ø˜i�.�Ø˜i�.’ñ +ô �iŠi˜ 'Ð*°Ô2ˆØ×+Ñ+¨CÓ0×3Ñ3°A·G±GÓ<×GÑGÓIÐØ�˜'ä.1°$×2IÑ2IÓ.JÐ˜NÑ+Ø,4Ð˜LÑ)Ü! $×"9Ñ"9Ö:‰HˆAˆuØ12Ð Ñ.Ø Ð" nÔ4÷ð % n°aÐ%8Ö9¸5À-ÐQVÐXmÐotÐvzð  }Að  CSð  Ujð  lð  ;@ð  CSð  U_ð  B`ó  a�Ø # E¥
�Ø(+¨E­
Ñ%á7<Ø"/Ø*?Ø/IØØ%5Ø(;Ø(;ô8Ñ4Ð%ð  Ô(Ø  ×0Ð0�AÙ˜U M°5Ð:OÐQTÐVWÐYfÐhið  lAð  CVð  Wó  X�CØ$'¨¥J�MØ,/°­JÒ)ñ 1ð
 Ô"Ø#ŸK™K¨Ó0�	Ø”s˜9“~Ö%Ø# A�,�CØ”Ø% a¨¨#¯)©)°A­,¨Ð&6×7¸3Õ>Õ7ñA ;ðD Ò*Ø—:‘:˜a Q�:Ó'¨Õ*ˆDàŸ™ m°TÓ:ˆØŸ™ mÓ4ˆà% a¨¨*¨ nÕ5×:Ñ:¸:Àa½=È*ÐUWÍ.ÐZdÐegÕZhÐlmÕZmÐoyÐz|Õo}ð  BCõ  pCð  EOð  PQõ  ERð  TUð  WXó  YˆØ%×-Ñ-¨a°°A°q¸!¸QÀÓBˆØ×$Ñ$ ZÓ0°°A°q¸,¸1¿7¹7À2½;¸,ÈÈÏÉÐQSÍÈÐ1TÕUÐUr   )r,  r    r1  rk   rI   r7  r"  r!  r.  r8  r/  r4  r2  r0  )r   rÎ   é   é<   é€   é   i   i   )r–  é8   rš  rI  NTFNNN)r   r   r   )r2   r3   r4   r5   r#   rV  r.   r\  r6   r7   r8   r9   s   @@r   r  r  Z  sO   ù‡ € ÷;Rõ ;RôzZð4 <@ÈTÐeið  @Bô wð ØØØØ÷DVô DVr   r  )"rN   Útorch.nnrD   Útorch.nn.functionalÚ
functionalr+   Útypingr   r   Úeinopsr   Úcomfy.ldm.lightricks.modelr   r   Úcomfy.ldm.modules.attentionr   r	   r
   Úcomfy.ldm.flux.layersr   Úcomfy.ldm.common_ditrC  Úcomfy.patcher_extensionÚcomfy.ldm.flux.mathr   ÚModuler   r;   rb   rd   r�   rÑ   r  r  rw  r   r   Ú<module>r§     s«   ðã Ý ß Ð ß "Ý ç Cß lÑ lÝ )Û Û Ý +ô	ˆ2�9‰9ô 	ô�"—)‘)ô ò<#ô §¡ô ô8r0�—	‘	ô r0ôjn4 §	¡	ô n4ôb�—	‘	ô ô,cV "§)¡)ö cVr   