+
    ƒ°ñjùp  ã                   óæ  € ^ RI t ^ RIt^ RIHt ^ RIHu Ht ^ RIt^ RI	t^ RI
t^ RIt^ RIt^ RIt^ RIt^ RIHtHt ^ RIHt ^ RIHt ^ RIHtHtHt ^ RIHtHt  ! R R]P:                  4      t ! R	 R
]P:                  4      t ! R R]P:                  4      t  ! R R]P:                  4      t! ! R R]P:                  4      t"R t#R t$R t% ! R R]P:                  4      t& ! R R]P:                  4      t' ! R R]&4      t( ! R R]P:                  4      t)/ R^ ^ 3R lt*/ 3R lt+R t, ! R R ]PZ                  4      t. ! R! R"]P:                  4      t/ ! R# R$4      t0 ! R% R&]P:                  4      t1R# )'é    N)ÚEmbedNDÚtimestep_embedding)Úapply_rope1)ÚTimestepEmbedding)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention)ÚPoseBranchCacheÚquantize_cachec                   ó<   a a€ ] tR t^t oRV 3R lltR tRtVtV ;t# )ÚZeroCenteredRMSNormc                ó’   <€ \         SV `  4        \        P                  ! \        P
                  ! WVR 7      4      V n        W n        R# )©ÚdtypeÚdeviceN)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚemptyÚweightÚeps)ÚselfÚdimr   r   r   Ú	__class__s   &&&&&€Ú@C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\qwen_image21\model.pyr   ÚZeroCenteredRMSNorm.__init__   s.   ø€ Ü‰ÑÔÜ—l’l¤5§;¢;¨sÈÔ#OÓPˆŒØŽó    c                ó:  € \         P                  P                  V P                  \        P
                  VP                  R 7      R,           p\         P                  P                  VP                  4       W P                  4      P                  VP                  4      # )r   g      ð?)ÚcomfyÚmodel_managementÚcast_tor   r   Úfloat32r   ÚrmsnormÚrms_normÚfloatr   Útor   )r   ÚxÚws   && r   ÚforwardÚZeroCenteredRMSNorm.forward   sd   € Ü×"Ñ"×*Ñ*¨4¯;©;¼e¿m¹mÐTU×T\ÑT\Ð*Ó]Ð`cÕcˆÜ�}‰}×%Ñ% a§g¡g£i°·H±HÓ=×@Ñ@ÀÇÁÓIÐIr   )r   r   )ç�íµ ÷Æ°>NN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r+   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   Ú__classdict__s   @@r   r   r      s   ù‡ € ÷÷
Jò Jr   r   c                   ó<   a a€ ] tR t^!t oRV 3R lltR tRtVtV ;t# )ÚTextProjectionc                ó¬   <€ \         SV `  4        \        WWER 7      V n        VP	                  WRWER7      V n        VP	                  W"RWER7      V n        R# )©r   r   r   F©Úbiasr   r   N)r   r   r   Ú	text_normÚLinearÚin_layerÚ	out_layer)r   Úin_dimÚhidden_sizer   r   r   Ú
operationsr   s   &&&&&&&€r   r   ÚTextProjection.__init__"   sP   ø€ Ü‰ÑÔÜ,¨VÀEÔYˆŒØ"×)Ñ)¨&ÀEÐQVÐ)ÓfˆŒØ#×*Ñ*¨;È%ÐW\Ð*ÓlˆŽr   c           	     óŒ   € V P                  \        P                  ! V P                  V P	                  V4      4      R R7      4      # )Útanh)Úapproximate)rA   ÚFÚgelur@   r>   ©r   r)   s   &&r   r+   ÚTextProjection.forward(   s/   € Ø�~‰~œaŸfšf T§]¡]°4·>±>À!Ó3DÓ%EÐSYÔZÓ[Ð[r   )r@   rA   r>   ©r-   NNNr.   r6   s   @@r   r9   r9   !   s   ù‡ € ÷m÷\ò \r   r9   c                   ó<   a a€ ] tR t^,t oRV 3R lltR tRtVtV ;t# )ÚTimestepProjEmbeddingsc           	     óN   <€ \         SV `  4        \        R VRW#VR7      V n        R# )é   F)Úin_channelsÚtime_embed_dimÚsample_proj_biasr   r   rD   N)r   r   r   Útimestep_embedder)r   Úembedding_dimr   r   rD   r   s   &&&&&€r   r   ÚTimestepProjEmbeddings.__init__-   s2   ø€ Ü‰ÑÔÜ!2¸sÐS`Ðsxð  AFð  blô  "mˆÖr   c                ór   € V P                  \        VP                  4       R 4      P                  V4      4      # )rQ   )rU   r   r'   r(   )r   Útimestepr   s   &&&r   r+   ÚTimestepProjEmbeddings.forward1   s.   € Ø×%Ñ%Ô&8¸¿¹Ó9IÈ3Ó&O×&RÑ&RÐSXÓ&YÓZÐZr   )rU   )NNNr.   r6   s   @@r   rO   rO   ,   s   ù‡ € ÷m÷[ò [r   rO   c                   ó<   a a€ ] tR t^5t oRV 3R lltR tRtVtV ;t# )ÚSwiGLUFeedForwardc                ó  <€ \         SV `  4        W0n        V'       d#   VP                  V^V,          RWER7      V n        M2VP                  WRWER7      V n        VP                  WRWER7      V n        VP                  W!RWER7      V n        R# )é   Fr<   N)r   r   Úfusedr?   Úgate_upÚprojÚ
gate_layerÚout)r   r   Ú
hidden_dimr_   r   r   rD   r   s   &&&&&&&€r   r   ÚSwiGLUFeedForward.__init__6   s‚   ø€ Ü‰ÑÔØŒ
ßà%×,Ñ,¨S°!°jµ.ÀuÐTYÐ,ÓiˆD�Là"×)Ñ)¨#ÀÈUÐ)ÓbˆDŒIØ(×/Ñ/°ÀeÐSXÐ/ÓhˆDŒOØ×$Ñ$ Z¸5ÈÐ$Ó]ˆŽr   c                ó.  € V P                   '       d;   \        P                  P                  V P                  V P                  V4      R 4      # V P	                  \        P                  ! V P                  V4      4      V P                  V4      ,          4      # )Úswiglu)
r_   r!   ÚopsÚlinear_input_actrc   r`   rI   Úsilurb   ra   rK   s   &&r   r+   ÚSwiGLUFeedForward.forwardA   s^   € Ø�:�:ˆ:Ü—9‘9×-Ñ-¨d¯h©h¸¿¹ÀQ»ÈÓRÐRØ�x‰xœŸš˜tŸ™¨qÓ1Ó2°T·Y±Y¸q³\ÕAÓBÐBr   )r_   rb   r`   rc   ra   )TNNNr.   r6   s   @@r   r\   r\   5   s   ù‡ € ÷	^÷Cò Cr   r\   c                   óB   a a€ ] tR t^Gt oRV 3R llt/ 3R ltRtVtV ;t# )Ú	Attentionc           
     ó°  <€ \         S	V `  4        \        4       V n        W n        W#,          pVP                  WR WVR7      V n        VP                  WR WVR7      V n        VP                  WR WVR7      V n        \        P                  ! VP                  W�R WVR7      .4      V n        VP                  W4WVR7      V n        VP                  W4WVR7      V n        R# )Fr<   r;   N)r   r   r   Úcomfy_attentionÚheadsr?   Úto_qÚto_kÚto_vr   Ú
ModuleListÚto_outÚRMSNormÚnorm_qÚnorm_k)
r   r   rp   Údim_headr   r   r   rD   Ú	inner_dimr   s
   &&&&&&&& €r   r   ÚAttention.__init__H   sÅ   ø€ Ü‰ÑÔÜ-Ó/ˆÔØŒ
ØÕ$ˆ	Ø×%Ñ% c¸5ÈÐ%Ó]ˆŒ	Ø×%Ñ% c¸5ÈÐ%Ó]ˆŒ	Ø×%Ñ% c¸5ÈÐ%Ó]ˆŒ	Ü—m’m Z×%6Ñ%6°yÈEÐY^Ð%6Ó%nÐ$oÓpˆŒØ ×(Ñ(¨À%Ð(ÓWˆŒØ ×(Ñ(¨À%Ð(ÓWˆŽr   c           
     óD  € VP                   w  rgpV P                  V4      P                  WgV P                  R4      p	V P	                  V4      P                  WgV P                  R4      p
V P                  V4      P                  WgV P                  R4      pVP                  R/ 4      P                  R. 4      p\        P                  P                  '       g
   V'       Eds   V P                  V	4      V P                  V
4      r©V'       Ed2   V	P                  ^^4      V
P                  ^^4      VP                  ^^4      VP                  ^^4      3w  ršr²/ VCRWG./CpV FX  pV! WšW²RVR7      pVP                  RV	4      VP                  RV
4      VP                  RV4      VP                  R	V4      3w  ršr²KZ  	  V	P                  ^^4      P                  4       V
P                  ^^4      P                  4       VP                  ^^4      P                  4       VP                  ^^4      P                  4       3w  ršr²\        W’4      p	\        W¢4      p
Mû\        P                  P!                  V P                  V	R
R7      w  ppp\        P                  P!                  V P                  V
R
R7      w  ppp\        P"                  P$                  P'                  WšVVVV P                  P(                  4      w  rš\        P                  P+                  V P                  VRV4       \        P                  P+                  V P                  VRV4       \-        V	4      \-        V
4      \-        V4      rºp	V P.                  ^ ,          ! V! WšW°P                  V P0                  R7      4      # )é   ÚpatchesÚattn1_patchÚ	img_sliceN)ÚpeÚ	attn_maskÚextra_optionsÚqÚkÚvr�   T)Úoffloadable)Úpreferred_attentionéÿÿÿÿ)Úshaperq   Úviewrp   rr   rs   Úgetr!   r"   Úin_trainingrw   rx   Ú	transposeÚ
contiguousr   rh   Úcast_bias_weightÚ	quant_opsÚckÚrms_roper   Úuncast_bias_weightr   ru   ro   )r   r)   r�   Úattn_fnÚ
prefix_lenÚtransformer_optionsÚBÚNÚ_r„   r…   r†   r~   rƒ   Úprc   Úq_scaleÚq_streamÚk_scaleÚk_streams   &&&&&&              r   r+   ÚAttention.forwardT   s  € à—'‘'‰ˆˆaØ�I‰I�a‹L×Ñ˜a D§J¡J°Ó3ˆØ�I‰I�a‹L×Ñ˜a D§J¡J°Ó3ˆØ�I‰I�a‹L×Ñ˜a D§J¡J°Ó3ˆØ%×)Ñ)¨)°RÓ8×<Ñ<¸]ÈBÓOˆÜ×!Ñ!×-×-Ð-·°Ø—;‘;˜q“> 4§;¡;¨q£>ˆqßˆwàŸk™k¨!¨QÓ/°·±¸QÀÓ1BÀAÇKÁKÐPQÐSTÓDUÐWY×WcÑWcÐdeÐghÓWiÐi‘��aØ UÐ#6Ð U¸ÀjÀ_Ñ U�Û �AÙ˜A !°dÈ-ÔX�CØ"%§'¡'¨#¨q£/°3·7±7¸3À³?ÀCÇGÁGÈCÐQRÃOÐUX×U\ÑU\Ð]aÐceÓUfÐ"f‘K�A˜!™Rñ !ð  Ÿk™k¨!¨QÓ/×:Ñ:Ó<¸a¿k¹kÈ!ÈQÓ>O×>ZÑ>ZÓ>\Ð^_×^iÑ^iÐjkÐmnÓ^o×^zÑ^zÓ^|ð  A÷  Kñ  Kð  LMð  OPó  Q÷  \ñ  \ó  ^ð  ^‘��aÜ˜AÓ"ˆAÜ˜AÓ"‰Aä#(§9¡9×#=Ñ#=¸d¿k¹kÈ1ÐZ^Ð#=Ó#_Ñ ˆG�Q˜Ü#(§9¡9×#=Ñ#=¸d¿k¹kÈ1ÐZ^Ð#=Ó#_Ñ ˆG�Q˜Ü—?‘?×%Ñ%×.Ñ.¨q°R¸À'È4Ï;É;Ï?É?Ó[‰DˆAÜ�I‰I×(Ñ(¨¯©°g¸tÀXÔNÜ�I‰I×(Ñ(¨¯©°g¸tÀXÔNÜ*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆØ�{‰{˜1Ž~™g a¨A¯z©zÈt×OcÑOcÔdÓeÐer   )ro   rp   rx   rw   rr   ru   rq   rs   rM   r.   r6   s   @@r   rm   rm   G   s   ù‡ € ÷
Xð GI÷ fô fr   rm   c                 óR   € V RR P                  ^4      V RR P                  ^4      3# )r}   Nr‰   )Ú	unsqueeze)r›   s   &r   Ú_split_rowsr£   q   s/   € àˆRˆSˆ6×Ñ˜AÓ  # 2 × 0Ñ 0°Ó 3Ð3Ð3r   c                 óü  € Vw  rV\         P                  P                  '       dU   V ! V4      p\        P                  ! VR RV13,          ^V,           ,          VR VR13,          ^V,           ,          .^R7      # \         P
                  P                  P                  WW@P                  4      pV'       dH   \         P
                  P                  P                  VR RV13,          WTV P                  4      VR RV13&   V# )ºNNNN©r   )	r!   r"   r�   r   Úcatr‘   r’   Úadalnr   )Únormr)   Úscaler–   ÚzeroÚs_prefixÚs_targetrc   s   &&&&&   r   Ú_modulated_normr®   v   sÐ   € àÑ€HÜ×Ñ×)×)Ð)Ù�1‹gˆÜ�yŠy˜#˜a  * ˜nÕ-°°XµÕ>ÀÀAÀzÁ{ÀNÕ@SÐWXÐ[cÕWcÕ@dÐeÐklÔmÐmÜ
�/‰/×
Ñ
×
"Ñ
" 1°·h±hÓ
?€CßÜ#Ÿo™o×0Ñ0×6Ñ6°q¸¸K¸Z¸K¸Õ7HÈ(ÐZ^×ZbÑZbÓcˆˆAˆ{�
ˆ{ˆNÑØ€Jr   c                 ó¾   € Vw  rEV R VR13,          P                  VR VR13,          V4       V'       d)   V R RV13,          P                  VR RV13,          V4       V # ©r¥   N)Úaddcmul_)r)   ÚyÚgater–   Úg_prefixÚg_targets   &&&&  r   Ú_gated_residualr¶   ‚   sb   € ØÑ€HØ€aˆ‰€nÕ×Ñ˜q  J¡K Õ0°(Ô;ßØ	ˆ!ˆ[ˆjˆ[ˆ.Õ×"Ñ" 1 Q¨¨¨ ^Õ#4°hÔ?Ø€Hr   c                   óB   a a€ ] tR t^Št oRV 3R llt/ 3R ltRtVtV ;t# )ÚQwenImage21TransformerBlockc
           
     óè   <€ \         S
V `  4        V	P                  VR WWVR7      V n        \	        WW5WxV	R7      V n        V	P                  VR WWVR7      V n        \        WV,          WgW‰R7      V n        R# )F)Úelementwise_affiner   r   r   ©r   r   r   rD   )r_   r   r   rD   N)	r   r   Ú	LayerNormÚ	img_norm1rm   ÚattnÚ	img_norm2r\   Úimg_mlp)r   r   Únum_attention_headsÚattention_head_dimÚ	mlp_ratior   Ú	fused_mlpr   r   rD   r   s   &&&&&&&&&&€r   r   Ú$QwenImage21TransformerBlock.__init__‹   s|   ø€ Ü‰ÑÔØ#×-Ñ-¨cÀeÐQTÐjpÐ-ÓqˆŒÜ˜cÐ8JÐ[`ð  }Gô  HˆŒ	Ø#×-Ñ-¨cÀeÐQTÐjpÐ-ÓqˆŒÜ(¨°I­oÀYÐdjô  CˆŽr   c                ó8  € Vw  rxršp\        WP                  \        V P                  WW[4      W4WV4      W…4      p\        WP	                  \        V P
                  WW[4      4      W¥4      pVP                  \        P                  8X  d   VP                  RR 4       V# )iàÿ  i  ÿÿ)
r¶   r¾   r®   r½   rÀ   r¿   r   r   Úfloat16Úclamp_)r   r)   Úmodr�   r•   r–   r—   Úscale1Úgate1Úscale2Úgate2r«   s   &&&&&&&     r   r+   Ú#QwenImage21TransformerBlock.forward’   s‰   € Ø-0Ñ*ˆ�v dÜ˜AŸy™y¬¸¿¹ÈÐT^Ó)eÐgiÐt~ó   Uð  W\ó  iˆÜ˜AŸ|™|¬O¸D¿N¹NÈAÐWaÓ,hÓiÐkpÓ}ˆØ�7‰7”e—m‘mÔ#Ø�H‰H�V˜UÔ#Øˆr   )r¾   rÀ   r½   r¿   )é   r-   TNNNr.   r6   s   @@r   r¸   r¸   Š   s   ù‡ € ÷Cð LN÷ ô r   r¸   c                   ó<   a a€ ] tR t^›t oRV 3R lltR tRtVtV ;t# )Ú	LastLayerc                óˆ   <€ \         SV `  4        VP                  WR W4R7      V n        VP	                  WR W4R7      V n        R# )Fr<   )rº   r   r   N)r   r   r?   Úlinearr¼   r©   )r   r   r   r   r   rD   r   s   &&&&&&€r   r   ÚLastLayer.__init__�   sA   ø€ Ü‰ÑÔØ ×'Ñ'¨°uÀEÐ'ÓYˆŒØ×(Ñ(¨ÀeÐSXÐ(ÓhˆŽ	r   c                óœ  € V P                  \        P                  ! V4      4      P                  ^4      p\        P
                  P                  '       d    V P                  V4      ^V,           ,          # \        P                  P                  P                  W\        P                  ! VR,          4      V P                  P                  4      # )r}   ºNr}   N)rÓ   rI   rj   r¢   r!   r"   r�   r©   r‘   r’   r¨   r   Ú
zeros_liker   )r   r)   Útembrª   s   &&& r   r+   ÚLastLayer.forward¢   s‚   € Ø—‘œAŸFšF 4›LÓ)×3Ñ3°AÓ6ˆÜ×!Ñ!×-×-Ð-Ø—9‘9˜Q“< 1 u¥9Õ-Ð-Ü�‰×!Ñ!×'Ñ'¨´%×2BÒ2BÀ5ÈÅ9Ó2MÈtÏyÉyÏ}É}Ó]Ð]r   )rÓ   r©   rM   r.   r6   s   @@r   rÑ   rÑ   ›   s   ù‡ € ÷i÷
^ò ^r   rÑ   c                   ó6   a a€ ] tR t^©t oRV 3R lltRtVtV ;t# )ÚQwenImage21FunControlBlockc                ó    <€ \         SV `  WW4WVW‰V
R 7	       V'       d   V
P                  WW‰R7      V n        V
P                  WW‰R7      V n        R# )©r   r   rD   r   N)r   r   r?   Úbefore_projÚ
after_proj)r   r   rÁ   rÂ   rÃ   r   rÄ   Úfirstr   r   rD   r   s   &&&&&&&&&&&€r   r   Ú#QwenImage21FunControlBlock.__init__ª   sX   ø€ Ü‰Ñ˜Ð3EÐRUÐhmð  JTÐô  	UßØ)×0Ñ0°ÀÐ0ÓVˆDÔØ$×+Ñ+¨C¸EÐ+ÓQˆŽr   )rß   rÞ   )rÏ   r-   TFNNN©r/   r0   r1   r2   r   r3   r4   r5   r6   s   @@r   rÛ   rÛ   ©   s   ù‡ € ÷R÷ Rr   rÛ   c                   óH   a a€ ] tR t^±t oRV 3R lltR t/ 3R ltRtVtV ;t	# )ÚQwenImage21FunControlc                óø   <€ \         SV `  4        V
P                  W#W‰R 7      V n        \        P
                  ! \        V4       Uu. uF  p\        W3V,          WEWgV^ 8H  W‰V
R7
      NK   	  up4      V n        R# u upi )r   )rà   r   r   rD   N)	r   r   r?   Úcontrol_img_inr   rt   ÚrangerÛ   Úcontrol_blocks)r   Ú
num_blocksÚcontrol_in_dimrz   rÂ   rÃ   r   rÄ   r   r   rD   Úir   s   &&&&&&&&&&& €r   r   ÚQwenImage21FunControl.__init__³   s”   ø€ Ü‰ÑÔØ(×/Ñ/°ÐQVÐ/ÓfˆÔÜ Ÿmšmä˜:Ô&ó-
á&�ô ' yÐ?QÕ2QÐSeÐruð  IJð  NOñ  IOð  W\ð  xB÷  CÙ&ñ-
ó ˆÖùò -
s   Á$A7c                ó°   € \         P                  ! V4      pV P                  V4      VR VR13&   V P                  ^ ,          P	                  V4      V,           # r°   )r   r×   ræ   rè   rÞ   )r   r)   Úcontrolr–   Újoints   &&&& r   Úinit_streamÚ!QwenImage21FunControl.init_stream»   sN   € ä× Ò  Ó#ˆØ $× 3Ñ 3°GÓ <ˆˆa�‘ˆnÑØ×"Ñ" 1Õ%×1Ñ1°%Ó8¸1Õ<Ð<r   c                ó`   € V P                   V,          pV! W#WEWg4      pW(P                  V4      3# ©N)rè   rß   )	r   ÚindexÚcrÉ   r�   r•   r–   r—   Úblocks	   &&&&&&&& r   ÚstepÚQwenImage21FunControl.stepÁ   s4   € Ø×#Ñ# EÕ*ˆÙ�!˜" zÓGˆØ×"Ñ" 1Ó%Ð%Ð%r   )rè   ræ   )
é   é�   é   é€   rÏ   r-   TNNN)
r/   r0   r1   r2   r   rð   r÷   r3   r4   r5   r6   s   @@r   rä   rä   ±   s    ù‡ € ÷ò=ð PR÷ &ô &r   rä   c                 ó(   a aaaa€ RVVVV V3R llpV# )Nc                 óÎ  <€ V P                  4       VP                  4       VP                  4       r!p SeA   SP                  S\        P                  ! VRR S13,          VRR S13,          .^R7      4       . p\	        S4       F°  w  pw  rxp	\        X RWx13,          P                  ^4      4      p
\        XRR V13,          P                  ^4      4      p\        XRR V13,          P                  ^4      4      pV\        S4      ^,
          8X  d   ? ??VP                  \        W«WÃV	SVR7      4       K²  	  \        V4      ^8”  d   \        P                  ! V^R7      # V^ ,          # )Nr¥   r¦   )Úmaskr—   rˆ   )ÚtakeÚputr   ÚstackÚ	enumerater   ÚflattenÚlenÚappendr	   r§   )r„   r…   r†   rp   rˆ   Úoutsrë   ÚstartÚendrÿ   ÚqueryÚkeyÚvalueÚblock_indexÚcacher–   Úsegmentsr—   s   &&&&&        €€€€€r   r¾   Ú$block_causal_attention.<locals>.attnÉ   sB  ø€ Ø—&‘&“(˜AŸF™F›H a§f¡f£hˆaˆØÒà�I‰I�k¤5§;¢;°°!°[°j°[°.Õ0AÀ1ÀQÈÈÈÀ^ÕCTÐ/UÐ[\Ô#]Ô^ØˆÜ%.¨xÖ%8Ñ!ˆAÑ!�˜DÜ,¨Q¨q°%°)¨|­_×-DÑ-DÀQÓ-GÓHˆEÜ*¨1¨Q°°°¨W­:×+=Ñ+=¸aÓ+@ÓAˆCÜ,¨Q¨q°$°3°$¨w­Z×-?Ñ-?ÀÓ-BÓCˆEØ”C˜“M AÕ%Ô%Ø�q˜!Ø�K‰KÔ+¨E¸È4Ðexð  Obô  cö  dñ &9ô *-¨T«°Q¬Œu�yŠy˜ 1Ô%ÐC¸DÀ½GÐCr   ró   © )r  r—   r  r  r–   r¾   s   fffff r   Úblock_causal_attentionr  Ç   s   ü€ ÷Dó Dð €Kr   c                 ó    a aa€ RV VV3R llpV# )Nc           	      ór  <€ \        V P                  4       P                  ^4      4      p \        \        P                  ! SVP                  4       .^R7      P                  ^4      4      p\        \        P                  ! SVP                  4       .^R7      P                  ^4      4      p\        WW#SVR7      # )r^   r¦   )r—   rˆ   )r   r   r  r   r§   r	   )r„   r…   r†   rp   rˆ   Úprefix_kÚprefix_vr—   s   &&&&&€€€r   r¾   Ú%prefix_cached_attention.<locals>.attnÜ   s“   ø€ Ü$ Q§V¡V£X×%5Ñ%5°aÓ%8Ó9ˆÜ$¤U§Y¢Y°¸!¿&¹&»(Ð/CÈÔ%K×%SÑ%SÐTUÓ%VÓWˆÜ$¤U§Y¢Y°¸!¿&¹&»(Ð/CÈÔ%K×%SÑ%SÐTUÓ%VÓWˆÜ" 1¨ÐGZð  qDô  Eð  	Er   ró   r  )r  r  r—   r¾   s   fff r   Úprefix_cached_attentionr  Ú   s   ú€ ÷Eñ Eð
 €Kr   c           	      ó¢  € VP                   ^ ,          VP                   ^,          \        V4      .\        V P                   RR 4      ,           \        V4      ,           V UUu. uF  qDP                   RR  F  qUNK  	  K  	  upp,           p\        P                  ! V\        P
                  VP                  R7      P                  VP                   ^ ,          R4      p\        P                  ! WaP                  4       P                  ^4      .V Uu. uF!  qDP                  4       P                  ^4      NK#  	  up,           ^R7      # u uppi u upi )r   Nr   r¦   éþÿÿÿr‰   )rŠ   r  Úlistr   Útensorr$   r   Úexpandr§   r'   r  )r)   ÚcontextÚrefsÚslotsÚrÚsÚheaders   &&&&   r   Úprefix_cache_keyr$  ä   s&  € ð �m‰m˜AÕ §¡¨aÕ 0´#°d³)Ð<¼tÀAÇGÁGÈBÈCÀLÓ?QÕQÔTXÐY^ÓT_Õ_Ñnrô  cJÑnrÐij÷  }Dñ  }Dð  EGð  EHó  }IÐwxÒcdñ  }IÑcdÑnrò  cJõ  J€FÜ�\Š\˜&¬¯©¸g¿n¹nÔM×TÑTÐU\×UbÑUbÐcdÕUeÐgiÓj€FÜ�9Š9�fŸm™m›o×5Ñ5°aÓ8Ð9Ñ[_Ó<`Ñ[_ÐVW¿W¹W»Y×=NÑ=NÈqÖ=QÑ[_Ñ<`Õ`ÐfgÔhÐhùó cJùâ<`s   Á"EÄ'E
c                   ó2   a a€ ] tR t^ít oV 3R ltRtVtV ;t# )ÚPrefixCacheSlotc           	     óÔ  <€ \        \        P                  ! VR VR7      V4      w  rVVe,   VR8X  d   RMRp\        P                  P                  WWV4      p\        P                  P                  V4      p\        S
T `%  \        V4       U	u. uF2  p	\        P                  P                  V3\        P                  4      NK4  	  up	4       WPn        Wn        W@n        RV n        R# u up	i )Úmeta©r   r   NÚint4ÚTensorCoreConvRotW4A4LayoutÚTensorWiseINT8Layout)r   r   r   r!   r‘   ÚQuantizedTensorÚmemory_managementÚvram_aligned_sizer   r   rç   rh   ÚWritableWeightÚuint8ÚtemplateÚcache_shapeÚcache_dtyper  )r   rŠ   ré   r   r4  r2  ÚparamsÚlayoutÚsizerš   r   s   &&&&&     €r   r   ÚPrefixCacheSlot.__init__î   s½   ø€ Ü)¬%¯+ª+°eÀFÐRWÔ*XÐZeÓfÑˆØÒØ6AÀVÔ6KÑ2ÐQgˆFÜ—‘×6Ñ6°xÈÓPˆHÜ×&Ñ&×8Ñ8¸ÓBˆÜ‰ÑÔRWÐXbÔRcÓdÑRcÈQœ%Ÿ)™)×2Ñ2°D°7¼E¿K¹KÖHÑRcÑdÔeØ ŒØ ÔØ&ÔØˆŽùò	 es   Â8C%)r4  r3  r  r2  râ   r6   s   @@r   r&  r&  í   s   ù‡ € ÷
õ 
r   r&  c                   ó<   a a€ ] tR t^ût oRtV 3R ltR tRtVtV ;t	# )ÚKVCacheModelTc                óZ   <€ \         SV `  4        \        P                  ! V4      V n        R # ró   )r   r   r   rt   r   )r   r   r   s   &&€r   r   ÚKVCacheModel.__init__þ   s   ø€ Ü‰ÑÔÜ—]’] 5Ó)ˆŽ
r   c                ó\   € V P                    UUu. uF  q F  q"NK  	  K  	  upp. 3# u uppi ró   ©r   )r   ÚslotÚlayers   &  r   Úget_dynamic_vram__unitsÚ$KVCacheModel.get_dynamic_vram__units  s+   € Ø"&§*¢*Ô?¡*˜$º$°’¹$‘¡*Ò?ÀÐCÐCùÓ?s   �(r>  )
r/   r0   r1   r2   Úcomfy_writabler   rA  r3   r4   r5   r6   s   @@r   r:  r:  û   s   ù‡ € Ø€Nõ*÷Dò Dr   r:  c                   óf   a € ] tR tRt o RtR tR tR t]P                  R 4       t
R tR tR	tV tR
# )ÚPrefixCachei  zUForward-local views; every layer of a selected prefix is retained or filled together.c                ó   € Wn         R # ró   r>  )r   r   s   &&r   r   ÚPrefixCache.__init__  s   € ØŽ
r   c                ó:  € TP                  V P                   Uu. uF  q"P                  ^ ,          NK  	  up4      p\        ;QJ d0    R \	        V P                  V4       4       F  '       d   K   R# 	  R# ! R \	        V P                  V4       4       4      # u upi )r   c              3   ó”   "  € T F>  w  rVP                   R J;'       d"    \        P                  ! VP                   V4      x € K@  	  R # 5iró   )r  r   Úequal)Ú.0r?  r…   s   &  r   Ú	<genexpr>Ú%PrefixCache.filled.<locals>.<genexpr>  s7   é € ÐjÑTiÉÈ�4—8‘8 4Ð'×DÐD¬E¯KªK¸¿¹À!Ó,DÔDÓTiùs
   ‚Až*AFT)Úsplitr   r3  ÚallÚzip)r   r  r?  Úkeyss   &&  r   ÚfilledÚPrefixCache.filled  s~   € Ø�y‰y¸$¿*º*ÓE¹*°$×*Ñ*¨1×-Ð-¹*ÑEÓFˆß‹sÑjÔTWÐX\×XbÑXbÐdhÔTiÓj�sŒsÐjŠsÐjˆsÑjÔTWÐX\×XbÑXbÐdhÔTiÓjÓjÐjùò Fs   šBc           
     óê   € \        V P                  TP                  V P                   Uu. uF  q"P                  ^ ,          NK  	  up4      4       F  w  r4VP	                  4       Vn        K  	  R# u upi ©r   N)rP  r   rN  r3  Úcloner  )r   r  r"  r?  r…   s   &&   r   ÚfinishÚPrefixCache.finish  sP   € Ü˜4Ÿ:™: s§y¡yÈDÏJÊJÓ1WÉJÀq·-±-À×2BÐ2BÉJÑ1WÓ'XÖY‰GˆDØ—w‘w“yˆDŽHó ZùÒ1Ws   ªA0c           
   #  ó  "  € \         P                  ! 4       ;_uu_ 4       pV P                   Uu. uF?  qUV,          P                  '       g   K  \	        WQ,          R 4      '       d   K7  WQ,          NKA  	  ppV'       dJ   \
        P                  P                  Wb4       VP                  \
        P                  P                  W4       . V n
        V P                   Fy  pWQ,          pVP                  VP                  W#R7      4      w  r‰V P                  P                  \
        P                  P                  VP                   .V4      ^ ,          4       K{  	   Rx € V =
 RRR4       R# u upi   T =
i ; i  + '       g   i     R# ; i5i)Ú	_prefetch©ÚwriteN)Ú
contextlibÚ	ExitStackr   Úcomfy_cast_weightsÚhasattrr!   Úmodel_prefetchÚpin_modulesÚcallbackÚcleanup_prefetched_modulesÚvaluesÚenter_contextÚaccessr  r.  Úinterpret_gathered_liker2  )
r   rë   r   r\  r  r?  Úunprefetchedr@  r  rš   s
   &&&&      r   r@  ÚPrefixCache.layer  s  é € ä×!Ò!×#Ô# uØ04·
²
Ó±
¨À1½g×>XÕ>XœGÔahÐimÕipÐr}×a~œG˜DŸG˜G±
ˆLÐßÜ×$Ñ$×0Ñ0°ÔFØ—‘œu×3Ñ3×NÑNÐPTÔcØˆDŒKØŸ
œ
�Ø��Ø ×.Ñ.¨u¯|©|¸F¨|Ó/PÓQ‘�Ø—‘×"Ñ"¤5×#:Ñ#:×#RÑ#RÐTX×TaÑTaÐSbÐdiÓ#jÐklÕ#mÖnñ #ð Ûà‘K÷ $Ñ#ùÚøð ‘Kú÷ $×#Ð#üsR   ‚F	 E5¯E)ÁE)Á(E)Á4C#E5ÅE.ÅE5Å
F	Å)E5Å.E2Å2E5Å5F	Æ 	F	c                ó¦  € \        V P                  V P                  TP                  V P                   Uu. uF  q3P                  ^ ,          NK  	  up4      4       Fm  w  rEp\        WdP                  4      w  rgVf   VP                  V4       K4  VP                  \        P                  P                  WeP                  V4      4       Ko  	  R# u upi rU  )rP  r   re  rN  r3  r   r4  Úcopy_r!   r‘   r-  Ú_layout_cls)r   rë   r  r"  r?  ÚdestÚtr5  s   &&&     r   r  ÚPrefixCache.put$  s˜   € Ü  §¡¨T¯[©[¸&¿,¹,Ðbf×blÒblÓGmÑblÐ]^ÏÉÐVW×HXÐHXÑblÑGmÓ:nÖo‰MˆD˜Ü& q×*:Ñ*:Ó;‰IˆAØŠ~Ø—
‘
˜1–à—
‘
œ5Ÿ?™?×:Ñ:¸1×>NÑ>NÐPVÓWÖXó pùÒGms   µCc                ó–  € \        V P                  V P                  4       UUu. uFi  w  rV\        V\        P
                  P                  4      '       d   VP                  4       MTP                  VP                  4      P                  V4      NKk  	  ppp\        V4      ^8X  d
   V^ ,          # \        P                  ! V4      # u uppi )r}   )rP  r   re  Ú
isinstancer!   r‘   r-  Ú
dequantizeÚreshaper3  r(   r  r   r§   )r   rë   r   r   Ú
batch_sizer?  ro  re  s   &&&&&   r   r   ÚPrefixCache.take,  sÍ   € ô PSð  TX÷  T^ñ  T^ð  `d÷  `kñ  `kô  Plô  mñ  Plñ  ELð  EI¤Z°´5·?±?×3RÑ3R×%SÒ%S�1—<‘<”>ÐYZ×cÑcÐdh×dtÑdtÓu×xÑxÐy~Öñ  Plˆñ  mÜ ›K¨1Ô,ˆv�a�yÐC´%·)²)¸FÓ2CÐCùó ms   ¤A/C)r   re  N)r/   r0   r1   r2   Ú__doc__r   rR  rW  r]  Úcontextmanagerr@  r  r   r3   r4   )r7   s   @r   rE  rE    sD   ø‡ € Ù_òòkò!ð ×Ññ ó ð ò Y÷Dð Dr   rE  c                   óp   a a€ ] tR tRt oRV 3R lltR tR tR tR tRR/ 3R lt	RR/ R3R	 lt
R
tVtV ;t# )ÚQwenImage21Transformer2DModeli1  Nc                óB  <€ \         SV `  4        WÀn        W n        WT,          V n        \        VR \        V4      R7      V n        \        V P                  WÍVR7      V n	        \        W`P                  WœWÞR7      V n        VP                  WP                  RWÍR7      V n        \        P                  ! \        P                   ! 4       VP                  V P                  ^V P                  ,          RWÍR7      4      V n        \        P$                  ! \'        V4       Uu. uF  p\)        V P                  WTWyW¬WÞR7	      NK  	  up4      V n        \-        V P                  WœWÞR7      V n        VP                  V P                  VRWÍR7      V n        RV n        RV n        R# u upi )i'  )r   ÚthetaÚaxes_dimrÝ   r»   Fr<   )rÃ   r   rÄ   r   r   rD   N)r   r   r   Úout_channelsrz   r   r  Úpe_embedderrO   Útime_text_embedr9   Útxt_inr?   Úimg_inr   Ú
SequentialÚSiLUÚ
modulationrt   rç   r¸   Útransformer_blocksrÑ   Únorm_outÚproj_outÚprefix_cacheÚprefix_cache_enabled)r   rR   r~  Ú
num_layersrÂ   rÁ   Úcontext_in_dimrÃ   Úaxes_dims_roper   rÄ   Úimage_modelr   r   rD   rš   r   s   &&&&&&&&&&&&&&& €r   r   Ú&QwenImage21Transformer2DModel.__init__2  ss  ø€ ô" 	‰ÑÔØŒ
Ø(ÔØ,ÕAˆŒä"Ð'9ÀÔQUÐVdÓQeÔfˆÔÜ5°d·n±nÈEÐmwÔxˆÔÜ$ ^·^±^ÈÐbhô  AˆŒØ ×'Ñ'¨·^±^È%ÐW\Ð'ÓlˆŒô Ÿ-š-¬¯ª«	°:×3DÑ3DÀTÇ^Á^ÐUVÐY]×YgÑYgÕUgÐnsð  |AÐ3Dó  4Qó  RˆŒä"$§-¢-ä˜:Ô&ó1
á&�ô (¨¯©Ð8KÐktð  JSð  io÷  GÙ&ñ1
ó #ˆÔô
 " $§.¡.°cÈvÔmˆŒØ"×)Ñ)¨$¯.©.¸,ÈUÐZ_Ð)ÓoˆŒð !ˆÔØ$)ˆÖ!ùò1
s   Ä#Fc                óp   € V P                   e   V P                   P                  4        R V n         Wn        R # ró   )r‰  ÚfreerŠ  )r   Úenableds   &&r   Úreset_prefix_cacheÚ0QwenImage21Transformer2DModel.reset_prefix_cache\  s.   € Ø×ÑÒ(Ø×Ñ×"Ñ"Ô$Ø ˆÔØ$+Ö!r   c                ó4  € VP                  R / 4      pVP                  R/ 4      P                  R4      ;'       gD    VP                  R4      ;'       g+    VP                  R4      ;'       g    VP                  R4      pV P                  ;'       d    V'       * # )r~   Úpatches_replaceÚditÚ
post_inputÚsingle_blockr   )rŒ   rŠ  )r   r—   r~   Úhookeds   &&  r   Úcan_cache_prefixÚ.QwenImage21Transformer2DModel.can_cache_prefixb  s“   € à%×)Ñ)¨)°RÓ8ˆØ%×)Ñ)Ð*;¸RÓ@×DÑDÀUÓK÷ Oð OÈwÏ{É{Ð[gÓOh÷ Oð OØ—[‘[ Ó0÷Oð OØ4;·K±KÀÓ4Nð 	à×(Ñ(×7Ð7°¬ZÐ7r   c                óŽ  € VP                  R 4      R8X  d   R# V P                  pVe?   VP                  VRR7      '       d&   WUP                  \	        V P
                  4      4      3# VP                  RR4      pVR^R^/P                  V^4      ,          pVfµ   VP                  R R4      pVR8X  dk   V P                  P                  V4      ^V,          8”  d   TpM_\        P                  P                  VRR	7      '       d   \        P                  ! R
4      pM!R# VR8X  d   TM\        P                  ! R
4      p\        WvR7      ;qPn        VP                  V8X  d+   V P                  P                  V4      ^V,          8”  g-    R# \        P                  P                  VRR	7      '       g   R# VP                  V4       VR3# )r   ÚoffF)Úcreater   ÚdefaultÚint8r*  Úauto)Úevict_activeÚcpuÚgpu)Ústore_devicer   )NF)rŒ   r‰  ÚselectrR  r  r†  Úcurrent_patcherÚget_free_memoryr!   r"   Úensure_pin_budgetr   r   r
   r¦  )r   r  Úcache_bytesr   Úoptionsr  r   Ústores   &&&&&   r   Úselect_prefix_cacheÚ1QwenImage21Transformer2DModel.select_prefix_cachei  s±  € à�;‰;�xÓ  EÔ)ØÐØ×!Ñ!ˆØÒ §¡¨c¸% ×!@Ò!@ØŸ,™,¤s¨4×+BÑ+BÓ'CÓDÐDÐDØ—‘˜G YÓ/ˆØ˜  F¨AÐ.×2Ñ2°5¸!Ó<Õ<ˆØŠ=Ø—K‘K ¨&Ó1ˆEØ˜Œà×'Ñ'×7Ñ7¸Ó?À!ÀkÅ/ÔQØ"‘EÜ×+Ñ+×=Ñ=¸kÐX]Ð=×^Ò^Ü!ŸLšL¨Ó/‘Eà&Ð&à"'¨5¤.™´e·l²lÀ5Ó6I�Ü(7ÀUÔ(XÐXˆEÔ%ØRW×RdÑRdÐhnÔRn�×$Ñ$×4Ñ4°VÓ<¸qÀ;½ÕNàÐô uz÷  uKñ  uK÷  u]ñ  u]ð  ^ið  x}ð  u]÷  u~ò  u~àÐØ�‰�SÔØ�eˆ|Ðr   c                óx  € V P                  V4      pWEP                  ^,          .\        V4      ,          ,           R\        V4       p^ .V,           VP                  ^,          .,           p. . . r©p^ ^ rË\        \        VRR VR,          4      W1.,           4       EF.  w  w  rÞpWí,
          pV^ 8”  dõ   VP	                  VRWÞ13,          4       V	P	                  \
        P                  ! W»V,           VP                  \
        P                  R7      P                  ^4      P                  V^4      4       V
P	                  WÌV,           \
        P                  ! VVV,           3\
        P                  VP                  R7      P                  V4      34       VV,          pVV,          pVP                  R	R w  ppVP	                  V P                  VP                  ^4      P!                  ^^4      4      4       \
        P                  ! VVP                  \
        P                  R7      VV^,          ,
          ,
          RV^,          VP                  R	,          ^,          ,
          ,          ,           p\
        P                  ! VVP                  \
        P                  R7      VV^,          ,
          ,
          RV^,          VP                  R,          ^,          ,
          ,          ,           pV	P	                  \
        P"                  ! \
        P$                  ! VV3W±P                  \
        P                  R7      VR
,          P                  VV4      VR,          P                  VV4      .RR7      P                  ^ ^4      4       V
P	                  WÌVV,          ,           R34       V\'        VV4      ,          pVVV,          ,          pEK1  	  V P)                  \
        P*                  ! V	^ R7      P                  ^ 4      4      P!                  ^^4      P-                  4       p\
        P*                  ! V^R7      VV
3# )r}   N:r}   NNr¥   r)  r   g      à?r¦   r‰   r  r°   )Nr¥   )r�  rŠ   r  rP  r  r   Úaranger   r$   r¢   r  ÚonesÚboolÚtrilr‚  r  rŽ   r  ÚfullÚmaxr  r§   r�   )r   r)   r  Úref_latentsÚimage_slotsÚtxtr   ÚboundsÚpartsÚidsr  ÚposÚlengthr  r	  ÚimgÚnÚhr*   ÚhhÚwwr�   s   &&&&&                 r   Úbuild_sequenceÚ,QwenImage21Transformer2DModel.build_sequence…  sv  € à�k‰k˜'Ó"ˆØ§	¡	¨!¥˜~´°KÓ0@Õ@Õ@ÐBSÄ3À{ÓCSÐTˆØ��u• §	¡	¨!¥˜~Õ-ˆà! 2 r�HˆØ˜ˆVÜ!$¤S¨°°¨°f¸RµjÓ%AÀ;ÐQTÕCT×!UÑ‰LˆU˜#Ø•ˆAØ�1ŒuØ—‘˜S  E I Õ.Ô/Ø—
‘
œ5Ÿ<š<¨°1­W¸Q¿X¹XÌUÏ]É]Ô[×eÑeÐfgÓh×oÑoÐpqÐstÓuÔvØ—‘ °!­´U·Z²ZÀÀFÈQÅJÀÔW\×WaÑWaÐjk×jrÑjrÔ5s×5xÑ5xÐyó  6Að  !Bô  CØ�q•�Ø˜!•�Ø—9‘9˜R˜S�>‰DˆAˆqØ�L‰L˜Ÿ™ S§[¡[°£^×%=Ñ%=¸aÀÓ%CÓDÔEä—’˜a¨¯©¼¿¹ÔFÈ!ÈaÐSTÍfÍ*ÕUÐX[Ð_`ÐcdÕ_dÐgh×gnÑgnÐoqÕgrÐuvÕgvÕ_vÕXwÕwˆBÜ—’˜a¨¯©¼¿¹ÔFÈ!ÈaÐSTÍfÍ*ÕUÐX[Ð_`ÐcdÕ_dÐgh×gnÑgnÐoqÕgrÐuvÕgvÕ_vÕXwÕwˆBØ�J‰J”u—{’{¤E§J¢J°°1¨v°sÇ8Á8ÔSX×S`ÑS`Ô$aÐceÐfmÕcn×cuÑcuÐvwÐyzÓc{Ð}ð  AHõ  ~I÷  ~Pñ  ~Pð  QRð  TUó  ~Vð  $Wð  ]_ô  `÷  hñ  hð  ijð  lmó  nô  oØ�O‰O˜V¨a°!­e¥^°TÐ:Ô;Ø”3�q˜!“9ÕˆCØ�a˜!•e�O‹Fñ! "Vð& ×ÑœeŸiši¨°Ô3×=Ñ=¸aÓ@ÓA×KÑKÈAÈqÓQ×\Ñ\Ó^ˆÜ�yŠy˜ AÔ&¨¨HÐ4Ð4r   c                ó  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  ! WW4WV3/ VB # ró   )	r!   Úpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)r   r)   rY   r  r·  r¸  r—   Úkwargss   &&&&&&&,r   r+   Ú%QwenImage21Transformer2DModel.forward£  s}   € Ü×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4´U×5LÑ5L×5WÑ5W×5gÑ5gÐi|Ó}ó
÷ ‰'ð	ð ˜w°[ñ	að Z`ñ	að 	ar   c                ój  a$a%a-a.a/€ VP                   w  ršr¼VP                  p\        T;'       g    . 4      p\        T;'       g    . 4      pV P                  WWE4      w  rïpVP                   ^,          W¼,          ,
          o/VP	                  R/ 4      pVP	                  R. 4       F+  pV! RVRVRV/4      pVR,          VP	                  RV4      rþK-  	  VR,          P                  V4      R,          P                  V4      pV P                  \        P                  ! VVP                  ^4      .4      V4      pV P                  V4      P                  ^R R7      w  pppp\        V4      \        VP                  4       4      \        V4      \        VP                  4       4      \        P                  ! VR!,          4      3o.VP	                  R	/ 4      P	                  R
/ 4      pRRppS/^ 8”  EdD   V P                  V4      '       Ed,   \!        WWE4      pVP	                  R/ 4      pVeb   VP"                  V9   dQ   \%        WqP"                  ,          4      pVP'                  V4      pVP(                   F  pV F
  p VV n        K  	  K  	  M¨VP	                  RR4      R"9   g!   V P,                  P/                  4       '       gr   ^\1        V P2                  4      ,          V	,          S/,          V P4                  ,          VP7                  4       ,          p!V P9                  VV!VP"                  V4      w  ppV'       d   VRS/R13,          VRS/R13,          ^ urïo/M:Ve7   VRRS/13,          VRS/R13,          pp"VRRS/13,          VRS/R13,          pp#^ o/\1        V P2                  4      VR&   RVR&   \;        V\$        4      '       dD   \=        V P2                  4       U$a$U%u. uF   w  o$p%V%.V$3R lVP(                   4       O5NK"  	  up%p$M\        V P2                  4      p&\>        P@                  PC                  V&VP"                  V4      p'\>        P@                  PE                  VP"                  4       \=        V P2                  4       EF%  w  o$o%\>        P@                  PG                  V'VP"                  S%VRR7       S$VR&   \;        V\$        4      '       d$   VPI                  S$VP"                  V'       * R7      M\J        PL                  ! 4       ;_uu_ 4        Veø   V'       gq   \>        P@                  PO                  4       ;_uu_ 4        \Q        VRR  VVS$X"P                   ^,          4      p(S%! V"S.X#V(V"P                   ^,          V4      p"RRR4       VPS                  S$VP"                  WÙ4      PU                  ^4      w  p)p*V'       d;   \;        V\$        4      '       g%   VPW                  S$^,           VP"                  V4       \Y        V)V*V4      o-M\Q        VVVS$S/4      o-RS$3V9   d6   V-V%V.V/3R lp+RVRVRVRS.RS-RS/RV/p,VRS$3,          ! V,RV+/4      R,          pMS%! VS.VS-S/V4      pVP	                  R. 4       F  pV! RVRVRS$RV/4      R,          pK  	  Ve   >-?)?*RRR4       EK(  	  \>        P@                  PG                  V'VP"                  RRR7       \>        P@                  P[                  4        \;        V\$        4      '       d   V'       g   VP]                  X4       V P_                  VRS/R13,          VRR  4      pV Pa                  V4      pVPc                  ^^4      Pe                  W�Pf                  W¼4      # u up%p$i   + '       g   i     EL; i  + '       g   i     EK-  ; i)#r}   r~   r˜  r¿  r�   r—   iè  r¦   Nr–  r—  FÚqwen_image21_cacher   r¢  r¥   Útotal_blocksÚsingleÚ
block_typec              3   ó4   <"  € T F  qS,          x € K  	  R # 5iró   r  )rK  r?  rë   s   & €r   rL  Ú9QwenImage21Transformer2DModel._forward.<locals>.<genexpr>Ô  s   øé € Ð;©{ t Ÿ7š7«{ùs   ƒrö   )Úmalloc_scoper  r[  r™  c           
      óL   <€ R S! V R ,          SV R,          SSV R,          4      /# )r¿  r�   r—   r  )Úargsr•   rö   rÉ   r–   s   &€€€€r   Ú
block_wrapÚ:QwenImage21Transformer2DModel._forward.<locals>.block_wrapç  s-   ø€ Ø %¡u¨T°%­[¸#¸tÀD½zÈ7ÐT^Ð`dÐezÕ`{Ó'|Ð}Ð}r   ÚvecrÉ   r•   r–   Úoriginal_blockr)   r‰   )rÖ   N)r¤  r¥  )4rŠ   r   r  rÄ  rŒ   r(   r€  r   r§   Ú	new_zerosr…  Úchunkr£   rG   r×   r›  r$  r   rE  rR  r   Úread_before_writer¨  Ú
is_dynamicr  r†  rz   Úelement_sizer®  rr  r  r!   ra  Úmake_prefetch_queueÚmalloc_graph_beginÚprefetch_queue_popr@  r]  ÚnullcontextÚpause_malloc_graphr  r   ÚunbindÚprefetchr  Úmalloc_graph_endrW  r‡  rˆ  rŽ   rt  r~  )0r   r)   Ú	timestepsr  r·  r¸  r—   r‰  rÏ  r˜   ÚCÚHÚWr   Úhidden_statesr�   r  r~   r›   rc   ro  rØ   rÊ   rË   rÌ   rÍ   Úblocks_replacer  Úcachedr  r¬  r?  r@  r«  Úprefix_statesÚ	prefix_perë   rö   ÚunitsÚprefetch_queueÚprefix_attnr  r  rÛ  rÚ  r•   rÉ   r–   s0   &&&&&&&&,                           ``       @@@r   rÊ  Ú&QwenImage21Transformer2DModel._forwardª  s?  ü€ Ø—W‘W‰
ˆˆaØ—‘ˆÜ˜;×,Ð,¨"Ó-ˆÜ˜;×,Ð,¨"Ó-ˆà&*×&9Ñ&9¸!ÀkÓ&_Ñ#ˆ˜8Ø"×(Ñ(¨Õ+¨a­eÕ3ˆ
Ø%×)Ñ)¨)°RÓ8ˆØ—‘˜\¨2Ö.ˆAÙ�U˜M¨4°Ð5JÐL_Ð`ÓaˆCØ # E¥
¨C¯G©G°D¸"Ó,=š2ñ /ð
 ˜$Õ×"Ñ" 5Ó)¨DÕ0×4Ñ4°UÓ;ˆØ×#Ñ#¤E§I¢I¨q°!·+±+¸a³.Ð.AÓ$BÀEÓJˆØ'+§¡°tÓ'<×'BÑ'BÀ1È"Ð'BÓ'MÑ$ˆ��v˜uÜ˜6Ó"¤K°·
±
³Ó$=¼{È6Ó?RÔT_Ð`e×`jÑ`jÓ`lÓTmÔot×oÒoð  AGð  HPõ  AQó  pRð  Sˆà,×0Ñ0Ð1BÀBÓG×KÑKÈEÐSUÓVˆØ˜eˆvˆØ˜�>˜d×3Ñ3Ð4G×HÓHÜ" 1¨{ÓHˆCØ)×-Ñ-Ð.BÀBÓGˆGØÒ'¨A¯H©H¸Ô,DÜ# L·±Õ$:Ó;�ØŸ™ cÓ*�Ø!ŸKœK�DÛ!%˜Ø28˜Ö/ó "&ò (ð —‘˜X vÓ.°.Ô@È×H\ÑH\×HgÑHg×HiÒHiØ¤# d×&=Ñ&=Ó">Õ>ÀÕBÀZÕOÐRV×R`ÑR`Õ`Ðcp×c}Ñc}ÓcÕ�Ø $× 8Ñ 8¸¸kÈ1Ï8É8ÐU\Ó ]‘��vßØ,9¸!¸Z¹[¸.Õ,IÈ2ÈaÐQ[ÑQ\ÈnÕK]Ð_`Ð)ˆM™zØÒØ+8¸¸K¸Z¸K¸Õ+HÈ-ÐXYÐ[eÑ[fÐXfÕJg˜=ˆMØ˜q + : +˜~Õ.°°1°j±k°>Õ0B�rˆIØˆJä.1°$×2IÑ2IÓ.JÐ˜NÑ+Ø,4Ð˜LÑ)Üt~ð  @Eô  GR÷  uSò  uSÌYÐW[×WnÑWnÔMoÕpÑMoÁÀÀE�%Ð<Ô;¨u¯{ª{Ó;Ô<ÑMoÓpô  Y]ð  ^b÷  ^uñ  ^uó  YvˆÜ×-Ñ-×AÑAÀ%ÈÏÉÐSfÓgˆÜ×Ñ×/Ñ/°·±Ô9Ü! $×"9Ñ"9×:‰HˆAˆuÜ× Ñ ×3Ñ3°NÀAÇHÁHÈeÐUZÐipÐ3ÔqØ12Ð Ñ.Ü?IÈ%ÔQ\×?]Ò?]�—‘˜Q §¡°F´
�Ô;Ôcm×cyÒcyÓc{×{Ñ{ØÒ$ß!Ü"×1Ñ1×DÑD×FÕFÜ*@ÀÈ#È2ÀÐPcÐejÐlmÐo|÷  pCñ  pCð  DEõ  pFó  +G˜KÙ,1°-ÀÀiÐQ\Ð^k×^qÑ^qÐrsÕ^tð  wJó  -K˜M÷ Gð */¯©°A°q·x±xÀÓ)J×)QÑ)QÐRSÓ)TÑ&�H˜hß¤j°¼×&DÒ&DØŸ™ q¨1¥u¨a¯h©h¸Ô>Ü5°hÀÐJ]Ó^‘Gä4°XÐ?RÐTYÐ[\Ð^hÓi�GØ" AÐ&¨.Ô8÷~ð ~à! =°%¸¸tÀRÈÐPSÐU^Ð`gÐiuð  xBð  DYð  [nð  o�DØ$2°NÀAÐ3FÖ$GÈÐO_ÐakÐNlÓ$mÐnsÕ$t‘Má$)¨-¸¸bÀ'È:ÐWjÓ$k�MØ Ÿ™ ^°RÖ8�AÙ$% u¨m¸SÀ!À]ÐTUÐWlð  oBð  'Có  %Dð  EJõ  %K’Mñ 9àÒ$à ¨8÷/ |Ò{ñ ;ô8 	×Ñ×/Ñ/°ÀÇÁÈ$Ð]dÐ/ÔeÜ×Ñ×-Ñ-Ô/Ü�eœ[×)Ò)·&Ø�L‰L˜ÔØŸ™ m°A°z±{°NÕ&CÀTÈ#È2ÀYÓOˆØŸ™ mÓ4ˆØ×&Ñ& q¨!Ó,×4Ñ4°Q×8IÑ8IÈ1ÓPÐPùóK q÷ G×FÐFú÷ |×{Ñ{ús2   Ï&^Ô^ Ô"&^ ÕA^Ö
D^ Þ^Þ	^ Þ ^2)r   r‚  rz   r…  r‡  r~  r  r‰  rŠ  rˆ  r€  r†  r�  )é@   rù  é    rü   rú  rû   rÏ   )rù   é8   rû  r-   TNNNN)r/   r0   r1   r2   r   r“  r›  r®  rÄ  r+   rÊ  r3   r4   r5   r6   s   @@r   rz  rz  1  sJ   ù‡ € ÷(*òT,ò8òò85ð< 9=È$Ðdfô að ;?ÈDÐfhÐw{÷ OQô OQr   rz  )2r]  r   Útorch.nnr   Útorch.nn.functionalÚ
functionalrI   Úcomfy.memory_managementr!   Úcomfy.model_managementÚcomfy.model_prefetchÚ	comfy.opsÚcomfy.patcher_extensionÚcomfy.quant_opsÚcomfy.rmsnormÚcomfy.ldm.flux.layersr   r   Úcomfy.ldm.flux.mathr   Úcomfy.ldm.lightricks.modelr   Úcomfy.ldm.modules.attentionr   r   r	   Úcomfy.ldm.wan.model_animate2r
   r   ÚModuler   r9   rO   r\   rm   r£   r®   r¶   r¸   rÑ   rÛ   rä   r  r  r$  rt   r&  r:  rE  rz  r  r   r   Ú<module>r     sM  ðã Û Ý ß Ð ã Û Û Û Û Û Û ß =Ý +Ý 8ß eÑ eß Hô	J˜"Ÿ)™)ô 	Jô\�R—Y‘Yô \ô[˜RŸY™Yô [ôC˜Ÿ	™	ô Cô$'f�—	‘	ô 'fòT4ò
	òô "§)¡)ô ô"^�—	‘	ô ^ôRÐ!<ô Rô&˜BŸI™Iô &ð, :<À4ÐUVÐcdô ð& EGô òiô�b—m‘mô ôD�2—9‘9ô D÷(Dñ (DôVHQ B§I¡Iö HQr   