+
    ƒ°ñjŽ  ã                   ó
  € ^ RI HtHtHt ^ RIt^ RIHt ^ RIt^ RIHt ^ RI	H
t
Ht ^ RIHu Ht ^ RIHtHt ^ RIHt ^ RIHtHtHt ^ RIt^ RIt^ RIt ! R R	]P8                  4      t ! R
 R]P8                  4      t ! R R]P8                  4      t ! R R]P8                  4      t / R3R R llt! ! R R4      t" ! R R]P8                  4      t# ! R R]P8                  4      t$ ! R R]P8                  4      t% ! R R]P8                  4      t& ! R R]P8                  4      t' ! R R4      t( ! R  R!]P8                  4      t) ! R" R#]P8                  4      t* ! R$ R%]P8                  4      t+ ! R& R']P8                  4      t,R# )(é    )ÚOptionalÚTupleÚListN)Úrepeat)ÚTimestepEmbeddingÚ	Timesteps)Ú
apply_ropeÚrope)Ú	LastLayer)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc                   óP   a a€ ] tR t^t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚEmbedNDc                ó6   <€ V ^8„  d   QhRS[ RS[S[ ,          /# )é   ÚthetaÚaxes_dim)Úintr   )ÚformatÚ__classdict__s   "€Ú;C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\hidream\model.pyÚ__annotate__ÚEmbedND.__annotate__   s   ø€ ÷ !ñ !™cð !©T±#­Yñ !ó    c                ó<   <€ \         SV `  4        Wn        W n        R # ©N)ÚsuperÚ__init__r   r   )Úselfr   r   Ú	__class__s   &&&€r   r   ÚEmbedND.__init__   s   ø€ Ü‰ÑÔØŒ
Ø Žr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   ÚidsÚreturn©ÚtorchÚTensor)r   r   s   "€r   r   r      s#   ø€ ÷  ñ  ™5Ÿ<™<ð  ©E¯L©Lñ  r   c           
     ó  € VP                   R,          p\        P                  ! \        V4       Uu. uF4  p\	        VRV3,          V P
                  V,          V P                  4      NK6  	  upRR7      pVP                  ^4      # u upi )é   .©Údiméÿÿÿÿéýÿÿÿ)Úshaper'   ÚcatÚranger
   r   r   Ú	unsqueeze)r    r$   Ún_axesÚiÚembs   &&   r   ÚforwardÚEmbedND.forward   sk   € Ø—‘˜2•ˆÜ�iŠiÜFKÈFÄmÓTÁmÀŒT�#�c˜1�f•+˜tŸ}™}¨QÕ/°·±Ö<ÁmÑTØô
ˆð �}‰}˜QÓÐùò Us   ±:B)r   r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r6   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r!   r   s   @@r   r   r      s   ù‡ € ÷!ó !÷
 ÷  ð  r   r   c                   ó<   a a€ ] tR t^$t oRV 3R lltR tRtVtV ;t# )Ú
PatchEmbedc                óŠ   <€ \         SV `  4        Wn        W0n        VP	                  W!,          V,          VR WER7      V n        R# )T©ÚbiasÚdtypeÚdeviceN)r   r   Ú
patch_sizeÚout_channelsÚLinearÚproj)r    rH   Úin_channelsrI   rF   rG   Ú
operationsr!   s   &&&&&&&€r   r   ÚPatchEmbed.__init__%   sC   ø€ ô 	‰ÑÔØ$ŒØ(ÔØ×%Ñ% kÕ&>ÀÕ&KÈ\Ð`dÐlqÐ%ó  BˆŽ	r   c                ó(   € V P                  V4      pV# r   )rK   )r    Úlatents   &&r   r6   ÚPatchEmbed.forward1   s   € Ø—‘˜6Ó"ˆØˆr   )rI   rH   rK   )r   é   i   NNNr8   r@   s   @@r   rB   rB   $   s   ù‡ € ÷
B÷ò r   rB   c                   ó<   a a€ ] tR t^6t oRV 3R lltR tRtVtV ;t# )ÚPooledEmbedc                óJ   <€ \         SV `  4        \        WW4VR 7      V n        R# )©rL   Útime_embed_dimrF   rG   rM   N)r   r   r   Úpooled_embedder)r    Útext_emb_dimÚhidden_sizerF   rG   rM   r!   s   &&&&&&€r   r   ÚPooledEmbed.__init__7   s(   ø€ Ü‰ÑÔÜ0¸\Ðmrð  OYô   ZˆÖr   c                ó$   € V P                  V4      # r   ©rX   )r    Úpooled_embeds   &&r   r6   ÚPooledEmbed.forward;   s   € Ø×#Ñ# LÓ1Ð1r   r]   ©NNNr8   r@   s   @@r   rT   rT   6   s   ù‡ € ÷Z÷2ò 2r   rT   c                   ó<   a a€ ] tR t^?t oRV 3R lltR tRtVtV ;t# )ÚTimestepEmbedc                óp   <€ \         SV `  4        \        VR ^ R7      V n        \	        W!W4VR7      V n        R# )T)Únum_channelsÚflip_sin_to_cosÚdownscale_freq_shiftrV   N)r   r   r   Ú	time_projr   Útimestep_embedder)r    rZ   Úfrequency_embedding_sizerF   rG   rM   r!   s   &&&&&&€r   r   ÚTimestepEmbed.__init__@   s@   ø€ Ü‰ÑÔÜ"Ð0HÐZ^ÐuvÔwˆŒÜ!2Ð?Wð  |Að  ]gô  "hˆÖr   c                ój   € V P                  V4      P                  VR 7      pV P                  V4      pV# )©rF   )rg   Útorh   )r    Ú	timestepsÚwdtypeÚt_embs   &&& r   r6   ÚTimestepEmbed.forwardE   s3   € Ø—‘˜yÓ)×,Ñ,°6Ð,Ó:ˆØ×&Ñ& uÓ-ˆØˆr   )rg   rh   )é   NNNr8   r@   s   @@r   rb   rb   ?   s   ù‡ € ÷h÷
ò r   rb   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r   ÚqueryÚkeyÚvaluer&   )r   s   "r   r   r   K   s2   € ÷ Kñ K”U—\‘\ð K¬¯©ð K¼U¿\¹\ñ Kr   c           	      óh  € \        V \        4      '       d0   V P                  4       VP                  4       VP                  4       r!p V P                  ^,          p\        V P	                  ^4      4      \        VP	                  ^4      4      \        VP	                  ^4      4      r!p \        WW%WCR7      # )r   )Úpreferred_attentionÚtransformer_options)Ú
isinstancer   Útaker/   Úflattenr   )rt   ru   rv   ry   rx   Úheadss   &&&&& r   Ú	attentionr~   K   s«   € Ü�%Ô1×2Ò2Ø!ŸJ™J›L¨#¯(©(«*°e·j±j³l�EˆØ�K‰K˜�N€EÜ0°·±¸qÓ1AÓBÔD\Ð]`×]hÑ]hÐijÓ]kÓDlô  oGð  HM÷  HUñ  HUð  VWó  HXó  oY�€EÜ˜u¨5ÐM`ô  Kð  Kr   c                   óB   a € ] tR t^St o RtRRR/ 3V 3R lR lltRtV tR# )ÚHiDreamAttnProcessor_flashattnzYAttention processor used typically in processing the SD3-like self-attention projections.Nc          
      ó¼   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  RS[ P                  /# )r   Úimage_tokensÚimage_tokens_masksÚtext_tokensr
   r%   ©r'   ÚFloatTensorr   )r   r   s   "€r   r   Ú+HiDreamAttnProcessor_flashattn.__annotate__V   sn   ø€ ÷ D!ñ D!ñ ×'Ñ'ðD!ñ %¡U×%6Ñ%6Õ7ð	D!ñ
 ™e×/Ñ/Õ0ðD!ñ ×ÑðD!ñ 
×	Ñ	ñD!r   c                óÊ  € VP                   p	VP                  ^ ,          p
VP                  VP                  V4      4      P	                  V	R7      pVP                  VP                  V4      4      P	                  V	R7      pVP                  V4      pVP                  R,          pWáP                  ,          pVP                  V
RVP                  V4      pVP                  V
RVP                  V4      pVP                  V
RVP                  V4      pVe   WÃP                  V
R^^4      ,          pVP                  '       EgD   VP                  VP                  V4      4      P	                  V	R7      pVP                  VP                  V4      4      P	                  V	R7      pVP                  V4      pVP                  V
RVP                  V4      pVP                  V
RVP                  V4      pVP                  V
RVP                  V4      pVP                  ^,          pVP                  ^,          p\         P"                  ! VV.^R7      p\         P"                  ! VV.^R7      p\         P"                  ! VV.^R7      p???MTpTpTp???VP                  R,          VP                  R,          ^,          8X  d   \%        VVV4      w  ppMtVP'                  ^RR7      w  ppVP'                  ^RR7      w  pp\%        VVV4      w  pp\         P"                  ! VV.RR7      p\         P"                  ! VV.RR7      p????\)        V4      \)        V4      \)        V4      ppp\+        VVVWaP,                  R7      pVP                  '       gE   \         P.                  ! VXX.^R7      w  ppVP1                  V4      pVP3                  V4      pVV3# VP1                  V4      pV# )r   rl   r+   )ry   rx   r-   r.   )rF   r/   Ú
q_rms_normÚto_qrm   Ú
k_rms_normÚto_kÚto_vr}   ÚviewÚsingleÚq_rms_norm_tÚto_q_tÚk_rms_norm_tÚto_k_tÚto_v_tr'   r0   r	   Úchunkr   r~   Úcomfy_attentionÚsplitÚto_outÚto_out_t)r    Úattnr‚   rƒ   r„   r
   ry   ÚargsÚkwargsrF   Ú
batch_sizeÚquery_iÚkey_iÚvalue_iÚ	inner_dimÚhead_dimÚquery_tÚkey_tÚvalue_tÚnum_image_tokensÚnum_text_tokensrt   ru   rv   Úquery_1Úquery_2Úkey_1Úkey_2Úhidden_statesÚhidden_states_iÚhidden_states_ts   &&&&&&&*,                      r   Ú__call__Ú'HiDreamAttnProcessor_flashattn.__call__V   sa  € ð ×"Ñ"ˆØ!×'Ñ'¨Õ*ˆ
à—/‘/ $§)¡)¨LÓ"9Ó:×=Ñ=ÀEÐ=ÓJˆØ—‘ §	¡	¨,Ó 7Ó8×;Ñ;À%Ð;ÓHˆØ—)‘)˜LÓ)ˆà—K‘K •Oˆ	Ø§
¡
Õ*ˆà—,‘,˜z¨2¨t¯z©z¸8ÓDˆØ—
‘
˜: r¨4¯:©:°xÓ@ˆØ—,‘,˜z¨2¨t¯z©z¸8ÓDˆØÒ)Ø×3Ñ3°JÀÀAÀqÓIÕIˆEà�{�{‰{Ø×'Ñ'¨¯©°KÓ(@ÓA×DÑDÈ5ÐDÓQˆGØ×%Ñ% d§k¡k°+Ó&>Ó?×BÑBÈÐBÓOˆEØ—k‘k +Ó.ˆGà—l‘l :¨r°4·:±:¸xÓHˆGØ—J‘J˜z¨2¨t¯z©z¸8ÓDˆEØ—l‘l :¨r°4·:±:¸xÓHˆGà&Ÿ}™}¨QÕ/ÐØ%Ÿm™m¨AÕ.ˆOÜ—I’I˜w¨Ð0°aÔ8ˆEÜ—)’)˜U E˜N°Ô2ˆCÜ—I’I˜w¨Ð0°aÔ8ˆEØ˜¡àˆEØˆCØˆEà�U˜GØ�;‰;�r�?˜dŸj™j¨�n¨qÕ0Ô0Ü# E¨3°Ó5‰JˆE‘3à$Ÿ{™{¨1°"˜{Ó5ÑˆG�WØŸ9™9 Q¨B˜9Ó/‰LˆE�5Ü'¨°¸Ó=‰NˆG�UÜ—I’I˜w¨Ð0°bÔ9ˆEÜ—)’)˜U E˜N°Ô3ˆCØ˜ %¨ä4°UÓ;Ô=UÐVYÓ=ZÔ\tÐuzÓ\{�EˆsˆÜ! %¨¨eÐI\÷  sGñ  sGô  Hˆà�{�{ˆ{Ü/4¯{ª{¸=ÐK[Ð]lÐJmÐstÔ/uÑ,ˆO˜_Ø"Ÿk™k¨/Ó:ˆOØ"Ÿm™m¨OÓ<ˆOØ" OÐ3Ð3à ŸK™K¨Ó6ˆMØ Ð r   © )r9   r:   r;   r<   Ú__doc__r¯   r=   r>   )r   s   @r   r€   r€   S   s(   ø‡ € Ùcð ;?Ø37Ø"&Ø÷D!÷ D!ð D!r   r€   c                   ó`   a a€ ] tR t^œt oRV3R lV 3R llltRRR/ 3V3R lR lltRtVtV ;t# )ÚHiDreamAttentionNc                óP   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[RS[RS[RS[ R	S[/	# )
r   Ú	query_dimr}   Údim_headÚupcast_attentionÚupcast_softmaxÚscale_qkÚepsÚout_dimr�   )r   ÚboolÚfloat)r   r   s   "€r   r   ÚHiDreamAttention.__annotate__�   se   ø€ ÷ /#ñ /#áð/#ñ ð/#ñ ð	/#ñ
 ð/#ñ ð/#ñ ð/#ñ ð/#ñ ð/#ñ ñ/#r   c                ó   <€ \         SV `  4        \        4       V n        V	e   T	MW2,          V n        Wn        W@n        WPn        V	e   T	MTV n        W`n	        V P                  '       d
   VR,          MRV n
        V	e	   W“,          MTV n        W n        W n        VP                  pWàn        V! WP                  W¼R7      V n        V! V P                  V P                  W¼R7      V n        V! V P                  V P                  W¼R7      V n        V! V P                  V P                  W¼R7      V n        VP)                  V P                  W{VR7      V n        VP)                  V P                  W{VR7      V n        V
'       gÌ   V! WP                  W¼R7      V n        V! V P                  V P                  W¼R7      V n        V! V P                  V P                  W¼R7      V n        V! V P                  V P                  W¼R7      V n        VP)                  V P                  W{VR7      V n        VP)                  V P                  W{VR7      V n        W€n        R # )Ng      ð?©rF   rG   g      à¿)r   r   r   r–   r¡   r¶   r¸   r¹   r¼   rº   Úscaler}   Úsliceable_head_dimr�   rJ   Ú
linear_clsrŠ   rŒ   r�   r˜   ÚRMSNormr‰   r‹   r‘   r“   r”   r™   r�   r’   Ú	processor)r    r¶   r}   r·   r¸   r¹   rº   r»   rÆ   r¼   r�   rF   rG   rM   rÄ   r!   s   &&&&&&&&&&&&&& €r   r   ÚHiDreamAttention.__init__�   sÇ  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØ$+Ò$7™¸XÕ=MˆŒØ"ŒØ 0ÔØ,ÔØ")Ò"5‘w¸9ˆŒà ŒØ'+§}§} }�X˜t–^¸#ˆŒ
à,3Ò,?�WÖ(ÀUˆŒ
Ø"'ÔØŒà×&Ñ&ˆ
Ø$ŒÙ˜y¯.©.ÀÔUˆŒ	Ù˜tŸ~™~¨t¯~©~ÀUÔZˆŒ	Ù˜tŸ~™~¨t¯~©~ÀUÔZˆŒ	Ù  §¡°·±ÀUÔZˆŒØ$×,Ñ,¨T¯^©^¸SÐV\Ð,Ó]ˆŒØ$×,Ñ,¨T¯^©^¸SÐV\Ð,Ó]ˆŒçÙ$ Y·±ÀeÔ[ˆDŒKÙ$ T§^¡^°T·^±^È5Ô`ˆDŒKÙ$ T§^¡^°T·^±^È5Ô`ˆDŒKÙ& t§~¡~°t·|±|È5Ô`ˆDŒMØ *× 2Ñ 2°4·>±>À3Ð\bÐ 2Ó cˆDÔØ *× 2Ñ 2°4·>±>À3Ð\bÐ 2Ó cˆDÔà"Žr   c          
      óœ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r   Únorm_image_tokensrƒ   Únorm_text_tokensr
   r%   )r'   r†   r(   )r   r   s   "€r   r   r¿   Î   sY   ø€ ÷ 
ñ 
á ×,Ñ,ð
ñ "×-Ñ-ð
ñ  ×+Ñ+ð	
ñ
 ×Ñð
ñ 
�‰ñ
r   c           	     ó0   € V P                  V VVVVVR 7      # ))r‚   rƒ   r„   r
   ry   )rÆ   )r    rÉ   rƒ   rÊ   r
   ry   s   &&&&&&r   r6   ÚHiDreamAttention.forwardÎ   s,   € ð �~‰~ØØ,Ø!3Ø*ØØ 3ð ó 
ð 	
r   )r–   r}   r¡   r‹   r’   rÄ   r¼   rÆ   r‰   r�   r¶   rÂ   rº   r�   rÃ   rŒ   r“   r˜   r™   rŠ   r‘   r�   r”   r¸   r¹   )é   é@   FFTgñhãˆµøä>NNFNNNr8   r@   s   @@r   r´   r´   œ   s-   ù‡ € ÷/#õ /#ðh 15Ø.2Ø"&Ø÷
÷ 
ò 
r   r´   c                   óH   a a€ ] tR t^àt oRV3R lV 3R llltR tRtVtV ;t# )ÚFeedForwardSwiGLUc          	      óB   <€ V ^8„  d   QhRS[ RS[ RS[ RS[S[,          /# )r   r,   Ú
hidden_dimÚmultiple_ofÚffn_dim_multiplier)r   r   r¾   )r   r   s   "€r   r   ÚFeedForwardSwiGLU.__annotate__á   s=   ø€ ÷ ]ñ ]áð]ñ ð]ñ ð	]ñ
 %¡U�Oñ]r   c                óP  <€ \         SV `  4        \        ^V,          ^,          4      pVe   \        WB,          4      pVW#,           ^,
          V,          ,          pVP                  WRWVR7      V n        VP                  W!RWVR7      V n        VP                  WRWVR7      V n        R# )r   NFrD   )r   r   r   rJ   Úw1Úw2Úw3)	r    r,   rÒ   rÓ   rÔ   rF   rG   rM   r!   s	   &&&&&&&&€r   r   ÚFeedForwardSwiGLU.__init__á   sž   ø€ ô 	‰ÑÔÜ˜˜Z�¨!Õ+Ó,ˆ
àÒ)ÜÐ/Õ<Ó=ˆJØ ØÕ%¨Õ)¨kÕ9õ
ˆ
ð ×#Ñ# C¸%ÀuÐ#Ó\ˆŒØ×#Ñ# J¸%ÀuÐ#Ó\ˆŒØ×#Ñ# C¸%ÀuÐ#Ó\ˆŽr   c                ó¼   € V P                  \        P                  P                  P	                  V P                  V4      4      V P                  V4      ,          4      # r   )rØ   r'   ÚnnÚ
functionalÚsilur×   rÙ   )r    Úxs   &&r   r6   ÚFeedForwardSwiGLU.forwardö   s;   € Ø�w‰w”u—x‘x×*Ñ*×/Ñ/°·±¸³
Ó;¸d¿g¹gÀa»jÕHÓIÐIr   )r×   rØ   rÙ   )rr   NNNNr8   r@   s   @@r   rÐ   rÐ   à   s   ù‡ € ÷]õ ]÷*Jò Jr   rÐ   c                   óN   a a€ ] tR t^ût oRV 3R lltV3R lR ltR tRtVtV ;t	# )ÚMoEGatec                ó,  <€ \         SV `  4        W0n        W n        R V n        W@n        RV n        RV n        Wn        \        P                  ! \        P                  ! V P                  V P                  3WVR7      4      V n        V P                  4        R# )ÚsoftmaxFrÁ   N)r   r   Útop_kÚn_routed_expertsÚscoring_funcÚalphaÚseq_auxÚnorm_topk_probÚ
gating_dimrÜ   Ú	Parameterr'   ÚemptyÚweightÚreset_parameters)	r    Ú	embed_dimÚnum_routed_expertsÚnum_activated_expertsÚaux_loss_alpharF   rG   rM   r!   s	   &&&&&&&&€r   r   ÚMoEGate.__init__ü   ss   ø€ Ü‰ÑÔØ*Œ
Ø 2Ôà%ˆÔØ#Œ
ØˆŒð $ˆÔØ#ŒÜ—l’l¤5§;¢;°×0EÑ0EÀtÇÁÐ/WÐ_dÔ#tÓuˆŒØ×ÑÖr   c                ó   <€ V ^8„  d   QhRR/# )r   r%   Nr±   )r   r   s   "€r   r   ÚMoEGate.__annotate__  s   ø€ ÷ ñ  $ñ r   c                ó   € R # r   r±   )r    s   &r   rï   ÚMoEGate.reset_parameters  s   € Ùr   c           	     óH  € VP                   w  r#pVP                  RV4      p\        P                  ! V\        P
                  P                  V P                  VP                  VP                  R7      R4      pV P                  R8X  d   VP                  RR7      pM\        RV P                   24      h\        P                  ! W`P                  RRR7      w  rxV P                  ^8”  d5   V P                   '       d#   VP#                  RRR	7      R
,           p	Wy,          pRp
W‡V
3# )r*   rÁ   Nrä   r+   z/insupportable scoring function for MoE gating: F)Úkr,   ÚsortedT)r,   Úkeepdimg#B’¡œÇ;r-   )r/   rŽ   ÚFÚlinearÚcomfyÚmodel_managementÚcast_torî   rF   rG   rç   rä   ÚNotImplementedErrorr'   Útopkrå   rê   Úsum)r    r¬   ÚbszÚseq_lenÚhÚlogitsÚscoresÚtopk_weightÚtopk_idxÚdenominatorÚaux_losss   &&         r   r6   ÚMoEGate.forward  s  € Ø'×-Ñ-‰ˆ�að &×*Ñ*¨2¨qÓ1ˆÜ—’˜-¬×)?Ñ)?×)GÑ)GÈÏÉÐ[h×[nÑ[nð  xE÷  xLñ  xLÐ)Gó  *Mð  OSó  TˆØ×Ñ 	Ô)Ø—^‘^¨�^Ó+‰Fä%Ð(WÐX\×XiÑXiÐWjÐ&kÓlÐlô !&§
¢
¨6·Z±ZÀRÐPUÔ VÑˆð �:‰:˜Œ>˜d×1×1Ð1Ø%Ÿ/™/¨b¸$˜/Ó?À%ÕGˆKØ%Õ3ˆKàˆØ hÐ.Ð.r   )rè   rë   ræ   rê   rç   ré   rå   rî   )rR   r   g{®Gáz„?NNN)
r9   r:   r;   r<   r   rï   r6   r=   r>   r?   r@   s   @@r   râ   râ   û   s   ù‡ € ÷ ÷ð ÷
/ò /r   râ   c                   óv   a a€ ] tR tRt oRV3R lV 3R llltR t]P                  ! 4       R 4       tRt	Vt
V ;t# )ÚMOEFeedForwardSwiGLUi(  c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )r   r,   rÒ   rñ   rò   ©r   )r   r   s   "€r   r   Ú!MOEFeedForwardSwiGLU.__annotate__)  s3   ø€ ÷ ;ñ ;áð;ñ ð;ñ  ð	;ñ
  #ñ;r   c                ó  <€ \         S	V `  4        \        W^,          WVVR7      V n        \        P
                  ! \        V4       Uu. uF  p\        WWVVR7      NK  	  up4      V n        \        VVVWVVR7      V n	        W@n
        R# u upi )r   ©rF   rG   rM   )rð   rñ   rò   rF   rG   rM   N)r   r   rÐ   Úshared_expertsrÜ   Ú
ModuleListr1   Úexpertsrâ   Úgaterò   )
r    r,   rÒ   rñ   rò   rF   rG   rM   r4   r!   s
   &&&&&&&& €r   r   ÚMOEFeedForwardSwiGLU.__init__)  s£   ø€ ô 	‰ÑÔÜ/°À1µ_ÈEÐmwÔxˆÔÜ—}’}ô  FKð  L^ô  F_ó  &`ñ  F_ð  ABÔ&7¸ÈuÐpz×&{ñ  F_ñ  &`ó  aˆŒÜØØ!3Ø$9Ø°:ô	
ˆŒ	ð &;Ö"ùò &`s   ÁBc                ó´  € VP                   pTpVP                  pV P                  V4      w  rVpVP                  RVP                  R,          4      pVP                  R4      p VP	                  V P
                  ^ R7      p\        P                  ! WR7      p	\        V P                  4       F+  w  r«V! WV
8H  ,          4      P                  VR7      W˜V
8H  &   K-  	  V	P                  ! . VP                  ORN5!  VP                  R4      ,          P                  ^R7      p	V	P                  ! V!  P                  VR7      p	Y�P                  T4      ,           p	T	# )r*   r+   rl   r-   )rF   r/   r  rŽ   Úrepeat_interleaverò   r'   Ú
empty_likeÚ	enumerater  rm   r2   r  Ú	moe_inferr  )r    rß   ÚwtypeÚidentityÚ
orig_shaper  r
  r  Úflat_topk_idxÚyr4   Úexperts   &&          r   r6   ÚMOEFeedForwardSwiGLU.forward<  s-  € Ø—‘ˆØˆØ—W‘Wˆ
Ø*.¯)©)°A«,Ñ'ˆ˜xØ�F‰F�2�q—w‘w˜r•{Ó#ˆØ Ÿ™ bÓ)ˆØØ×#Ñ# D×$>Ñ$>ÀAÐ#ÓFˆAÜ× Ò  Ô0ˆAÜ& t§|¡|Ö4‘	�Ù(.¨qÀ!Ñ1CÕ/DÓ(E×(HÑ(HÈuÐ(HÓ(U� 1Ñ$Ó%ñ 5à—’Ð/˜×*Ñ*Ð/¨BÓ/°+×2GÑ2GÈÓ2KÕK×PÑPÐUVÐPÓWˆAØ—’˜Ñ$×'Ñ'¨eÐ'Ó4ˆAð ×#Ñ# HÓ-Õ-ˆØˆr   c           	     ó–  € \         P                  ! V4      pVP                  4       pVP                  4       P	                  4       P                  4       P                  ^ 4      pWPP                  ,          p\        V4       FÆ  w  r‰V^ 8X  d   ^ MWh^,
          ,          p
W©8X  d   K%  V P                  V,          pWzV	 pW,          pV! V4      pVP                  W5W© ,          4       VP                  VP                  4      pVP                  ^ VP                  R^4      P                  ^VP                   R,          4      VRR7       KÈ  	  V# )r   r  )Úreducer-   )r'   Ú
zeros_likeÚargsortÚbincountÚcpuÚnumpyÚcumsumrò   r  r  Úmul_rm   rF   Úscatter_reduce_rŽ   r   r/   )r    rß   Úflat_expert_indicesÚflat_expert_weightsÚexpert_cacheÚidxsÚtokens_per_expertÚ
token_idxsr4   Úend_idxÚ	start_idxr%  Úexp_token_idxÚexpert_tokensÚ
expert_outs   &&&&           r   r  ÚMOEFeedForwardSwiGLU.moe_inferP  s'  € ä×'Ò'¨Ó*ˆØ"×*Ñ*Ó,ˆØ/×8Ñ8Ó:×>Ñ>Ó@×FÑFÓH×OÑOÐPQÓRÐØ×7Ñ7Õ7ˆ
Ü#Ð$5Ö6‰JˆAØ !œV™Ð):¸Q½3Õ)?ˆIØÔ#ÙØ—\‘\ !•_ˆFØ&°Ð9ˆMØÕ,ˆMÙ Ó.ˆJØ�O‰OÐ/°YÐ0GÕHÔIð (Ÿ?™?¨:×+;Ñ+;Ó<ˆLØ×(Ñ(¨¨M×,>Ñ,>¸rÀ1Ó,E×,LÑ,LÈQÐPQ×PWÑPWÐXZÕP[Ó,\Ð^hÐqvÐ(Öwñ 7ð Ðr   )r  r  rò   r  r`   )r9   r:   r;   r<   r   r6   r'   Úno_gradr  r=   r>   r?   r@   s   @@r   r  r  (  s3   ù‡ € ÷;õ ;ò&ð( ‡]‚]ƒ_ñó ÷ð r   r  c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚTextProjectionif  c                óV   <€ \         SV `  4        VP                  WR W4R7      V n        R# )F)Úin_featuresÚout_featuresrE   rF   rG   N)r   r   rJ   rþ   )r    rA  rZ   rF   rG   rM   r!   s   &&&&&&€r   r   ÚTextProjection.__init__g  s,   ø€ Ü‰ÑÔØ ×'Ñ'°KÐ`eÐmrÐ'ó  CˆŽr   c                ó(   € V P                  V4      pV# r   ©rþ   )r    Úcaptionr¬   s   && r   r6   ÚTextProjection.forwardk  s   € ØŸ™ GÓ,ˆØÐr   rE  r`   r8   r@   s   @@r   r?  r?  f  s   ù‡ € ÷C÷ò r   r?  c                   ó   € ] tR tRt^t^tRtR# )Ú	BlockTypeip  r±   N)r9   r:   r;   r<   ÚTransformerBlockÚSingleTransformerBlockr=   r±   r   r   rI  rI  p  s   † ØÐØÔr   rI  c                   ób   a a€ ] tR tRt oRV3R lV 3R llltRRRR/ 3V3R lR lltRtVtV ;t# )	Ú"HiDreamImageSingleTransformerBlockiu  Nc          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ /# ©r   r,   Únum_attention_headsÚattention_head_dimrñ   rò   r  )r   r   s   "€r   r   Ú/HiDreamImageSingleTransformerBlock.__annotate__v  sD   ø€ ÷ &~ñ &~áð&~ñ !ð&~ñ  ð	&~ñ
  ð&~ñ  #ñ&~r   c	                óê  <€ \         S	V `  4        W n        \        P                  ! \        P
                  ! 4       VP                  V^V,          RWgR7      4      V n        VP                  VRRWgR7      V n	        \        VVV\        4       RWgVR7      V n        VP                  VRRWgR7      V n        V^ 8”  d    \        V^V,          VVWgVR7      V n        R	# \!        V^V,          WgVR7      V n        R	# )
é   TrD   ç�íµ ÷Æ°>F©r»   Úelementwise_affinerF   rG   ©r¶   r}   r·   rÆ   r�   rF   rG   rM   ©r,   rÒ   rñ   rò   rF   rG   rM   ©r,   rÒ   rF   rG   rM   N)r   r   rP  rÜ   Ú
SequentialÚSiLUrJ   ÚadaLN_modulationÚ	LayerNormÚnorm1_ir´   r€   Úattn1Únorm3_ir  Úff_irÐ   ©
r    r,   rP  rQ  rñ   rò   rF   rG   rM   r!   s
   &&&&&&&&&€r   r   Ú+HiDreamImageSingleTransformerBlock.__init__v  sï   ø€ ô 	‰ÑÔØ#6Ô Ü "§¢Ü�GŠG‹IØ×Ñ˜c 1 s¥7°¸UÐÓRó!
ˆÔð "×+Ñ+¨C°uÐSXÐ`eÐ+ÓuˆŒÜ%ØØ%Ø'Ü6Ó8ØØ°:ô
ˆŒ
ð "×+Ñ+¨C°uÐSXÐ`eÐ+ÓuˆŒØ Ô!Ü,ØØ �WØ%7Ø(=Ø°zôˆDŽIô *°À!ÀcÅ'ÐQVÐr|Ô}ˆDŽIr   c                óæ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  RS[ P                  /# ©r   r‚   rƒ   r„   Úadaln_inputr
   r%   r…   )r   r   s   "€r   r   rR  ž  sz   ø€ ÷ ñ á×'Ñ'ðñ %¡U×%6Ñ%6Õ7ðñ ™e×/Ñ/Õ0ð	ñ
 ™e×/Ñ/Õ0ðñ ×Ñðñ 
×	Ñ	ñr   c                ó   € VP                   pV P                  V4      R,          P                  ^RR7      w  r‰r«rÍV P                  V4      P	                  VR7      pV^V	,           ,          V,           pV P                  VVVVR7      pW¯,          V,           pV P                  V4      P	                  VR7      pV^V,           ,          V,           pWÐP                  VP	                  VR7      4      ,          pVV,           pV# ©ºNNNr+   rl   )r
   ry   ©rj  Nr-   )rF   r]  r•   r_  rm   r`  ra  rb  )r    r‚   rƒ   r„   rg  r
   ry   r   Úshift_msa_iÚscale_msa_iÚ
gate_msa_iÚshift_mlp_iÚscale_mlp_iÚ
gate_mlp_irÉ   Úattn_output_iÚff_output_is   &&&&&&&          r   r6   Ú*HiDreamImageSingleTransformerBlock.forwardž  s  € ð ×"Ñ"ˆà×!Ñ! +Ó.¨vÕ6×<Ñ<¸QÀBÐ<ÓGñ 	Sˆ *¸;ð !ŸL™L¨Ó6×9Ñ9ÀÐ9ÓFÐØ-°°[µÕAÀKÕOÐØŸ
™
ØØØØ 3ð	 #ó 
ˆð "Õ1°LÕ@ˆð !ŸL™L¨Ó6×9Ñ9ÀÐ9ÓFÐØ-°°[µÕAÀKÕOÐØ §9¡9Ð->×-AÑ-AÈÐ-AÓ-NÓ#OÕOˆØ" \Õ1ˆØÐr   )r]  r`  rb  r_  ra  rP  ©rR   r   NNNr8   r@   s   @@r   rM  rM  u  s2   ù‡ € ÷&~õ &~ðV ;?Ø37Ø37Ø"&Ø÷÷ ò r   rM  c                   ób   a a€ ] tR tRt oRV3R lV 3R llltRRRR/ 3V3R lR lltRtVtV ;t# )	ÚHiDreamImageTransformerBlocki¾  Nc          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ /# rO  r  )r   r   s   "€r   r   Ú)HiDreamImageTransformerBlock.__annotate__¿  sD   ø€ ÷ +zñ +záð+zñ !ð+zñ  ð	+zñ
  ð+zñ  #ñ+zr   c	                ó„  <€ \         S	V `  4        W n        \        P                  ! \        P
                  ! 4       VP                  V^V,          RWgR7      4      V n        VP                  VRRWgR7      V n	        VP                  VRRWgR7      V n
        \        VVV\        4       RWgVR7      V n        VP                  VRRWgR7      V n        V^ 8”  d   \        V^V,          VVWgVR7      V n        M\#        V^V,          WgVR7      V n        VP                  VRRR	7      V n        \#        V^V,          WgVR7      V n        R
# )é   TrD   rU  FrV  rX  rY  rZ  )r»   rW  N)r   r   rP  rÜ   r[  r\  rJ   r]  r^  r_  Únorm1_tr´   r€   r`  ra  r  rb  rÐ   Únorm3_tÚff_trc  s
   &&&&&&&&&€r   r   Ú%HiDreamImageTransformerBlock.__init__¿  s=  ø€ ô 	‰ÑÔØ#6Ô Ü "§¢Ü�GŠG‹IØ×Ñ˜c 2¨¥8°$¸eÐÓSó!
ˆÔð "×+Ñ+¨C°uÐSXÐ`eÐ+ÓuˆŒØ!×+Ñ+¨C°uÐSXÐ`eÐ+ÓuˆŒÜ%ØØ%Ø'Ü6Ó8ØØ°:ô
ˆŒ
ð "×+Ñ+¨C°uÐSXÐ`eÐ+ÓuˆŒØ Ô!Ü,ØØ �WØ%7Ø(=Ø°zôˆD�Iô *°À!ÀcÅ'ÐQVÐr|Ô}ˆDŒIØ!×+Ñ+¨C°uÐSXÐ+ÓYˆŒÜ%¨C¸aÀ#½gÈUÐnxÔyˆŽ	r   c                óæ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  RS[ P                  /# rf  r…   )r   r   s   "€r   r   ry  ì  sz   ø€ ÷ ))ñ ))á×'Ñ'ð))ñ %¡U×%6Ñ%6Õ7ð))ñ ™e×/Ñ/Õ0ð	))ñ
 ™e×/Ñ/Õ0ð))ñ ×Ñð))ñ 
×	Ñ	ñ))r   c                ó8  € VP                   pV P                  V4      R,          P                  ^RR7      w  r‰r«rÍrïppppV P                  V4      P	                  VR7      pV^V	,           ,          V,           pV P                  V4      P	                  VR7      pV^V,           ,          V,           pV P                  VVVVVR7      w  ppV
V,          V,           pVV,          V,           pV P                  V4      P	                  VR7      pV^V,           ,          V,           pV P                  V4      P	                  VR7      pV^V,           ,          V,           pWÐP                  V4      ,          pVV P                  V4      ,          pVV,           pVV,           pW3# ri  )rF   r]  r•   r_  rm   r|  r`  ra  r}  rb  r~  )r    r‚   rƒ   r„   rg  r
   ry   r   rl  rm  rn  ro  rp  rq  Úshift_msa_tÚscale_msa_tÚ
gate_msa_tÚshift_mlp_tÚscale_mlp_tÚ
gate_mlp_trÉ   rÊ   rr  Úattn_output_trs  Úff_output_ts   &&&&&&&                   r   r6   Ú$HiDreamImageTransformerBlock.forwardì  s�  € ð ×"Ñ"ˆð ×!Ñ! +Ó.¨vÕ6×<Ñ<¸RÀRÐ<ÓHñ	Sˆ *¸;Ø *¨k¸;È
ð !ŸL™L¨Ó6×9Ñ9ÀÐ9ÓFÐØ-°°[µÕAÀKÕOÐØŸ<™<¨Ó4×7Ñ7¸eÐ7ÓDÐØ+¨q°;­Õ?À+ÕMÐà'+§z¡zØØØØØ 3ð (2ó (
Ñ$ˆ�}ð " MÕ1°LÕ@ˆØ  =Õ0°;Õ>ˆð !ŸL™L¨Ó6×9Ñ9ÀÐ9ÓFÐØ-°°[µÕAÀKÕOÐØŸ<™<¨Ó4×7Ñ7¸eÐ7ÓDÐØ+¨q°;­Õ?À+ÕMÐà §9¡9Ð->Ó#?Õ?ˆØ  4§9¡9Ð-=Ó#>Õ>ˆØ" \Õ1ˆØ! KÕ/ˆØÐ(Ð(r   )	r]  r`  rb  r~  r_  r|  ra  r}  rP  ru  r8   r@   s   @@r   rw  rw  ¾  s2   ù‡ € ÷+zõ +zð` ;?Ø37Ø37Ø"&Ø÷))÷ ))ò ))r   rw  c                   ó‚   a a€ ] tR tRt o^^]P
                  RRR3V3R lV 3R llltRRRR/ 3V3R lR lltRtVt	V ;t
# )ÚHiDreamImageBlocki  Nc                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[/# )r   r,   rP  rQ  rñ   rò   Ú
block_type)r   rI  )r   r   s   "€r   r   ÚHiDreamImageBlock.__annotate__  sG   ø€ ÷ 
ñ 
áð
ñ !ð
ñ  ð	
ñ
  ð
ñ  #ð
ñ ñ
r   c
                ó¬   <€ \         SV `  4        \        P                  \        \        P
                  \        /p
W¦,          ! VVVVVWxV	R 7      V n        R# )r  N)r   r   rI  rJ  rw  rK  rM  Úblock)r    r,   rP  rQ  rñ   rò   rŽ  rF   rG   rM   Úblock_classesr!   s   &&&&&&&&&& €r   r   ÚHiDreamImageBlock.__init__  sT   ø€ ô 	‰ÑÔä×&Ñ&Ô(DÜ×,Ñ,Ô.Pð
ˆð #Ö.ØØØØØ!Ø°:ô
ˆŽ
r   c                óÖ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  RS[ P                  RS[ P                  /# rf  r…   )r   r   s   "€r   r   r�  1  su   ø€ ÷ 
ñ 
á×'Ñ'ð
ñ %¡U×%6Ñ%6Õ7ð
ñ ™e×/Ñ/Õ0ð	
ñ
 ×&Ñ&ð
ñ ×Ñð
ñ 
×	Ñ	ñ
r   c           	     ó0   € V P                  VVVVVVR 7      # ))ry   ©r‘  )r    r‚   rƒ   r„   rg  r
   ry   s   &&&&&&&r   r6   ÚHiDreamImageBlock.forward1  s,   € ð �z‰zØØØØØØ 3ð ó 
ð 	
r   r–  )r9   r:   r;   r<   rI  rJ  r   r6   r=   r>   r?   r@   s   @@r   rŒ  rŒ    sG   ù‡ € ð #$Ø%&Ø )× :Ñ :Ø˜4¨D÷
õ 
ð6 ;?Ø37Ø)-Ø"&Ø÷
÷ 
ò 
r   rŒ  c                   ó¨   a a€ ] tR tRt oRV3R lV 3R llltR tV3R lR ltRR ltRRRRR/ 3V3R	 lR
 lltRRRRR/ 3V3R lR llt	Rt
VtV ;t# )ÚHiDreamImageTransformer2DModeliD  Nc                óÚ   <€ V ^8„  d   QhRS[ S[,          RS[RS[ S[,          RS[RS[RS[RS[RS[S[,          R	S[R
S[RS[RS[S[S[3,          RS[S[S[3,          RS[S[,          /# )r   rH   rL   rI   Ú
num_layersÚnum_single_layersrQ  rP  Úcaption_channelsrY   rñ   rò   Úaxes_dims_ropeÚmax_resolutionÚllama_layers)r   r   r   r   )r   r   s   "€r   r   Ú+HiDreamImageTransformer2DModel.__annotate__E  s×   ø€ ÷ PZñ PZá™S•MðPZñ ðPZñ ™s•mð	PZñ
 ðPZñ ðPZñ  ðPZñ !ðPZñ ™s�)ðPZñ ðPZñ  ðPZñ  #ðPZñ ™c¡3˜h�ðPZñ ™c¡3˜h�ðPZñ ™3•iñPZr   c                ó\  <€ Wn         Wpn        W`n        W@n        WPn        R V n        \        SV `  4        VV n        T;'       g    TV n	        V P                  V P                  ,          V n
        Wàn        \        V P                  VVVR7      V n        \        W�P                  VVVR7      V n        \!        VVV P                  VVVR7      V n        \%        RVR7      V n        \(        P*                  ! \-        V P                  4       Uu. uFC  p\/        V P                  V P                  V P                  V
V\0        P2                  VVVR7	      NKE  	  up4      V n        \(        P*                  ! \-        V P                  4       Uu. uFC  p\/        V P                  V P                  V P                  V
V\0        P6                  VVVR7	      NKE  	  up4      V n        \;        V P                  WP                  VVVR7      V n        V^,          .WE,           ,          V^ ,          .,           p. pV F,  pVP?                  \A        VV P                  VVVR7      4       K.  	  \(        P*                  ! V4      V n!        V^ ,          V^,          ,          W,          ,          V n"        R# u upi u upi )Fr  )rH   rL   rI   rF   rG   rM   i'  )r   r   )	r,   rP  rQ  rñ   rò   rŽ  rF   rG   rM   )rA  rZ   rF   rG   rM   N)#rH   rP  rQ  r›  rœ  Úgradient_checkpointingr   r   rF   rI   r¡   r   rb   Ú
t_embedderrT   Ú
p_embedderrB   Ú
x_embedderr   Úpe_embedderrÜ   r  r1   rŒ  rI  rJ  Údouble_stream_blocksrK  Úsingle_stream_blocksr   Úfinal_layerÚappendr?  Úcaption_projectionÚmax_seq)r    rH   rL   rI   r›  rœ  rQ  rP  r�  rY   rñ   rò   rž  rŸ  r   Úimage_modelrF   rG   rM   r4   r¬  Úcaption_channelr!   s   &&&&&&&&&&&&&&&&&&&   €r   r   Ú'HiDreamImageTransformer2DModel.__init__E  sy  ø€ ð& %ŒØ#6Ô Ø"4ÔØ$ŒØ!2Ôà&+ˆÔ#ä‰ÑÔØˆŒ
Ø(×7Ð7¨KˆÔØ×1Ñ1°D×4KÑ4KÕKˆŒØ(Ôä'¨¯©¸eÈFÐ_iÔjˆŒÜ% l·N±NÈ%ÐX^ÐkuÔvˆŒÜ$Ø#Ø%ØŸ>™>Ø °:ô	
ˆŒô #¨¸ÔHˆÔä$&§M¢Mô ˜tŸ™Ô/óñ 0�Aô "ØŸ.™.Ø*.×*BÑ*BØ)-×)@Ñ)@Ø);Ø,AÜ!*×!;Ñ!;Ø¨¸:÷ñ 0ñó%
ˆÔ!ô %'§M¢Mô ˜t×5Ñ5Ô6óñ 7�Aô "ØŸ.™.Ø*.×*BÑ*BØ)-×)@Ñ)@Ø);Ø,AÜ!*×!AÑ!AØ¨¸:÷ñ 7ñó%
ˆÔ!ô % T§^¡^°Z×ARÑARÐZ_Ðhnð  |Fô  GˆÔà,¨QÕ/Ð2°jÕ6TÕUÐYiÐjkÕYlÐXoÕoÐØÐÛ/ˆOØ×%Ñ%¤nÀÐ^b×^lÑ^lÐtyð  CIð  V`ô  'aö  bñ  0ä"$§-¢-Ð0BÓ"CˆÔØ% aÕ(¨>¸!Õ+<Õ<ÀÕAXÕYˆŽùòIùòs   Ä
A	J$ÆA	J)c                óä  € \         P                  ! V4      '       g‘   VP                  R 8H  p\        V\        4      '       d*   V'       d   \         P
                  M\         P                  pM(V'       d   \         P                  M\         P                  p\         P                  ! V.WSR7      pM2\        VP                  4      ^ 8X  d   VR,          P                  V4      pVP                  V4      pV# )ÚmpsrÁ   N)r'   Ú	is_tensorÚtyperz   r¾   Úfloat32Úfloat64Úint32Úint64ÚtensorÚlenr/   rm   Úexpand)r    rn   r�   rG   Úis_mpsrF   s   &&&&  r   Úexpand_timestepsÚ/HiDreamImageTransformer2DModel.expand_timesteps—  s›   € Ü�Š˜y×)Ò)Ø—[‘[ EÑ)ˆFÜ˜)¤U×+Ò+ß)/œŸš´U·]±]‘ç'-œŸš´5·;±;�ÜŸš i [¸ÔM‰IÜ�—‘Ó! QÔ&Ø! $�×*Ñ*¨6Ó2ˆIà×$Ñ$ ZÓ0ˆ	ØÐr   c                óŠ   <€ V ^8„  d   QhRS[ P                  RS[S[S[S[3,          ,          RS[S[ P                  ,          /# )r   rß   Ú	img_sizesr%   )r'   r(   r   r   r   )r   r   s   "€r   r   r¡  ¥  s?   ø€ ÷ 	ñ 	™EŸL™Lð 	±T¹%ÁÁSÀ½/Õ5Jð 	ÉtÑTY×T`ÑT`ÕOañ 	r   c                ó&  € . p\        V4       Fg  w  rEVw  rgVP                  \        P                  ! WR Wg,          13,          P	                  ^WgR4      RV P
                  V P
                  R7      4       Ki  	  \        P                  ! V^ R7      pV# )Nz$B H W (p1 p2 C) -> B C (H p1) (W p2)©Úp1Úp2r+   r-   )r  r«  ÚeinopsÚ	rearrangeÚreshaperH   r'   r0   )r    rß   rÀ  Úx_arrr4   Úimg_sizeÚpHÚpWs   &&&     r   Ú
unpatchifyÚ)HiDreamImageTransformer2DModel.unpatchify¥  s}   € ØˆÜ$ YÖ/‰KˆAØ‰FˆBØ�L‰LÜ× Ò   f r¥u f 9¥×!5Ñ!5°a¸ÀÓ!DÐFlØ—‘¨4¯?©?ô<öñ 0ô �IŠI�e Ô#ˆØˆr   c                ó`  € V P                   V P                   ,          p\        V\        P                  4      '       d-   VP                  ^ ,          pVP
                  pVP                  pM1\        V4      pV^ ,          P
                  pV^ ,          P                  p\        P                  ! WR3WvR7      pVeL   \        V4       F"  w  rš^W‰^ V
^ ,          V
^,          ,          13&   K$  	  \        P                  ! VRVR7      pM©\        V\        P                  4      '       d„   VP                  R,          V P                   ,          VP                  R,          V P                   ,          rË\        P                  ! VRV P                   V P                   R7      pW¼..V,          pRpM\        hWV3# )	r   rÁ   NzB C S p -> B S (p C))Úpz&B C (H p1) (W p2) -> B (H W) (p1 p2 C)rÂ  éþÿÿÿr-   )rH   rz   r'   r(   r/   rG   rF   rº  Úzerosr  rÅ  rÆ  r  )r    rß   r­  rÀ  Úpz2ÚBrG   rF   Úx_masksr4   rÉ  rÊ  rË  s   &&&&         r   ÚpatchifyÚ'HiDreamImageTransformer2DModel.patchify°  sF  € Ø�o‰o §¡Õ/ˆÜ�aœŸ™×&Ò&Ø—‘˜•
ˆAØ—X‘XˆFØ—G‘G‰Eä�A“ˆAØ�q•T—[‘[ˆFØ�a•D—J‘JˆEÜ—+’+˜q˜l°%ÔGˆàÒ Ü(¨Ö3‘�Ø:;�˜1˜X a�[¨8°A­;Õ6Ð6Ð6Ó7ñ  4ä× Ò  Ð$:¸cÔB‰AÜ˜œ5Ÿ<™<×(Ò(Ø—W‘W˜R•[ D§O¡OÕ3°Q·W±W¸Rµ[ÀDÇOÁOÕ5S�Ü× Ò  Ð$LÐQU×Q`ÑQ`Ðei×etÑetÔuˆAØ˜˜
 Q�ˆIØ‰Gä%Ð%Ø˜9Ð$Ð$r   c          	      ó¢   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# )r   rß   Útr$  Úcontext©r'   r(   r   )r   r   s   "€r   r   r¡  É  sQ   ø€ ÷ lñ lÙ�<‰<ðlá�<‰<ðlñ ‘E—L‘LÕ!ðlñ ™%Ÿ,™,Õ'ñ	lr   c	           
     ó  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  WW4WVWx4      # r   )	rÿ   Úpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)	r    rß   rØ  r$  rÙ  Úencoder_hidden_states_llama3Ú
image_condÚcontrolry   s	   &&&&&&&&&r   r6   Ú&HiDreamImageTransformer2DModel.forwardÉ  sj   € ô ×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4´U×5LÑ5L×5WÑ5W×5gÑ5gÐi|Ó}ó
÷ ‰'�!˜Ð$@ÈgÓ
kð		lr   c          
      ó¼   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  /# )r   rß   rØ  r$  rÙ  r%   rÚ  )r   r   s   "€r   r   r¡  Ù  sb   ø€ ÷ m%ñ m%á�<‰<ðm%ñ �<‰<ðm%ñ ‘E—L‘LÕ!ð	m%ñ
 ™%Ÿ,™,Õ'ðm%ñ 
�‰ñm%r   c	           
     óê
  € VP                   w  ršr¼Ve   \        P                  ! W.R	R7      p\        P                  P
                  P                  WP                  V P                  34      pTpTpTpR pVP                   ^ ,          pVP                  pV P                  VVVP                  4      pV P                  VV4      pV P                  V4      pVV,           pV P                  WÐP                  V4      w  pppVf³   V^ ,          w  pp\        P                  ! VV^VP                  R7      pVR
,          \        P                   ! VVP                  R7      R,          ,           VR
&   VR,          \        P                   ! VVP                  R7      R,          ,           VR&   \#        VRVR7      pV P%                  V4      pVP'                  ^^ 4      pV P(                   Uu. uF  pVV,          NK  	  ppV P*                  e¶   . p\-        V4       FT  w  ppV P*                  V,          ! V4      pVP/                  VR	VP                   R	,          4      pVP1                  V4       KV  	  TpV P*                  R	,          ! V4      pVP/                  VR	VP                   R	,          4      pVP1                  V4       \        P                  ! VVR	,          P                   ^,          VR,          P                   ^,          ,           V^ ,          P                   ^,          ,           ^XP                  VP                  R7      p\        P                  ! VV3^R7      p V P3                  V 4      p!^ p"\        P                  ! VR	,          VR,          .^R7      p#V#P                   ^,          p$\-        V P4                  4       FP  w  p%p&VV",          p'\        P                  ! V#V'.^R7      p(V&! VVV(VV!VR7      w  pp#V#RR V$13,          p#V"^,          p"KR  	  VP                   ^,          p)\        P                  ! VV#.^R7      pVP                   ^,          p*Ves   \        P6                  ! VV#P                   ^,          X'P                   ^,          ,           3VP                  VP                  R7      p+\        P                  ! VV+.^R7      p\-        V P8                  4       FM  w  p%p&VV",          p'\        P                  ! VV'.^R7      pV&! VVR VV!VR7      pVRR V*13,          pV"^,          p"KO  	  VRR V)1R3,          pV P;                  VV4      p,V P=                  V,V4      p,V,RRR V1R V13,          ) # u upi )Nr+   )rG   .rj  zh w c -> b (h w) c)Úb)rG   rF   )r‚   rƒ   r„   rg  r
   ry   r-   ).r*   rk  ).r   )Nrj  rÐ  )r/   r'   r0   rÿ   ÚldmÚ
common_ditÚpad_to_patch_sizerH   rF   r½  rG   r¤  r¥  rÕ  r­  rÑ  Úaranger   r¦  Úmovedimr   r¬  r  rŽ   r«  r§  r¨  Úonesr©  rª  rÌ  )-r    rß   rØ  r$  rÙ  rä  rå  ræ  ry   ÚbsÚcr  Úwr¬   rn   Úpooled_embedsÚT5_encoder_hidden_statesrÀ  r�   Úhidden_states_typer¥  rg  rƒ   rÊ  rË  Úimg_idsÚencoder_hidden_statesrú   Únew_encoder_hidden_statesr4   Úenc_hidden_stateÚtxt_idsr$   r
   Úblock_idÚinitial_encoder_hidden_statesÚ%initial_encoder_hidden_states_seq_lenÚbidr‘  Ú!cur_llama31_encoder_hidden_statesÚcur_encoder_hidden_statesÚimage_tokens_seq_lenÚhidden_states_seq_lenÚencoder_attention_mask_onesÚoutputs-   &&&&&&&&&                                    r   rß  Ú'HiDreamImageTransformer2DModel._forwardÙ  sG  € ð —g‘g‰ˆˆqØÒ!Ü—	’	˜1˜/¨rÔ2ˆAÜŸ	™	×,Ñ,×>Ñ>¸qÇ?Á?ÐTX×TcÑTcÐBdÓeˆØˆ	ØˆØ#*Ð àˆ	ð #×(Ñ(¨Õ+ˆ
Ø*×0Ñ0Ðð ×)Ñ)¨)°ZÀ×AUÑAUÓVˆ	Ø—O‘O IÐ/AÓBˆ	Ø—_‘_ ]Ó3ˆ
Ø *Õ,ˆà7;·}±}À]×T`ÑT`ÐbkÓ7lÑ4ˆÐ)¨9ØÒ%Ø˜q•\‰FˆB�Ü—k’k " b¨!°M×4HÑ4HÔIˆGØ% f�o´·²¸RÈ×H\ÑH\Ô0]Ð^eÕ0fÕfˆG�F‰OØ% f�o´·²¸RÈ×H\ÑH\Ô0]Ð^eÕ0fÕfˆG�F‰OÜ˜WÐ&:¸jÔIˆGØŸ™¨Ó6ˆð !=× DÑ DÀQÈÓ JÐØCG×CTÒCTÓ UÑCT¸aÐ!6°q×!9Ð!9ÑCTÐÐ Uà×"Ñ"Ò.Ø(*Ð%Ü'0Ð1FÖ'GÑ#�Ð#Ø#'×#:Ñ#:¸1Ö#=Ð>NÓ#OÐ Ø#3×#8Ñ#8¸ÀRÈ×I\ÑI\Ð]_ÕI`Ó#aÐ Ø)×0Ñ0Ð1AÖBñ (Hð %>Ð!Ø'+×'>Ñ'>¸rÖ'BÐC[Ó'\Ð$Ø'?×'DÑ'DÀZÐQSÐUb×UhÑUhÐikÕUlÓ'mÐ$Ø!×(Ñ(Ð)AÔBä—+’+ØØ! "Õ%×+Ñ+¨AÕ.Ð1FÀrÕ1J×1PÑ1PÐQRÕ1SÕSÐVkÐlmÕVn×VtÑVtÐuvÕVwÕwØØ—>‘>¨¯©ô	
ˆô �iŠi˜ 'Ð*°Ô2ˆØ×Ñ Ó$ˆð ˆÜ(-¯	ª	Ð3HÈÕ3LÐNcÐdfÕNgÐ2hÐnoÔ(pÐ%Ø0M×0SÑ0SÐTUÕ0VÐ-Ü# D×$=Ñ$=Ö>‰JˆC�Ø0EÀhÕ0OÐ-Ü(-¯	ª	Ð3PÐRsÐ2tÐz{Ô(|Ð%Ù;@Ø,Ø%7Ø7Ø)ØØ$7ô<Ñ8ˆMÐ8ð -JÈ!ÐMsÐNsÐMsÐJsÕ,tÐ)Ø˜�MŠHñ ?ð  -×2Ñ2°1Õ5ÐÜŸ	š	 =Ð2OÐ"PÐVWÔXˆØ -× 3Ñ 3°AÕ 6ÐØÒ)Ü*/¯*ª*ØÐ:×@Ñ@ÀÕCÐFg×FmÑFmÐnoÕFpÕpÐqØ)×0Ñ0Ð8J×8PÑ8Pô+Ð'ô "'§¢Ð,>Ð@[Ð+\ÐbcÔ!dÐä# D×$=Ñ$=Ö>‰JˆC�Ø0EÀhÕ0OÐ-Ü!ŸIšI }Ð6WÐ&XÐ^_Ô`ˆMÙ!Ø*Ø#5Ø Ø'ØØ$7ôˆMð *¨!Ð-CÐ.CÐ-CÐ*CÕDˆMØ˜�MŠHñ ?ð & aÐ)>Ð*>Ð)>ÀÐ&CÕDˆØ×!Ñ! -°Ó=ˆØ—‘ ¨Ó3ˆØ�q˜!˜R˜a˜R  ! �|Õ$Ð$Ð$ùòG !Vs   Ç#U0)rQ  r¬  r¨  rF   rª  r£  r¡   r   r­  rP  r›  rœ  rI   r¥  rH   r§  r©  r¤  r¦  )NrÎ   Né   é    é€   é   Ni   rR   r   )r  r  )r	  r	  NNNNNr   )r9   r:   r;   r<   r   r½  rÌ  rÕ  r6   rß  r=   r>   r?   r@   s   @@r   r™  r™  D  sn   ù‡ € ÷PZõ PZòd÷	ð 	ô%ð8 %)Ø*.Ø%)ØØØ ÷lò lð( %)Ø*.Ø%)ØØØ ÷m%÷ m%ò m%r   r™  )-Útypingr   r   r   r'   Útorch.nnrÜ   rÅ  r   Úcomfy.ldm.lightricks.modelr   r   Útorch.nn.functionalrÝ   rý   Úcomfy.ldm.flux.mathr	   r
   Úcomfy.ldm.flux.layersr   Úcomfy.ldm.modules.attentionr   r   r   Úcomfy.model_managementrÿ   Úcomfy.patcher_extensionÚcomfy.ldm.common_ditÚModuler   rB   rT   rb   r~   r€   r´   rÐ   râ   r  r?  rI  rM  rw  rŒ  r™  r±   r   r   Ú<module>r     s<  ðß (Ñ (ã Ý Û Ý ç Cß Ð ç 0Ý +ç eÑ eÛ Û Û ô ˆb�i‰iô  ô�—‘ô ô$2�"—)‘)ô 2ô	�B—I‘Iô 	ð `bÐw{÷ K÷G!ñ G!ôRA
�r—y‘yô A
ôHJ˜Ÿ	™	ô Jô6)/ˆb�i‰iô )/ôZ;˜2Ÿ9™9ô ;ô|�R—Y‘Yô ÷ñ ô
F¨¯©ô FôRW) 2§9¡9ô W)ôt)
˜Ÿ	™	ô )
ôXB% R§Y¡Yö B%r   