+
    ƒ°ñjÌ\  ã                   ó   € ^ RI HtHt ^ RIt^ RIHt ^ RIHu Ht ^ RI	H
t
Ht ^ RIHt ^ RIHt ^ RIHt ^ RIHtHtHt ^ RIt^ RItR tR	 R
 lt ! R R]P6                  4      t ! R R]P6                  4      t ! R R]P6                  4      t ! R R]P6                  4      t ! R R]P6                  4      t  ! R R]P6                  4      t! ! R R]P6                  4      t" ! R R]P6                  4      t# ! R R]P6                  4      t$R# )é    )ÚOptionalÚTupleN)Ú	rearrangeÚrepeat)Ú	Timesteps)ÚEmbedND)Úapply_rope1)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention_maskedc                 óL   € V P                   ^,          ^ 8X  d   V # \        W4      # )é   )Úshaper	   )ÚxÚ	freqs_ciss   &&Ú>C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\omnigen\omnigen2.pyÚapply_rotary_embr      s!   € Ø‡w�wˆq…z�Q„Øˆä�qÓ$Ð$ó    c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )é   r   ÚyÚreturn©ÚtorchÚTensor)Úformats   "r   Ú__annotate__r      s-   € ÷ +ñ +Œe�l‰lð +œuŸ|™|ð +´·±ñ +r   c                 óP   € \         P                  ! V R R7      P                  V4      # )T)Úinplace)ÚFÚsiluÚmul_)r   r   s   &&r   Úswiglur#      s   € Ü�6Š6�!˜TÔ"×'Ñ'¨Ó*Ð*r   c                   óT   a a€ ] tR t^t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚTimestepEmbeddingc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Úin_channelsÚtime_embed_dim©Úint)r   Ú__classdict__s   "€r   r   ÚTimestepEmbedding.__annotate__   s   ø€ ÷ fñ f¡Cð f¹ñ fr   c                ó¸   <€ \         SV `  4        VP                  WW4R 7      V n        \        P
                  ! 4       V n        VP                  W"W4R 7      V n        R# )©ÚdtypeÚdeviceN)ÚsuperÚ__init__ÚLinearÚlinear_1ÚnnÚSiLUÚactÚlinear_2)Úselfr'   r(   r/   r0   Ú
operationsÚ	__class__s   &&&&&&€r   r2   ÚTimestepEmbedding.__init__   sJ   ø€ Ü‰ÑÔØ"×)Ñ)¨+ÈUÐ)ÓbˆŒÜ—7’7“9ˆŒØ"×)Ñ)¨.ÐPUÐ)ÓeˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úsampler   r   )r   r+   s   "€r   r   r,   #   s#   ø€ ÷ ñ ™eŸl™lð ©u¯|©|ñ r   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# ©N)r4   r7   r8   )r9   r>   s   &&r   ÚforwardÚTimestepEmbedding.forward#   s1   € Ø—‘˜vÓ&ˆØ—‘˜&Ó!ˆØ—‘˜vÓ&ˆØˆr   )r7   r4   r8   )NNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r2   rA   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r;   r+   s   @@r   r%   r%      s    ù‡ € ÷fõ f÷÷ ð r   r%   c                   óT   a a€ ] tR t^*t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚLuminaRMSNormZeroc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   Úembedding_dimÚnorm_eps©r*   Úfloat)r   r+   s   "€r   r   ÚLuminaRMSNormZero.__annotate__+   s   ø€ ÷ `ñ `¡cð `±Uñ `r   c                óÜ   <€ \         SV `  4        \        P                  ! 4       V n        VP                  \        VR 4      ^V,          W4R7      V n        VP                  WW4R7      V n	        R# )é   r.   ©Úepsr/   r0   N)
r1   r2   r5   r6   r!   r3   ÚminÚlinearÚRMSNormÚnorm)r9   rO   rP   r/   r0   r:   r;   s   &&&&&&€r   r2   ÚLuminaRMSNormZero.__init__+   sX   ø€ Ü‰ÑÔÜ—G’G“IˆŒ	Ø ×'Ñ'¬¨M¸4Ó(@À!ÀmÕBSÐ[`Ð'ÓpˆŒØ×&Ñ& }È%Ð&Ó_ˆŽ	r   c          
      óÂ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  S[ P                  S[ P                  3,          /# )r   r   Úembr   )r   r   r   )r   r+   s   "€r   r   rS   1   sQ   ø€ ÷ 0ñ 0™Ÿ™ð 0©E¯L©Lð 0¹UÁ5Ç<Á<ÑQV×Q]ÑQ]Ñ_d×_kÑ_kÑmr×myÑmyÐCyÕ=zñ 0r   c                óÂ   € V P                  V P                  V4      4      pVP                  ^^R7      w  r4rVV P                  V4      ^VR,          ,           ,          pWWV3# )é   ©Údim)ºNNNN)rY   r!   Úchunkr[   )r9   r   r^   Ú	scale_msaÚgate_msaÚ	scale_mlpÚgate_mlps   &&&    r   rA   ÚLuminaRMSNormZero.forward1   sV   € Ø�k‰k˜$Ÿ)™) C›.Ó)ˆØ36·9±9¸QÀA°9Ó3FÑ0ˆ	˜YØ�I‰I�a‹L˜A 	¨'Õ 2Õ2Õ3ˆØ˜IÐ/Ð/r   )rY   r[   r!   )çñhãˆµøä>NNNrC   rK   s   @@r   rM   rM   *   s    ù‡ € ÷`õ `÷0÷ 0ð 0r   rM   c                   óT   a a€ ] tR t^8t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚLuminaLayerNormContinuousc                óH   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[S[ ,          /# )r   rO   Úconditioning_embedding_dimÚelementwise_affinerW   Úout_dim)r*   ÚboolrR   r   )r   r+   s   "€r   r   Ú&LuminaLayerNormContinuous.__annotate__9   sL   ø€ ÷ Jñ J¡cð JÁsð JÑ`dð JÑsxð Jñ  KSñ  TWõ  KXñ Jr   c	                ó  <€ \         S	V `  4        \        P                  ! 4       V n        VP                  W!WgR 7      V n        VP                  WW6VR 7      V n        Ve   VP                  WRWgR7      V n	        R# RV n	        R# )r.   NT©Úbiasr/   r0   )
r1   r2   r5   r6   r!   r3   r4   Ú	LayerNormr[   r8   )
r9   rO   rn   ro   rW   rp   r/   r0   r:   r;   s
   &&&&&&&&&€r   r2   Ú"LuminaLayerNormContinuous.__init__9   sy   ø€ Ü‰ÑÔÜ—G’G“IˆŒ	Ø"×)Ñ)Ð*DÐ[`Ð)ÓpˆŒØ×(Ñ(¨Ð=OÐekÐ(ÓlˆŒ	ØlsÒl˜
×)Ñ)¨-ÀtÐSXÐ)ÓhˆŽð  FJˆŽr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   r   Úconditioning_embeddingr   r   )r   r+   s   "€r   r   rr   @   s1   ø€ ÷ ñ ™Ÿ™ð ¹u¿|¹|ð ÑPU×P\ÑP\ñ r   c                ó  € V P                  V P                  V4      P                  VP                  4      4      pV P	                  V4      ^V,           R,          ,          pV P
                  e   V P                  V4      pV# )r   )rc   Nrc   )r4   r!   Útor/   r[   r8   )r9   r   ry   r^   s   &&& r   rA   Ú!LuminaLayerNormContinuous.forward@   sa   € Ø�m‰m˜DŸI™IÐ&<Ó=×@Ñ@ÀÇÁÓIÓJˆØ�I‰I�a‹L˜A �G ZÕ0Õ0ˆØ�=‰=Ò$Ø—‘˜aÓ ˆAØˆr   )r4   r8   r[   r!   )Fç�íµ ÷Æ°>NNNNrC   rK   s   @@r   rl   rl   8   s    ù‡ € ÷Jõ J÷÷ ð r   rl   c                   óT   a a€ ] tR t^Ht oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚLuminaFeedForwardc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   rb   Ú	inner_dimÚmultiple_ofr)   )r   r+   s   "€r   r   ÚLuminaFeedForward.__annotate__I   s'   ø€ ÷ bñ b™Cð b©Cð b¹cñ br   c                óô   <€ \         SV `  4        W2V,           ^,
          V,          ,          pVP                  WRWER7      V n        VP                  W!RWER7      V n        VP                  WRWER7      V n        R# )r   Frt   N)r1   r2   r3   r4   r8   Úlinear_3)r9   rb   r�   r‚   r/   r0   r:   r;   s   &&&&&&&€r   r2   ÚLuminaFeedForward.__init__I   sp   ø€ Ü‰ÑÔØ°Õ$;¸aÕ$?ÀKÕ#OÕPˆ	Ø"×)Ñ)¨#¸uÈEÐ)ÓaˆŒØ"×)Ñ)¨)¸uÈEÐ)ÓaˆŒØ"×)Ñ)¨#¸uÈEÐ)ÓaˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r   r   r   )r   r+   s   "€r   r   rƒ   P   s#   ø€ ÷ -ñ -™Ÿ™ð -©%¯,©,ñ -r   c                óx   € V P                  V4      V P                  V4      r2V P                  \        W#4      4      # r@   )r4   r…   r8   r#   )r9   r   Úh1Úh2s   &&  r   rA   ÚLuminaFeedForward.forwardP   s.   € Ø—‘˜qÓ! 4§=¡=°Ó#3ˆBØ�}‰}œV B›^Ó,Ð,r   )r4   r8   r…   )é   NNNrC   rK   s   @@r   r   r   H   s    ù‡ € ÷bõ b÷-÷ -ð -r   r   c                   óT   a a€ ] tR t^Ut oRV3R lV 3R llltV3R lR ltRtVtV ;t# )Ú'Lumina2CombinedTimestepCaptionEmbeddingc          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[/# )r   Úhidden_sizeÚtext_feat_dimÚfrequency_embedding_sizerP   Útimestep_scalerQ   )r   r+   s   "€r   r   Ú4Lumina2CombinedTimestepCaptionEmbedding.__annotate__V   s7   ø€ ÷ 
ñ 
¡Cð 
¹sð 
Ñehð 
Ñzð 
ñ  Y^ñ 
r   c	                ó  <€ \         S	V `  4        \        VR RVR7      V n        \	        V\        VR4      WgVR7      V n        \        P                  ! VP                  W$WgR7      VP                  W!R WgR7      4      V n        R# )Tç        )Únum_channelsÚflip_sin_to_cosÚdownscale_freq_shiftÚscalerU   )r'   r(   r/   r0   r:   rV   rt   N)r1   r2   r   Ú	time_projr%   rX   Útimestep_embedderr5   Ú
SequentialrZ   r3   Úcaption_embedder)
r9   r�   r‘   r’   rP   r“   r/   r0   r:   r;   s
   &&&&&&&&&€r   r2   Ú0Lumina2CombinedTimestepCaptionEmbedding.__init__V   s�   ø€ Ü‰ÑÔÜ"Ð0HÐZ^Ðuxð  AOô  PˆŒÜ!2Ð?WÔhkÐlwÐy}Óh~ð  GLð  hrô  "sˆÔÜ "§¢Ø×Ñ˜}À%ÐÓWØ×Ñ˜m¸tÈ5ÐÓ`ó!
ˆÖr   c          
      ó¬   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# )r   ÚtimestepÚtext_hidden_statesr/   r   )r   r   r/   r   )r   r+   s   "€r   r   r”   _   se   ø€ ÷ )ñ )¡§¡ð )Á%Ç,Á,ð )ÑW\×WbÑWbð )ÑglÑmr×myÑmyñ  |A÷  |Hñ  |Hð  nHõ  hIñ )r   c                óŽ   € V P                  V4      P                  VR 7      pV P                  V4      pV P                  V4      pWV3# ))r/   )r›   r{   rœ   rž   )r9   r¡   r¢   r/   Útimestep_projÚ
time_embedÚcaption_embeds   &&&&   r   rA   Ú/Lumina2CombinedTimestepCaptionEmbedding.forward_   sH   € ØŸ™ xÓ0×3Ñ3¸%Ð3Ó@ˆØ×+Ñ+¨MÓ:ˆ
Ø×-Ñ-Ð.@ÓAˆØÐ(Ð(r   )rž   r›   rœ   )i   i   rŒ   rj   ç      ð?NNNrC   rK   s   @@r   rŽ   rŽ   U   s   ù‡ € ÷
õ 
÷)÷ )ð )r   rŽ   c                   ó^   a a€ ] tR t^ft oRV3R lV 3R llltRR/ 3V3R lR lltRtVtV ;t# )Ú	AttentionNc                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[RS[/# )r   Ú	query_dimÚdim_headÚheadsÚkv_headsrW   ru   ©r*   rR   rq   )r   r+   s   "€r   r   ÚAttention.__annotate__g   s9   ø€ ÷ 
ñ 
¡#ð 
±ð 
¹Sð 
ÉCð 
ÑV[ð 
Ñjnñ 
r   c
           
     ó2  <€ \         S
V `  4        \        4       V n        W0n        W@n        W n        VR,          V n        V	P                  WV,          WgVR7      V n	        V	P                  WV,          WgVR7      V n
        V	P                  WV,          WgVR7      V n        V	P                  W%WxR7      V n        V	P                  W%WxR7      V n        \        P                   ! V	P                  W2,          WWxR7      \        P"                  ! R4      4      V n        R# )g      à?rt   rV   r–   Ng      à¿)r1   r2   r   Úcomfy_attentionr®   r¯   r­   rš   r3   Úto_qÚto_kÚto_vrZ   Únorm_qÚnorm_kr5   r�   ÚDropoutÚto_out)r9   r¬   r­   r®   r¯   rW   ru   r/   r0   r:   r;   s   &&&&&&&&&&€r   r2   ÚAttention.__init__g   sò   ø€ Ü‰ÑÔÜ-Ó/ˆÔØŒ
Ø ŒØ ŒØ Õ%ˆŒ
à×%Ñ% i¸Õ1AÈÐbhÐ%ÓiˆŒ	Ø×%Ñ% i¸HÕ1DÈ4ÐekÐ%ÓlˆŒ	Ø×%Ñ% i¸HÕ1DÈ4ÐekÐ%ÓlˆŒ	à ×(Ñ(¨À%Ð(ÓWˆŒØ ×(Ñ(¨À%Ð(ÓWˆŒä—m’mØ×Ñ˜eÕ.°	ÈEÐÓaÜ�JŠJ�s‹Oó
ˆŽr   c          
      ó¼   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[ P                  /# )r   Úhidden_statesÚencoder_hidden_statesÚattention_maskÚimage_rotary_embr   ©r   r   r   )r   r+   s   "€r   r   r±   {   sy   ø€ ÷ ñ ¡U§\¡\ð É%Ï,É,ð ÑhpÑqv×q}Ñq}Õh~ð ñ  Zbñ  ch÷  coñ  coõ  Zpð ñ  TY÷  T`ñ  T`ñ r   c           	     óX  € VP                   w  rgpV P                  V4      p	V P                  V4      p
V P                  V4      pV	P	                  VRV P
                  V P                  4      p	V
P	                  VRV P                  V P                  4      p
VP	                  VRV P                  V P                  4      pV P                  V	4      p	V P                  V
4      p
Ve   \        W”4      p	\        W¤4      p
\        V	P                  ^^4      4      p	\        V
P                  ^^4      4      p
\        VP                  ^^4      4      pV P                  V P
                  8  d   RR/M/ p\        WšW°P
                  V3RRRV P                  RV/VB pV P                  ^ ,          ! V4      pV# )r   Ú
enable_gqaTÚskip_reshapeÚpreferred_attentionÚtransformer_optionséÿÿÿÿ)r   r´   rµ   r¶   Úviewr®   r­   r¯   r·   r¸   r   r
   Ú	transposer   r³   rº   )r9   r½   r¾   r¿   rÀ   rÆ   Ú
batch_sizeÚsequence_lengthÚ_ÚqueryÚkeyÚvalueÚ
gqa_kwargss   &&&&&&       r   rA   ÚAttention.forward{   sŽ  € Ø)6×)<Ñ)<Ñ&ˆ
 Qà—	‘	˜-Ó(ˆØ�i‰iÐ-Ó.ˆØ—	‘	Ð/Ó0ˆà—
‘
˜: r¨4¯:©:°t·}±}ÓEˆØ�h‰h�z 2 t§}¡}°d·m±mÓDˆØ—
‘
˜: r¨4¯=©=¸$¿-¹-ÓHˆà—‘˜EÓ"ˆØ�k‰k˜#ÓˆàÒ'Ü$ UÓ=ˆEÜ" 3Ó9ˆCä(¨¯©¸¸AÓ)>Ó?ˆÜ& s§}¡}°Q¸Ó':Ó;ˆÜ(¨¯©¸¸AÓ)>Ó?ˆà-1¯]©]¸T¿Z¹ZÔ-G�l DÑ)ÈRˆ
Ü2°5¸uÇjÁjÐR`ñ  VÐosð  Vð  JN÷  J^ñ  J^ð  Vð  tGð  Vð  KUñ  VˆØŸ™ Až }Ó5ˆØÐr   )r³   r­   r®   r¯   r¸   r·   rš   rµ   rº   r´   r¶   )rj   FNNNrC   rK   s   @@r   rª   rª   f   s3   ù‡ € ÷
õ 
ð( BFð  swð  MO÷ ÷ ò r   rª   c                   ó\   a a€ ] tR t^—t oRV3R lV 3R llltR/ 3V3R lR lltRtVtV ;t# )ÚOmniGen2TransformerBlockNc                óD   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[RS[RS[/# )r   rb   Únum_attention_headsÚnum_kv_headsr‚   Úffn_dim_multiplierrP   Ú
modulationr°   )r   r+   s   "€r   r   Ú%OmniGen2TransformerBlock.__annotate__˜   sS   ø€ ÷ [ñ [™Cð [±cð [Éð [Ñ[^ð [Ñtyð [ñ  FKð [ñ  Y]ñ [r   c                óœ  <€ \         SV `  4        Wpn        \        VW,          VVR RW‰V
R7	      V n        \        V^V,          VW‰V
R7      V n        V'       d   \        WW‰V
R7      V n        MV
P                  WW‰R7      V n        V
P                  WW‰R7      V n
        V
P                  WW‰R7      V n        V
P                  WW‰R7      V n        R# )rj   F)	r¬   r­   r®   r¯   rW   ru   r/   r0   r:   )rb   r�   r‚   r/   r0   r:   )rO   rP   r/   r0   r:   rV   N)r1   r2   rØ   rª   Úattnr   Úfeed_forwardrM   Únorm1rZ   Ú	ffn_norm1Únorm2Ú	ffn_norm2)r9   rb   rÕ   rÖ   r‚   r×   rP   rØ   r/   r0   r:   r;   s   &&&&&&&&&&&€r   r2   Ú!OmniGen2TransformerBlock.__init__˜   sÔ   ø€ Ü‰ÑÔØ$ŒäØØÕ/Ø%Ø!ØØØ°:ô
ˆŒ	ô .ØØ˜#•gØ#Ø°:ô	
ˆÔ÷ Ü*¸ÐW\ð  yCô  DˆD�Jà#×+Ñ+¨CÀUÐ+ÓZˆDŒJà#×+Ñ+¨CÀUÐ+ÓZˆŒØ×'Ñ'¨ÀÐ'ÓVˆŒ
Ø#×+Ñ+¨CÀUÐ+ÓZˆŽr   c          
      ó¬   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          RS[ P                  /# )r   r½   r¿   rÀ   Útembr   rÁ   )r   r+   s   "€r   r   rÙ   ¶   so   ø€ ÷ ñ ¡U§\¡\ð Á5Ç<Á<ð Ñch×coÑcoð Ñwñ  AF÷  AMñ  AMõ  xNð ñ  rw÷  r~ñ  r~ñ r   c                óÊ  € V P                   '       dÞ   V P                  W4      w  rgr‰V P                  WfW#VR 7      p
WP                  ^4      P	                  4       V P                  V
4      ,          ,           pV P                  V P                  V4      ^VP                  ^4      ,           ,          4      pWP                  ^4      P	                  4       V P                  V4      ,          ,           pV# V P                  V4      pV P                  WfW#VR 7      p
WP                  V
4      ,           pV P                  V P                  V4      4      pWP                  V4      ,           pV# )©rÆ   )	rØ   rÝ   rÛ   Ú	unsqueezeÚtanhrß   rÜ   rÞ   rà   )r9   r½   r¿   rÀ   rã   rÆ   Únorm_hidden_statesrf   rg   rh   Úattn_outputÚ
mlp_outputs   &&&&&&      r   rA   Ú OmniGen2TransformerBlock.forward¶   s9  € Ø�?�?ˆ?Ø@DÇ
Á
È=Ó@_Ñ=Ð¨)ØŸ)™)Ð$6ÈNð  CV˜)ó  WˆKØ)×,>Ñ,>¸qÓ,A×,FÑ,FÓ,HÈ4Ï:É:ÐVaÓKbÕ,bÕbˆMØ×*Ñ*¨4¯>©>¸-Ó+HÈAÐPY×PcÑPcÐdeÓPfÕLfÕ+gÓhˆJØ)×,>Ñ,>¸qÓ,A×,FÑ,FÓ,HÈ4Ï>É>ÐZdÓKeÕ,eÕeˆMð Ðð "&§¡¨MÓ!:ÐØŸ)™)Ð$6ÈNð  CV˜)ó  WˆKØ)¯J©J°{Ó,CÕCˆMØ×*Ñ*¨4¯>©>¸-Ó+HÓIˆJØ)¯N©N¸:Ó,FÕFˆMØÐr   )rÛ   rÜ   rÞ   rà   rØ   rÝ   rß   )TNNNrC   rK   s   @@r   rÓ   rÓ   —   s.   ù‡ € ÷[õ [ð< QUð  km÷ ÷ ò r   rÓ   c                   óH   a a€ ] tR t^Æt oRV3R lV 3R llltR tRtVtV ;t# )ÚOmniGen2RotaryPosEmbedc          	      óf   <€ V ^8„  d   QhRS[ RS[S[ S[ S[ 3,          RS[S[ S[ S[ 3,          RS[ /# )r   ÚthetaÚaxes_dimÚ	axes_lensÚ
patch_size)r*   r   )r   r+   s   "€r   r   Ú#OmniGen2RotaryPosEmbed.__annotate__Ç   sN   ø€ ÷ ]ñ ]™cð ]©U±3¹¹S°=Õ-Að ]ÉeÑTWÑY\Ñ^aÐTaÕNbð ]ñ  CFñ ]r   c                ó    <€ \         SV `  4        Wn        W n        W0n        W@n        \        \        V4      V P                  VR 7      V n        R# ))rb   rï   rð   N)	r1   r2   rï   rð   rñ   rò   r   ÚsumÚrope_embedder)r9   rï   rð   rñ   rò   r;   s   &&&&&€r   r2   ÚOmniGen2RotaryPosEmbed.__init__Ç   s<   ø€ Ü‰ÑÔØŒ
Ø ŒØ"ŒØ$ŒÜ$¬¨X«¸d¿j¹jÐS[Ô\ˆÖr   c	           
     óÆ  € V P                   p	\        W4V4       U
UUu. uF  w  r«qÊ\        V4      ,           V,           NK   	  ppp
p\        V4      p\        V Uu. uF  p\        V4      NK  	  up4      p\        V4      p\        P
                  ! W^\        P                  VR7      p\        \        W=4      4       EF  w  pw  pp\        \        P                  ! V\        P                  VR7      R4      VVRV13&   TpTpVV,          Ee   \        VV,          VV,          4       Fî  w  ppVw  ppVV	,          VV	,          pp\        \        P                  ! V\        P                  VR7      RVR7      P                  4       p\        \        P                  ! V\        P                  VR7      RVR7      P                  4       pVVVVVV,           1^ 3&   VVVVVV,           1^3&   VVVVVV,           1^3&   V\        VV4      ,          pVV,          pKð  	  VV,          w  ppVV	,          VV	,          pp\        \        P                  ! V\        P                  VR7      RVR7      P                  4       p\        \        P                  ! V\        P                  VR7      RVR7      P                  4       pVVVVV1^ 3&   VVVVV1^3&   VVVVV1^3&   EK  	  V P                  V4      P                  ^^4      p \        V P                  4      p!VV!^&   \        P
                  ! V!RVR	V P                  / p"\        V P                  4      p#VV#^&   \        P
                  ! V#RVR	V P                  / p$\        V P                  4      p%VV%^&   \        P
                  ! V%RVR	V P                  / p&\        \        W4W]4      4       F†  w  pw  pr¼pV VRV13,          V"VRV13&   V VVV\        V4      ,           13,          V$VR\        V4      13&   V VV\        V4      ,           V\        V4      ,           V,           13,          V&VRV13&   Kˆ  	  V"V$V&V W=3# u uppp
i u upi )
é   r.   zl -> l 3Nzh -> h w)Úwzw -> h w)Úhr0   r/   )rò   Úziprõ   Úmaxr   ÚzerosÚint32Ú	enumerater   ÚarangeÚflattenrö   ÚmovedimÚlistr   r/   )'r9   rÊ   Úencoder_seq_lenÚl_effective_cap_lenÚl_effective_ref_img_lenÚl_effective_img_lenÚref_img_sizesÚ	img_sizesr0   ÚpÚcap_lenÚref_img_lenÚimg_lenÚseq_lengthsÚmax_seq_lenÚmax_ref_img_lenÚmax_img_lenÚposition_idsÚiÚcap_seq_lenÚseq_lenÚpe_shiftÚpe_shift_lenÚref_img_sizeÚHÚWÚref_H_tokensÚref_W_tokensÚrow_idsÚcol_idsÚH_tokensÚW_tokensr   Úcap_freqs_cis_shapeÚcap_freqs_cisÚref_img_freqs_cis_shapeÚref_img_freqs_cisÚimg_freqs_cis_shapeÚimg_freqs_ciss'   &&&&&&&&&                              r   rA   ÚOmniGen2RotaryPosEmbed.forwardÏ   s  € Ø�O‰OˆäadÐexð  Tgô  bhõ  iñ  bhÑ@]ÀÐV]¤ [Ó!1Õ1°G×;Ð;ñ  bhˆò  iä˜+Ó&ˆÜÑCZÓ[ÑCZ°Kœs ;Ö/ÑCZÑ[Ó\ˆÜÐ-Ó.ˆä—{’{ :¸AÄUÇ[Á[ÐY_Ô`ˆä)2´3Ð7JÓ3X×)YÑ%ˆAÑ%�˜WÜ,2´5·<²<ÀÔSX×S^ÑS^ÐgmÔ3nÐpzÓ,{ˆL˜˜L˜[˜L˜Ñ)à"ˆHØ&ˆLà˜QÕÓ+Ü14°]À1Õ5EÐG^Ð_`ÕGaÖ1bÑ-�L +Ø'‘D�A�qØ12°aµ¸¸a½ ,�Lä$¤U§\¢\°,ÄeÇkÁkÐZ`Ô%aÐcmÐq}Ô~÷  Gñ  Gó  I�GÜ$¤U§\¢\°,ÄeÇkÁkÐZ`Ô%aÐcmÐq}Ô~÷  Gñ  Gó  I�GØRZ�L  L°ÀÕ1KÐ$KÈQÐ!NÑOØRY�L  L°ÀÕ1KÐ$KÈQÐ!NÑOØRY�L  L°ÀÕ1KÐ$KÈQÐ!NÑOà¤ L°,Ó ?Õ?�HØ  KÕ/’Lñ 2cð ˜Q•<‰DˆAˆqØ!" a¥¨¨a­�hˆHäœUŸ\š\¨(¼%¿+¹+ÈfÔUÐWaÐemÔn×vÑvÓxˆGÜœUŸ\š\¨(¼%¿+¹+ÈfÔUÐWaÐemÔn×vÑvÓxˆGà8@ˆL˜˜L¨'Ð1°1Ð4Ñ5Ø8?ˆL˜˜L¨'Ð1°1Ð4Ñ5Ø8?ˆL˜˜L¨'Ð1°1Ð4Ô5ñ9 *Zð< ×&Ñ& |Ó4×<Ñ<¸QÀÓBˆ	ä" 9§?¡?Ó3ÐØ!0Ð˜AÑÜŸšÐ%8Ð_ÀÐ_ÈyÏÉÑ_ˆä"& y§¡Ó"7ÐØ%4Ð Ñ"Ü!ŸKšKÐ)@ÐgÈÐgÐW`×WfÑWfÑgÐä" 9§?¡?Ó3ÐØ!,Ð˜AÑÜŸšÐ%8Ð_ÀÐ_ÈyÏÉÑ_ˆä?HÌÐM`ð  |Oó  J]ö  @^Ñ;ˆAÑ;�˜[°7Ø-6°q¸,¸;¸,°Õ-GˆM˜!˜\˜k˜\˜/Ñ*Ø6?ÀÀ;È{Ô]`ÐalÓ]mÕOmÐCmÐ@mÕ6nÐ˜aÐ!2¤# kÓ"2Ð!2Ð2Ñ3Ø)2°1°kÄCÈÓDTÕ6TÐU`ÔcfÐgrÓcsÕUsÐv}ÕU}Ð6}Ð3}Õ)~ˆM˜!˜X˜g˜X˜+Ó&ñ @^ð
 Ð/°À	ÐK^ÐkÐkùôs iùò \s   �$QÁQ)rð   rñ   rò   rö   rï   )©i,  é   r*  r   rC   rK   s   @@r   rí   rí   Æ   s   ù‡ € ÷]õ ]÷<lò <lr   rí   c                   ód   a a€ ] tR tRt oR	V3R lV 3R llltR t/ 3R ltRR/ 3R ltRtVt	V ;t
# )
ÚOmniGen2Transformer2DModeli  Nc                óÈ   <€ V ^8„  d   QhRS[ RS[ RS[S[ ,          RS[ RS[ RS[ RS[ RS[ R	S[ R
S[S[,          RS[RS[S[ S[ S[ 3,          RS[S[ S[ S[ 3,          RS[ RS[/# )r   rò   r'   Úout_channelsr�   Ú
num_layersÚnum_refiner_layersrÕ   rÖ   r‚   r×   rP   Úaxes_dim_roperñ   r‘   r“   )r*   r   rR   r   )r   r+   s   "€r   r   Ú'OmniGen2Transformer2DModel.__annotate__  sá   ø€ ÷ Rkñ RkáðRkñ ðRkñ ™s•mð	Rkñ
 ðRkñ ðRkñ  ðRkñ !ðRkñ ðRkñ ðRkñ %¡U�OðRkñ ðRkñ ™S¡#¡s˜]Õ+ðRkñ ™™c¡3˜Õ'ðRkñ ðRkñ  ñ!Rkr   c                ó†  <€ \         SV `  4        Wn        T;'       g    TV n        W@n        VV n        \        R VVVR7      V n        VP                  W,          V,          VVVR7      V n	        VP                  W,          V,          VVVR7      V n
        \        VVVVVVVR7      V n        \        P                  ! \        V4       Uu. uF  p\!        WGVWšVRVVVR7
      NK  	  up4      V n        \        P                  ! \        V4       Uu. uF  p\!        WGVWšVRVVVR7
      NK  	  up4      V n        \        P                  ! \        V4       Uu. uF  p\!        WGVWšVRVVVR7
      NK  	  up4      V n        \        P                  ! \        V4       Uu. uF  p\!        WGVWšVRVVVR7
      NK  	  up4      V n        \+        V\-        VR4      RRW,          V P                  ,          VVVR	7      V n        \        P0                  ! \2        P4                  ! ^VVVR
7      4      V n        R# u upi u upi u upi u upi )i'  )rï   rð   rñ   rò   r.   )r�   r‘   rP   r“   r/   r0   r:   T)rØ   r/   r0   r:   FrU   r}   )rO   rn   ro   rW   rp   r/   r0   r:   )r0   r/   N)r1   r2   rò   r.  r�   r/   rí   rö   r3   Ú
x_embedderÚref_image_patch_embedderrŽ   Útime_caption_embedr5   Ú
ModuleListÚrangerÓ   Únoise_refinerÚref_image_refinerÚcontext_refinerÚlayersrl   rX   Únorm_outÚ	Parameterr   ÚemptyÚimage_index_embedding)r9   rò   r'   r.  r�   r/  r0  rÕ   rÖ   r‚   r×   rP   r1  rñ   r‘   r“   Úimage_modelr0   r/   r:   rÌ   r;   s   &&&&&&&&&&&&&&&&&&&& €r   r2   Ú#OmniGen2Transformer2DModel.__init__  sƒ  ø€ ô, 	‰ÑÔà$ŒØ(×7Ð7¨KˆÔØ&ÔØˆŒ
ä3ØØ"ØØ!ô	
ˆÔð %×+Ñ+¨JÕ,CÀkÕ,QÐS^ÐfkÐtzÐ+Ó{ˆŒØ(2×(9Ñ(9¸*Õ:QÐT_Õ:_ÐalÐtyð  CIÐ(9ó  )JˆÔ%ä"IØ#Ø'ØØ)°¸vÐR\ô	#
ˆÔô  Ÿ]š]ô Ð/Ô0ó	,
ñ 1�!ô %Ø°,Ø°ÀdÐRWÐ`fÐs}÷ñ 1ñ	,
ó ˆÔô "$§¢ô Ð/Ô0ó	0
ñ 1�!ô %Ø°,Ø°ÀdÐRWÐ`fÐs}÷ñ 1ñ	0
ó "ˆÔô  "Ÿ}š}ô Ð/Ô0ó	.
ñ 1�!ô %Ø°,Ø°ÀeÐSXÐagÐt~÷ñ 1ñ	.
ó  ˆÔô —m’mô ˜ZÔ(ó	%
ñ )�!ô %Ø°,Ø°ÀdÐRWÐ`fÐs}÷ñ )ñ	%
ó ˆŒô 2Ø%Ü'*¨;¸Ó'=Ø$ØØÕ+¨d×.?Ñ.?Õ?ÀuÐU[Ðhrô
ˆŒô &(§\¢\´%·+²+¸aÀÐU[ÐchÔ2iÓ%jˆÖ"ùòI,
ùò0
ùò.
ùò%
s   Ã
H/ÄH4ÅH9ÆH>c           
     óþ  a€ \        V4      pV P                  oV Uu. uF$  qDP                  ^4      VP                  ^4      3NK&  	  ppV UUu. uF  w  rgVS,          VS,          ,          NK  	  pppVe¥   \        \	        V3R lV4      4      pV U	u. uF*  q™e#   V	P                  ^4      V	P                  ^4      3MRNK,  	  up	.V,          p
V
 UUu. uF=  q»e5   V Uu. uF'  qÌ^ ,          S,          V^,          S,          ,          NK)  	  upM^ .NK?  	  pppM5\        V4       Uu. uF  pRNK  	  p
p\        V4       Uu. uF  p^ .NK  	  ppRpVeX   . p	V F7  pVP                  4       w  pprg\        VRSSR7      pV	P                  V4       K9  	  \        P                  ! V	^R7      pTpVP                  4       w  pprg\        VRSSR7      pVVRRWØW¥3# u upi u uppi u up	i u upi u uppi u upi u upi )r   Nc                 ó\   <€ \         P                  P                  P                  V SS34      # r@   )ÚcomfyÚldmÚ
common_ditÚpad_to_patch_size)Úrefr  s   &€r   Ú<lambda>Ú@OmniGen2Transformer2DModel.flat_and_pad_to_seq.<locals>.<lambda>k  s$   ø€ ¼5¿9¹9×;OÑ;O×;aÑ;aÐbeÐhiÐklÐgmÔ;nr   z&b c (h p1) (w p2) -> b (h w) (p1 p2 c))Úp1Úp2ra   )
Úlenrò   Úsizer  Úmapr8  r   Úappendr   Úcat)r9   r½   Úref_image_hidden_statesrÊ   Úimgr
  r  r  r  Úimgsr	  Ú_ref_img_sizesr  r  rÌ   Úflat_ref_img_hidden_statesÚref_imgÚBÚCÚflat_hidden_statesr  s   &&&                 @r   Úflat_and_pad_to_seqÚ.OmniGen2Transformer2DModel.flat_and_pad_to_seqc  sA  ø€ Ü˜Ó'ˆ
Ø�O‰Oˆá;HÓI¹=°C—h‘h˜q“k 3§8¡8¨A£;Ó/¹=ˆ	ÐIÙAJÔKÁ±v¸  Q¥¨1°­6×2Ð2ÁÐÑKà"Ò.Ü&*¬3Ô/nð  qHó  ,Ió  'JÐ#Ùe|Ó}Ñe|Ð]aÒ>N˜tŸy™y¨›|¨T¯Y©Y°q«\Ñ:ÐTXÒXÑe|Ñ}Ð~ð  BLõ  LˆMñ zGô  'Hñ  zGð  hvò  @ZÑlzÓ'{ÑlzÐ\h°a­¸AÕ)=À,ÈqÅ/ÐUVÕBV×(WÐ(WÑlzÒ'{ð  abð  `cò  (cñ  zGÐ#ñ  'HÐ#ä+0°Ô+<Ó=Ñ+< a›TÑ+<ˆMÐ=Ü49¸*Ô4EÓ&FÑ4E¨q¨£sÑ4EÐ#Ð&Fà%)Ð"Ø"Ò.ØˆDÛ2�Ø$Ÿ\™\›^‘
��1�aÜ# GÐ-UÐZ[Ð`aÔb�Ø—‘˜GÖ$ñ 3ô */¯ª°4¸QÔ)?Ð&àˆØ—X‘X“Z‰
ˆˆ1ˆaÜ& sÐ,TÐYZÐ_`ÔaÐð Ð :Ø�$Ø#Øð	
ð 	
ùò1 JùÛKùò ~ùÚ'{ùó  'Hùâ=ùÚ&Fs5   �*GÁ"GÂ0G%ÃG/Ã -G*Ä
G/Ä(G5ÅG:Ç*G/c           	     ód  € \        V4      pV P                  V4      pVe·   V P                  V4      p\        P                  P                  V P                  VP                  VP                  R7      p\        V4       FV  p^ p\        W},          4       F<  w  ppW-WîV,           1R3,          WÏ,          ,           W-WîV,           1R3&   VV,          pK>  	  KX  	  V P                   F  pV! WWYV
R7      pK  	  Ve8   V P                   F  pV! W$WiV
R7      pK  	  \        P                  ! W!.^R7      pV# )Nr.   rc   rå   ra   )rN  r4  r5  rE  Úmodel_managementÚcast_tor@  r/   r0   r8  r   r9  r:  r   rR  )r9   r½   rS  Úpadded_img_maskÚpadded_ref_img_maskÚnoise_rotary_embÚref_img_rotary_embr  r  rã   rÆ   rÊ   r@  r  ÚshiftÚjr  Úlayers   &&&&&&&&&&&       r   Úimg_patch_embed_and_refineÚ5OmniGen2Transformer2DModel.img_patch_embed_and_refine†  sz  € Ü˜Ó'ˆ
àŸ™¨Ó6ˆØ"Ò.Ø&*×&CÑ&CÐD[Ó&\Ð#Ü$)×$:Ñ$:×$BÑ$BÀ4×C]ÑC]Ðer×exÑexð  BO÷  BVñ  BVÐ$Bó  %WÐ!ä˜:Ö&�Ø�Ü&/Ð0GÕ0JÖ&K‘N�A�{ØOfÐjoð  yDõ  qDð  kDð  FGð  hGõ  PHð  K`õ  Kcõ  PcÐ+¨u¸[Õ5HÐ/HÈ!Ð,KÑLØ˜[Õ(’Eó 'Lñ 'ð ×'Ô'ˆEÙ! -ÐBRð  oBô  CŠMñ (ð #Ò.Ø×/Ô/�Ù*/Ð0GÐ^pð  M`ô  +aÒ'ñ 0ô "ŸIšIÐ'>Ð&NÐTUÔVˆMàÐr   c                óò  € VP                   w  ršr¼\        P                  P                  P	                  WP
                  V P
                  34      pVP                   w   rïpR V,
          pTpTpTpVP                  pV P                  VVV^ ,          P                  4      w  ppV P                  VV4      w  ppppppppV P                  VP                   ^ ,          VP                   ^,          V.VP                   ^ ,          ,          VVVVV4      w  pppp p!p"V P                   F  p#V#! VVVVR7      pK  	  VP                   ^,          p$V P                  VVVVVVVVVVR7
      p%\        P                  ! VV%.^R7      pRpV P                   F  p#V#! WÖV VVR7      pK  	  V P!                  VV4      pV P
                  p&\#        VRV$) R13,          RVV&,          VV&,          V&V&R7      RRRV1RV13,          p'V') # )r¨   rå   ra   Nrc   z&b (h w) (p1 p2 c) -> b c (h p1) (w p2))rû   rú   rL  rM  )r   rE  rF  rG  rH  rò   r0   r6  r/   r\  rö   r;  rh  r   rR  r<  r=  r   )(r9   r   Ú	timestepsÚcontextÚ
num_tokensÚref_latentsr¿   rÆ   ÚkwargsrY  rZ  r  r  r½   rÌ   ÚH_paddedÚW_paddedr¡   r¢   Útext_attention_maskrS  r0   rã   Úimg_maskÚref_img_maskr  r  r	  r
  Úcontext_rotary_embrd  rc  Ú
rotary_embÚencoder_seq_lengthsr  rg  r  Úcombined_img_hidden_statesr  Úoutputs(   &&&&&&&&,                               r   rA   Ú"OmniGen2Transformer2DModel.forwardŸ  sŠ  € Ø—W‘W‰
ˆˆaÜŸ	™	×,Ñ,×>Ñ>¸qÇ?Á?ÐTX×TcÑTcÐBdÓeˆØ#0×#6Ñ#6Ñ ˆˆ1˜Ø˜•?ˆØ$ÐØ,ÐØ"-ÐØ×%Ñ%ˆà#'×#:Ñ#:¸8ÐEWÐYfÐghÕYi×YoÑYoÓ#pÑ ˆÐ ð ×$Ñ$ ]Ð4KÓLñ	
ØÐ2Ø�lØ#Ð%8Ø˜9ð ×ÑØ×Ñ Õ"Ð$6×$<Ñ$<¸QÕ$?À*ÀÐPb×PhÑPhÐijÕPkÕAkØ#Ð%8Ø˜9 fó
ñ	
ØÐ 2Ð4DØÐ+¨[ð ×)Ô)ˆEÙ!&Ð'9Ð;NÐPbð  yLô  "MÒñ *ð  ×%Ñ% aÕ(ˆØ%)×%DÑ%DØÐ2Ø�lØÐ0Ø#Ð%8ØØ 3ð &Eó &
Ð"ô Ÿ	š	Ð#5Ð7QÐ"RÐXYÔZˆØˆà—[”[ˆEÙ! -ÀÈTÐgzÔ{ŠMñ !ð Ÿ™ m°TÓ:ˆà�O‰OˆÜ˜=¨¨W¨H©I¨Õ6Ð8`ÐemÐqrÕerÐv~ð  BCõ  wCð  HIð  NOô  Pð  QRð  TUð  WYð  XYð  WYð  []ð  \]ð  []ð  Q]õ  ^ˆàˆwˆr   )r;  r/   r�   r@  r<  r9  r=  r.  rò   r5  r:  rö   r6  r4  )r   é   Ni 	  é   r   é   é   rŒ   Nrj   )é    r  r  r)  rU   r¨   NNNN)rD   rE   rF   rG   r2   r\  rh  rA   rH   rI   rJ   rK   s   @@r   r,  r,    s=   ù‡ € ÷Rkõ Rkòh!
ðF fhô ð2 FJÐZ^Ðtv÷ 4ô 4r   r,  )%Útypingr   r   r   Útorch.nnr5   Útorch.nn.functionalÚ
functionalr    Úeinopsr   r   Úcomfy.ldm.lightricks.modelr   Úcomfy.ldm.flux.layersr   Úcomfy.ldm.flux.mathr	   Úcomfy.ldm.modules.attentionr
   r   r   Úcomfy.model_managementrE  Úcomfy.ldm.common_ditr   r#   ÚModuler%   rM   rl   r   rŽ   rª   rÓ   rí   r,  © r   r   Ú<module>r�     sÑ   ð÷ #ã Ý ß Ð ß $Ý 0Ý )Ý +ß lÑ lÛ Û ò%õ+ô˜Ÿ	™	ô ô0˜Ÿ	™	ô 0ô §	¡	ô ô 
-˜Ÿ	™	ô 
-ô)¨b¯i©iô )ô".�—	‘	ô .ôb,˜rŸy™yô ,ô^El˜RŸY™Yô ElôPE §¡ö Er   