+
    ƒ°ñj¶Å  ã                   ó�  € ^ RI Ht ^ RIHtHtHtHtHtHtH	t	 ^ RI
Hu Ht ^ RIHtHt ^ RIt^ RIHtHt ^ RIHtHt ^ RIHt ^ RIHt ^ R	IHt ^ R
IHt ^ RIt^ RIHt ^ RI H!t!H"t"H#t#H$t$H%t%H&t&H't'H(t(H)t) ^ RI*t+^ RI,t+^ RI-t+ ! R R4      t.R R lt/R t0] ! R R4      4       t1R R lt2R R lt3R R lt4R R lt5R R lt6 ! R R]Pn                  4      t8 ! R  R!]Pn                  4      t9 ! R" R#]94      t: ! R$ R%]:4      t; ! R& R']94      t<R( t=R) R* lt>R+ R, lt? ! R- R.]<4      t@ ! R/ R0]Pn                  4      tAR1 R2 ltB ! R3 R4]Pn                  4      tCR5 R6 ltDR7 R8 ltE ! R9 R:]C4      tF ! R; R<]Pn                  4      tG ! R= R>]Pn                  4      tHRWR? R@ lltI ! RA RB]Pn                  4      tJ ! RC RD]Pn                  4      tK ! RE RF]K4      tL ! RG RH]Pn                  4      tM ! RI RJ]M4      tNRK RL ltO ! RM RN]Pn                  4      tP ! RO RP]Pn                  4      tQRQ RR ltRRS RT ltS ! RU RV]Pn                  4      tTR# )Xé    )Ú	dataclass)ÚOptionalÚTupleÚUnionÚListÚDictÚAnyÚCallableN)ÚceilÚpi)ÚAttentionTensorContainerÚComfyAttention)Ú
accumulateÚchain)Úget_timestep_embedding)Úoptimized_var_attention)Ú_triple)Únn)Úapply_rope1)	ÚBYTEDANCE_720P_REF_AREAÚBYTEDANCE_MAX_TEMPORAL_WINDOWÚBYTEDANCE_ROPE_MAX_FREQÚBYTEDANCE_SINUSOIDAL_DIMÚ
ROPE_THETAÚSEEDVR2_MLP_CHUNKÚSEEDVR2_7B_VID_DIMÚSEEDVR2_LATENT_CHANNELSÚ!SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENSc                   óN   a € ] tR t^t o RR ltV 3R lR ltV 3R lR ltRtV tR# )	ÚCacheNc                ó8   € Ve   TM/ V n         Wn        W n        R # ©N©ÚcacheÚdisableÚprefix)Úselfr%   r&   r$   s   &&&&Ú:C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\seedvr\model.pyÚ__init__ÚCache.__init__   s   € Ø#Ò/‘U°RˆŒ
ØŒØŽó    c                ó&   <€ V ^8„  d   QhRS[ RS[/# )é   ÚkeyÚfn)Ústrr
   )ÚformatÚ__classdict__s   "€r(   Ú__annotate__ÚCache.__annotate__#   s   ø€ ÷ ñ ™Cð ¡Xñ r+   c                óÊ   € V P                   '       d   V! 4       # V P                  V,           pWP                  9  d   V! 4       pW0P                  V&   V P                  V,          # r"   ©r%   r&   r$   )r'   r.   r/   Úresults   &&& r(   Ú__call__ÚCache.__call__#   sJ   € Ø�<�<ˆ<Ù“4ˆKà�k‰k˜CÕˆØ—j‘jÔ Ù“TˆFØ$�J‰J�s‰OØ�z‰z˜#�Ðr+   c                ó    <€ V ^8„  d   QhRS[ /# )r-   Ú	namespace©r0   )r1   r2   s   "€r(   r3   r4   -   s   ø€ ÷ 
ñ 
¡3ñ 
r+   c                óv   € \        V P                  V P                  V,           R ,           V P                  R7      # )Ú.r6   )r    r%   r&   r$   )r'   r;   s   &&r(   r;   ÚCache.namespace-   s.   € ÜØ—L‘LØ—;‘; Õ*¨SÕ0Ø—*‘*ô
ð 	
r+   r#   )FÚ N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r)   r8   r;   Ú__static_attributes__Ú__classdictcell__©r2   s   @r(   r    r       s   ø‡ € ô÷
ð ÷
ö 
r+   r    c                ó°   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \        \        \        3,          /# )r-   Úvid_lenÚtxt_lenÚ
txt_repeatÚreturn)ÚtorchÚ
LongTensorr   r
   r   )r1   s   "r(   r3   r3   4   sU   € ÷ #ñ #Ü×Ñð#ä×Ñð#ô × Ñ ð#ô ÜÜÜð
õñ	#r+   c                 óT  aaaaaaa€ V P                   p\        P                  ! V P                  4       VR 7      o\        P                  ! \	        S4      \	        S4      VP                  4       ,           VR 7      pVP                  4       o\        P                  ! SV P                  4       4      p\        \        R \        \        P                  ! WAP                  4       4      S4       4       !  4      p\        WV4       UUu. uF*  w  rxVP                  4       VP                  4       ,           NK,  	  p	pp\        \        \	        V	4      4      V	P                  R7      p
\        P                  ! V
 UUu. uF  qµV,          Wk,          3 F  qÌNK  	  K  	  upp4      o\        P                  ! S4      o\	        S4      \	        S4      ,
          oW,          P                  4       oVVVVV3R loV3R lV3R l\!        V
 Uu. uF  q¹V,          NK  	  up4      3# u uppi u uppi u upi )©Údevicec              3   ó:   "  € T F  w  rV.V,          x € K  	  R # 5ir"   © )Ú.0ÚxÚns   &  r(   Ú	<genexpr>Ú$repeat_concat_idx.<locals>.<genexpr>B   s   é € ÐoÑ2n©$¨!˜q˜c AŸgšgÓ2nùs   ‚)r.   c                 óV  <€ V S,          P                  \        S
4      S.4      w  r. p\        VP                  S4      S	4       FK  w  rEVP                  ! RV.VP                  R,          O5!  P                  ^4      pVP                  V4       KM  	  V\        P                  ! V4      3# )é   ºrZ   NNéÿÿÿÿ)	ÚsplitÚlenÚzipÚreshapeÚshapeÚmeanÚappendrM   Úcat)ÚallÚvid_outÚtxt_outÚtxt_out_coalescedÚtxtÚrepeat_timeÚrepeat_txt_lenÚsrc_idxÚtxt_idx_lenÚtxt_repeat_listÚvid_idxs   &     €€€€€r(   Úunconcat_coalesceÚ,repeat_concat_idx.<locals>.unconcat_coalesceK   s”   ø€ Ø˜w�<×-Ñ-¬s°7«|¸[Ð.IÓJÑˆØÐÜ # G§M¡M°.Ó$AÀ?Ö SÑˆCØ—+’+˜b +Ð>°·	±	¸"µÓ>×CÑCÀAÓFˆCØ×$Ñ$ SÖ)ñ !Tð œŸ	š	Ð"3Ó4Ð4Ð4r+   c                 ó@   <€ \         P                  ! W.4      S,          # r"   )rM   rd   )Úvidri   Útgt_idxs   &&€r(   Ú<lambda>Ú#repeat_concat_idx.<locals>.<lambda>T   s   ø€ œŸš C :Ó.¨wÖ7r+   c                 ó   <€ S! V 4      # r"   rS   )re   rp   s   &€r(   ru   rv   U   s   ø€ Ñ% cÔ*r+   )rQ   rM   ÚarangeÚsumr^   Útolistr]   Úlistr   r_   ÚnumelÚsortedÚrangeÚ__getitem__rd   ÚargsortÚcumulative_lengths)rI   rJ   rK   rQ   Útxt_idxÚ
vid_blocksÚ
txt_blocksrs   ri   Úseq_lenÚorderÚiÚblockrk   rl   rt   rm   rn   rp   ro   s   &&&          @@@@@@@r(   Úrepeat_concat_idxr‰   4   s�  þ€ ð �^‰^€FÜ�lŠl˜7Ÿ;™;›=°Ô8€GÜ�lŠlœ3˜w›<¬¨W«¸¿¹»Õ)EÈfÔU€GØ ×'Ñ'Ó)€OÜ—’˜W g§n¡nÓ&6Ó7€JÜ”eÑo´#´e·k²kÀ'Ï>É>ÓK[Ó6\Ð^mÔ2nÓoÑpÓq€JÜ9<¸ZÔ9TÔUÑ9T©X¨Sˆs�y‰y‹{˜SŸY™Y›[×(Ð(Ñ9T€GÑUä”5œ˜W›Ó&¨G×,?Ñ,?Ô@€EÜ�iŠi©ÔZ© 1Àq½MÈ:Í=Ó;Y°%šÑ;Y™©ÒZÓ[€GÜ�mŠm˜GÓ$€GÜ�g“,¤ W£Õ-€KØÕ*×2Ñ2Ó4€N÷5ñ 5ô 	8Ü*Ü±Ó6±¨1 AŸJ˜J±Ñ6Ó7ðð ùó! Vùó [ùò  7s   Ä0HÅ3!H
È H%c                 ó   € ^ .\        V 4      O# ©r   )r   )Úlengthss   &r(   r�   r�   Y   s   € ØÐ$”
˜7Ó#Ð$Ð$r+   c                   ó,   a € ] tR t^]t o V 3R ltRtV tR# )ÚMMArgc                ó2   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   # )r-   rs   ri   )r	   )r1   r2   s   "€r(   r3   ÚMMArg.__annotate__]   s   ø‡ ‚ á	�Hñ ñ 
�Hò r+   rS   N)rA   rB   rC   rD   Ú__annotate_func__rE   rF   rG   s   @r(   rŽ   rŽ   ]   s   ø‡ ‡ „ r+   rŽ   c                óh   € V ^8„  d   QhR\         R\        \        ,          R\        \        ,          /# )r-   r.   ÚargsrL   )r0   r   r	   )r1   s   "r(   r3   r3   b   s.   € ÷ Jñ J”#ð JœT¤#�Yð J¬4´­9ñ Jr+   c                 ór   € V Uu. uF&  p\        V\        4      '       d   \        W 4      MTNK(  	  up# u upi r"   )Ú
isinstancerŽ   Úgetattr)r.   r“   Úvs   && r(   Úget_argsr˜   b   s/   € ÙDHÓIÁD¸qœz¨!¬U×3Ò3ŒG�AŒO¸Ò:ÁDÑIÐIùÒIs   …,4c                ó€   € V ^8„  d   QhR\         R\        \         \        3,          R\        \         \        3,          /# )r-   r.   ÚkwargsrL   )r0   r   r	   )r1   s   "r(   r3   r3   f   s6   € ÷ Zñ Z”Cð Z¤¤c¬3 h¥ð Z´D¼¼c¸µNñ Zr+   c           	      óš   € VP                  4        UUu/ uF)  w  r#T\        V\        4      '       d   \        W04      MTbK+  	  upp# u uppi r"   )Úitemsr•   rŽ   r–   )r.   rš   Úkr—   s   &&  r(   Ú
get_kwargsrž   f   s9   € ØJPÏ,É,Ì.ÔYÉ.Á$À!ˆA¤*¨Q´×"6Ò"6Œw�qŒ¸AÒ=É.ÒYÐYùÓYs   ”/Ac                ó$   € V ^8„  d   QhR\         /# )r-   Únamer<   )r1   s   "r(   r3   r3   j   s   € ÷ :ñ :œñ :r+   c                 óR   € V R 8X  d   \         # V R8X  d   \        # \        RV  24      h)Ú720pwin_by_size_bysizeÚ720pswin_by_size_bysizezUnknown windowing method: )Úmake_720Pwindows_bysizeÚmake_shifted_720Pwindows_bysizeÚ
ValueError)r    s   &r(   Úget_window_opr§   j   s3   € ØÐ'Ô'Ü&Ð&ØÐ(Ô(Ü.Ð.Ü
Ð1°$°Ð8Ó
9Ð9r+   c                óˆ   € V ^8„  d   QhR\         \        \        \        3,          R\         \        \        \        3,          /# ©r-   ÚsizeÚnum_windows©r   Úint)r1   s   "r(   r3   r3   r   s2   € ÷ ñ ¤%¬¬S´#¨Õ"6ð ÄUÌ3ÔPSÔUXÈ=ÕEYñ r+   c                 ó  € V w  r#pVw  rVp\         P                  ! \        W4,          ,          4      p\        W8,          4      \        WH,          4      r©\	        W–,          4      \	        W§,          4      rË\	        \        V\        4      V,          4      p\	        W-,          4      \	        W;,          4      \	        WL,          4      prþ\        V4       UUUu. uEF   p\        V^,           V,          V4      VV,          8”  g   K,  \        V4       Få  p\        V^,           V,          V4      VV,          8”  g   K+  \        V4       F«  p\        V^,           V,          V4      VV,          8”  g   K+  \        VV,          \        V^,           V,          V4      4      \        VV,          \        V^,           V,          V4      4      \        VV,          \        V^,           V,          V4      4      3NK­  	  Kç  	  EK#  	  uppp# u upppi ©rZ   )	ÚmathÚsqrtr   Úroundr   Úminr   r~   Úslice)rª   r«   ÚtÚhÚwÚ
resized_ntÚ
resized_nhÚ
resized_nwÚscaleÚ	resized_hÚ	resized_wÚwhÚwwÚwtÚntÚnhÚnwÚiwÚihÚits   &&                  r(   r¤   r¤   r   s›  € Ø�G€Aˆ!Ø)4Ñ&€J˜JÜ�IŠIÔ-°µÕ7Ó8€EÜ  ¥Ó+¬U°1µ9Ó-=ˆyÜ�)Õ(Ó)¬4°	Õ0FÓ+GˆÜ	Œc�!Ô2Ó3°jÕ@Ó	A€BÜ�a•f“œt A¥F›|¬T°!µ&«\ˆBˆô ˜”)õò ˆBÜ��Q•˜"�}˜aÓ  2¨¥7Ñ*ô	
ô ˜–)ˆBÜ��Q•˜"�}˜aÓ  2¨¥7Ñ*ô	
ô ˜–)ˆBÜ��Q•˜"�}˜aÓ  2¨¥7Ñ*ô	
Ü�"�r•'œ3  Q¥¨"�}¨aÓ0Ó1Ü�"�r•'œ3  Q¥¨"�}¨aÓ0Ó1Ü�"�r•'œ3  Q¥¨"�}¨aÓ0Ó1ó	
ñ ñ	
ñ ò	
ñ
 óð ùô s   Ã)HÄ1HÄ91HÅ/BHc                óˆ   € V ^8„  d   QhR\         \        \        \        3,          R\         \        \        \        3,          /# r©   r¬   )r1   s   "r(   r3   r3   ˆ   s4   € ÷ ñ ¬%´´S¼#°Õ*>ð ÌUÔSVÔX[Ô]`ÐS`ÕMañ r+   c                 óä  € V w  r#pVw  rVp\         P                  ! \        W4,          ,          4      p\        W8,          4      \        WH,          4      r©\	        W–,          4      \	        W§,          4      rË\	        \        V\        4      V,          4      pWÒ8  d   R M^ W³8  d   R M^ WÄ8  d   R M^ prþ\	        W.,
          V,          4      \	        W?,
          V,          4      \	        VV,
          V,          4      pppV^ 8”  d
   V^,           M^V^ 8”  d
   V^,           M^V^ 8”  d
   V^,           M^ppp\        V4       UUUu. uEF   p\        \        VV,
          ^,           V,          4      V4      \        \        VV,
          V,          4      ^ 4      8”  g   KV  \        V4       EFº  p\        \        VV,
          ^,           V,          4      V4      \        \        VV,
          V,          4      ^ 4      8”  g   KV  \        V4       EFT  p\        \        VV,
          ^,           V,          4      V4      \        \        VV,
          V,          4      ^ 4      8”  g   KV  \        \        \        VV,
          V,          4      ^ 4      \        \        VV,
          ^,           V,          4      V4      4      \        \        \        VV,
          V,          4      ^ 4      \        \        VV,
          ^,           V,          4      V4      4      \        \        \        VV,
          V,          4      ^ 4      \        \        VV,
          ^,           V,          4      V4      4      3NEKW  	  EK½  	  EK#  	  uppp# u upppi )g      à?)r°   r±   r   r²   r   r³   r   r~   r­   Úmaxr´   )rª   r«   rµ   r¶   r·   r¸   r¹   rº   r»   r¼   r½   r¾   r¿   rÀ   ÚstÚshÚswrÁ   rÂ   rÃ   rÄ   rÅ   rÆ   s   &&                     r(   r¥   r¥   ˆ   s¦  € Ø�G€Aˆ!Ø)4Ñ&€J˜JÜ�IŠIÔ-°µÕ7Ó8€EÜ  ¥Ó+¬U°1µ9Ó-=ˆyÜ�)Õ(Ó)¬4°	Õ0FÓ+GˆÜ	Œc�!Ô2Ó3°jÕ@Ó	A€Bð Œv‰˜1ØŒv‰˜1ØŒv‰˜1ð ˆô
 �q•v •mÓ$¤d¨A­F°b­=Ó&9¼4ÀÀRÅÈ2ÅÓ;NˆBˆ€Bà�q”&ˆˆQŽ˜aØ�q”&ˆˆQŽ˜aØ�q”&ˆˆQŽ˜að ˆ€Bô ˜”)õò ˆBÜŒs�B˜•G˜a•K 2Õ%Ó&¨Ó*¬S´°b¸2µgÀµ^Ó1DÀaÓ-HÑHô	
ô ˜—)ˆBÜŒs�B˜•G˜a•K 2Õ%Ó&¨Ó*¬S´°b¸2µgÀµ^Ó1DÀaÓ-HÑHô	
ô ˜—)ˆBÜŒs�B˜•G˜a•K 2Õ%Ó&¨Ó*¬S´°b¸2µgÀµ^Ó1DÀaÓ-HÑHô	
Ü”#”c˜2 �7 b�.Ó)¨1Ó-¬s´3¸¸R½À!½ÀrÕ7IÓ3JÈAÓ/NÓOÜ”#”c˜2 �7 b�.Ó)¨1Ó-¬s´3¸¸R½À!½ÀrÕ7IÓ3JÈAÓ/NÓOÜ”#”c˜2 �7 b�.Ó)¨1Ó-¬s´3¸¸R½À!½ÀrÕ7IÓ3JÈAÓ/NÓOô	
ñ ò	
ñ ò	
ñ
 óð ùô s   Ä?AM+ÆAM+Ç8AM+ÉDM+c                   óZ   a a€ ] tR t^©t oRV 3R llt]R 4       tRR/R ltR tRt	Vt
V ;t# )	ÚRotaryEmbeddingc                ó|  <€ \         SV `  4        W n        VR 8X  dG   RV\        P                  ! ^ V^4      RV^,           P                  4       V,          ,          ,          pMGVR8X  d3   \        P                  ! RV^,          V^,          4      \        ,          pM\        RV 24      hV P                  RV4       R# )Úlangç      ð?NÚpixelzUnknown rotary frequency type: Úfreqs)
Úsuperr)   Ú	freqs_forrM   rx   ÚfloatÚlinspacer   r¦   Úregister_buffer)r'   ÚdimrÕ   ÚthetaÚmax_freqrÓ   Ú	__class__s   &&&&& €r(   r)   ÚRotaryEmbedding.__init__ª   s�   ø€ ô 	‰ÑÔà"Œà˜ÔØ˜%¤E§L¢L°°C¸Ó$;¸K¸cÀQ½hÐ$H×$NÑ$NÓ$PÐSVÕ$VÕWÕX‰EØ˜'Ô!Ü—N’N 2 x°!¥|°S¸AµXÓ>ÄÕC‰EäÐ>¸y¸kÐJÓKÐKà×Ñ˜W eÖ,r+   c                ó.   € V P                   P                  # r"   )rÓ   rQ   )r'   s   &r(   rQ   ÚRotaryEmbedding.device¾   s   € à�z‰z× Ñ Ð r+   ÚoffsetsNc               óÀ  € \        R 4      p. p\        V4      '       d>   \        V4      \        V4      8w  d%   \        R\        V4       R\        V4       R24      h\	        V4       FÉ  w  rV^ p\        V4      '       d	   W,          pV P
                  R8X  d%   \        P                  ! R^W`P                  R7      pM!\        P                  ! W`P                  R7      pW‡,           pV P                  V4      p	R .\        V4      ,          p
W:V&   \        .V
OVN5pVP                  W›,          4       KË  	  \        P                  ! V!  p\        P                  ! VRR7      # )	Nz:SeedVR2 rotary offsets length must match dims length, got z and r>   rÒ   )ÚstepsrQ   rP   ©rÙ   r\   )r´   Úexistsr^   r¦   Ú	enumeraterÕ   rM   r×   rQ   rx   ÚforwardÚEllipsisrc   Úbroadcast_tensorsrd   )r'   rà   ÚdimsÚColonÚ	all_freqsÚindrÙ   ÚoffsetÚposrÓ   Úall_axisÚnew_axis_slices   &$*         r(   Úget_axial_freqsÚRotaryEmbedding.get_axial_freqsÂ   s#  € ô
 �d“ˆØˆ	ä�'�?Š?Ü�7‹|œs 4›yÔ(Ü Ð#]Ô^aÐbiÓ^jÐ]kÐkpÔqtÐuyÓqzÐp{Ð{|Ð!}Ó~Ð~ä! $ž‰HˆCàˆFÜ�g�ŠØ ��à�~‰~ Ô(Ü—n’n R¨°CÇ+Á+ÔN‘ä—l’l 3·±Ô=�à•,ˆCà—L‘L Ó%ˆEà�v¤ D£	Õ)ˆHØ!�S‰Mä&Ð9¨Ð9°5Ñ9ˆNØ×Ñ˜UÕ2Ö3ñ' (ô* ×+Ò+¨YÑ7ˆ	Ü�yŠy˜¨"Ô-Ð-r+   c                óú   € V P                   p\        P                  ! R VP                  VP                  4      V4      pVP                  R4      P                  ! . VP                  O^N5!  P                  R4      pV# )z..., f -> ... fr\   éþÿÿÿ)	rÓ   rM   ÚeinsumÚtypeÚdtypeÚ	unsqueezeÚexpandra   Úflatten)r'   rµ   rÓ   s   && r(   ræ   ÚRotaryEmbedding.forwardæ   sb   € ð —
‘
ˆä—’Ð.°·±°u·{±{Ó0CÀUÓKˆØ—‘ Ó#×*Ò*Ð;¨E¯K©KÐ;¸Ó;×CÑCÀBÓGˆàˆr+   )rÕ   )rÐ   i'  é
   )rA   rB   rC   rD   r)   ÚpropertyrQ   rñ   ræ   rE   rF   Ú__classcell__©rÜ   r2   s   @@r(   rÎ   rÎ   ©   s8   ù‡ € ÷-ð( ñ!ó ð!ð".ð ô".÷H	ò 	r+   rÎ   c                   óD   a a€ ] tR t^ñt oV3R lV 3R lltR tRtVtV ;t# )ÚRotaryEmbeddingBasec                ó&   <€ V ^8„  d   QhRS[ RS[ /# ©r-   rÙ   Úrope_dim©r­   )r1   r2   s   "€r(   r3   Ú RotaryEmbeddingBase.__annotate__ò   s   ø€ ÷ 
ñ 
™Cð 
©3ñ 
r+   c                ó^   <€ \         SV `  4        \        W,          R \        R7      V n        R# )rÒ   ©rÙ   rÕ   rÛ   N)rÔ   r)   rÎ   r   Úrope©r'   rÙ   r  rÜ   s   &&&€r(   r)   ÚRotaryEmbeddingBase.__init__ò   s%   ø€ Ü‰ÑÔÜ#Ø•ØÜ,ô
ˆŽ	r+   c                ó6   € V P                   P                  ! V!  # r"   )r	  rñ   )r'   ré   s   &*r(   rñ   Ú#RotaryEmbeddingBase.get_axial_freqsú   s   € Ø�y‰y×(Ò(¨$Ñ/Ð/r+   )r	  )	rA   rB   rC   rD   r)   rñ   rE   rF   rþ   rÿ   s   @@r(   r  r  ñ   s   ù‡ € ÷
ó 
÷0ò 0r+   r  c                   ó>   a a€ ] tR t^þt oV3R lV 3R lltRtVtV ;t# )ÚRotaryEmbedding3dc                ó    <€ V ^8„  d   QhRS[ /# ©r-   rÙ   r  )r1   r2   s   "€r(   r3   ÚRotaryEmbedding3d.__annotate__ÿ   s   ø€ ÷ ñ ™Cñ r+   c                ó8   <€ \         SV `  V^R7       RV n        R# )é   ©r  FN)rÔ   r)   Úmm©r'   rÙ   rÜ   s   &&€r(   r)   ÚRotaryEmbedding3d.__init__ÿ   s   ø€ Ü‰Ñ˜ qÐÔ)ØˆŽr+   )r  ©rA   rB   rC   rD   r)   rE   rF   rþ   rÿ   s   @@r(   r  r  þ   s   ù‡ € ÷÷ ó r+   r  c                   óv   a € ] tR tRt o V 3R lR lt]P                  P                  V 3R lR l4       tRt	V t
R# )ÚNaRotaryEmbedding3di  c                ó²   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[RS[S[ P                  S[ P                  3,          /# )r-   Úqr�   ra   r$   rL   ©rM   ÚFloatTensorrN   r    r   )r1   r2   s   "€r(   r3   Ú NaRotaryEmbedding3d.__annotate__  sk   ø€ ÷ ñ á×Ñðñ ×Ñðñ ×Ñð	ñ
 ðñ 
Ù×ÑÙ×Ñð	õ
ñr+   c                óŒ   a a€ V! R V V3R l4      pVP                  VP                  R7      p\        W4      p\        W%4      pW3# )Úrope_freqs_3dc                  ó&   <€ S P                  S4      # r"   ©Ú	get_freqs)r'   ra   s   €€r(   ru   Ú-NaRotaryEmbedding3d.forward.<locals>.<lambda>  s   ø€ ¨t¯~©~¸eÔ/Dr+   rP   )ÚtorQ   Ú_apply_rope1_partial)r'   r  r�   ra   r$   rÓ   s   f&&f& r(   ræ   ÚNaRotaryEmbedding3d.forward  sA   ù€ ñ �oÕ'DÓEˆØ—‘ §¡�Ó)ˆÜ  Ó*ˆÜ  Ó*ˆØˆtˆr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r-   ra   rL   )rM   rN   ÚTensor)r1   r2   s   "€r(   r3   r     s)   ø€ ÷ ?ñ ?á×Ñð?ñ 
�‰ñ?r+   c           	     óÀ  € \        V P                  P                  P                  4       ^,          R\        R7      pVP                  V P                  P                  4      p. pVP                  4        FH  w  rEpVP                  WEV4      pVP                  VP                  RVP                  R4      4      4       KJ  	  \        \        P                  ! V^ R7      4      # )r-   rÒ   r  rã   r\   )rÎ   r	  rÓ   r|   r   r'  rQ   rz   rñ   rc   Úviewrª   Ú_to_flux_freqs_cisrM   rd   )r'   ra   Ú
plain_ropeÚ	freq_listÚfr¶   r·   rÓ   s   &&      r(   r%  ÚNaRotaryEmbedding3d.get_freqs  s«   € ô %Ø—	‘	—‘×%Ñ%Ó'¨!Õ+ØÜ,ô
ˆ
ð
  —]‘] 4§9¡9×#3Ñ#3Ó4ˆ
Øˆ	Ø—|‘|–~‰GˆA�!Ø×.Ñ.¨q°QÓ7ˆEØ×Ñ˜UŸZ™Z¨¨E¯J©J°r«NÓ;Ö<ñ &ô "¤%§)¢)¨I¸1Ô"=Ó>Ð>r+   rS   N)rA   rB   rC   rD   ræ   rM   Ú_dynamor%   r%  rE   rF   rG   s   @r(   r  r    s-   ø‡ € ÷ð ð  ‡]�]×Ñ÷?ó ö?r+   r  c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚMMRotaryEmbeddingBasei*  c                ó&   <€ V ^8„  d   QhRS[ RS[ /# r  r  )r1   r2   s   "€r(   r3   Ú"MMRotaryEmbeddingBase.__annotate__+  s   ø€ ÷ ñ ™Cð ©3ñ r+   c                ón   <€ \         SV `  W4       \        W,          R \        R7      V n        RV n        R# )rÐ   )rÙ   rÕ   rÚ   TN)rÔ   r)   rÎ   r   r	  r  r
  s   &&&€r(   r)   ÚMMRotaryEmbeddingBase.__init__+  s0   ø€ Ü‰Ñ˜Ô'Ü#Ø•ØÜô
ˆŒ	ð
 ˆŽr+   )r  r	  r  rÿ   s   @@r(   r5  r5  *  s   ù‡ € ÷÷ ó r+   r5  c                 ó
   € V R J# r"   rS   )Úvals   &r(   rä   rä   4  s   € Ø�dˆ?Ðr+   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r-   Úfreqs_interleavedrL   ©rM   r+  )r1   s   "r(   r3   r3   8  s"   € ÷ .ñ .¬%¯,©,ð .¼5¿<¹<ñ .r+   c                 ó  € V R,          P                  4       p\        P                  ! V4      p\        P                  ! V4      p\        P                  ! W#) W2.RR7      pVP
                  ! . VP                  RR O^N^N5!  # ).rã   N).:NNr-   r\   )rÖ   rM   ÚcosÚsinÚstackr`   ra   )r=  Úanglesr@  rA  Úouts   &    r(   r.  r.  8  sn   € Ø˜xÕ(×.Ñ.Ó0€FÜ
�)Š)�FÓ
€CÜ
�)Š)�FÓ
€CÜ
�+Š+�s˜D #Ð+°Ô
4€CØ�;Š;Ð-˜Ÿ	™	 # 2˜Ð-¨Ð-¨1Ó-Ð-r+   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r-   rµ   Ú	freqs_cisrL   r>  )r1   s   "r(   r3   r3   @  s-   € ÷ ñ œEŸL™Lð ´U·\±\ð ÄeÇlÁlñ r+   c                ó’  € \         P                  P                  ;'       g    V P                  pV'       d   V P	                  4       MT p^VP
                  R,          ,          pVP                  ^ 4      pWEP
                  R,          8w  d   VRRV13,          pVR,          pVP
                  ^ ,          p\        ^ V\        4       F~  p\        V\        ,           V4      pVRWx13,          VRWx13,          r©V'       d   V	P                  \        Wš4      4       KU  \         P                  P                  P                  Wš4       K€  	  V# )zöRotate the leading ``2 * freqs_cis.shape[-3]`` head features of an (L, heads, dim) tensor.

The 4-D token-major view with 6-D frequencies is what the fused kernel dispatches on;
a 3-D tensor silently falls back to the elementwise reference path.
.NºNNNéýÿÿÿr\   )NrH  N)ÚcomfyÚmodel_managementÚin_trainingÚrequires_gradÚclonera   rø   r~   r   r³   Úcopy_r   Ú	quant_opsÚckÚapply_rope1_)rµ   rF  ÚtrainingrD  Úrot_dr-  r…   ÚstartÚendÚchunkÚfreqs_chunks   &&         r(   r(  r(  @  sÿ   € ô ×%Ñ%×1Ñ1×DÐD°Q·_±_€Hßˆ!�'‰'Œ) Q€CØ�	—‘ Õ#Õ#€EØ�=‰=˜Ó€DØ—
‘
˜2•ÔØ�C˜˜%˜�KÕ ˆØ˜-Õ(€IØ�i‰i˜�l€GÜ�q˜'Ô#DÖEˆÜ�%Ô;Õ;¸WÓEˆØ! ! U Y ,Õ/°¸1¸e¸i¸<Õ1Hˆ{ßØ�K‰Kœ EÓ7Ö8ä�O‰O×Ñ×+Ñ+¨EÖ?ñ Fð €Jr+   c                   ó”   a a€ ] tR tRt oV3R lV 3R lltV3R lR lt]P                  P                  V3R lR l4       t	Rt
VtV ;t# )	ÚNaMMRotaryEmbedding3diX  c                ó    <€ V ^8„  d   QhRS[ /# r  r  )r1   r2   s   "€r(   r3   Ú"NaMMRotaryEmbedding3d.__annotate__Y  s   ø€ ÷ *ñ *™Cñ *r+   c                ó*   <€ \         SV `  V^R7       R# )r  r  N)rÔ   r)   r  s   &&€r(   r)   ÚNaMMRotaryEmbedding3d.__init__Y  s   ø€ Ü‰Ñ˜ qÐÖ)r+   c                óJ  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[R	S[S[ P                  S[ P                  S[ P                  S[ P                  3,          /	# )
r-   Úvid_qÚvid_kÚ	vid_shapeÚtxt_qÚtxt_kÚ	txt_shapeÚvid_txt_shaper$   rL   r  )r1   r2   s   "€r(   r3   r\  \  s½   ø€ ÷ *ñ *á× Ñ ð*ñ × Ñ ð*ñ ×#Ñ#ð	*ñ
 × Ñ ð*ñ × Ñ ð*ñ ×#Ñ#ð*ñ ×'Ñ'ð*ñ ð*ñ 
Ù×ÑÙ×ÑÙ×ÑÙ×Ñð	õ
ñ*r+   c	                ó.  a aaa€ V! R V VVV3R l4      w  ršVP                   pV	P                   V8w  d   V	P                  V4      p	V
P                   V8w  d   V
P                  V4      p
\        W4      p\        W)4      p\        WJ4      p\        WZ4      pWWE3# )Úmmrope_freqs_3dc                  ó*   <€ S P                  SSS4      # r"   r$  )r'   re  rb  rf  s   €€€€r(   ru   Ú/NaMMRotaryEmbedding3d.forward.<locals>.<lambda>n  s   ø€ �D—N‘N 9¨i¸ÔGr+   )rQ   r'  r(  )r'   r`  ra  rb  rc  rd  re  rf  r$   Ú	vid_freqsÚ	txt_freqsÚtarget_devices   f&&f&&ff&   r(   ræ   ÚNaMMRotaryEmbedding3d.forward\  sŽ   û€ ñ   %ØßGó 
Ñˆ	ð Ÿ™ˆØ×Ñ˜}Ô,Ø!Ÿ™ ]Ó3ˆIØ×Ñ˜}Ô,Ø!Ÿ™ ]Ó3ˆIÜ$ UÓ6ˆÜ$ UÓ6ˆÜ$ UÓ6ˆÜ$ UÓ6ˆØ˜UÐ)Ð)r+   c          
      ó¬   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# )r-   rb  re  rf  rL   )rM   rN   r   r+  )r1   r2   s   "€r(   r3   r\  |  sc   ø€ ÷ )dñ )dá×#Ñ#ð)dñ ×#Ñ#ð)dñ ×'Ñ'ð	)dñ
 
Ù�‰Ù�‰ð	õ
ñ)dr+   c                óô  € ^ p^ p^ p\        VR,          P                  4       4      p\        VP                  4       VR,          P                  4       4       F;  w  w  r‰r«\        WKV,           4      p\        WY4      p\        Wj4      p\        W{4      pK=  	  \        P                  P                  4       '       d   RMRp\        P                  P                  VRR7      ;_uu_ 4        V P                  V^,           V^,           V^,           4      P                  4       pV P                  V^,           4      pRRR4       \        VP                  4       VR,          P                  4       4       UU	U
Uu. uF?  w  w  r‰r«XW»V,           1RV	1RV
13,          P                  RVP                  R4      4      NKA  	  pp
p	ppVR,          P                  4        Uu. uF7  pXRV P                  ^^4      P                  RXP                  R4      4      NK9  	  pp\        P                  ! V^ R7      p\        P                  ! V^ R7      p\        V4      \        V4      3#   + '       g   i     EL-; iu upp
p	pi u upi )	r   ÚcudaÚcpuF)ÚenabledNrã   )rH  r   r\   )rÉ   rz   r_   rM   rq  Úis_availableÚampÚautocastrñ   rÖ   r`   rª   Úrepeatrd   r.  )r'   rb  re  rf  Úmax_temporalÚ
max_heightÚ	max_widthÚmax_txt_lenr1  r¶   r·   ÚlÚautocast_devicerk  rl  Úvid_freq_listÚtxt_freq_listÚvid_freqs_interleavedÚtxt_freqs_interleaveds   &&&&               r(   r%  ÚNaMMRotaryEmbedding3d.get_freqs{  s)  € ð ˆØˆ
Øˆ	Ü˜) D�/×0Ñ0Ó2Ó3ˆä 	× 0Ñ 0Ó 2°MÀ$Õ4G×4NÑ4NÓ4PÖQ‰L‰IˆQ�1Ü˜|°­UÓ3ˆLÜ˜ZÓ+ˆJÜ˜IÓ)ˆIÜ˜kÓ-ŠKñ	 Rô %*§J¡J×$;Ñ$;×$=Ò$=™&À5ˆÜ�Y‰Y×Ñ ¸Ð×?Õ?Ø×,Ñ,Ø˜rÕ!Ø˜Q•Ø˜A•ó÷ ‰e‹gð	 ð
 ×,Ñ,¨[¸2Õ-=Ó>ˆI÷ @ô !$ I×$4Ñ$4Ó$6¸ÀdÕ8K×8RÑ8RÓ8TÔ Uö
á U‘‘	��qð �a˜a�%�i  !  R a RÐ'Õ(×0Ñ0°°Y·^±^ÀBÓ5GÖHÙ Uð 	ó 
ð ˜t•_×+Ñ+Ô-ó
á-�ð �b�qˆM× Ñ   AÓ&×.Ñ.¨r°9·>±>À"Ó3EÖFÙ-ð 	ð 
ô !&§	¢	¨-¸QÔ ?ÐÜ %§	¢	¨-¸QÔ ?Ðä!Ð"7Ó8Ô:LÐMbÓ:cÐcÐc÷' @×?Ð?üõ
ùò
s   Ã&AIÅ1AI-
Ç=I5ÉI*	rS   )rA   rB   rC   rD   r)   ræ   rM   r3  r%   r%  rE   rF   rþ   rÿ   s   @@r(   rZ  rZ  X  s?   ù‡ € ÷*ó *÷*ð *ð> ‡]�]×Ñ÷)dó ÷)dð )dr+   rZ  c                   ó\   a a€ ] tR tRt oRRRR/V3R lV 3R llltV3R lR ltR	tVtV ;t# )
ÚMMModulei§  Úshared_weightsFÚvid_onlyc                óT   <€ V ^8„  d   QhRS[ RS[P                  3,          RS[RS[/# )r-   Úmodule.r…  r†  )r
   r   ÚModuleÚbool)r1   r2   s   "€r(   r3   ÚMMModule.__annotate__¨  s6   ø€ ÷ ñ á˜™bŸi™i˜Õ(ðñ ð	ñ
 ñr+   c               óÖ  <€ \         SV `  4        W n        W0n        V P                  '       dp   \	        R V4      \	        RV4      8w  d   \        R4      h\        R V4      \        RV4      8w  d   \        R4      hV! \	        R V4      / \        R V4      B V n        R# V! \	        R V4      / \        R V4      B V n        V'       g   V! \	        RV4      / \        RV4      B MRV n	        R# )rs   ri   z7SeedVR2 shared MMModule requires matching vid/txt args.z9SeedVR2 shared MMModule requires matching vid/txt kwargs.N)
rÔ   r)   r…  r†  r˜   r¦   rž   re   rs   ri   )r'   rˆ  r…  r†  r“   rš   rÜ   s   &&$$*,€r(   r)   ÚMMModule.__init__¨  sÕ   ø€ ô 	‰ÑÔØ,ÔØ ŒØ××ÐÜ˜˜tÓ$¬°¸Ó(=Ô=Ü Ð!ZÓ[Ð[Ü˜% Ó(¬J°u¸fÓ,EÔEÜ Ð!\Ó]Ð]Ùœx¨¨tÓ4ÐR¼
À5È&Ó8QÑRˆDŽHáœx¨¨tÓ4ÐR¼
À5È&Ó8QÑRˆDŒH÷  ñ œ ¨Ó-ÐK´¸EÀ6Ó1JÒKàð ŽHr+   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# ©r-   rs   ri   rL   ©rM   r  r   )r1   r2   s   "€r(   r3   r‹  Á  sQ   ø€ ÷ ñ á×Ñðñ ×Ñðñ 
Ù×ÑÙ×Ñð	õ
ñr+   c                ó¢  € V P                   '       g   V P                  MV P                  pV! V.\        R V4      O5/ \	        R V4      B pV P
                  '       gr   V P                   '       g   V P                  MV P                  pVP                  VP                  VP                  R7      pV! V.\        RV4      O5/ \	        RV4      B pW3# )rs   ©rQ   r÷   ri   )
r…  rs   re   r˜   rž   r†  ri   r'  rQ   r÷   )r'   rs   ri   r“   rš   Ú
vid_moduleÚ
txt_modules   &&&*,  r(   ræ   ÚMMModule.forwardÁ  s¦   € ð &*×%8×%8Ð%8�T—X’X¸d¿h¹hˆ
Ù˜ÐRœx¨¨tÓ4ÒR¼
À5È&Ó8QÑRˆØ�}�}ˆ}Ø)-×)<×)<Ð)<˜ŸšÀ$Ç(Á(ˆJØ—&‘& §
¡
°#·)±)�&Ó<ˆCÙ˜SÐV¤8¨E°4Ó#8ÒV¼JÀuÈfÓ<UÑVˆCØˆxˆr+   )re   r…  ri   rs   r†  ©	rA   rB   rC   rD   r)   ræ   rE   rF   rþ   rÿ   s   @@r(   r„  r„  §  s2   ù‡ € ðð  %ð	ð
 ÷õ ÷2÷ ð r+   r„  c                óF   € V ^8„  d   QhR\         \        ,          R\        /# )r-   Ú	rope_typerÙ   )r   r0   r­   )r1   s   "r(   r3   r3   Ó  s"   € ÷ @ñ @œ8¤C�=ð @¬sñ @r+   c                 óv   € V f   R # V R8X  d   \        VR7      # V R8X  d   \        VR7      # \        RV  24      h)NÚrope3drã   Úmmrope3dzUnknown SeedVR2 rope type: )r  rZ  r¦   ©r˜  rÙ   s   &&r(   Úget_na_roper�  Ó  sD   € ØÒÙØ�HÔÜ" sÔ+Ð+Ø�JÔÜ$¨Ô-Ð-Ü
Ð2°9°+Ð>Ó
?Ð?r+   c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚNaMMAttentioniÜ  c                ó`   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[RS[RS[S[,          RS[ R	S[/	# )
r-   Úvid_dimÚtxt_dimÚheadsÚhead_dimÚqk_biasÚqk_norm_epsr˜  r  r…  )r­   rŠ  rÖ   r   r0   )r1   r2   s   "€r(   r3   ÚNaMMAttention.__annotate__Ý  st   ø€ ÷ +Cñ +Cáð+Cñ ð+Cñ ð	+Cñ
 ð+Cñ ð+Cñ ð+Cñ ™C•=ð+Cñ ð+Cñ ñ+Cr+   c           
     ó†  <€ \         SV `  4        \        4       V n        \	        W4      pW0n        W4,          pV^,          pW@n        \        VP                  VVWZW¼R7      V n	        \        VP                  WþW«VR7      V n
        \        VVVRV
W¼R7      V n        \        VVVRV
W¼R7      V n        \        W‰R7      V n        R# )r  )Úbiasr…  rQ   r÷   )r…  rQ   r÷   T©Únormalized_shapeÚepsÚelementwise_affiner…  rQ   r÷   rœ  N)rÔ   r)   r   Úcomfy_attentionrŽ   r£  r¤  r„  ÚLinearÚproj_qkvÚproj_outÚnorm_qÚnorm_kr�  r	  )r'   r¡  r¢  r£  r¤  r¥  Úqk_normr¦  r˜  r  r…  rQ   r÷   Ú
operationsrÙ   Ú	inner_dimÚqkv_dimrÜ   s   &&&&&&&&&&&&&&   €r(   r)   ÚNaMMAttention.__init__Ý  sÆ   ø€ ô 	‰ÑÔÜ-Ó/ˆÔÜ�GÓ%ˆØŒ
ØÕ$ˆ	Ø˜a•-ˆØ ŒÜ Ø×Ñ˜s G°'Ðagô
ˆŒô ! ×!2Ñ!2°IÐSaÐx}Ô~ˆŒÜØØ%ØØ#Ø)Øô
ˆŒô ØØ%ØØ#Ø)Øô
ˆŒô  ¨)ÔBˆŽ	r+   )r®  r¤  r£  r³  r²  r±  r°  r	  r  rÿ   s   @@r(   rŸ  rŸ  Ü  s   ù‡ € ÷+C÷ +Có +Cr+   rŸ  c          	      óÆ   € V ^8„  d   QhR\         P                  R\         P                  R\        \         P                  .\
        \         P                  ,          3,          /# )r-   ÚhidÚ	hid_shapeÚ	window_fn)rM   r  rN   r
   r+  r   )r1   s   "r(   r3   r3   
  sP   € ÷ Gñ GÜ	×	Ñ	ðGä×ÑðGô œŸ™˜¬¬U¯\©\Õ(:Ð:Õ;ñGr+   c                 óv  € \        W4      p \        \        W 4      4      p V  Uu. uF  p\        V4      NK  	  pp\        P
                  ! WAP                  R 7      p\        \        V !  4      p V  Uu. uF&  p\        P                  ! VP                  RR 4      NK(  	  pp\        V 4      w  rWWVV3# u upi u upi )rP   Nr\   )Ú	unflattenr{   Úmapr^   rM   Ú	as_tensorrQ   r   r°   Úprodra   rú   )rº  r»  r¼  rU   Úhid_windows_listÚhid_windowsÚhid_len_lists   &&&    r(   ÚwindowrÅ  
  s£   € ô
 �CÓ
#€CÜ
Œs�9Ó"Ó
#€CÙ(+Ó,© 1œ˜Až©ÐÐ,Ü—/’/Ð"2×;KÑ;KÔL€KÜ
Œu�c‰{Ó
€CÙ58Ó9±S°”D—I’I˜aŸg™g c r˜lÖ+±S€LÐ9Ü˜S“\�N€CØ˜;Ð6FÐFÐFùò -ùò :s   ¤B1Á1,B6c                ó¦   € V ^8„  d   QhR\         P                  R\        \         P                  .\        \         P                  ,          3,          /# )r-   r»  r¼  )rM   rN   r
   r+  r   )r1   s   "r(   r3   r3     s;   € ÷ ñ Ü×ÑðäœŸ™˜¬¬U¯\©\Õ(:Ð:Õ;ñr+   c                 ó.  aa€ \         P                  ! V P                  R4      P                  4       V P                  R7      P                  R4      p\        W V4      w  or4 pSP                  R4      o\         P                  ! S4      oV3R lV3R lVV3# )rZ   rP   c                 ó4   <€ \         P                  ! V ^ S4      # r‹   ©rM   Úindex_select)rº  rt   s   &€r(   ru   Úwindow_idx.<locals>.<lambda>!  ó   ø€ ”E×&Ò& s¨A¨wÔ7r+   c                 ó4   <€ \         P                  ! V ^ S4      # r‹   rÉ  )rº  rl   s   &€r(   ru   rË  "  rÌ  r+   r\   )	rM   rx   rÁ  ry   rQ   rø   rÅ  Úsqueezer€   )r»  r¼  Úhid_idxÚ	tgt_shapeÚtgt_windowsÚ_rl   rt   s   &&    @@r(   Ú
window_idxrÓ    s€   ù€ ô �lŠl˜9Ÿ>™>¨"Ó-×1Ñ1Ó3¸I×<LÑ<LÔM×WÑWÐXZÓ[€GÜ,2°7ÀyÓ,QÑ)€GˆY Q¨Ø�o‰o˜bÓ!€GÜ�mŠm˜GÓ$€Gä7Ü7ØØð	ð r+   c                   óX   a a€ ] tR tRt oRR/V3R lV 3R llltV3R lR ltRtVtV ;t# )	ÚNaSwinAttentioni'  ÚversionFc                ó\   <€ V ^8„  d   QhRS[ S[S[S[S[S[3,          3,          RS[RS[/# )r-   rÅ  Úwindow_methodrÖ  )r   r­   r   r0   rŠ  )r1   r2   s   "€r(   r3   ÚNaSwinAttention.__annotate__(  s@   ø€ ÷ 6ñ 6ñ ‘c™5¡¡c©3 Õ/Ð/Õ0ð6ñ ð	6ñ
 ñ6r+   c               óN  <€ \         SV `  ! V/ VB  W0n        \        V4      V n        W n        \        ;QJ d&    R  V P                   4       F  '       d   K   RM	  RM! R  V P                   4       4      '       g   \        RV P                   R24      h\        V4      V n	        R# )c              3   ó\   "  € T F"  p\        V\        4      ;'       d    V^ 8¬  x € K$  	  R# 5i)r   N)r•   r­   )rT   r—   s   & r(   rW   Ú+NaSwinAttention.__init__.<locals>.<genexpr>4  s&   é € ÐF¹+°Q”:˜a¤Ó%×0Ð0¨!¨q©&Ô0»+ùs   ‚,ž,FTz7SeedVR2 window must contain non-negative integers, got r>   N)
rÔ   r)   Ú
version_7br   rÅ  rØ  re   r¦   r§   Ú	window_op)r'   rÅ  rØ  rÖ  r“   rš   rÜ   s   &$$$*,€r(   r)   ÚNaSwinAttention.__init__(  s€   ø€ ô 	‰Ò˜$Ð) &Ò)Ø!ŒÜ˜f“oˆŒØ*Ôß‹sÑF¸$¿+º+ÓF�s�sŠsÑF¸$¿+º+ÓF×FÒFÜÐVÐW[×WbÑWbÐVcÐcdÐeÓfÐfä& }Ó5ˆŽr+   c                óÌ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[RS[S[ P                  S[ P                  3,          /# )r-   rs   ri   rb  re  r$   rL   r  )r1   r2   s   "€r(   r3   rÙ  9  sƒ   ø€ ÷ S ñ S á×ÑðS ñ ×ÑðS ñ ×#Ñ#ð	S ñ
 ×#Ñ#ðS ñ ðS ñ 
Ù×ÑÙ×Ñð	õ
ñS r+   c                óˆ  a aaaaaaa € S P                  W4      w  rgVP                  S P                   R S P                   R24      pR V 3R lloV! RVV3R l4      w  ršo oV	! V4      p?VP	                  VP
                  ^ ,          ^S P                  S P                  4      pVP	                  VP
                  ^ ,          ^S P                  S P                  4      pVP                  ^4      w  rÍpVP                  ^4      w  ppp??S P                  WÏ4      w  rÏS P                  VV4      w  ppV! RV3R l4      oV! RV 3R	 l4      oSP                  SP                  4      oS P                  '       d„   S P                  '       d   S P                  WÍS V4      w  rÍM[S P                  P                  '       d+   V! R
VV3R l4      pS P                  WÍS VVSVV4      w  rÍppMS P                  WÍS V4      w  rÍV! RVVV3R l4      w  ppp\!        V! WÏ4      4      p??\!        V! VV4      4      p??\!        V! VV4      4      p??\#        VVVS P$                  S P                  RRVVR7	      p???V! V4      w  pp?VP'                  ^^4      pVP'                  ^^4      pV
! V4      pS P)                  VV4      w  ppVV3# )rÒ  Ú_sd3c                ó8   € V ^8„  d   QhR\         P                  /# )r-   rU   r>  )r1   s   "r(   r3   Ú-NaSwinAttention.forward.<locals>.__annotate__I  s   € ÷ 	Eñ 	Eœ5Ÿ<™<ñ 	Er+   c           	      ó®   <€ V P                   w  rr4S	P                  WV3S	P                  4      pV UUUu. uF  w  rgq€WgV3,          NK  	  uppp# u upppi r"   )ra   rÞ  rÅ  )
rU   rµ   r¶   r·   rÒ  Úwindow_slicesrÊ   rË   rÌ   r'   s
   &        €r(   Úmake_windowÚ,NaSwinAttention.forward.<locals>.make_windowI  sL   ø€ ØŸ™‰JˆA�!Ø ŸN™N¨A°!¨9°d·k±kÓBˆMÙ6CÕD±m¡l r¨r�b˜b�j—M�M±mÓDÐDùÔDs   µAÚwin_transformc                  ó   <€ \        SS 4      # r"   )rÓ  )rç  rb  s   €€r(   ru   Ú)NaSwinAttention.forward.<locals>.<lambda>P  s   ø€ ”J˜y¨+Ô6r+   rJ   c                  ó&   <€ S P                  R4      # ©rZ   r\   ©rÁ  ©re  s   €r(   ru   rë  _  s   ø€ ¨9¯>©>¸"Ô+=r+   rI   c                  ó&   <€ S P                  R4      # rí  rî  )Úwindow_shapes   €r(   ru   rë  a  s   ø€ °<×3DÑ3DÀRÔ3Hr+   Útxt_shape_winc                  óh   <€ \         P                  ! S SP                  S P                  4      ^ R7      # )r   rã   )rM   Úrepeat_interleaver'  rQ   )re  Úwindow_counts   €€r(   ru   rë  k  s$   ø€ œE×3Ò3°I¸|¿¹Èy×O_ÑO_Ó?`ÐfgÕhr+   Úmm_pnpc                  ó   <€ \        SS S4      # r"   )r‰   )rJ   Úvid_len_winrõ  s   €€€r(   ru   rë  t  s   ø€ Ô/°¸WÀlÔSr+   T)	r  r�   r—   Úpreferred_attentionr£  Úskip_reshapeÚskip_output_reshapeÚcu_seqlens_qÚcu_seqlens_k)r°  r;   rØ  rÅ  r`   ra   r£  r¤  Úunbindr²  r³  r'  rQ   r	  rÝ  r  r   r   r®  rú   r±  )!r'   rs   ri   rb  re  r$   Úvid_qkvÚtxt_qkvÚ	cache_winÚwindow_partitionÚwindow_reverseÚvid_qkv_winr`  ra  Úvid_vrc  rd  Útxt_vrò  Ú
concat_winÚunconcat_winÚcu_seqlens_winr  r�   r—   rD  rf   rg   rç  rJ   rø  rõ  rñ  s!   f&&ff&                      @@@@@r(   ræ   ÚNaSwinAttention.forward9  s­  ÿ€ ð  Ÿ=™=¨Ó2Ñˆà—O‘O t×'9Ñ'9Ð&:¸!¸D¿K¹K¸=ÈÐ$MÓNˆ	÷	Eð 	Eñ
 HQØÝ6óH
ÑDÐ¨,¸ñ ' wÓ/ˆØà!×)Ñ)¨+×*;Ñ*;¸AÕ*>ÀÀ4Ç:Á:ÈtÏ}É}Ó]ˆØ—/‘/ '§-¡-°Õ"2°A°t·z±zÀ4Ç=Á=ÓQˆà)×0Ñ0°Ó3Ñˆ�eØ%Ÿn™n¨QÓ/Ñˆˆu�eØ�[à—{‘{ 5Ó0‰ˆØ—{‘{ 5¨%Ó0‰ˆˆuá˜	Ô#=Ó>ˆá 	Ô+HÓIˆØ—*‘*˜\×0Ñ0Ó1ˆà�9�9ˆ9Ø��ˆØ#Ÿy™y¨°|ÀYÓO‘��uØ—‘——�á )Ø#Ýhó!�ð .2¯Y©YØ ,°°u¸iÈÐXaó.Ñ*�˜e¡Uð  $Ÿy™y¨°|ÀYÓO‘�á3<ØÖSó4
Ñ0ˆ
�L .ô %¡Z°Ó%=Ó>ˆØ�5Ü$¡Z°°uÓ%=Ó>ˆØ�5Ü$¡Z°°uÓ%=Ó>ˆØ�5Ü%Ø�1˜¨t×/CÑ/CØ—*‘*¨4ÀTØ'Ø'ô	
ˆð ˆq�!Ù'¨Ó,Ñˆ�Øà—/‘/ ! QÓ'ˆØ—/‘/ ! QÓ'ˆÙ  Ó)ˆàŸ=™=¨°'Ó:Ñˆ�à˜ÐÐr+   )rÝ  rÅ  rØ  rÞ  r–  rÿ   s   @@r(   rÕ  rÕ  '  s+   ù‡ € ð6ð
 ÷6õ 6÷"S ÷ S ð S r+   rÕ  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPiŽ  c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r-   rÙ   Úexpand_ratior  )r1   r2   s   "€r(   r3   ÚMLP.__annotate__�  s#   ø€ ÷ 	_ñ 	_áð	_ñ ñ	_r+   c                óÖ   <€ \         SV `  4        VP                  WV,          W4R 7      V n        \        P
                  ! R4      V n        VP                  W,          WVR 7      V n        R# )r’  ÚtanhN)rÔ   r)   r¯  Úproj_inr   ÚGELUÚactr±  )r'   rÙ   r  rQ   r÷   rµ  rÜ   s   &&&&&&€r(   r)   ÚMLP.__init__�  sX   ø€ ô 	‰ÑÔØ!×(Ñ(¨°LÕ.@ÈÐ(Ó]ˆŒÜ—7’7˜6“?ˆŒØ"×)Ñ)¨#Õ*<¸cÐX]Ð)Ó^ˆŽr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r-   rU   rL   ©rM   r  )r1   r2   s   "€r(   r3   r  š  s'   ø€ ÷ ñ ™×*Ñ*ð ©u×/@Ñ/@ñ r+   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r"   )r  r  r±  )r'   rU   s   &&r(   ræ   ÚMLP.forwardš  s/   € Ø�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�M‰M˜!ÓˆØˆr+   )r  r  r±  r–  rÿ   s   @@r(   r  r  Ž  s    ù‡ € ÷	_ó 	_÷÷ ð r+   r  c                   óT   a a€ ] tR tRt oRV3R lV 3R llltV3R lR ltRtVtV ;t# )Ú	SwiGLUMLPi¡  c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r-   rÙ   r  Úmultiple_ofr  )r1   r2   s   "€r(   r3   ÚSwiGLUMLP.__annotate__¢  s.   ø€ ÷ bñ báðbñ ðbñ ñ	br+   c                ó4  <€ \         SV `  4        \        ^V,          V,          ^,          4      pW7V,           ^,
          V,          ,          pVP                  WRWER7      V n        VP                  WqRWER7      V n        VP                  WRWER7      V n        R# )r-   F)r©  rQ   r÷   N)rÔ   r)   r­   r¯  Úproj_in_gater±  r  )	r'   rÙ   r  r  rQ   r÷   rµ  Ú
hidden_dimrÜ   s	   &&&&&&& €r(   r)   ÚSwiGLUMLP.__init__¢  sŒ   ø€ ô 	‰ÑÔÜ˜˜S� <Õ/°!Õ3Ó4ˆ
Ø °+Õ%=ÀÕ%AÀkÕ$QÕRˆ
Ø&×-Ñ-¨cÀEÐRXÐ-ÓfˆÔØ"×)Ñ)¨*ÀÈfÐ)ÓbˆŒØ!×(Ñ(¨¸uÈVÐ(ÓaˆŽr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r  r  )r1   r2   s   "€r(   r3   r  °  s+   ø€ ÷ Bñ B™×*Ñ*ð B©u×/@Ñ/@ñ Br+   c                ón  € V P                  V4      pV P                  V4      p\        P                  P                  '       g   VP
                  '       d-   V P                  \        P                  ! V4      V,          4      # V P                  \        P                  ! VR R7      P                  V4      4      # )T)Úinplace)
r!  r  rJ  rK  rL  rM  r±  ÚFÚsiluÚmul_)r'   rU   ÚgateÚups   &&  r(   ræ   ÚSwiGLUMLP.forward°  sz   € Ø× Ñ  Ó#ˆØ�\‰\˜!‹_ˆÜ×!Ñ!×-×-Ð-°··°Ø—=‘=¤§¢¨£°Õ!2Ó3Ð3Ø�}‰}œQŸVšV D°$Ô7×<Ñ<¸RÓ@ÓAÐAr+   )r  r!  r±  )é   NNNr–  rÿ   s   @@r(   r  r  ¡  s#   ù‡ € ÷bõ b÷B÷ Bð Br+   r  c                ó:   € V ^8„  d   QhR\         \        ,          /# )r-   Úmlp_type©r   r0   )r1   s   "r(   r3   r3   ·  s   € ÷ >ñ >”hœs•mñ >r+   c                 óR   € V R 8X  d   \         # V R8X  d   \        # \        RV  24      h)ÚnormalÚswigluzUnknown SeedVR2 MLP type: )r  r  r¦   )r/  s   &r(   Úget_mlpr4  ·  s0   € Ø�8ÔÜˆ
Ø�8ÔÜÐÜ
Ð1°(°Ð<Ó
=Ð=r+   c                   óh   a a€ ] tR tRt oV3R lV 3R lltR tV3R lR ltV3R lR ltR	tVt	V ;t
# )
ÚNaMMSRTransformerBlocki¾  c          !      óª   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[ RS[RS[R	S[R
S[RS[RS[ RS[RS[S[ S[S[ S[ S[ 3,          3,          RS[RS[/# )r-   r¡  r¢  Úemb_dimr£  r¤  r  Únorm_epsr¥  r/  r…  r˜  r  Úis_last_layerrÅ  rØ  rÖ  )r­   rÖ   rŠ  r0   r   r   )r1   r2   s   "€r(   r3   Ú#NaMMSRTransformerBlock.__annotate__¿  sÂ   ø€ ÷ 9!ñ 9!ñ ð9!ñ ð	9!ñ
 ð9!ñ ð9!ñ ð9!ñ ð9!ñ ð9!ñ ð9!ñ ð9!ñ ð9!ñ  ð!9!ñ" ð#9!ñ$ ð%9!ñ& ‘c™5¡¡c©3 Õ/Ð/Õ0ð'9!ñ( ð)9!ñ* ñ+9!r+   c               ó²  <€ \         SV `  4        \        W4      p\        VVVR VVVR7      V n        \        R/ RVbRVbRVbRVbRV
bRVbRVbR	VbR
VbRVbRVbRVbRVbRVbRVbRVb V n        \        VVVR VVVVR7      V n        \        \        V4      VVVVVVVR7      V n	        \        V	VVRR.VVVVR7      V n
        VV n        VV n        W€n        R# )Frª  r¡  r¢  r£  r¤  r¥  r´  r¦  r˜  r  r…  rÅ  rØ  rÖ  rQ   r÷   rµ  )r«  r¬  r­  r…  r†  rQ   r÷   )rÙ   r  r…  r†  rQ   r÷   rµ  ÚattnÚmlp)rÙ   r8  Úlayersr…  r†  rQ   r÷   NrS   )rÔ   r)   rŽ   r„  Ú	attn_normrÕ  r=  Úmlp_normr4  r>  Úadar:  rÖ  r9  )r'   r¡  r¢  r8  r£  r¤  r  Únormr9  rB  r¥  r´  r/  r…  r˜  r  r:  rÅ  rØ  rÖ  rQ   r÷   rµ  rÙ   rÜ   s   &$$$$$$$$$$$$$$$$$$$$$$ €r(   r)   ÚNaMMSRTransformerBlock.__init__¿  s�  ø€ ô0 	‰ÑÔÜ�GÓ%ˆÜ! $¸À(Ð_dð  vDð  MSð  [`ô  aˆŒä#ò 
Ùð
áð
ñ ð
ñ ð	
ñ
 ð
ñ ð
ñ !ð
ñ  ð
ñ ð
ñ *ð
ñ ð
ñ (ð
ñ ð
ñ ð
ñ "'ð
ñ 4>ð
ˆŒ	ô" ! ¸ÀÐ^cð  uCð  N[ð  djð  rwô  xˆŒÜÜ�HÓØØ%Ø)Ø"Ø °:ô
ˆŒô ˜C S°'À6È5À/Ðbpð  |Ið  RXð  `eô  fˆŒØ*ˆÔØˆŒØ Žr+   c           	     óV  € \         P                  P                  ;'       g    VP                  '       * ;'       d$    VP                  V P
                  P                  8H  pV'       g"   V! W#4      w  r#V P
                  ! W#3RVRR/VB # V P
                  ! W#3RVRRRV P                  /VB # )z•Normalize then modulate as one fused kernel, only where norm and ada own the same
branches (the last block's attn_norm covers txt, its ada does not).ÚlayerÚmodeÚinr9  )rJ  rK  rL  rM  r†  rB  r9  )r'   rC  rs   ri   rF  Ú
ada_kwargsÚfusables   &&&&&& r(   Ú_norm_ada_inÚ#NaMMSRTransformerBlock._norm_ada_inú  sŸ   € ô ×'Ñ'×3Ñ3×HÐH°s×7HÑ7HÔI÷ 3ð 3Ø—‘ §¡×!2Ñ!2Ñ2ð 	÷ Ù˜C“~‰HˆCØ—8’8˜CÑK¨EÐK¸ÐKÀ
ÑKÐKØ�xŠx˜Ñ_¨Ð_°DÐ_À4Ç=Á=Ð_ÐT^Ñ_Ð_r+   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# r�  r�  )r1   r2   s   "€r(   r3   r;    sQ   ø€ ÷ ñ á×Ñðñ ×Ñðñ 
Ù×ÑÙ×Ñð	õ
ñ	r+   c                ó4  € V P                   P                  '       g   V P                   P                  MV P                   P                  pVP                  ^ ,          \
        8:  d   V! V4      pEM\        P                  P                  '       g   VP                  '       dD   \        P                  ! VP                  \
        ^ R7       Uu. uF
  qC! V4      NK  	  up^ R7      pM›Rp^ pVP                  \
        ^ R7       Fz  pV! V4      pVf8   VP                  VP                  ^ ,          .VP                  R,          O54      pWuWfVP                  ^ ,          ,           % WgP                  ^ ,          ,          pK|  	  TpV P                   P                  '       gx   V P                   P                  '       g   V P                   P                  MV P                   P                  pVP!                  VP"                  VP$                  R7      pV! V4      pW3# u upi )r   rã   Nr[   r’  )r>  r…  rs   re   ra   r   rJ  rK  rL  rM  rM   rd   r]   Ú	new_emptyr†  ri   r'  rQ   r÷   )	r'   rs   ri   r“  rW  rf   rí   Ú	chunk_outr”  s	   &&&      r(   Ú_chunked_mlpÚ#NaMMSRTransformerBlock._chunked_mlp  s†  € ð *.¯©×)@×)@Ð)@�T—X‘X—\’\ÀdÇhÁhÇlÁlˆ
Ø�9‰9�Q�<Ô,Ô,Ù˜S“/ŠCÜ×#Ñ#×/×/Ð/°3×3D×3DÐ3DÜ—)’)¸C¿I¹IÔFWÐ]^¸IÔ<_Ó`Ñ<_°5˜Z¨Ö.Ñ<_Ñ`ÐfgÔh‰CàˆGØˆFØŸ™Ô#4¸!˜Ö<�Ù& uÓ-�	Ø’?Ø'×1Ñ1°3·9±9¸Qµ<Ð2VÀ)Ç/Á/ÐRTÕBUÑ2VÓW�GØ>G˜¨	¯©¸Õ(:Õ:Ð;ØŸ/™/¨!Õ,Õ,’ñ =ð ˆCØ�x‰x× × Ð Ø-1¯X©X×-D×-DÐ-D˜Ÿ™ŸšÈ$Ï(É(Ï,É,ˆJØ—&‘& §
¡
°#·)±)�&Ó<ˆCÙ˜S“/ˆCØˆxˆùò as   Ã
Hc                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[RS[S[ P                  S[ P                  S[ P                  S[ P                  3,          /# )r-   rs   ri   rb  re  Úembr$   rL   r  )r1   r2   s   "€r(   r3   r;  #  s�   ø€ ÷ #6ñ #6á×Ñð#6ñ ×Ñð#6ñ ×#Ñ#ð	#6ñ
 ×#Ñ#ð#6ñ ×Ñð#6ñ ð#6ñ 
Ù×ÑÙ×ÑÙ×ÑÙ×Ñð	õ
ñ#6r+   c                óî  aa€ \        V! R V3R l4      V! RV3R l4      4      pRVRVRVR\        RR	4      /pV P                  V P                  WR
V4      w  ršV P                  WšSSV4      w  ršV P                  ! Wš3RR
RR/VB w  ršW‘,           W¢,           r©V P                  V P
                  WšRV4      w  r¼V P                  W¼4      w  r¼V P                  ! W¼3RRRR/VB w  r¼W¹,           WÊ,           rËW¼SS3# )rI   c                  ó&   <€ S P                  R4      # rí  rî  ©rb  s   €r(   ru   Ú0NaMMSRTransformerBlock.forward.<locals>.<lambda>2  ó   ø€  Y§^¡^°BÔ%7r+   rJ   c                  ó&   <€ S P                  R4      # rí  rî  rï  s   €r(   ru   rX  3  rY  r+   rT  Úhid_lenr$   Ú
branch_tagrs   ri   r=  rF  rG  rD  r>  )rŽ   rK  r@  r=  rB  rA  rQ  )r'   rs   ri   rb  re  rT  r$   r[  rI  Úvid_attnÚtxt_attnÚvid_mlpÚtxt_mlps   &&&ff&&      r(   ræ   ÚNaMMSRTransformerBlock.forward#  s  ù€ ô Ù�)Ô7Ó8Ù�)Ô7Ó8ó
ˆð
 �3Ø�wØ�UØœ%  uÓ-ð	
ˆ
ð "×.Ñ.¨t¯~©~¸sÈÐQ[Ó\ÑˆØ!ŸY™Y x¸9ÀiÐQVÓWÑˆØ!ŸXšX hÑaÀÐaÈUÐaÐV`ÑaÑˆØ&�n°µ�(à×,Ñ,¨T¯]©]¸HÐPUÐWaÓbÑˆØ×,Ñ,¨WÓ>ÑˆØŸ8š8 GÑ\¸EÐ\ÈÐ\ÐQ[Ñ\ÑˆØ#Õ.°'Õ2D�à ¨IÐ5Ð5r+   )rB  r=  r@  r:  r>  rA  r9  rÖ  )rA   rB   rC   rD   r)   rK  rQ  ræ   rE   rF   rþ   rÿ   s   @@r(   r6  r6  ¾  s/   ù‡ € ÷9!ó 9!òv
`÷ð ÷:#6÷ #6ð #6r+   r6  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚPatchOutiH  c          	      ó\   <€ V ^8„  d   QhRS[ RS[S[ S[S[ S[ S[ 3,          3,          RS[ /# )r-   Úout_channelsÚ
patch_sizerÙ   ©r­   r   r   )r1   r2   s   "€r(   r3   ÚPatchOut.__annotate__I  sC   ø€ ÷ 
añ 
aáð
añ ™#™u¡S©#©s ]Õ3Ð3Õ4ð
añ ñ	
ar+   c                ó¬   <€ \         S
V `  4        \        V4      w  rxp	WxV	3V n        VP	                  W1V,          V,          V	,          WER 7      V n        R# ©r’  N©rÔ   r)   r   rf  r¯  Úproj)r'   re  rf  rÙ   rQ   r÷   rµ  rµ   r¶   r·   rÜ   s   &&&&&&&   €r(   r)   ÚPatchOut.__init__I  sO   ø€ ô 	‰ÑÔÜ˜*Ó%‰ˆˆaØ ˜'ˆŒØ×%Ñ% c¸!Õ+;¸aÕ+?À!Õ+CÈFÐ%Ó`ˆŽ	r+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r-   rs   rL   r>  )r1   r2   s   "€r(   r3   rh  U  s'   ø€ ÷ ñ á�\‰\ðñ 
�‰ñr+   c           
     óf  € V P                   w  r#pV P                  V4      pVP                  w  rVrxp	W’V,          V,          ,          p
VP                  WVWxW#WJ4      P	                  ^ ^^^^^^^4      P                  WZWb,          Ws,          W„,          4      pV^8”  d   VRRV^,
          R13,          pV# )r   rH  N)rf  rl  ra   r-  Úpermuter`   )r'   rs   rµ   r¶   r·   ÚbÚTÚHÚWÚchannelsÚcs   &&         r(   ræ   ÚPatchOut.forwardU  s«   € ð —/‘/‰ˆˆaØ�i‰i˜‹nˆØ"Ÿy™yÑˆˆa�HØ˜Q� �Õ#ˆØ�h‰h�q˜Q 1¨Ó.×6Ñ6°q¸!¸QÀÀ1ÀaÈÈAÓN×VÑVÐWXÐ]^Õ]bÐdeÕdiÐklÕkpÓqˆØˆqŒ5Ø�a˜˜Q �U™I�oÕ&ˆCØˆ
r+   ©rf  rl  r–  rÿ   s   @@r(   rc  rc  H  s    ù‡ € ÷
aó 
a÷÷ ð r+   rc  c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )Ú
NaPatchOutib  Nc          
      ó¨   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[S[ P                  S[ P                  3,          /# ©r-   rs   rb  r$   rL   )rM   r  rN   r   r    r   )r1   r2   s   "€r(   r3   ÚNaPatchOut.__annotate__c  s_   ø€ ÷ ñ á×Ñðñ ×#Ñ#ðñ ™�ð	ñ 
Ù×ÑÙ×Ñð	õ
ñr+   c           
     ó†  € Vf   \        RR7      pV P                  w  rVpV P                  V4      pYVu;8X  d   Tu;8X  d   ^8X  gú   M \        W4      p\	        \        V4      4       FÇ  pW,          P                  w  ršr¼WÅV,          V,          ,          pW,          P                  WšWµWgV4      P                  ^ ^^^^^^4      P                  W•,          W¦,          W·,          V4      W&   V^8”  g   KŒ  WH^ 3,          V,          ^ 8w  g   K¤  W,          WTV^ 3,          V,          ,
          R  W&   KÉ  	  \        V4      w  rW3# )NT©r%   )r    rf  rl  r¾  r~   r^   ra   r-  rq  r`   rú   )r'   rs   rb  r$   Úvid_shape_before_patchifyrµ   r¶   r·   r‡   rs  rt  ru  rv  rw  s   &&&&&         r(   ræ   ÚNaPatchOut.forwardc  s  € ð Š=Ü $Ô'ˆEà—/‘/‰ˆˆaØ�i‰i˜‹nˆàÖ ˜!Ö ˜qÖ Ü˜CÓ+ˆCÜœ3˜s›8–_�Ø$'¥F§L¡LÑ!��aØ Q¥¨¥Õ+�Ø�Ÿ™ Q¨1°°qÓ9×AÑAÀ!ÀQÈÈ1ÈaÐQRÐTUÓV×^Ñ^Ð_`Õ_dÐfgÕfkÐmnÕmrÐtuÓv�‘Ø�q–5Ð6¸!°tÕ<¸qÕ@ÀAÖEØ �V QÀ1ÀaÀ4Õ)HÈ1Õ)LÕ%LÐ$OÐP�C“Fñ %ô % S›\‰NˆCàˆ~Ðr+   rS   )NN©rA   rB   rC   rD   ræ   rE   rF   rG   s   @r(   r{  r{  b  s   ø‡ € ÷÷ ð r+   r{  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚPatchIni  c          	      ó\   <€ V ^8„  d   QhRS[ RS[S[ S[S[ S[ S[ 3,          3,          RS[ /# )r-   Úin_channelsrf  rÙ   rg  )r1   r2   s   "€r(   r3   ÚPatchIn.__annotate__€  sC   ø€ ÷ 
`ñ 
`áð
`ñ ™#™u¡S©#©s ]Õ3Ð3Õ4ð
`ñ ñ	
`r+   c                ó¬   <€ \         S
V `  4        \        V4      w  rxp	WxV	3V n        VP	                  W,          V,          V	,          W4VR 7      V n        R# rj  rk  )r'   r‡  rf  rÙ   rQ   r÷   rµ  rµ   r¶   r·   rÜ   s   &&&&&&&   €r(   r)   ÚPatchIn.__init__€  sO   ø€ ô 	‰ÑÔÜ˜*Ó%‰ˆˆaØ ˜'ˆŒØ×%Ñ% k¥o¸Õ&9¸AÕ&=¸sÐY^Ð%Ó_ˆŽ	r+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ro  r>  )r1   r2   s   "€r(   r3   rˆ  Œ  s'   ø€ ÷ ñ á�\‰\ðñ 
�‰ñr+   c           
     óL  € V P                   w  r#pV^8”  du   VP                  ^4      V,          ^8w  d"   \        RV RVP                  ^4       R24      h\        P                  ! VR,          .V^,
          ,          V.,           ^R7      pVP
                  w  rVrxp	VP                  WVWr,          W(V,          W9V,          V4      P                  ^ ^^^^^^^4      P                  WWV,          Wƒ,          W”,          W#,          V,          V,          4      pV P                  V4      pV# )rZ   z3SeedVR2 patch input temporal size must satisfy T % z == 1, got r>   rã   )rH  rH  ºNrZ   N)
rf  rª   r¦   rM   rd   ra   r-  rq  r`   rl  )
r'   rs   rµ   r¶   r·   rr  rw  ÚTtÚHhÚWws
   &&        r(   ræ   ÚPatchIn.forwardŒ  s5  € ð —/‘/‰ˆˆaØˆqŒ5Ø�x‰x˜‹{˜Q� !Ô#Ü ØIÈ!ÈÈKÐX[×X`ÑX`ÐabÓXcÐWdÐdeÐfóð ô —)’)˜S �]˜O¨q°1­uÕ5¸¸Õ=À1ÔEˆCØŸ9™9Ñˆˆb�bØ�h‰h�q˜R�W a¨q­°!¸1µW¸aÓ@×HÑHÈÈAÈqÐRSÐUVÐXYÐ[\Ð^_Ó`×hÑhÐijÐrsÕlsÐuwÕu|ð  Aõ  Fð  HIõ  HMð  PQõ  HQð  TUõ  HUó  VˆØ�i‰i˜‹nˆØˆ
r+   ry  r–  rÿ   s   @@r(   r…  r…    s    ù‡ € ÷
`ó 
`÷÷ ð r+   r…  c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )Ú	NaPatchIniœ  Nc                ó~   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[,          RS[ P                  /# r}  )rM   r+  rN   r   r    )r1   r2   s   "€r(   r3   ÚNaPatchIn.__annotate__�  sE   ø€ ÷ ñ á�\‰\ðñ ×#Ñ#ðñ ™�ð	ñ
 
�‰ñr+   c           
     óf  a€ Vf   \        RR7      pVP                  R4      pV! RV3R l4      pV P                  w  rVpYVu;8X  d   Tu;8X  d	   ^8X  EgI   M \        VS4      p\	        \        V4      4       EF  pV^8”  dw   WH^ 3,          V,          ^ 8w  da   \        P                  ! W,          R,          .WQV,          P                  ^ 4      V,          ,
          ,          W,          .,           ^ R7      W&   W,          P                  w  ršr¼W,          P                  W•,          WZV,          WkV,          W|4      P                  ^ ^^^^^^4      P                  W•,          W¦,          W·,          WV,          V,          V,          4      W&   EK  	  \        V4      w  poV P                  V4      pVS3# )NTr€  Úpatchr�  c                  ó   <€ S # r"   rS   rW  s   €r(   ru   Ú#NaPatchIn.forward.<locals>.<lambda>¦  s   ø€ Éyr+   r�  rã   )r    r;   rf  r¾  r~   r^   rM   rd   rª   ra   r-  rq  r`   rú   rl  )r'   rs   rb  r$   r�  rµ   r¶   r·   r‡   rŽ  r�  r�  rw  s   &&f&         r(   ræ   ÚNaPatchIn.forward�  s�  ø€ ð Š=Ü $Ô'ˆEØ—‘ Ó(ˆÙ$)Ð*EÔGXÓ$YÐ!Ø—/‘/‰ˆˆaØÖ ˜!Ö ˜q× Ü˜C Ó+ˆCÜœ3˜s›8—_�Ø�q”5Ð6¸!°tÕ<¸qÕ@ÀAÔEÜ"ŸYšY¨­¨r­
 |°q¸q½6¿;¹;Àq»>ÈAÕ;MÕ7MÕ'NÐRUÕRXÐQYÕ'YÐ_`Ôa�C‘FØ #¥§¡‘�˜Ø�Ÿ™ R¥W¨a°qµ¸!À1½WÀaÓK×SÑSÐTUÐWXÐZ[Ð]^Ð`aÐcdÐfgÓh×pÑpÐqsÕqxÐz|õ  {Bð  DFõ  DKð  MNõ  MRð  UVõ  MVð  YZõ  MZó  [�”ñ	 %ô
 % S›\‰NˆC�à�i‰i˜‹nˆØ�Iˆ~Ðr+   rS   r"   rƒ  rG   s   @r(   r“  r“  œ  s   ø‡ € ÷÷ ð r+   r“  c                óP   € V ^8„  d   QhR\         P                  R\        R\        /# )r-   rU   rÙ   Úndim)rM   r+  r­   )r1   s   "r(   r3   r3   ´  s%   € ÷ ñ ”5—<‘<ð ¤cð ´ñ r+   c                 ó”   € V P                   pVR V RV\        V4      ,
          ,          ,           W1R  ,           pV P                  V4      # )Nr¯   )ra   r^   r`   )rU   rÙ   rœ  ra   s   &&& r(   Úexpand_dimsrž  ´  sA   € Ø�G‰G€EØ�$�3ˆK˜$ $¬¨U«Õ"3Õ4Õ4°u¸T°{ÕB€EØ�9‰9�UÓÐr+   c                   ón   a a€ ] tR tRt oR	V3R lV 3R llltR
V3R lR lltRV3R lR lltRtVtV ;t	# )Ú	AdaSingleiº  c          
      óV   <€ V ^8„  d   QhRS[ RS[ RS[S[,          RS[S[R3,          /# )r-   rÙ   r8  r?  Úmodes.)r­   r   r0   r   )r1   r2   s   "€r(   r3   ÚAdaSingle.__annotate__»  sE   ø€ ÷ eñ eáðeñ ðeñ ‘S•	ð	eñ
 ‘S˜#�X�ñer+   c                ó:  <€ V^V,          8w  d   \        RV RV R24      h\        S	V `	  4        Wn        W n        W0n        RVRV/pV FÍ  pRV9   d}   V P                  V R2\        P                  ! \        P                  ! V3/ VB 4      4       V P                  V R2\        P                  ! \        P                  ! V3/ VB 4      4       R	V9   g   K�  V P                  V R
2\        P                  ! \        P                  ! V3/ VB 4      4       KÏ  	  R# )é   z;SeedVR2 AdaSingle requires emb_dim == 6 * dim, got emb_dim=z, dim=r>   rQ   r÷   rH  Ú_shiftÚ_scalerD  Ú_gateN)r¦   rÔ   r)   rÙ   r8  r?  Úregister_parameterr   Ú	ParameterrM   Úempty)
r'   rÙ   r8  r?  r¢  rQ   r÷   Úparam_kwargsr|  rÜ   s
   &&&&&&&  €r(   r)   ÚAdaSingle.__init__»  sü   ø€ ð �a˜#•gÔÜÐZÐ[bÐZcÐciÐjmÐinÐnoÐpÓqÐqÜ‰ÑÔØŒØŒØŒà  &¨'°5Ð9ˆãˆAØ�uŒ}Ø×'Ñ'¨1¨#¨V¨´b·l²lÄ5Ç;Â;ÈsÑCcÐVbÑCcÓ6dÔeØ×'Ñ'¨1¨#¨V¨´b·l²lÄ5Ç;Â;ÈsÑCcÐVbÑCcÓ6dÔeØ˜Ž~Ø×'Ñ'¨1¨#¨U¨´R·\²\Ä%Ç+Â+ÈcÑBbÐUaÑBbÓ5cÖdó r+   c                óÐ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[RS[S[,          RS[RS[S[ P
                  ,          RS[S[,          R	S[ P                  /	# )
r-   rº  rT  rF  rG  r$   r\  r[  r9  rL   )rM   r  r0   r   r    rN   rÖ   )r1   r2   s   "€r(   r3   r£  Ó  sŠ   ø€ ÷ )ñ )á×Ñð)ñ ×Ñð)ñ ð	)ñ
 ð)ñ ™�ð)ñ ð)ñ ™%×*Ñ*Õ+ð)ñ ™5•/ð)ñ 
×	Ñ	ñ)r+   c	                ó^  a aaa€ Vf   \        RR7      pS P                  P                  V4      oVVVV 3R lp	Vf	   V	! 4       p
MV! RS RV 2V	4      p
V
P                  R4      w  r¼pVe   V
P                  ^ ,          ^8X  d   S P                  SW4W¼WØR7      # Ve   \        P                  ! S4      MSp^ p\        V! RV 2VP                  4      4       FS  w  ppS P                  SWÿV,            W4VV,          VV,          VV,          VR7      pVe   VWïVV,           % VV,          pKU  	  V# )	NTr€  c                  óÆ   <€ SP                  SP                  ^ ,          R\        SP                  4      ^4      RRSR3,          p \	        V ^SP
                  ^,           4      # )r   rH  r\   )r`   ra   r^   r?  rž  rœ  )ÚerT  rº  Úidxr'   s    €€€€r(   ÚslicedÚ!AdaSingle.forward.<locals>.slicedâ  sL   ø€ Ø—‘˜CŸI™I a�L¨"¬c°$·+±+Ó.>ÀÓBÀ1ÀaÈÈaÀ<ÕPˆAÜ˜q ! S§X¡X°¥\Ó2Ð2r+   Úemb_repeat_rÒ  )r9  Úhid_len_list_r\   )
r    r?  Úindexrþ  ra   Ú	_modulaterM   Ú
empty_likerå   rz   )r'   rº  rT  rF  rG  r$   r\  r[  r9  r³  ÚmodÚshiftAÚscaleAÚgateArD  rí   r‡   ÚlengthÚ	modulatedr²  s   fff&&&&&&          @r(   ræ   ÚAdaSingle.forwardÓ  s<  û€ ð Š=Ü $Ô'ˆEØ�k‰k×Ñ Ó&ˆ÷	3ð 	3ð Š?Ù“(‰Cñ ˜+ c U¨!¨J¨<Ð8¸&ÓAˆCà #§
¡
¨2£Ñˆ˜ØŠ?˜cŸi™i¨�l¨aÔ/à—>‘> # u°FÀE�>Ó]Ð]ð (0Ò';Œe×Ò˜sÔ#ÀˆØˆÜ"¡5¨=¸¸Ð)EÀwÇ~Á~Ó#VÖW‰IˆAˆvØŸ™Ø�F F�?Ð+¨U¸&À½)ÀVÈAÅYÐPUÐVWÕPXÐckð 'ó ˆIð Ò#Ø.7�˜6 F�?Ð+Ø�fÕŠFñ Xð ˆ
r+   c                ó¾   <€ V ^8„  d   QhRS[ P                  RS[RS[RS[ P                  RS[ P                  RS[ P                  RS[S[,          RS[ P                  /# )	r-   rº  rF  rG  r»  r¼  r½  r9  rL   )rM   r  r0   r   rÖ   )r1   r2   s   "€r(   r3   r£  þ  s}   ø€ ÷ <ñ <á×Ñð<ñ ð<ñ ð	<ñ
 ×!Ñ!ð<ñ ×!Ñ!ð<ñ × Ñ ð<ñ ™5•/ð<ñ 
×	Ñ	ñ<r+   c                óz  € VR 8X  dÈ   \         P                  P                  \        W R24      V4      p\         P                  P                  \        W R24      V4      p	Ve@   \         P                  P
                  P                  WV	,           R,
          WH,           V4      # VP                  WY,           4      P                  WH,           4      # VR8X  dZ   \        W R2R4      p
V
e6   VP                  V\         P                  P                  W¡4      ,           4      # VP                  V4      # \        RV 24      h)rH  r¦  r§  NrÑ   rD  r¨  zUnknown AdaSingle mode: )
rJ  ÚopsÚcast_to_inputr–   rP  rQ  Ú	rms_adalnr)  Úadd_r¦   )r'   rº  rF  rG  r»  r¼  r½  r9  ÚshiftBÚscaleBÚgateBs   &&&&&&&&   r(   r¸  ÚAdaSingle._modulateþ  s  € ð �4Œ<Ü—Y‘Y×,Ñ,¬W°T¸WÀFÐ;KÓ-LÈcÓRˆFÜ—Y‘Y×,Ñ,¬W°T¸WÀFÐ;KÓ-LÈcÓRˆFØÒ#ä—‘×)Ñ)×3Ñ3°CÀ&½È3Õ9NÐPVÕP_ÐaiÓjÐjØ—8‘8˜F�OÓ,×1Ñ1°&µ/ÓBÐBØ�5Œ=Ü˜D G¨5 /°4Ó8ˆEØÒ Ø—x‘x ¬¯	©	×(?Ñ(?ÀÓ(KÕ KÓLÐLØ—8‘8˜E“?Ð"äÐ3°D°6Ð:Ó;Ð;r+   )rÙ   r8  r?  ))rH  rD  NN)Nr@   NNr"   )
rA   rB   rC   rD   r)   ræ   r¸  rE   rF   rþ   rÿ   s   @@r(   r   r   º  s+   ù‡ € ÷eõ e÷0)ò )÷V<÷ <ò <r+   r   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚTimeEmbeddingi  c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r-   Úsinusoidal_dimr"  Ú
output_dimr  )r1   r2   s   "€r(   r3   ÚTimeEmbedding.__annotate__  s)   ø€ ÷ ñ áðñ ðñ ñ	r+   c                óô   <€ \         SV `  4        Wn        VP                  WWER 7      V n        VP                  W"WER 7      V n        VP                  W#WER 7      V n        \        P                  ! 4       V n	        R# rj  )
rÔ   r)   rÎ  r¯  r  Úproj_hidr±  r   ÚSiLUr  )r'   rÎ  r"  rÏ  rQ   r÷   rµ  rÜ   s   &&&&&&&€r(   r)   ÚTimeEmbedding.__init__  sh   ø€ ô 	‰ÑÔØ,ÔØ!×(Ñ(¨ÈFÐ(Ó`ˆŒØ"×)Ñ)¨*ÈÐ)Ó]ˆŒØ"×)Ñ)¨*ÈÐ)Ó]ˆŒÜ—7’7“9ˆŽr+   c                ó´   <€ V ^8„  d   QhRS[ S[S[S[P                  S[P
                  3,          RS[P                  RS[P                  RS[P
                  /# )r-   ÚtimesteprQ   r÷   rL   )r   r­   rÖ   rM   Ú	IntTensorr  rQ   r÷   )r1   r2   s   "€r(   r3   rÐ  '  sY   ø€ ÷ ñ á™™U¡E§O¡O±U×5FÑ5FÐFÕGðñ —‘ðñ �{‰{ð	ñ
 
×	Ñ	ñr+   c                óž  € \         P                  ! V4      '       g   \         P                  ! V.W#R 7      pVP                  ^ 8X  d
   VR,          p\	        VV P
                  R^ R7      P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# )r’  NF)Ú	timestepsÚembedding_dimÚflip_sin_to_cosÚdownscale_freq_shift)rM   Ú	is_tensorÚtensorrœ  r   rÎ  r'  r  r  rÒ  r±  )r'   rÖ  rQ   r÷   rT  s   &&&& r(   ræ   ÚTimeEmbedding.forward'  s¯   € ô �Š˜x×(Ò(Ü—|’| X J°vÔKˆHØ�=‰=˜AÔØ •~ˆHä$ØØ×-Ñ-Ø!Ø!"ô	
÷
 ‰"ˆU‹)ð 	ð �l‰l˜3ÓˆØ�h‰h�s‹mˆØ�m‰m˜CÓ ˆØ�h‰h�s‹mˆØ�m‰m˜CÓ ˆØˆ
r+   )r  rÒ  r  r±  rÎ  r–  rÿ   s   @@r(   rÌ  rÌ    s   ù‡ € ÷ó ÷÷ ð r+   rÌ  c                ó¤   € V ^8„  d   QhR\         \        P                  ,          R\        \        P                  \        P                  3,          /# )r-   rº  rL   )r   rM   r  r   rN   )r1   s   "r(   r3   r3   ?  sC   € ÷ 
ñ 
Ü	Œe×ÑÕ	 ð
ä
Ü	×ÑÜ	×Ñðõñ
r+   c           	      óF  € \        V 4      ^ 8X  d   \        R4      h\        P                  ! V  Uu. uF  qP                  RR NK  	  upV ^ ,          P
                  R7      p\        P                  ! V  Uu. uF  qP                  ^ R4      NK  	  up4      p W3# u upi u upi )r   z-SeedVR2 flatten requires at least one tensor.NrP   r\   rô   )r^   r¦   rM   rÀ  ra   rQ   rd   rú   )rº  rU   ra   s   &  r(   rú   rú   ?  s€   € ô ˆ3ƒx�1„}ÜÐHÓIÐIÜ�OŠO±3Ó7±3¨aŸW™W S b›\±3Ñ7ÀÀAÅÇÁÔN€EÜ
�)Š)©sÓ3©s¨!—Y‘Y˜q "Ö%©sÑ3Ó
4€CØˆ:Ðùò 8ùÚ3s   °BÁ5Bc                óŽ   € V ^8„  d   QhR\         P                  R\         P                  R\        \         P                  ,          /# )r-   rº  r»  rL   )rM   r  rN   r   r+  )r1   s   "r(   r3   r3   L  s;   € ÷ ñ Ü	×	Ñ	ðä×Ñðô 
Œ%�,‰,Õñr+   c           	      óê   € VP                  R4      pV P                  VP                  4       4      p \        W4       UUu. uF%  w  r4VP	                  ^ VP                  4       4      NK'  	  p ppV # u uppi rí  )rÁ  r]   rz   r_   r¾  )rº  r»  r[  rU   Úss   &&   r(   r¾  r¾  L  s`   € ð �n‰n˜RÓ €GØ
�)‰)�G—N‘NÓ$Ó
%€CÜ25°cÔ2EÔ
FÑ2E©$¨!ˆ1�;‰;�q˜!Ÿ(™(›*Ö%Ñ2E€CÑ
FØ€Jùó Gs   ¿+A/c                   ó®   a a€ ] tR tRt o^!]RR^^€^
^RR^€RRRRRR3V3R lV 3R llltRR	 lt]R
 4       t]R 4       t	RR lt
RV3R lR lltRtVtV ;t# )ÚNaDiTiU  i 
  é   Fr›  Nc                ó0   <€ V ^8„  d   QhRS[ S[,          /# )r-   Úvid_out_normr0  )r1   r2   s   "€r(   r3   ÚNaDiT.__annotate__W  s   ø€ ÷ qñ qñ" ™s•mñ#qr+   c                ó¦  <€ VR'9  d   \        RV: R24      hV\        8H  V n        V P                  '       d   RpVV n        RVRV/pV^,          RR.,          pTpV^,          pVR(.,          p\        pVP
                  pVP
                  p\        SV `  4        V P                  R\        P                  ! R)VVR	7      4       V P                  R
\        P                  ! R*VVR	7      4       \        VVVVVVR7      V n        V'       d   VV8w  d   VP                  ! VV3/ VB M\        P                  ! 4       V n        \#        \$        \'        VV4      VVVVR7      V n        Ve   \+        V^ ,          \,        4      '       d   V.V,          pVe   TMV	^,          p\        P.                  ! \1        V4       Uu. uF¬  p\3        R+/ RVbRVbRVbRVbRV	bRVbRVbRVbRVbRVbRVbRVbRVbRVV,          bRVV,          bRV P                  bRVV^,
          8H  ;'       d    V P                  '       * bRVbR\+        V
\,        4      '       d   VV
8  MV
V,          '       * bR VbVB NK®  	  up4      V n        \7        VVVVVVR!7      V n        R V n        Ve3   VP                  VVR"VVR#7      V n        V! VVR$.R%.VVR&7      V n        R # R # u upi ),Nz2SeedVR2 NaDiT expected image_model='seedvr2', got r>   rš  rQ   r÷   r¢   r£   Úpositive_conditioningr’  Únegative_conditioning)r‡  rf  rÙ   rQ   r÷   rµ  )rÎ  r"  rÏ  rQ   r÷   rµ  r¡  r¢  r8  r£  r¤  r  rC  r9  rB  r¥  r´  r/  r  rÅ  rØ  rÖ  r:  r˜  r…  rµ  )re  rf  rÙ   rQ   r÷   rµ  T)r«  r¬  r­  rQ   r÷   rD  rH  )rÙ   r8  r?  r¢  rQ   r÷   )NÚseedvr2)é   r  r  )é:   rç  )é@   rç  rS   )r¦   r   Ú_7b_versionr÷   r   ÚRMSNormrÔ   r)   rØ   rM   r«  r“  Úvid_inr¯  r   ÚIdentityÚtxt_inrÌ  r   rÉ   Úemb_inr•   r­   Ú
ModuleListr~   r6  Úblocksr{  rf   ré  Úvid_out_ada)r'   r9  Ú
num_layersr/  Úvid_in_channelsÚvid_out_channelsr¡  Ú
txt_in_dimr£  r¤  Ú	mm_layersr  r¥  rf  r  r˜  ré  Úimage_modelrQ   r÷   rµ  Úfactory_kwargsrØ  r¢  r8  rÅ  rB  rC  r´  r‡   rÜ   s   &&&&&&&&&&&&&&&&&&&&&         €r(   r)   ÚNaDiT.__init__W  sa  ø€ ð. Ð/Ô/ÜÐQÐR]ÑQ`Ð`aÐbÓcÐcØ"Ô&8Ñ8ˆÔØ××ÐØ ˆIØˆŒ
Ø" F¨G°UÐ;ˆØ" a�Ð+CÐD]Ð*^Õ^ˆØˆØ˜A•+ˆØ˜w˜iÕ'ˆÜˆØ×!Ñ!ˆØ×$Ñ$ˆÜ‰ÑÔØ×ÑÐ4´e·k²kÀ*ÐU[ÐchÔ6iÔjØ×ÑÐ4´e·k²kÀ*ÐU[ÐchÔ6iÔjÜØ'Ø!ØØ °:ô	
ˆŒ÷ ˜j¨GÔ3ð ×Ò˜j¨'ÑD°^ÒDä—’“ð 	Œô
 $Ü3Ü˜7 GÓ,ØØ °:ô	
ˆŒð Š>œZ¨¨q­	´3×7Ò7Ø�X 
Õ*ˆFà'Ò3‘8¸ÀQ½ˆÜ—m’mô6 ˜zÔ*ó5ñ4 +�Aô3 'ò Ù#ðá#ðñ $ðñ  ð	ñ
 &ðñ ".ðñ ðñ &ðñ ðñ $ðñ $ðñ &ðñ  (ðð " !ž9ðð #0°Ö"2ðð  #×.Ò.ð!ð" $%¨
°Q­Ñ#6×"PÐ"PÀ×@PÑ@PÔ<Pøð#ñ$ !*ð%ô( ,6°iÄ×+EÒ+E˜˜YšÈ9ÐUVÍ<õ$ð'ñ, ",Ø$ô/ñ2 +ñ5ó
ˆŒô< "Ø)Ø!ØØ °:ô	
ˆŒð !ˆÔØÒ#Ø *× 2Ñ 2Ø!(ØØ#'Ø Uð	 !3ó !ˆDÔñ  #ØØØ�wØ�fØ Uô ˆDÖñ $ùòKs   Æ;A'KÈ#A
Kc                óÈ  € Ve   VP                  4       ^ 8X  d   V P                  p\        V.4      # \        P	                  V4      '       dP   VP
                  ^ ,          ^8X  d   VP                  ^ 4      p\        V.4      # \        VP                  ^ 4      4      # VP
                  ^ ,          ^,          ^ 8w  d"   \        R\        VP
                  4       24      hVP                  ^^ R7      w  r4VP
                  ^ ,          ^8X  d.   VP                  ^ 4      VP                  ^ 4      r4\        WC.4      # \        . VP                  ^ 4      OVP                  ^ 4      O54      # )Nz<SeedVR2 expected an even text-conditioning batch, got shape rã   )r|   rì  rú   ræ  Ú&_seedvr2_is_single_conditioning_branchra   rÎ  rþ  r¦   ÚtuplerW  )r'   ÚcontextÚcond_or_uncondÚneg_condÚpos_conds   &&&  r(   Ú_resolve_text_conditioningÚ NaDiT._resolve_text_conditioningÊ  s-  € ØŠ?˜gŸm™m›o°Ô2Ø×0Ñ0ˆGÜ˜G˜9Ó%Ð%Ü×7Ñ7¸×GÒGØ�}‰}˜QÕ 1Ô$Ø!Ÿ/™/¨!Ó,�Ü ˜yÓ)Ð)Ü˜7Ÿ>™>¨!Ó,Ó-Ð-Ø�=‰=˜Õ˜aÕ 1Ô$ÜÐ[Ô\aÐbi×boÑboÓ\pÐ[qÐrÓsÐsØ$Ÿ]™]¨1°!˜]Ó4ÑˆØ�>‰>˜!Õ Ô!Ø!)×!1Ñ!1°!Ó!4°h×6FÑ6FÀqÓ6I�hÜ˜HÐ/Ó0Ð0ÜÐA˜Ÿ™¨Ó+ÐA¨h¯o©o¸aÓ.@ÑAÓBÐBr+   c                ó¶   a€ V e   \        V 4      ^ 8X  d   R# V ^ ,          o\        ;QJ d    V3R lV  4       F  '       d   K   R# 	  R# ! V3R lV  4       4      # )NFc              3   ó,   <"  € T F	  qS8H  x € K  	  R # 5ir"   rS   )rT   ÚentryÚfirsts   & €r(   rW   Ú?NaDiT._seedvr2_is_single_conditioning_branch.<locals>.<genexpr>à  s   øé € Ð>©~ e˜E–>«~ùs   ƒT)r^   re   )r  r  s   &@r(   r  Ú,NaDiT._seedvr2_is_single_conditioning_branchÛ  sK   ø€ àÒ!¤S¨Ó%8¸AÔ%=ÙØ˜qÕ!ˆß‹sÔ>©~Ó>�sŒsÐ>ŠsÐ>ˆsÔ>©~Ó>Ó>Ð>r+   c           
     óò   € V P                   ^8w  d&   \        RV R\        V P                  4       R24      hV P                  ^,          V8w  d)   \        RV RV R\        V P                  4       R24      hV # )é   zSeedVR2 expected z$ to be 5-D native latent, got shape r>   z channels to be z, got shape )rœ  r¦   r  ra   )rU   rv  r    s   &&&r(   Ú_check_seedvr2_video_latentÚ!NaDiT._check_seedvr2_video_latentâ  s   € à�6‰6�QŒ;ÜÐ0°°Ð6ZÔ[`Ðab×ahÑahÓ[iÐZjÐjkÐlÓmÐmØ�7‰7�1�:˜Ô!ÜÐ0°°Ð6FÀxÀjÐP\Ô]bÐcd×cjÑcjÓ]kÐ\lÐlmÐnÓoÐoØˆr+   c                ó˜   € \         P                  V4      '       d   V# VP                  ^^ R7      w  r4\        P                  ! WC.^ R7      # )r-   rã   )ræ  r  rW  rM   rd   )r'   rD  r  rî   Únegs   &&&  r(   Ú_swap_pos_neg_halvesÚNaDiT._swap_pos_neg_halvesê  s?   € Ü×7Ñ7¸×GÒGØˆJØ—9‘9˜Q A�9Ó&‰ˆÜ�yŠy˜#˜¨Ô+Ð+r+   c                ó    <€ V ^8„  d   QhRS[ /# )r-   Údisable_cache)rŠ  )r1   r2   s   "€r(   r3   rê  ð  s   ø€ ÷ WYñ WYñ
 ñWYr+   c                ó@  aa€ VP                  R / 4      pVP                  R/ 4      pVP                  R/ 4      pVP                  R4      p	V	f   \        R4      hV P                  V\        R4      pV P                  V	\        ^,           R4      p	VP                  w  r«rÍpV	P                  ^ ,          V
8w  g   V	P                  R,          WÍV38w  d9   \        R\        VP                  4       R	\        V	P                  4       R
24      hVP                  ^R 4      pV	P                  ^R 4      p	\        VR7      pV P                  W6P                  R4      4      w  pp\        V4      w  po\        V	4      w  pp\        P                  ! VV.R R7      pV P                  V4      pSpV P                  VSVR7      w  poV P                  VVP                  VP                   R7      p\#        V P$                  4       Fq  w  poRV3V9   dP   V3R lpVRV3,          ! RVRVRSRVRVRV/RV/4      pVR,          VR,          VR,          VR,          3w  ppopK^  S! VVSVVVR7      w  ppopKs  	  V P&                  '       d5   V P'                  V4      pV P)                  VVRRV! RV3R l4      VRR7      pV P+                  VSVVR7      w  po\-        VS4      p\        P.                  ! V4      pVP                  R ^4      pV P1                  VVP                  R4      4      # )!Útransformer_optionsÚpatches_replaceÚditÚ	conditionzHSeedVR2 requires conditioning latents from the SeedVR2Conditioning node.ÚlatentÚconditioning:r-   NNz[SeedVR2 conditioning shape must match latent batch/temporal/spatial dimensions; got latent z and conditioning r>   r€  r  rã   )r$   r’  rˆ   c           	      óž   <€ / pS! V R ,          V R,          V R,          V R,          V R,          V R,          R7      w  VR &   VR&   VR&   VR&   V# )rs   ri   rb  re  rT  r$   ©rs   ri   rb  re  rT  r$   rS   )r“   rD  rˆ   s   & €r(   Ú
block_wrapÚ!NaDiT.forward.<locals>.block_wrap  sa   ø€ Ø�CÙQVØ $ U¥Ø $ U¥Ø&*¨;Õ&7Ø&*¨;Õ&7Ø $ U¥Ø"& w¥-ôRÑN�C˜‘J  E¡
¨C°Ñ,<¸cÀ+Ñ>Nð �Jr+   rs   ri   rb  re  rT  r$   Úoriginal_blockr$  rD  rH  rI   c                  ó&   <€ S P                  R4      # rí  rî  rW  s   €r(   ru   ÚNaDiT.forward.<locals>.<lambda>>  s   ø€ °·±ÀÔ1Cr+   )rT  rF  rG  r[  r$   r\  )r�  r\   )Úgetr¦   r  r   ra   r  Úmovedimr    r
  rú   rM   rd   rö  rô  r÷  rQ   r÷   rå   rù  ré  rú  rf   r¾  rB  r  )r'   rU   rÖ  r  r  rš   r  r  Úblocks_replaceÚ
conditionsrr  rÒ  rµ   r¶   r·   r$   ri   re  rs   Úcond_latentr�  rT  r‡   r%  rD  rˆ   rb  s   &&&&&,                   @@r(   ræ   ÚNaDiT.forwardð  s_  ù€ ð %Ÿj™jÐ)>ÀÓCÐØ-×1Ñ1Ð2CÀRÓHˆØ(×,Ñ,¨U°BÓ7ˆØ—Z‘Z Ó,ˆ
ØÒÜÐgÓhÐhØ×,Ñ,¨QÔ0GÈÓRˆØ×5Ñ5°jÔBYÐ\]ÕB]Ð_mÓnˆ
ØŸ™‰ˆˆa�AØ×Ñ˜AÕ !Ô# z×'7Ñ'7¸Õ';ÀÀa¸yÔ'HÜØmÔnsÐtu×t{Ñt{Ón|Ðm}ð  ~Pô  QVð  Wa÷  Wgñ  Wgó  Qhð  Pið  ijð  kóð ð �I‰I�a˜ÓˆØ×'Ñ'¨¨2Ó.ˆ
Ü˜mÔ,ˆà×8Ñ8¸×BYÑBYÐZjÓBkÓl‰ˆˆYä  ›‰ˆˆYÜ  Ó,‰ˆ�Qä�iŠi˜˜kÐ*°Ô3ˆà�k‰k˜#Óˆà$-Ð!ØŸ™ S¨)¸5˜ÓA‰ˆˆYà�k‰k˜(¨3¯:©:¸S¿Y¹YˆkÓGˆä! $§+¡+Ö.‰HˆAˆuØ˜ˆ|˜~Ô-õ
ð % g¨q \Ö2Ø˜cØ˜cØ# IØ# IØ˜cØ ð4ð )¨*Ð5ó7�ð 25°Uµ¸SÀ½ZÈÈ[ÕIYÐ[^Ð_jÕ[kÐ1kÑ.��S˜)¢Yá16ØØØ'Ø'ØØô2Ñ.��S˜)¢Yñ/ /ð@ ××ÐØ×#Ñ# CÓ(ˆCØ×"Ñ"ØØØØÙ˜iÔ)CÓDØØ ð #ó ˆCð Ÿ™ c¨9°eÐYr˜Ós‰ˆˆYÜ˜˜YÓ'ˆÜ�kŠk˜#ÓˆØ�k‰k˜"˜aÓ ˆØ×(Ñ(¨Ð.A×.EÑ.EÐFVÓ.WÓXÐXr+   )	rò  rù  r÷   r÷  rö  rô  rf   rú  ré  )rZ   r-   r-   r"   )F)rA   rB   rC   rD   r   r)   r
  Ústaticmethodr  r  r  ræ   rE   rF   rþ   rÿ   s   @@r(   ræ  ræ  U  s�   ù‡ € ð Ø2ØØØØØØØØØØØ&*ØØØØ÷+qõ qôfCð" ñ?ó ð?ð ñó ðô,÷WY÷ WYò WYr+   ræ  )r2  )UÚdataclassesr   Útypingr   r   r   r   r   r	   r
   Útorch.nn.functionalr   Ú
functionalr'  r°   r   r   rM   Úcomfy.ldm.modules.attentionr   r   Ú	itertoolsr   r   Ú(comfy.ldm.modules.diffusionmodules.modelr   Úcomfy.ldm.seedvr.attentionr   Útorch.nn.modules.utilsr   Úcomfy.ldm.flux.mathr   Úcomfy.ldm.seedvr.constantsr   r   r   r   r   r   r   r   r   Úcomfy.model_managementrJ  Ú	comfy.opsÚcomfy.quant_opsr    r‰   r�   rŽ   r˜   rž   r§   r¤   r¥   r‰  rÎ   r  r  r  r5  rä   r.  r(  rZ  r„  r�  rŸ  rÅ  rÓ  rÕ  r  r  r4  r6  rc  r{  r…  r“  rž  r   rÌ  rú   r¾  ræ  rS   r+   r(   Ú<module>r?     s  ðÝ !ß D× DÑ Dß Ð ß Û ß Pß 'Ý KÝ >Ý *Ý Û Ý +÷
÷ 
õ 
ó Û Û ÷
ñ 
õ.#òJ%ð ÷ð ó ðõJõZõ:õõ,ôBF�b—i‘iô FôP
0˜"Ÿ)™)ô 
0ôÐ+ô ô#?Ð+ô #?ôLÐ/ô òõ.õô0MdÐ1ô Mdô^*ˆr�y‰yô *õX@ô,C�B—I‘Iô ,Cõ\Gõôe �mô e ôNˆ"�)‰)ô ô&B�—	‘	ô B÷,>ôH6˜RŸY™Yô H6ôTˆr�y‰yô ô4�ô ô:ˆb�i‰iô ô:�ô õ0ô[<�—	‘	ô [<ô|%�B—I‘Iô %õN
õôrYˆB�I‰Iö rYr+   