+
    ƒ°ñj%   ã                   ó¢  € ^ RI HtHtHt ^ RIt^ RIt^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIt^ RIt^RIHt R t ! R R	]	P&                  4      t ! R
 R]	P&                  4      t ! R R]	P&                  4      t ! R R]	P&                  4      t ! R R]	P&                  4      t ! R R]	P&                  4      t ! R R]	P&                  4      tR tR$R lt ! R R]	P&                  4      t ! R R]	P&                  4      t ! R R]	P&                  4      t ! R R]	P&                  4      t  ! R  R!]	P&                  4      t! ! R" R#]	P&                  4      t"R# )%é    )ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionN©Ú	rearrange)Únn)Ú
functional)ÚExpoFourierFeaturesc                 ó¨   € V P                   R,          pW!8  d$   \        P                  ! V ^ ^ W,
          ^ 3RR7      # W!8”  d   V RV) R1R3,          # V # )é   ç        )ÚvalueºNNNNéþÿÿÿ)ÚshapeÚFÚpad)ÚembÚ
target_lenÚemb_lens   && Ú7C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\audio\dit.pyÚ_left_pad_to_matchr      sU   € Ø�i‰i˜�m€GØÔÜ�uŠu�S˜1˜a Õ!5°qÐ9ÀÔDÐDØ	Ô	Ø�1�z�k‘l AÐ%Õ&Ð&Ø€Jó    c                   ó<   a a€ ] tR t^t oRV 3R lltR tRtVtV ;t# )ÚFourierFeaturesc                ó¸   <€ \         SV `  4        V^,          ^ 8X  g   Q h\        P                  ! \        P
                  ! V^,          V.WER7      4      V n        R# )r   ©ÚdtypeÚdeviceN)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚemptyÚweight)ÚselfÚin_featuresÚout_featuresÚstdr   r   Ú	__class__s   &&&&&&€r   r!   ÚFourierFeatures.__init__   sM   ø€ Ü‰ÑÔØ˜aÕ 1Ô$Ð$Ð$Ü—l’l¤5§;¢;Ø˜QÕ Ð,°Eô$Jó KˆŽr   c                ó  € ^\         P                  ,          V,          \        P                  P	                  V P
                  P                  V4      ,          p\        P                  ! VP                  4       VP                  4       .RR7      # )r   ©Údiméÿÿÿÿ)ÚmathÚpiÚcomfyÚopsÚcast_to_inputr%   ÚTr#   ÚcatÚcosÚsin)r&   ÚinputÚfs   && r   ÚforwardÚFourierFeatures.forward    sT   € Ø”—‘�K˜%Õ¤%§)¡)×"9Ñ"9¸$¿+¹+¿-¹-ÈÓ"OÕOˆÜ�yŠy˜!Ÿ%™%›' 1§5¡5£7Ð+°Ô4Ð4r   ©r%   )ç      ð?NN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r!   r;   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r*   Ú__classdict__s   @@r   r   r      s   ù‡ € ÷K÷5ò 5r   r   c                   ó<   a a€ ] tR t^&t oRV 3R lltR tRtVtV ;t# )Ú	LayerNormc                ó
  <€ \         SV `  4        \        P                  ! \        P
                  ! WVR7      4      V n        V'       d4   \        P                  ! \        P
                  ! WVR7      4      V n        R# RV n        R# )zt
bias-less layernorm has been shown to be more stable. most newer models have moved towards rmsnorm, also bias-less
r   N)r    r!   r   r"   r#   r$   ÚgammaÚbeta)r&   r.   ÚbiasÚ	fix_scaler   r   r*   s   &&&&&&€r   r!   ÚLayerNorm.__init__'   sP   ø€ ô 	‰ÑÔä—\’\¤%§+¢+¨cÀvÔ"NÓOˆŒ
çÜŸš¤U§[¢[°È&Ô%QÓRˆDŽIàˆDŽIr   c                óü   € V P                   pVe    \        P                  P                  W!4      p\        P
                  ! WP                  RR  \        P                  P                  V P                  V4      VR7      # )N)r%   rN   r/   )rM   r2   r3   r4   r   Ú
layer_normr   rL   )r&   ÚxrM   s   && r   r;   ÚLayerNorm.forward4   sY   € Ø�y‰yˆØÒÜ—9‘9×*Ñ*¨4Ó3ˆDÜ�|Š|˜AŸw™w r s˜|´E·I±I×4KÑ4KÈDÏJÉJÐXYÓ4ZÐaeÔfÐfr   )rM   rL   )FFNNr?   rG   s   @@r   rJ   rJ   &   s   ù‡ € ÷÷gò gr   rJ   c                   ó<   a a€ ] tR t^;t oRV 3R lltR tRtVtV ;t# )ÚRMSNormc                ó†   <€ \         SV `  4        \        P                  ! \        P
                  ! WVR 7      4      V n        R# )r   N)r    r!   r   r"   r#   r$   rL   )r&   r.   r   r   r*   s   &&&&€r   r!   ÚRMSNorm.__init__<   s)   ø€ Ü‰ÑÔÜ—\’\¤%§+¢+¨cÀvÔ"NÓOˆŽ
r   c                óœ   € \         P                  ! WP                  RR \        P                  P                  V P                  V4      R7      # )é   Nr=   r/   )r   Úrms_normr   r2   r3   r4   rL   ©r&   rS   s   &&r   r;   ÚRMSNorm.forward@   s3   € Ü�zŠz˜!ŸW™W R S˜\´%·)±)×2IÑ2IÈ$Ï*É*ÐVWÓ2XÔYÐYr   )rL   ©NNr?   rG   s   @@r   rV   rV   ;   s   ù‡ € ÷P÷Zò Zr   rV   c                   ó<   a a€ ] tR t^Dt oRV 3R lltR tRtVtV ;t# )ÚGLUc	           	     óÎ   <€ \         S	V `  4        W0n        V'       g   VP                  W^,          WgR7      M!VP	                  W^,          WU^,          WgR7      V n        W@n        R# )r   r   )Úpaddingr   r   N)r    r!   ÚactÚLinearÚConv1dÚprojÚuse_conv)
r&   Údim_inÚdim_outÚ
activationrg   Úconv_kernel_sizer   r   Ú
operationsr*   s
   &&&&&&&&&€r   r!   ÚGLU.__init__E   s{   ø€ ô 	‰ÑÔØŒß^f�J×%Ñ% f¸­kÀÐ%ÔVÐlv×l}Ñl}ð  Eð  QRõ  GRð  Tdð  EFõ  qFð  OTÐl}ó  mdˆŒ	Ø Žr   c                óô   € V P                   '       d+   \        VR 4      pV P                  V4      p\        VR4      pMV P                  V4      pVP                  ^RR7      w  rWP	                  V4      ,          # )úb n d -> b d núb d n -> b n dr-   r/   )rg   r   rf   Úchunkrc   )r&   rS   Úgates   && r   r;   ÚGLU.forwardU   sd   € Ø�=�=ˆ=Ü˜!Ð-Ó.ˆAØ—	‘	˜!“ˆAÜ˜!Ð-Ó.‰Aà—	‘	˜!“ˆAà—'‘'˜! 2�'Ó&‰ˆØ—8‘8˜D“>Õ!Ð!r   )rc   rf   rg   )Fé   NNNr?   rG   s   @@r   r`   r`   D   s   ù‡ € ÷!÷ 	"ò 	"r   r`   c                   ó<   a a€ ] tR t^`t oV 3R ltRR ltRtVtV ;t# )ÚAbsolutePositionalEmbeddingc                ó‚   <€ \         SV `  4        VR,          V n        W n        \        P
                  ! W!4      V n        R# )ç      à?Nç      à¿)r    r!   ÚscaleÚmax_seq_lenr   Ú	Embeddingr   )r&   r.   r{   r*   s   &&&€r   r!   Ú$AbsolutePositionalEmbedding.__init__a   s/   ø€ Ü‰ÑÔØ˜D•[ˆŒ
Ø&ÔÜ—<’< Ó1ˆŽr   c                óR  € VP                   ^,          VP                  rTW@P                  8:  g   Q RV RV P                   24       hVf   \        P                  ! WER7      pVe    W#R,          ,
          P                  ^ R7      pV P                  V4      pW`P                  ,          pV# )rZ   z(you are passing in a sequence length of zE but your absolute positional embedding has a max sequence length of ©r   )Úmin©.N)r   r   r{   r#   ÚarangeÚclampr   rz   )r&   rS   ÚposÚseq_start_posÚseq_lenr   Úpos_embs   &&&&   r   r;   Ú#AbsolutePositionalEmbedding.forwardg   s´   € ØŸ'™' !�* a§h¡h�Ø×*Ñ*Ô*ð  	xÐ.VÐW^ÐV_ð  `eð  fj÷  fvñ  fvð  ewð  -xó  	xÐ*àŠ;Ü—,’,˜wÔ8ˆCàÒ$Ø yÕ1Õ1×8Ñ8¸qÐ8ÓAˆCà—(‘(˜3“-ˆØŸJ™JÕ&ˆØˆr   )r   r{   rz   r^   r?   rG   s   @@r   rv   rv   `   s   ù‡ € õ2÷ô r   rv   c                   ó@   a a€ ] tR t^ut oRV 3R lltRR ltRtVtV ;t# )ÚScaledSinusoidalEmbeddingc                ój  <€ \         SV `  4        V^,          ^ 8X  g   Q R4       h\        P                  ! \        P
                  ! ^4      VR,          ,          4      V n        V^,          p\        P                  ! V4      P                  4       V,          pW$) ,          pV P                  RVRR7       R# )r   z dimension must be divisible by 2Úinv_freqF)Ú
persistentNry   )
r    r!   r   r"   r#   Úonesrz   r‚   ÚfloatÚregister_buffer)r&   r.   ÚthetaÚhalf_dimÚfreq_seqrŒ   r*   s   &&&   €r   r!   Ú"ScaledSinusoidalEmbedding.__init__v   s‰   ø€ Ü‰ÑÔØ�a•˜AŒ~ÐAÐAÓAˆ~Ü—\’\¤%§*¢*¨Q£-°#¸µ+Õ"=Ó>ˆŒ
à˜!•8ˆÜ—<’< Ó)×/Ñ/Ó1°HÕ<ˆØ˜IÕ%ˆØ×Ñ˜Z¨ÀÐÖFr   c                ól  € VP                   ^,          VP                  rTVf   \        P                  ! WER7      pVe   W#R,          ,
          p\        P                  ! RW P
                  4      p\        P                  ! VP                  4       VP                  4       3RR7      pW`P                  ,          # )rZ   r   zi, j -> i jr-   r�   r/   )
r   r   r#   r‚   ÚeinsumrŒ   r6   r8   r7   rz   )r&   rS   r„   r…   r†   r   r   s   &&&&   r   r;   Ú!ScaledSinusoidalEmbedding.forward€   s}   € ØŸ'™' !�* a§h¡h�àŠ;Ü—,’,˜wÔ8ˆCàÒ$Ø iÕ0Õ0ˆCä�lŠl˜=¨#¯}©}Ó=ˆÜ�iŠi˜Ÿ™› C§G¡G£IÐ.°bÔ9ˆØ—Z‘ZÕÐr   )rz   )é'  r^   r?   rG   s   @@r   rŠ   rŠ   u   s   ù‡ € ÷G÷ ô  r   rŠ   c                   óB   a a€ ] tR t^�t oRV 3R lltR tR tRtVtV ;t	# )ÚRotaryEmbeddingc	           	     óª  <€ \         S
V `  4        WVW^,
          ,          ,          ,          pV P                  R\        P                  ! V^,          3W‡R7      4       VR8¼  g   Q hW@n        V'       g   V P                  RR4       R# \        P                  ! ^ V^4      RV,          ,           RV,          ,          p	W0n        V P                  RV	4       R# )r   rŒ   ©r   r   r>   rz   Ngš™™™™™Ù?gffffffö?)r    r!   r�   r#   r$   Úinterpolation_factorr‚   Ú
scale_base)r&   r.   Úuse_xposrž   r�   ÚbaseÚbase_rescale_factorr   r   rz   r*   s   &&&&&&&&& €r   r!   ÚRotaryEmbedding.__init__Ž   s«   ø€ ô 	‰ÑÔð 	¨°Q­w­Õ8Õ8ˆð 	×Ñ˜Z¬¯ª°c¸Qµh°[ÈÔ)]Ô^à# rÔ)Ð)Ð)Ø$8Ô!çØ× Ñ  ¨$Ô/Ùä—’˜a  aÓ(¨3°­9Õ4¸¸s½ÕCˆà$ŒØ×Ñ˜W eÖ,r   c                óT   € \         P                  ! WVR 7      pV P                  V4      # )rœ   )r#   r‚   r;   )r&   r†   r   r   Úts   &&&& r   Úforward_from_seq_lenÚ$RotaryEmbedding.forward_from_seq_len®   s"   € ô �LŠL˜°uÔ=ˆØ�|‰|˜A‹Ðr   c                ó@  € VP                   pWP                  ,          p\        P                  ! R V\        P
                  P                  V P                  V4      4      p\        P                  ! W33RR7      pV P                  f   VR3# \        P                  ! \        VR7      \        ^,          ,
          V P                  ,          p\        P
                  P                  V P                  V4      \        VR4      ,          p\        P                  ! WU3RR7      pW53# )zi , j -> i jr-   r>   r   zn -> n 1r/   )r   r�   r#   r–   r2   r3   r4   rŒ   r6   rz   r‚   r†   rž   r   )r&   r¤   r   ÚfreqsÚpowerrz   s   &&    r   r;   ÚRotaryEmbedding.forward´   sË   € à—‘ˆð ×)Ñ)Õ)ˆä—’˜^¨Q´·	±	×0GÑ0GÈÏÉÐWXÓ0YÓZˆÜ—	’	˜5˜.°Ô3ˆà�:‰:ÒØ˜"�9Ðä—’œg°Ô7¼7Àa½<ÕHÈDÏOÉOÕ[ˆÜ—	‘	×'Ñ'¨¯
©
°AÓ6¼)ÀEÈ:Ó:VÕVˆÜ—	’	˜5˜.°Ô3ˆàˆ|Ðr   )r�   rž   )Fi   r>   r˜   r>   NN)
r@   rA   rB   rC   r!   r¥   r;   rD   rE   rF   rG   s   @@r   rš   rš   �   s   ù‡ € ÷-ò@÷ò r   rš   c                 ó|   € \        V R ^R7      p V P                  RR7      w  r\        P                  ! V) V3RR7      # )z... (j d) -> ... j d)Újr-   r   r/   )r   Úunbindr#   r6   )rS   Úx1Úx2s   &  r   Úrotate_halfr°   È   s:   € Ü�!Ð+°Ô3€AØ�X‰X˜BˆXÓ�F€BÜ�9Š9�r�c˜2�Y bÔ)Ð)r   c                 ól  € V P                   pV P                   pVP                  R,          V P                  R,          reVP                  V4      V P                  V4      rW) R1R3,          pV P                  ^8X  d   VP                  ^8X  d   \	        VR4      pV RRV13,          V RVR13,          rpWP                  4       ,          V,          \        V 4      VP                  4       ,          V,          ,           p V P                  V4      VP                  V4      rp\        P                  ! W3RR7      # )rZ   Nr   zb n d -> b 1 n d.r-   r/   r   )
r   r   ÚtoÚndimr   r7   r°   r8   r#   r6   )r¤   r¨   rz   Ú	out_dtyper   Úrot_dimr†   Út_unrotateds   &&&     r   Úapply_rotary_pos_embr·   Í   sî   € Ø—‘€Ið �G‰G€EØ—{‘{ 2•¨¯©°­ˆWØ�x‰x˜‹ §¡ U£ˆ1Ø�(‘)˜Q�,Õ€Eà‡v�v�„{�u—z‘z Q”Ü˜%Ð!3Ó4ˆð �s˜H˜W˜H�}Õ% q¨¨g©h¨Õ'7€{Ø	
�Y‰Y‹[�˜5Õ	 ¤[°£^°e·i±i³kÕ%AÀEÕ%IÕJ€Aà—T‘T˜)“_ k§n¡n°YÓ&?€{ä�9Š9�aÐ%¨RÔ0Ð0r   c                   ó<   a a€ ] tR t^át oRV 3R lltR tRtVtV ;t# )ÚFeedForwardc                óP  <€ \         SV `  4        \        W,          4      p\        P                  ! 4       pVf   TMTpV'       d   \        WWÙW«R7      pM¥\        P                  ! V'       d   \        R4      M\        P                  ! 4       V'       g   VP                  WV'       * WšR7      M VP                  WWw^,          V'       * WšR7      V'       d   \        R4      M\        P                  ! 4       V4      pV'       g   VP                  WÂV'       * WšR7      M VP                  WÂWw^,          V'       * WšR7      p\        P                  ! TV'       d   \        R4      M\        P                  ! 4       TV'       d   \        R4      M\        P                  ! 4       4      V n        R # )N)r   r   rl   ro   ©rN   r   r   )rb   rN   r   r   rp   )r    r!   Úintr   ÚSiLUr`   Ú
Sequentialr   ÚIdentityrd   re   Úff)r&   r.   ri   ÚmultÚno_biasÚglurg   rk   Úzero_init_outputr   r   rl   Ú	inner_dimrj   Ú	linear_inÚ
linear_outr*   s   &&&&&&&&&&&&    €r   r!   ÚFeedForward.__init__â   s˜  ø€ ô 	‰ÑÔÜ˜�
“Oˆ	ô —W’W“Yˆ
à š‘#¨gˆçÜ˜C¨JÈFÔj‰IäŸšß/7”	Ð*Ô+¼R¿[º[»]ßiq�
×!Ñ! #¸W¼ÈEÐ!Ôað  xB÷  xIñ  xIð  JMð  Zjð  KLõ  wLð  Zaô  Vað  inð  xIó  x~ß/7”	Ð*Ô+¼R¿[º[»]Øó	ˆI÷ s{�Z×&Ñ& yÀgÄ+ÐUZÐ&Ôjð  AK÷  ARñ  ARð  S\ð  gwð  XYõ  DYð  gnô  cnð  v{ð  ARó  AKˆ
ä—-’-Øß+3ŒIÐ&Ô'¼¿º»Øß+3ŒIÐ&Ô'¼¿º»ó	
ˆŽr   c                ó$   € V P                  V4      # )N©rÀ   r\   s   &&r   r;   ÚFeedForward.forward  s   € Ø�w‰w�q‹zÐr   rÊ   )
Né   FTFrt   TNNNr?   rG   s   @@r   r¹   r¹   á   s   ù‡ € ÷(
÷Tò r   r¹   c                   óL   a a€ ] tR tRt oRV 3R lltRRRRR/ 3R ltRtVtV ;t# )Ú	Attentioni  Nc                óÆ  <€ \         SV `  4        \        4       V n        Wn        W n        W@n        Wpn        Ve   TMTpW,          V n        WÒ,          V n	        Ve„   V'       dB   VP                  W^,          RW«R7      V n        VP                  WÝ^,          RW«R7      V n        MƒVP                  WRW«R7      V n        VP                  WÝ^,          RW«R7      V n        MIV'       d"   VP                  W^,          RW«R7      V n        M VP                  W^,          RW«R7      V n        VP                  WRW«R7      V n        \        V\         4      '       d   V'       d   RMRpW`n        V P"                  R8X  d6   VP%                  VRRW«R7      V n        VP%                  VRRW«R7      V n        M5V P"                  R	8X  d%   \+        W*VR
7      V n        \+        W*VR
7      V n        W�n        V P,                  '       de   \.        P0                  ! \2        P4                  ! WVR
7      4      V n        \.        P0                  ! \2        P4                  ! WVR
7      4      V n        R # R # )NFr»   Úl2ÚnoneÚlnTg�íµ ÷Æ°>)Úelementwise_affineÚepsr   r   Úrmsr   )r    r!   r   Úcomfy_attentionr.   Ú	dim_headsÚcausalÚdifferentialÚ	num_headsÚkv_headsrd   Úto_qÚto_kvÚto_qkvÚto_outÚ
isinstanceÚboolÚqk_normrJ   Úq_normÚk_normrV   Ú
feat_scaler   r"   r#   r$   Ú	lambda_dcÚ	lambda_hf)r&   r.   r×   Údim_contextrØ   rÄ   râ   rÙ   Únatten_kernel_sizerå   r   r   rl   Údim_kvr*   s   &&&&&&&&&&&&& €r   r!   ÚAttention.__init__  s÷  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØŒØ"ŒØŒØ(Ôà +Ò 7‘¸SˆàÕ)ˆŒØÕ+ˆŒàÒ"ßØ&×-Ñ-¨c¸µ7ÀÈeÐ-Óc�”	Ø'×.Ñ.¨vÀµzÈÐUZÐ.Ój�•
à&×-Ñ-¨c¸UÈ%Ð-Ó_�”	Ø'×.Ñ.¨vÀµzÈÐUZÐ.Ój�•
çØ(×/Ñ/°¸1µWÀ5ÐPUÐ/Óe�•à(×/Ñ/°¸1µWÀ5ÐPUÐ/Óe�”à ×'Ñ'¨°uÀEÐ'ÓYˆŒô �gœt×$Ò$ß%‘d¨6ˆGØŒà�<‰<˜4ÔØ$×.Ñ.¨yÈTÐW]ÐejÐ.ÓzˆDŒKØ$×.Ñ.¨yÈTÐW]ÐejÐ.ÓzˆD�KØ�\‰\˜UÔ"Ü! )ÀÔHˆDŒKÜ! )ÀÔHˆDŒKà$Œà�?�?ˆ?ÜŸ\š\¬%¯+ª+°cÈvÔ*VÓWˆDŒNÜŸ\š\¬%¯+ª+°cÈvÔ*VÓWˆDŽNñ r   c                ó"  aa€ V P                   V P                  VR JuoopV'       d   TMTp	\        V R4      '       Ed-   V P                  '       dÄ   V P	                  V4      P                  ^RR7      w  r«\        V
RSR7      p
\        VRSR7      p\        P                  ! W«.^R7      p
V P                  V	4      P                  ^RR7      w  rÍp\        VRSR7      p\        VRSR7      p\        VRSR7      p\        P                  ! WÍ.^R7      pEMV P	                  V4      p
\        V
RSR7      p
V P                  V	4      P                  ^RR7      w  rÎ\        V3R lWÎ34      w  rÎM¼V P                  '       dq   V P                  V4      P                  ^RR7      w  r¬rëp\        V3R lW¬WëV34      w  r¬rëp\        P                  ! W«.^R7      p
\        P                  ! WÍ.^R7      pM:V P                  V4      P                  ^RR7      w  r¬p\        V3R lW¬V34      w  r¬pV P                  R8X  d2   \        P                  ! V
RR7      p
\        P                  ! VRR7      pM�V P                  R	8X  dZ   V
P                  VP                  ppV P!                  V
4      P#                  V4      p
V P%                  V4      P#                  V4      pM3V P                  R
8w  d#   V P!                  V
4      p
V P%                  V4      pVe½   V'       gµ   Vw  ppV
P                  pVP                  pV
P#                  \        P&                  4      p
VP#                  \        P&                  4      pVP#                  \        P&                  4      p\)        V
V4      p
\)        VV4      pV
P#                  V4      p
VP#                  V4      pTpVf   V'       g   Tp. pVe   \        VR4      pVP+                  V( 4       V
P,                  R,          pVf   V P.                  MTpV^8X  d   V'       d   RpSS8w  d   RR/M/ pV P                  '       d±   V
P1                  ^R7      w  r«VP1                  ^R7      w  rÍ\3        V
4      \3        V4      rÊ\5        W¬\3        V4      S3RRRRRV P6                  RV/VB p\3        V4      \3        V4      \3        V4      ríp\5        W½VS3RRRRRV P6                  RV/VB pVV,
          pM@\3        V
4      \3        V4      \3        V4      rìp
\5        W¬VS3RRRRRV P6                  RV/VB pV P9                  V4      pV P:                  '       d‹   VP=                  RRR7      pVV,
          pV\>        P@                  PC                  V PD                  V4      V,          ,           \>        P@                  PC                  V PF                  V4      V,          ,           pVe    \        VR4      pVPI                  V( R4      pV# )NrÜ   r-   úb n (h d) -> b h n d©Úhc                 ó    <€ \        V R SR7      # ©rí   rî   r   )r¤   Úkv_hs   &€r   Ú<lambda>Ú#Attention.forward.<locals>.<lambda>m  s   ø€ ¤Y¨qÐ2HÈdÕ%Sr   c                 ó    <€ \        V R SR7      # rñ   r   ©r¤   rï   s   &€r   ró   rô   s  s   ø€ œi¨Ð+AÀQÕGr   c                 ó    <€ \        V R SR7      # rñ   r   rö   s   &€r   ró   rô   {  s   ø€ ¬	°!Ð5KÐQRÕ(Sr   rÐ   rÕ   rÑ   zb j -> b 1 1 jFÚ
enable_gqaTÚskip_reshapeÚlow_precision_attentionÚpreferred_attentionÚtransformer_options)r.   Úkeepdimzb n -> b n 1r   r/   r   )%rÚ   rÛ   ÚhasattrrÙ   rÜ   rq   r   r#   ÚstackrÝ   ÚmaprÞ   râ   r   Ú	normalizer   rã   r²   rä   Úfloat32r·   Úappendr   rØ   r­   r   r   rÖ   rß   rå   Úmeanr2   r3   r4   ræ   rç   Úmasked_fill)r&   rS   ÚcontextÚmaskÚcontext_maskÚrotary_pos_embrØ   rü   Úhas_contextÚkv_inputÚqÚq_diffÚkÚk_diffÚvÚq_typeÚk_typer¨   Ú_Úq_dtypeÚk_dtypeÚ
input_maskÚmasksÚnÚ
gqa_kwargsÚoutÚout_diffÚout_dcÚout_hfrï   rò   s   &&&&&&&&                     @@r   r;   ÚAttention.forwardL  sß  ù€ ð  $Ÿ~™~¨t¯}©}¸gÈTÐ>QÐˆˆ4�ç)‘7¨qˆä�4˜× Ó Ø× × Ð à ŸI™I a›L×.Ñ.¨q°bÐ.Ó9‘	�Ü" 1Ð+AÀQÔG�Ü" 6Ð+AÀQÔG�Ü—K’K  °Ô3�Ø#Ÿz™z¨(Ó3×9Ñ9¸!ÀÐ9ÓD‘�˜1Ü" 1Ð+AÀTÔJ�Ü" 6Ð+AÀTÔJ�Ü" 1Ð+AÀTÔJ�Ü—K’K  °Ô3’ð —I‘I˜a“L�Ü˜aÐ!7¸QÔ?�à—z‘z (Ó+×1Ñ1°!¸Ð1Ó<‘�äÔSÐVWÐU[Ó\‘��1à× × Ð à*.¯+©+°a«.×*>Ñ*>¸qÀbÐ*>Ó*IÑ'��a Ü*-ÜGØ˜1 fÐ-ó+Ñ'��a ô —K’K  °Ô3�Ü—K’K  °Ô3‘ð Ÿ+™+ a›.×.Ñ.¨q°bÐ.Ó9‘��aÜÔSÐVWÐ\]ÐU^Ó_‘��að �<‰<˜4ÔÜ—’˜A 2Ô&ˆAÜ—’˜A 2Ô&‰AØ�\‰\˜UÔ"ØŸW™W a§g¡g�FˆFØ—‘˜A“×!Ñ! &Ó)ˆAØ—‘˜A“×!Ñ! &Ó)‰AØ�\‰\˜VÔ#Ø—‘˜A“ˆAØ—‘˜A“ˆAàÒ%¯kØ%‰HˆE�1à—g‘gˆGØ—g‘gˆGà—‘”U—]‘]Ó#ˆAØ—‘”U—]‘]Ó#ˆAØ—H‘HœUŸ]™]Ó+ˆEä$ Q¨Ó.ˆAÜ$ Q¨Ó.ˆAà—‘�W“ˆAØ—‘�W“ˆAà!ˆ
àÒ§kØˆJð ˆàÒ!Ü" :Ð/?Ó@ˆJØ�L‰L˜*˜Ô%ð �G‰G�B�Kˆà &¢�—’°Fˆà�Œ6—fØˆFà-.°$¬Y�l DÑ)¸Bˆ
à××ÐØŸ™ Q˜›‰IˆAØŸ™ Q˜›‰IˆAÜ+¨AÓ.Ô0HÈÓ0KˆqÜ*¨1Ô;SÐTUÓ;VÐXYñ  nÐhlð  nð  GLð  nð  bf÷  bvñ  bvð  nð  L_ð  nð  cmñ  nˆCÜ 8¸Ó @ÔBZÐ[aÓBbÔd|Ð}~Ód˜AˆFÜ*¨6¸1¸añ  TÈdð  TÐlqð  Tð  HL÷  H\ñ  H\ð  Tð  rEð  Tð  ISñ  TˆHØ˜•.‰Cä.¨qÓ1Ô3KÈAÓ3NÔPhÐijÓPk�!ˆAÜ% a¨A¨qñ  E¸tð  EÐ]bð  EÐx|÷  yMñ  yMð  Eð  cvð  Eð  zDñ  EˆCà�k‰k˜#Óˆà�?�?ˆ?Ø—X‘X "¨d�XÓ3ˆFØ˜6•\ˆFð œŸ	™	×/Ñ/°·±ÀÓDÀvÕMÕMÔPU×PYÑPY×PgÑPgÐhl×hvÑhvÐx{ÓP|ð  @Fõ  QFõ  FˆCàÒÜ˜T >Ó2ˆDØ—/‘/ 4 %¨Ó,ˆCàˆ
r   )rØ   rÖ   rÙ   r.   r×   rå   rä   rÛ   ræ   rç   rÚ   rã   râ   rÝ   rß   rÜ   rÞ   )é@   NFTrÑ   FNFNNNr?   rG   s   @@r   rÎ   rÎ     s,   ù‡ € ÷:Xð~ ØØØØØ÷|ô |r   rÎ   c                   ó>   a a€ ] tR tRt o/ 3V 3R lltR tRtVtV ;t# )ÚConformerModuleiÊ  c           	     ó¨  <€ \         SV `  4        Wn        \        V3/ VB V n        \
        P                  ! W^RR7      V n        \        W\
        P                  ! 4       4      V n
        \
        P                  ! W^V^RR7      V n        \        V3/ VB V n        \
        P                  ! 4       V n        \
        P                  ! W^RR7      V n        R# )rZ   F)Úkernel_sizerN   )r#  Úgroupsrb   rN   N)r    r!   r.   rJ   Úin_normr   re   Úpointwise_convr`   r½   rÃ   Údepthwise_convÚmid_normÚswishÚpointwise_conv_2)r&   r.   Únorm_kwargsr*   s   &&&€r   r!   ÚConformerModule.__init__Ë  s�   ø€ ô 	‰ÑÔàŒä  Ñ4¨Ñ4ˆŒÜ Ÿiši¨¸aÀeÔLˆÔÜ�s¤§¢£Ó+ˆŒÜ Ÿiši¨¸bÈÐVWÐ^cÔdˆÔÜ! #Ñ5¨Ñ5ˆŒÜ—W’W“YˆŒ
Ü "§	¢	¨#ÀÈÔ NˆÖr   c                ó„  € V P                  V4      p\        VR 4      pV P                  V4      p\        VR4      pV P                  V4      p\        VR 4      pV P	                  V4      p\        VR4      pV P                  V4      pV P                  V4      p\        VR 4      pV P                  V4      p\        VR4      pV# )ro   rp   )r%  r   r&  rÃ   r'  r(  r)  r*  r\   s   &&r   r;   ÚConformerModule.forwardÝ  sº   € Ø�L‰L˜‹OˆÜ�aÐ)Ó*ˆØ×Ñ Ó"ˆÜ�aÐ)Ó*ˆØ�H‰H�Q‹KˆÜ�aÐ)Ó*ˆØ×Ñ Ó"ˆÜ�aÐ)Ó*ˆØ�M‰M˜!ÓˆØ�J‰J�q‹MˆÜ�aÐ)Ó*ˆØ×!Ñ! !Ó$ˆÜ�aÐ)Ó*ˆàˆr   )r'  r.   rÃ   r%  r(  r&  r*  r)  r?   rG   s   @@r   r!  r!  Ê  s   ù‡ € ð ÷O÷$ò r   r!  c                   ór   a a€ ] tR tRt o^@RRRRRRRRR	RR/ / / RRR3V 3R lltRRRRRR/ 3R ltRtVtV ;t# )
ÚTransformerBlockiî  FNTrR   c                ó:  <aaaaaa€ \         SV `  4        SV n        W n        W0n        W@n        W€n        W`n        R \        R\        /pVP                  V\        4      oVVVVVV3R lpV! 4       V n        \        S3RVRVRV	RSRSRV/VB V n        V'       d,   V! 4       V n        \        S3RVR	VRVRV	RSRSRV/VB V n        V! 4       V n        \#        S3RV	RSRSRV/VB V n        W°n        V
'       d   \)        SSR
7      MRV n        VRJ;'       g    TV n        V'       d;   \.        P0                  ! \2        P4                  ! S^,          SSR7      4      V n        MNVeK   \.        P8                  ! \.        P:                  ! 4       VP=                  VS^,          RSSR7      4      V n        Wpn        Ve[   \.        P8                  ! VP=                  VSRSSR7      \.        P:                  ! 4       VP=                  SSRSSR7      4      V n         R# RV n         R# )rR   r[   c                  óX   <€ S'       d   \         P                  ! 4       # S! S3R SRS /SB # r   )r   r¿   )r   r.   r   Únorm_clsr+  Úremove_normss   €€€€€€r   Ú	make_normÚ,TransformerBlock.__init__.<locals>.make_norm  s,   ø€ ßÜ—{’{“}Ð$Ù˜CÑK uÐK°VÐK¸{ÑKÐKr   r×   rØ   rÄ   r   r   rl   rè   )r+  Nrœ   F)rN   r   r   Tr»   )!r    r!   r.   r×   Úcross_attendrè   rØ   Úglobal_cond_shared_embedrJ   rV   ÚgetÚpre_normrÎ   Ú	self_attnÚcross_attend_normÚ
cross_attnÚff_normr¹   rÀ   Úlayer_ixr!  Ú	conformerÚhas_global_condr   r"   r#   r$   Úto_scale_shift_gater¾   r½   rd   Úlocal_add_cond_dimÚto_local_embed)r&   r.   r×   r7  rè   Úglobal_cond_dimr8  rC  rØ   Úzero_init_branch_outputsr@  r?  r4  Ú	norm_typeÚattn_kwargsÚ	ff_kwargsr+  r   r   rl   Únorm_layer_mapr5  r3  r*   s   &f&&&&&&&&&&f&&&fff&  @€r   r!   ÚTransformerBlock.__init__ï  sv  þ€ ô. 	‰ÑÔØˆŒØ"ŒØ(ÔØ&ÔØŒØ(@Ô%ð œ)Øœð
ˆð "×%Ñ% i´Ó;ˆ÷	Lò 	Lñ
 "›ˆŒä"Øñ	
à!ð	
ð ð	
ð 6ð		
ð
 ð	
ð ð	
ð "ð	
ð ñ	
ˆŒ÷ Ù%.£[ˆDÔ"Ü'Øñ
à%ð
ð (ð
ð  ð	
ð
 ":ð
ð ð
ð ð
ð &ð
ð ñ
ˆDŒOñ !“{ˆŒÜ˜cñ  NÐ4Lð  NÐTYð  NÐbhð  NÐuð  Nð  DMñ  NˆŒà ŒçJSœ¨¸+ÕFÐY]ˆŒð !0°tÐ ;×XÐXÐ@XˆÔç#ä')§|¢|´E·K²KÀÀaÅÐPVÐ^cÔ4dÓ'eˆDÕ$ØÒ(ä')§}¢}Ü—’“	Ø×!Ñ! /°3¸µ7ÀÈvÐ]bÐ!Ócó(ˆDÔ$ð #5ÔØÒ)Ü"$§-¢-Ø×!Ñ!Ð"4°cÀÈEÐZ`Ð!ÓaÜ—’“	Ø×!Ñ! # s°¸UÈ6Ð!ÓRó#ˆDÖð #'ˆDÖr   c	                óì  € V P                   '       Edû   VEeö   V P                  '       dB   \        P                  P	                  V P
                  V4      V,           P                  ^4      p	M V P                  V4      P                  ^4      p	V	P                  ^RR7      w  r«rÍrïTpV P                  V4      pV^V
,           ,          V,           pV P                  WWxR7      pV\        P                  ! ^V,
          4      ,          pVV,           pVe*   WP                  V P                  V4      W&VR7      ,           pV P                  e   WP                  V4      ,           pVeB   V P                  e4   V\!        V P                  V4      VP"                  R,          4      ,           pTpV P%                  V4      pV^V,           ,          V,           pV P'                  V4      pV\        P                  ! ^V,
          4      ,          pVV,           pV# WP                  V P                  V4      WWVR7      ,           pVe*   WP                  V P                  V4      W&VR7      ,           pV P                  e   WP                  V4      ,           pVeB   V P                  e4   V\!        V P                  V4      VP"                  R,          4      ,           pWP'                  V P%                  V4      4      ,           pV# )Nr-   )r  r	  rü   )r  r  rü   r/   r   )rA  r8  r2   r3   r4   rB  Ú	unsqueezerq   r:  r;  r#   Úsigmoidr=  r<  r@  rD  r   r   r>  rÀ   )r&   rS   r  Úglobal_condÚlocal_add_condr  r  r	  rü   ÚssgÚ
scale_selfÚ
shift_selfÚ	gate_selfÚscale_ffÚshift_ffÚgate_ffÚresiduals   &&&&&&&&&        r   r;   ÚTransformerBlock.forwardS  s—  € ð ××Ñ KÓ$;Ø×,×,Ð,ä—y‘y×.Ñ.¨t×/GÑ/GÈÓUÐXcÕc×nÑnÐopÓq‘à×.Ñ.¨{Ó;×EÑEÀaÓH�àMPÏYÉYÐWXÐ`bÈYÓMcÑJˆJ I¸ð ˆHØ—‘˜aÓ ˆAØ�Q˜•^Õ$ zÕ1ˆAØ—‘˜qÀ�ÓxˆAØ”E—M’M ! i¥-Ó0Õ0ˆAØ�H•ˆAàÒ"ØŸ™¨×(>Ñ(>¸qÓ(AÈWð  H[˜ó  \õ  \�à�~‰~Ò)ØŸ™ qÓ)Õ)�àÒ)¨d×.AÑ.AÒ.MØÔ*¨4×+>Ñ+>¸~Ó+NÐPQ×PWÑPWÐXZÕP[Ó\Õ\�ð ˆHØ—‘˜Q“ˆAØ�Q˜•\Õ" XÕ-ˆAØ—‘˜“
ˆAØ”E—M’M ! g¥+Ó.Õ.ˆAØ�H•ˆAð  ˆð —N‘N 4§=¡=°Ó#3¸Dð  xK�Nó  Lõ  LˆAàÒ"ØŸ™¨×(>Ñ(>¸qÓ(AÈWð  H[˜ó  \õ  \�à�~‰~Ò)ØŸ™ qÓ)Õ)�àÒ)¨d×.AÑ.AÒ.MØÔ*¨4×+>Ñ+>¸~Ó+NÐPQ×PWÑPWÐXZÕP[Ó\Õ\�à—G‘G˜DŸL™L¨›OÓ,Õ,ˆAàˆr   )rØ   r@  r7  r<  r=  r.   rè   r×   rÀ   r>  r8  rA  r?  rC  r:  r;  rD  rB  r/   r?   rG   s   @@r   r0  r0  î  se   ù‡ € ð Ø ØØ"Ø',Ø!%ØØ'+ØØØ Ø$ØØØØØØ÷)b'ðN ØØØØØØ÷;ô ;r   r0  c            &       óŒ   a a€ ] tR tRt oRRRRR^@RRRRR	RR
RRRRRRRRRRRRRRRRRR^ RRRRRR/V 3R lltRR ltRtVtV ;t# )ÚContinuousTransformeri�  rh   Nri   r×   r7  FÚcond_token_dimrE  r8  rC  rØ   r	  TrF  r@  Úuse_sinusoidal_embÚuse_abs_pos_embÚabs_pos_emb_max_lengthr˜   Únum_memory_tokensr   r   rl   c                ód  <€ \         SV `  4        Wn        W n        W°n        \
        P                  ! . 4      V n        VV n        W�n	        Ve   VP                  W1RVVR7      M\
        P                  ! 4       V n        Ve   VP                  WRVVR7      M\
        P                  ! 4       V n        V'       d&   \        \        V^,          ^ 4      VVR7      V n        MR V n        Wðn        V'       d   \%        V4      V n        VV n        V'       d   \+        VVV,           4      V n        V^ 8”  d4   \
        P,                  ! \.        P0                  ! VVVVR7      4      V n        R V n        V	'       db   Ve^   \
        P6                  ! VP                  W�RVVR7      \
        P8                  ! 4       VP                  W^,          RVVR7      4      V n        V	'       d   R MTp\;        V4       FB  pV P                  P=                  \?        V3RVRVRVRVR	V	R
V
RVRVRVRVRVRVRV/VB 4       KD  	  R # )NFr»   rœ   Tr×   r7  rè   rE  r8  rC  rØ   rF  r@  r?  r   r   rl   ) r    r!   r.   ÚdepthrØ   r   Ú
ModuleListÚlayersr`  r8  rd   r¿   Ú
project_inÚproject_outrš   Úmaxr	  r]  rŠ   r‡   r^  rv   r"   r#   r$   Úmemory_tokensÚglobal_cond_embedderr¾   r½   Úranger  r0  )r&   r.   rb  rh   ri   r×   r7  r\  rE  r8  rC  rØ   r	  rF  r@  r]  r^  r_  r`  r   r   rl   ÚkwargsÚblock_global_cond_dimÚir*   s   &&&$$$$$$$$$$$$$$$$$$$,  €r   r!   ÚContinuousTransformer.__init__‘  s`  ø€ ô6 	‰ÑÔàŒØŒ
ØŒÜ—m’m BÓ'ˆŒØ!2ˆÔØ(@Ô%àdjÒdv˜*×+Ñ+¨F¸eÈ5ÐY_Ð+Ô`Ô|~÷  }Hò  }Hó  }JˆŒØfmÒfy˜:×,Ñ,¨SÀÈUÐ[aÐ,Ôbô  @B÷  @Kò  @Kó  @MˆÔçÜ"1´#°iÀ1µnÀbÓ2IÐRXÐ`eÔ"fˆDÕà"&ˆDÔà"4ÔßÜ4°SÓ9ˆDŒLà.ˆÔßÜ6°sÐ<RÐUfÕ<fÓgˆDŒLà˜qÔ Ü!#§¢¬e¯kªkÐ:KÈSÐY_ÐglÔ.mÓ!nˆDÔð %)ˆÔ!ß#¨Ò(CÜ(*¯ªØ×!Ñ! /¸TÈÐW]Ð!Ó^Ü—’“	Ø×!Ñ! #¨Q¥w°TÀÈvÐ!ÓVó)ˆDÔ%÷ )A¡ÀoÐä�u–ˆAØ�K‰K×ÑÜ Øñà )ðð $0ðð #1ð	ð
 '<ðð 0Hðð *<ðð $ðð 0Hðð (ðð ðð  ðð "ðð  *ðð ñöó r   c                óf  a€ VP                  R / 4      p	V	P                  R/ 4      p
. VP                  R,          OVP                  N5w  r¼pVR,          pR. /pV P                  V4      pVeÍ   VP                  R,          w  ppVVP                  R,          8X  g   Q R4       h\        P
                  ! W13RR7      pVf   Vew   Ve   TM'\        P                  ! W¼3V\        P                  R	7      pVe   TM(\        P                  ! VV3V\        P                  R	7      p\        P
                  ! WB3RR7      pV P                  ^ 8”  dV   \        P                  P                  V P                  V4      P                  VRR4      p\        P
                  ! VV3^R7      pV P                  eI   V P                  P                  VP                  ^,          \        P                   VP                  R
7      pMRpV P"                  '       g   V P$                  '       d   WP'                  V4      ,           pVe    V P(                  e   V P)                  V4      pV
P                  R/ 4      p\+        V P,                  4       Fl  w  poRV3V9   d/   V3R lpVRV3,          ! RVRVRVRVR V	/RV/4      pVR,          pMS! VVVWnV	R7      pV'       g   KT  VR,          P/                  V4       Kn  	  V P                  ^ 8”  d   VRV P                  R1R3,          pV P1                  V4      pV'       d   W3# V# )rü   Úpatches_replace:Nr   Nr  Úhidden_statesN:rZ   NNz/prepend dimension must match sequence dimensionr-   rœ   r   ÚditÚdouble_blockc                 ór   <€ / pS! V R ,          V R,          V R,          V R,          V R,          R7      VR &   V# )ÚimgÚpeÚvecÚtxtrü   )r	  rO  r  rü   © )Úargsr  Úlayers   & €r   Ú
block_wrapÚ1ContinuousTransformer.forward.<locals>.block_wrap   sO   ø€ Ø�CÙ!& t¨E¥{À4ÈÅ:Ð[_Ð`eÕ[fÐptÐuzÕp{ð  RVð  Wlõ  Rmô  "n�C˜‘JØ�Jr   ru  rx  rw  rv  Úoriginal_block)r	  rO  rP  r  rü   r   r/   r   )r9  r   r   re  r#   r6   rŽ   rá   r`  r2   r3   r4   rh  Úexpandr	  r¥   r�   r]  r^  r‡   ri  Ú	enumeraterd  r  rf  )r&   rS   r  Úprepend_embedsÚprepend_maskrO  rP  Úreturn_infork  rü   rp  ÚbatchÚseqr   r  ÚinfoÚprepend_lengthÚprepend_dimrh  r	  Úblocks_replacerm  r|  r  r{  s   &&&&&&&&,               @r   r;   ÚContinuousTransformer.forwardé  s*  ø€ ð %Ÿj™jÐ)>ÀÓCÐØ-×1Ñ1Ð2CÀRÓHˆØ3˜aŸg™g b�kÐ3¨1¯8©8Ñ3Ñˆ�FØ˜Õ#ˆð ˜Rð
ˆð �O‰O˜AÓˆàÒ%Ø*8×*>Ñ*>¸rÕ*BÑ'ˆN˜Kà !§'¡'¨"¥+Ô-Ð`Ð/`Ó`Ð-ä—	’	˜>Ð-°RÔ8ˆAàÒ'¨4Ò+;Ø#Ò/‘t´U·Z²ZÀÀÐW]Ôgl×gqÑgqÔ5r�Ø/;Ò/G™|ÌUÏZÊZÐY^Ð`nÐXoð  {Aô  KP÷  KUñ  KUô  NV�ä—y’y ,Ð!5¸RÔ@�à×!Ñ! AÔ%Ü!ŸI™I×3Ñ3°D×4FÑ4FÈÓJ×QÑQÐRWÐY[Ð]_Ó`ˆMÜ—	’	˜=¨!Ð,°!Ô4ˆAà×ÑÒ*Ø!×0Ñ0×EÑEÀaÇgÁgÈaÅjÔX]×XcÑXcÐlm×ltÑltÐEÓu‰Nà!ˆNà×"×"Ð" d×&:×&:Ð&:Ø—L‘L “OÕ#ˆAð Ò" t×'@Ñ'@Ò'LØ×3Ñ3°KÓ@ˆKà(×,Ñ,¨U°BÓ7ˆä! $§+¡+Ö.‰HˆAˆuØ Ð" nÔ4õð
 % n°aÐ%8Ö9¸5À!ÀUÈGÐUZÐ\gÐimÐo}ð  @Uð  Wjð  ;kð  n~ð  @Jð  mKó  L�Ø˜•J‘á˜!¨NÈØ)7Ø.AôC�÷ ‰{Ø�_Õ%×,Ñ,¨QÖ/ñ /ð$ ×!Ñ! AÔ%Ø�!�T×+Ñ+Ñ,¨aÐ/Õ0ˆAà×Ñ˜QÓˆçØ�7ˆNàˆr   )rØ   rb  r.   ri  r8  rd  rh  r`  r‡   re  rf  r	  r^  r]  )NNNNNFr?   rG   s   @@r   r[  r[  �  sé   ù‡ € ðVð
 ðVð ðVð ðVð ðVð ðVð ðVð "'ðVð  ðVð ðVð ðVð "&ðVð  ð!Vð" !ð#Vð$ ð%Vð&  %ð'Vð( ð)Vð* ð+Vð, ð-Vð. ÷/V÷pOô Or   r[  c                   óV   a a€ ] tR tRt oRV3R lV 3R llltRR ltR	R ltRtVtV ;t	# )
ÚAudioDiffusionTransformeri:  c                óp   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[/# )r   Útransformer_typeÚcontinuous_transformerÚglobal_cond_typeÚtimestep_features_type)ÚprependÚadaLN)ÚtpÚLiteralÚstr)ÚformatrH   s   "€r   Ú__annotate__Ú&AudioDiffusionTransformer.__annotate__;  sI   ø€ ÷ kwñ kwñ Ÿ*™*Ð%=Õ>ðkwñ Ÿ*™*Ð%7Õ8ðkwñ !$ñkwr   c                óp  <€ \         SV `  4        VV n        W@n        R pVR8X  d   \	        VRR4      V n        M\        ^VVVR7      V n        \        P                  ! VP                  VVRVVR7      \        P                  ! 4       VP                  W3RVVR7      4      V n        V^ 8”  df   V'       g   TMTp\        P                  ! VP                  VVRVVR7      \        P                  ! 4       VP                  VVRVVR7      4      V n        M^ pV^ 8”  de   V'       g   TMTp\        P                  ! VP                  VVRVVR7      \        P                  ! 4       VP                  VVRVVR7      4      V n        V	^ 8”  dW   \        P                  ! VP                  W“RVVR7      \        P                  ! 4       VP                  W3RVVR7      4      V n        W€n        WP                  ,           pW n        WÀn        WÐn        V P"                  R8X  dS   R	pV P$                  R
8X  d   Tp\'        RRVRV
RW;,          RVV,          RW,          RV^ 8„  RVRVRVRVRV/VB V n        M\+        RV P"                   24      hVP-                  VV^RVVR7      V n        VP-                  W^RVVR7      V n        R	# )é   Úexporx   g     ˆÃ@r   Tr»   Fr�  Nr“  r.   rb  r×   rh   ri   r7  r\  rE  r   r   rl   zUnknown transformer type: ry  )r    r!   r   r\  r
   Útimestep_featuresr   r   r¾   rd   r½   Úto_timestep_embedÚto_cond_embedÚto_global_embedÚto_prepend_embedÚinput_concat_dimÚ
patch_sizerŽ  r�  r[  ÚtransformerÚ
ValueErrorre   Úpreprocess_convÚpostprocess_conv)r&   Úio_channelsr£  Ú	embed_dimr\  Úproject_cond_tokensrE  Úproject_global_condr¢  Úprepend_cond_dimrb  rÚ   rŽ  r�  r‘  Úaudio_modelr   r   rl   rk  Útimestep_features_dimÚcond_embed_dimÚglobal_embed_dimrh   Ú
global_dimr*   s   &&&&&&&&&&&&&&&&&&&,     €r   r!   Ú"AudioDiffusionTransformer.__init__;  s  ø€ ô* 	‰ÑÔàˆŒ
Ø,Ôð !$Ðà! VÔ+Ü%8Ð9NÐPSÐU\Ó%]ˆDÕ"ä%4°QÐ8MÐUZÐciÔ%jˆDÔ"ä!#§¢Ø×ÑÐ3°YÀTÐQVÐ_eÐÓfÜ�GŠG‹IØ×Ñ˜i¸ÀUÐSYÐÓZó"
ˆÔð ˜AÔ÷ 4G™^ÈIˆNÜ!#§¢Ø×!Ñ! .°.ÀuÐTYÐbhÐ!ÓiÜ—’“	Ø×!Ñ! .°.ÀuÐTYÐbhÐ!Óió"ˆDÕð ˆNà˜QÔç6I™ÈyÐÜ#%§=¢=Ø×!Ñ! /Ð3CÈ%ÐW\ÐekÐ!ÓlÜ—’“	Ø×!Ñ!Ð"2Ð4DÈ5ÐX]ÐflÐ!Ómó$ˆDÔ ð ˜aÔä$&§M¢MØ×!Ñ!Ð"2ÀEÐQVÐ_eÐ!ÓfÜ—’“	Ø×!Ñ! )¸UÈ%ÐX^Ð!Ó_ó%ˆDÔ!ð !1Ôà×4Ñ4Õ4ˆà$Œð !1Ôà 0Ôà× Ñ Ð$<Ô<àˆJà×$Ñ$¨Ô/à&�
ä4ñ  Øð àð ð $Õ0ð ð  
Õ*ð	 ð
 $Õ0ð ð  .°Ñ1ð ð "0ð ð !+ð ð ð ð ð ð &ð ð ñ ˆDÕô Ð9¸$×:OÑ:OÐ9PÐQÓRÐRà)×0Ñ0°¸ÀÈÐV[ÐdjÐ0ÓkˆÔØ *× 1Ñ 1°+ÈAÐTYÐafÐouÐ 1Ó vˆÖr   c                ót  € Ve   V P                  V4      pVe   V P                  V4      pR pR p^ pV	e-   V P                  V	4      p	T	pV	P                  ^,          pV
e   T
pVe)   VP	                  4       ^8X  d   VP                  ^ ^^4      pVem   VP                  ^,          VP                  ^,          8w  d+   \        P                  ! WaP                  ^,          3RR7      p\        P                  ! W.^R7      pV P                  V P                  VR,          4      P                  VP                  4      4      pVe   VV,           pMTpV P                  R8X  dò   VfW   VP                  ^4      p\        P                   ! VP                  ^ ,          ^3VP"                  \        P$                  R7      pM„\        P                  ! WØP                  ^4      .^R7      p\        P                  ! V\        P                   ! VP                  ^ ,          ^3VP"                  \        P$                  R7      .^R7      pVP                  ^,          pV P'                  V4      V,           p\)        VR4      p/ pV P                  R8X  d   VVR	&   V P*                  ^8”  d   \)        VR
V P*                  R7      pV P,                  R8X  d!   V P.                  ! V3RVRVRVRVRV/VBVB pMnV P,                  R8X  d2   V P.                  ! V3RVRVRVRVRVRVRV/VBVB pV'       d   Vw  ppM,V P,                  R8X  d   V P.                  ! V3RVRVRV/VBVB p\)        XR4      RRVR 13,          pV P*                  ^8”  d   \)        VRV P*                  R7      pV P1                  V4      V,           pV'       d   VX3# V# )NÚnearest)Úmoder-   r   r’  rœ   zb c t -> b t cr“  rO  zb (t p) c -> b t (c p))Úpzx-transformersr�  r  r  r  r‚  r�  rƒ  rP  Úmm_transformerzb t c -> b c tzb (c p) t -> b c (t p))r   N)rŸ  r   r¡  r   r.   Úpermuter   Úinterpolater#   r6   rž  r�  r²   r   r�  rM  rŽ   r   rá   r¦  r   r£  rŽ  r¤  r§  )r&   rS   r¤   r  Úcross_attn_condÚcross_attn_cond_maskÚinput_concat_condrP  Úglobal_embedÚprepend_condÚprepend_cond_maskrƒ  rk  Úprepend_inputsr‚  r‡  Útimestep_embedÚ
extra_argsÚoutputr†  s   &&&&&&&&&&&&,       r   Ú_forwardÚ"AudioDiffusionTransformer._forward¨  s  € ð Ò&Ø"×0Ñ0°ÓAˆOàÒ#à×/Ñ/°Ó=ˆLàˆØˆØˆØÒ#à×0Ñ0°Ó>ˆLà)ˆNØ)×/Ñ/°Õ2ˆNØ Ò,Ø0�àÒ%¨.×*<Ñ*<Ó*>À!Ô*CØ+×3Ñ3°A°q¸!Ó<ˆNàÒ(ð !×&Ñ& qÕ)¨Q¯W©W°Q­ZÔ7Ü$%§M¢MÐ2CÇgÁgÈaÅjÀ^ÐZcÔ$dÐ!ä—	’	˜1Ð0°aÔ8ˆAð ×/Ñ/°×0FÑ0FÀqÈÅzÓ0R×0UÑ0UÐVW×V]ÑV]Ó0^Ó_ˆð Ò#Ø'¨.Õ8‰Là)ˆLð × Ñ  IÔ-ØÒ%à!-×!7Ñ!7¸Ó!:�Ü$Ÿzšz¨1¯7©7°1­:°q¨/À!Ç(Á(ÔRW×R\ÑR\Ô]‘ô "'§¢¨N×<RÑ<RÐSTÓ<UÐ+VÐ\]Ô!^�Ü$Ÿyšy¨,¼¿
º
ÀAÇGÁGÈAÅJÐPQÀ?Ð[\×[cÑ[cÔkp×kuÑkuÔ8vÐ)wÐ}~Ô�à+×1Ñ1°!Õ4ˆNà× Ñ  Ó# aÕ'ˆä�aÐ)Ó*ˆàˆ
à× Ñ  GÔ+Ø(4ˆJ�}Ñ%à�?‰?˜QÔÜ˜!Ð5¸¿¹ÔIˆAà× Ñ Ð$4Ô4Ø×%Ò% añ  B¸ð  BÐP_ð  Bð  oCð  Bð  JNð  Bð  ]ið  Bð  mwð  Bð  {Añ  B‰FØ×"Ñ"Ð&>Ô>Ø×%Ò% añ  z¸ð  zÐP_ð  zð  oCð  zð  JNð  zð  ]ið  zð  wBð  zð  Sað  zð  eoð  zð  syñ  zˆFçØ%‘�˜øØ×"Ñ"Ð&6Ô6Ø×%Ò% añ  H°ð  HÀtð  HÐZnð  HÐr|ð  Hð  AGñ  HˆFä˜6Ð#3Ó4°Q°q¸¹Ð5HÕIˆà�?‰?˜QÔÜ˜vÐ'?À4Ç?Á?ÔSˆFà×&Ñ& vÓ.°Õ7ˆçØ˜4�<Ðàˆr   c                óN   € V P                   ! VV3R VRVRVRVRVRV	RV
RVRV/	VB # )	rº  r»  r¼  rP  r½  r¾  r¿  r  rƒ  )rÄ  )r&   rS   Útimestepr  r  r¼  rP  r½  Únegative_global_embedr¾  r¿  r  rƒ  Úcontrolrk  s   &&&&&&&&&&&&&&,r   r;   Ú!AudioDiffusionTransformer.forward  s   € ð  —=’=ØØñð !(ðð &2ð	ð
 #4ðð  .ðð *ðð *ðð #4ðð ðð (ðð ñð r   )r\  r   r�  r¢  r£  r§  r¦  r�  rŸ  r   r¡  rž  r¤  rŽ  )r  rZ   é   i   FrË  Tr   r   é   rÌ  r�  r’  ÚlearnedÚ NNN)	NNNNNNNNF)NNNNNNNNNFN)
r@   rA   rB   rC   r!   rÄ  r;   rD   rE   rF   rG   s   @@r   rŒ  rŒ  :  s%   ù‡ € ÷kwõ kwôZc÷Jô r   rŒ  )rZ   )#Úcomfy.ldm.modules.attentionr   r   r   Útypingr”  r#   Úeinopsr   r   Útorch.nnr	   r   r0   Ú	comfy.opsr2   Ú	embeddersr
   r   ÚModuler   rJ   rV   r`   rv   rŠ   rš   r°   r·   r¹   rÎ   r!  r0  r[  rŒ  ry  r   r   Ú<module>rÖ     s  ð÷ fÑ eÛ ã å Ý Ý $Û Û Ý *òô	5�b—i‘iô 	5ôg�—	‘	ô gô*Zˆb�i‰iô Zô"ˆ"�)‰)ô "ô8 "§)¡)ô ô*  §	¡	ô  ô09�b—i‘iô 9òv*ô
1ô(,�"—)‘)ô ,ô\y�—	‘	ô yôv"�b—i‘iô "ôH`�r—y‘yô `ôDh˜BŸI™Iô hôTp §	¡	ö pr   