+
    ƒ°ñjñ§  ã                   ó
  € ^ RI t ^ RI Ht ^ RIHt ^ RIHt ^ RIt^ RIHtH	t	H
t
 ^ RIt^RIHtHt ^ RIHt ^ RIt^ RIHtHtHt ^ RIt^ RIt^ RIt ! R	 R
]P4                  4      t/ R3R R llt ! R R]P4                  4      t ! R R]P4                  4      t ! R R]P4                  4      t ! R R]P4                  4      t  ! R R]P4                  4      t! ! R R]P4                  4      t" ! R R]P4                  4      t#R# )é    N)Únn)Ú	rearrange)Ú	Rearrange)ÚCallableÚOptionalÚTuple)ÚVideoRopePosition3DEmbÚLearnablePosEmbAxis)Ú
transforms)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc                   óT   a a€ ] tR t^t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚGPT2FeedForwardc                ó*   <€ V ^8„  d   QhRS[ RS[ RR/# )é   Úd_modelÚd_ffÚreturnN©Úint)ÚformatÚ__classdict__s   "€Ú=C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\cosmos\predict2.pyÚ__annotate__ÚGPT2FeedForward.__annotate__   s#   ø€ ÷  ñ  ¡ð  ©3ð  Ð]añ  ó    c                óâ   <€ \         SV `  4        \        P                  ! 4       V n        VP                  WR W4R7      V n        VP                  W!R W4R7      V n        RV n        Wn	        W n
        R# )F©ÚbiasÚdeviceÚdtypeN)ÚsuperÚ__init__r   ÚGELUÚ
activationÚLinearÚlayer1Úlayer2Ú	_layer_idÚ_dimÚ_hidden_dim)Úselfr   r   r!   r"   Ú
operationsÚ	__class__s   &&&&&&€r   r$   ÚGPT2FeedForward.__init__   s_   ø€ Ü‰ÑÔÜŸ'š'›)ˆŒØ ×'Ñ'¨¸EÈ&Ð'Ó^ˆŒØ ×'Ñ'¨¸EÈ&Ð'Ó^ˆŒàˆŒØŒ	ØÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   Úxr   ©ÚtorchÚTensor)r   r   s   "€r   r   r   !   s#   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ñ r   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# ©N)r(   r&   r)   )r-   r3   s   &&r   ÚforwardÚGPT2FeedForward.forward!   s/   € Ø�K‰K˜‹Nˆà�O‰O˜AÓˆØ�K‰K˜‹NˆØˆr   )r+   r,   r*   r&   r(   r)   )NNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r$   r9   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r/   r   s   @@r   r   r      s   ù‡ € ÷ õ  ÷÷ ð r   r   c          
      óº   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \        ,          R\         P                  /# )r   Ú	q_B_S_H_DÚ	k_B_S_H_DÚ	v_B_S_H_DÚtransformer_optionsr   ©r5   r6   r   Údict)r   s   "r   r   r   )   sg   € ÷ uñ u¤%§,¡,ð u¼5¿<¹<ð uÔTY×T`ÑT`ð uÔwô  AEõ  xFð uô  jo÷  jvñ  jvñ ur   c           
     ó`  € \        V \        4      '       d0   V P                  4       VP                  4       VP                  4       r!p V P                  pVP                  p\	        V R4      P                  V^ ,          VR,          RVR,          4      p\	        VR4      P                  V^ ,          VR,          RVR,          4      p\	        VR4      P                  V^ ,          VR,          RVR,          4      p	\        V4      \        V4      \        V	4      r˜p? ??\        WxW•R,          RWCR7      # )a   Computes multi-head attention using PyTorch's native implementation.

This function provides a PyTorch backend alternative to Transformer Engine's attention operation.
It rearranges the input tensors to match PyTorch's expected format, computes scaled dot-product
attention, and rearranges the output back to the original format.

The input tensor names use the following dimension conventions:

- B: batch size
- S: sequence length
- H: number of attention heads
- D: head dimension

Args:
    q_B_S_H_D: Query tensor with shape (batch, seq_len, n_heads, head_dim)
    k_B_S_H_D: Key tensor with shape (batch, seq_len, n_heads, head_dim)
    v_B_S_H_D: Value tensor with shape (batch, seq_len, n_heads, head_dim)

Returns:
    Attention output tensor with shape (batch, seq_len, n_heads * head_dim)
zb ... h k -> b h ... kzb ... h v -> b h ... vT)Úskip_reshapeÚpreferred_attentionrH   éþÿÿÿéÿÿÿÿ)Ú
isinstancer   ÚtakeÚshaper   Úviewr   )
rE   rF   rG   rH   rM   Ú
in_q_shapeÚ
in_k_shapeÚ	q_B_H_S_DÚ	k_B_H_S_DÚ	v_B_H_S_Ds
   &&&&&     r   Útorch_attention_oprY   )   s)  € ô, �)Ô5×6Ò6Ø*3¯.©.Ó*:¸I¿N¹NÓ<LÈiÏnÉnÓN^˜iˆ	Ø—‘€JØ—‘€JÜ˜)Ð%=Ó>×CÑCÀJÈqÅMÐS]Ð^`ÕSaÐceÐgqÐrtÕguÓv€IÜ˜)Ð%=Ó>×CÑCÀJÈqÅMÐS]Ð^`ÕSaÐceÐgqÐrtÕguÓv€IÜ˜)Ð%=Ó>×CÑCÀJÈqÅMÐS]Ð^`ÕSaÐceÐgqÐrtÕguÓv€IÜ&>¸yÓ&IÔKcÐdmÓKnô  qIð  JSó  qT˜)€IØ�9˜iÜ˜y°YÈ2ÅÐ]að  xKô  uð  ur   c                   ó–   a a€ ] tR t^Kt oRtRV3R lV 3R llltRR/ 3V3R lR llt/ 3V3R lR lltRR/ 3V3R	 lR
 lltRt	Vt
V ;t# )Ú	Attentiona  
A flexible attention module supporting both self-attention and cross-attention mechanisms.

This module implements a multi-head attention layer that can operate in either self-attention
or cross-attention mode. The mode is determined by whether a context dimension is provided.
The implementation uses scaled dot-product attention and supports optional bias terms and
dropout regularization.

Args:
    query_dim (int): The dimensionality of the query vectors.
    context_dim (int, optional): The dimensionality of the context (key/value) vectors.
        If None, the module operates in self-attention mode using query_dim. Default: None
    n_heads (int, optional): Number of attention heads for multi-head attention. Default: 8
    head_dim (int, optional): The dimension of each attention head. Default: 64
    dropout (float, optional): Dropout probability applied to the output. Default: 0.0
    qkv_format (str, optional): Format specification for QKV tensors. Default: "bshd"
    backend (str, optional): Backend to use for the attention operation. Default: "transformer_engine"

Examples:
    >>> # Self-attention with 512 dimensions and 8 heads
    >>> self_attn = Attention(query_dim=512)
    >>> x = torch.randn(32, 16, 512)  # (batch_size, seq_len, dim)
    >>> out = self_attn(x)  # (32, 16, 512)

    >>> # Cross-attention
    >>> cross_attn = Attention(query_dim=512, context_dim=256)
    >>> query = torch.randn(32, 16, 512)
    >>> context = torch.randn(32, 8, 256)
    >>> out = cross_attn(query, context)  # (32, 16, 512)
Nc                óL   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[ RS[ RS[RR/# )r   Ú	query_dimÚcontext_dimÚn_headsÚhead_dimÚdropoutr   N)r   r   Úfloat)r   r   s   "€r   r   ÚAttention.__annotate__k   sK   ø€ ÷ +$ñ +$áð+$ñ ™c•]ð+$ñ ð	+$ñ
 ð+$ñ ð+$ð 
ñ+$r   c	                óH  <€ \         S
V `  4        \        4       V n        \        P
                  ! R V P                  P                   RV RV RV RV R24       VRJ V n        Vf   TMTpWC,          p	W0n	        W@n
        Wn        W n        VP                  WRWgR7      V n        VP                  V P                  R	WgR
7      V n        VP                  W)RWgR7      V n        VP                  V P                  R	WgR
7      V n        VP                  W)RWgR7      V n        \(        P*                  ! 4       V n        VP                  W‘RWgR7      V n        VR8”  d   \(        P0                  ! V4      M\(        P*                  ! 4       V n        \4        V n        Wn        W n        W�n        R# )zSetting up z. Query dim is z, context_dim is z and using z heads with a dimension of Ú.NFr   ç�íµ ÷Æ°>©Úepsr!   r"   g-Cëâ6?)r#   r$   r   Úcomfy_attentionÚloggingÚdebugr/   r<   Úis_selfattnr_   r`   r]   r^   r'   Úq_projÚRMSNormÚq_normÚk_projÚk_normÚv_projr   ÚIdentityÚv_normÚoutput_projÚDropoutÚoutput_dropoutrY   Úattn_opÚ
_query_dimÚ_context_dimÚ
_inner_dim)r-   r]   r^   r_   r`   ra   r!   r"   r.   Ú	inner_dimr/   s   &&&&&&&&& €r   r$   ÚAttention.__init__k   s|  ø€ ô 	‰ÑÔÜ-Ó/ˆÔÜ�ŠØ˜$Ÿ.™.×1Ñ1Ð2°/À)ÀÐL]Ð^iÐ]jÐjuØˆiÐ2°8°*¸Að?ô	
ð '¨$Ð.ˆÔà#.Ò#6‘i¸KˆØÕ&ˆ	àŒØ ŒØ"ŒØ&Ôà ×'Ñ'¨	À5ÐQWÐ'ÓeˆŒØ ×(Ñ(¨¯©¸DÈÐ(Ó]ˆŒà ×'Ñ'¨ÀUÐSYÐ'ÓgˆŒØ ×(Ñ(¨¯©¸DÈÐ(Ó]ˆŒà ×'Ñ'¨ÀUÐSYÐ'ÓgˆŒÜ—k’k“mˆŒà%×,Ñ,¨YÈÐV\Ð,ÓjˆÔØ5<¸t´^œbŸjšj¨Ô1ÌÏÊËˆÔä)ˆŒà#ŒØ'ÔØ#Žr   c                óú   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[S[ P                  S[ P                  S[ P                  3,          /# ©r   r3   ÚcontextÚrope_embrH   r   )r5   r6   r   rJ   Útuple)r   r   s   "€r   r   rc   ˜   so   ø€ ÷ 2ñ 2á�<‰<ð2ñ ™%Ÿ,™,Õ'ð2ñ ™5Ÿ<™<Õ(ð	2ñ
 &¡d�^ð2ñ 
‰u�|‰|™UŸ\™\©5¯<©<Ð7Õ	8ñ2r   c           
     ó`  a € Vf   TMTpTpTpTpVP                  R/ 4      pS P                  '       d   RMRp	W˜9   d’   VP                  4       p
S P                  V
R&   S P                  V
R&   W‰,           FW  pV! WVWsR V
R7      pVP                  RV4      pVP                  RV4      pVP                  R	V4      pVP                  R
V4      pKY  	  S P                  V4      pS P                  V4      pS P                  V4      p\        V 3R lWÞV34      w  rÞpR V 3R llpV! WÞWó4      w  rÞpWÞV3# )NÚpatchesÚattn1_patchÚattn2_patchr_   Údim_head)ÚpeÚ	attn_maskÚextra_optionsÚqÚkÚvrˆ   c                 óJ   <€ \        V R SP                  SP                  R7      # )zb ... (h d) -> b ... h d)ÚhÚd)r   r_   r`   )Útr-   s   &€r   Ú<lambda>Ú'Attention.compute_qkv.<locals>.<lambda>µ   s   ø€ ”i Ð#=ÀÇÁÐQU×Q^ÑQ^Õ_r   c                ó"  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  ,          R\        \         P                  \         P                  \         P                  3,          /# )r   r‹   rŒ   r�   r�   r   ©r5   r6   r   r   )r   s   "r   r   Ú+Attention.compute_qkv.<locals>.__annotate__¹   sd   € ÷ 	ñ 	Ü�|‰|ð	Ü %§¡ð	Ü16·±ð	ÜIQÔRW×R^ÑR^ÕI_ð	ä”5—<‘<¤§¡¬u¯|©|Ð;Õ<ñ	r   c                 ó�  <€ S	P                  V4      pS	P                  '       dþ   Veú   \        P                  P	                  S	P
                  V RR7      w  rEp\        P                  P	                  S	P                  VRR7      w  rup\        P                  P                  P                  WW4VS	P
                  P                  4      w  r\        P                  P                  S	P
                  VR V4       \        P                  P                  S	P                  VR V4       M"S	P                  V 4      p S	P                  V4      pWV3# )NT)Úoffloadable)rt   rl   ÚcomfyÚopsÚcast_bias_weightro   rq   Ú	quant_opsÚckÚrms_rope_split_halfrh   Úuncast_bias_weight)
r‹   rŒ   r�   r�   Úq_scaleÚ_Úq_offload_streamÚk_scaleÚk_offload_streamr-   s
   &&&&     €r   Úapply_norm_and_rotary_pos_embÚ<Attention.compute_qkv.<locals>.apply_norm_and_rotary_pos_emb¹   sþ   ø€ ð —‘˜A“ˆAØ××Ð HÒ$8Ü/4¯y©y×/IÑ/IÈ$Ï+É+ÐWXÐfjÐ/IÓ/kÑ,�Ð,Ü/4¯y©y×/IÑ/IÈ$Ï+É+ÐWXÐfjÐ/IÓ/kÑ,�Ð,Ü—‘×)Ñ)×=Ñ=¸aÀHÐW^Ð`d×`kÑ`k×`oÑ`oÓp‘�Ü—	‘	×,Ñ,¨T¯[©[¸'À4ÐIYÔZÜ—	‘	×,Ñ,¨T¯[©[¸'À4ÐIYÕZà—K‘K “N�Ø—K‘K “N�Ø˜�7ˆNr   )	Úgetrl   Úcopyr_   r`   rm   rp   rr   Úmap)r-   r3   r€   r�   rH   Úq_inputÚk_inputÚv_inputÚtransformer_patchesÚ
patch_namerŠ   ÚpatchÚoutr‹   rŒ   r�   r¥   s   f&&&&            r   Úcompute_qkvÚAttention.compute_qkv˜   s>  ø€ ð ’‘!¨GˆØˆØˆØˆà1×5Ñ5°iÀÓDÐØ&*×&6×&6Ð&6‘]¸Mˆ
ØÔ,Ø/×4Ñ4Ó6ˆMØ'+§|¡|ˆM˜)Ñ$Ø(,¯©ˆM˜*Ñ%Ø,×8Ð8�Ù˜G¨gÈdÐboÔp�ØŸ'™' # wÓ/�ØŸ'™' # wÓ/�ØŸ'™' # wÓ/�ØŸ7™7 4¨Ó2’ñ 9ð �K‰K˜Ó ˆØ�K‰K˜Ó ˆØ�K‰K˜Ó ˆÜÜ_Ø�1ˆIó
‰ˆˆa÷
	ð 	ñ 0°°aÓB‰ˆˆaà�Qˆwˆr   c          
      ó˜   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[,          RS[ P                  /# )r   r‹   rŒ   r�   rH   r   rI   )r   r   s   "€r   r   rc   Ì   sV   ø€ ÷ =ñ =¡5§<¡<ð =±E·L±Lð =ÁUÇ\Á\ð =ÑhpÑquÕhvð =ñ  AF÷  AMñ  AMñ =r   c                ó~   € V P                  WW4V P                  R 7      pV P                  V P                  V4      4      # ))rH   rM   )rx   ri   rw   ru   )r-   r‹   rŒ   r�   rH   Úresults   &&&&& r   Úcompute_attentionÚAttention.compute_attentionÌ   s9   € Ø—‘˜a AÐdh×dxÑdx�ÓyˆØ×"Ñ" 4×#3Ñ#3°FÓ#;Ó<Ð<r   c          
      ó¸   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[ P                  /# r   rI   )r   r   s   "€r   r   rc   Ð   sb   ø€ ÷ Xñ Xá�<‰<ðXñ ™%Ÿ,™,Õ'ðXñ ™5Ÿ<™<Õ(ð	Xñ
 &¡d�^ðXñ 
�‰ñXr   c                ó”   € V P                  WW4R7      w  rVp\        V4      \        V4      \        V4      rvpV P                  WVWtR7      # )z¬
Args:
    x (Tensor): The query tensor of shape [B, Mq, K]
    context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None
©r�   rH   ©rH   )r±   r   r¶   )r-   r3   r€   r�   rH   r‹   rŒ   r�   s   &&&&&   r   r9   ÚAttention.forwardÐ   sP   € ð ×"Ñ" 1¸Ð"Ój‰ˆˆaÜ*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆØ×%Ñ% a¨AÐ%ÓWÐWr   )rz   r{   ry   rx   ri   r^   r`   rl   rq   rp   r_   rw   ru   ro   rm   r]   rt   rr   )Né   é@   ç        NNN)r<   r=   r>   r?   Ú__doc__r$   r±   r¶   r9   r@   rA   rB   rC   s   @@r   r[   r[   K   sY   ù‡ € ñ÷>+$õ +$ð` +/Ø+/Ø.0÷2ò 2ðh z|÷ =ò =ð +/Ø+/Ø.0÷X÷ Xò Xr   r[   c                   óP   a a€ ] tR t^át oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú	Timestepsc                ó    <€ V ^8„  d   QhRS[ /# )r   Únum_channelsr   )r   r   s   "€r   r   ÚTimesteps.__annotate__â   s   ø€ ÷ )ñ )¡Sñ )r   c                ó0   <€ \         SV `  4        Wn        R # r8   )r#   r$   rÄ   )r-   rÄ   r/   s   &&€r   r$   ÚTimesteps.__init__â   s   ø€ Ü‰ÑÔØ(Ör   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Útimesteps_B_Tr   r4   )r   r   s   "€r   r   rÅ   æ   s'   ø€ ÷ fñ f¡U§\¡\ð f±e·l±lñ fr   c                óÈ  € VP                   ^8X  g   Q RVP                    24       hVP                  4       P                  4       pV P                  ^,          p\        P
                  ! R4      ) \        P                  ! V\        P                  VP                  R7      ,          pWCR,
          ,          p\        P                  ! V4      pVR,          P                  4       VR	,          ,          p\        P                  ! V4      p\        P                  ! V4      p\        P                  ! Wv.R
R7      p\        VRVP                  ^ ,          VP                  ^,          R7      # )r   zExpected 2D input, got i'  ©r"   r!   r¿   ©Údimz(b t) d -> b t d)Úbr‘   )ºNNNN)NrÏ   rO   )ÚndimÚflattenrb   rÄ   ÚmathÚlogr5   ÚarangeÚfloat32r!   ÚexpÚsinÚcosÚcatr   rR   )r-   rÉ   Ú	timestepsÚhalf_dimÚexponentÚembÚsin_embÚcos_embs   &&      r   r9   ÚTimesteps.forwardæ   s
  € Ø×!Ñ! QÔ&ÐVÐ*AÀ-×BTÑBTÐAUÐ(VÓVÐ&Ø!×)Ñ)Ó+×1Ñ1Ó3ˆ	Ø×$Ñ$¨Õ)ˆÜ—H’H˜U“OÐ#¤e§l¢l°8Ä5Ç=Á=ÐYb×YiÑYiÔ&jÕjˆØ¨#�~Õ.ˆä�iŠi˜Ó!ˆØ˜Õ ×&Ñ&Ó(¨3¨w­<Õ7ˆä—)’)˜C“.ˆÜ—)’)˜C“.ˆÜ�iŠi˜Ð*°Ô3ˆä˜Ð0°M×4GÑ4GÈÕ4JÈm×NaÑNaÐbcÕNdÔeÐer   )rÄ   r;   rC   s   @@r   rÂ   rÂ   á   s!   ù‡ € ÷)ó )÷f÷ fð fr   rÂ   c                   óT   a a€ ] tR t^÷t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚTimestepEmbeddingc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r   Úin_featuresÚout_featuresÚuse_adaln_lora©r   Úbool)r   r   s   "€r   r   ÚTimestepEmbedding.__annotate__ø   s'   ø€ ÷ rñ r¡Cð r±sð rÉDñ rr   c                ót  <€ \         SV `  4        \        P                  ! R V R24       Wn        W n        VP                  WV'       * WER7      V n        \        P                  ! 4       V n
        W0n        V'       d$   VP                  V^V,          RWER7      V n        R# VP                  W"RWER7      V n        R# )zUsing AdaLN LoRA Flag:  z=. We enable bias if no AdaLN LoRA for backward compatibility.r   FN)r#   r$   rj   rk   Úin_dimÚout_dimr'   Úlinear_1r   ÚSiLUr&   ræ   Úlinear_2)r-   rä   rå   ræ   r!   r"   r.   r/   s   &&&&&&&€r   r$   ÚTimestepEmbedding.__init__ø   s£   ø€ Ü‰ÑÔÜ�ŠØ& ~Ð&6Ð6sÐtô	
ð "ŒØ#ŒØ"×)Ñ)¨+ÈnÔJ\ÐekÐ)ÓyˆŒÜŸ'š'›)ˆŒØ,ÔßØ&×-Ñ-¨l¸AÀÕ<LÐSXÐagÐ-ÓuˆDŽMà&×-Ñ-¨lÈuÐ]cÐ-ÓqˆDŽMr   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[S[ P                  ,          3,          /# )r   Úsampler   )r5   r6   r   r   )r   r   s   "€r   r   ré     s6   ø€ ÷ ,ñ ,™eŸl™lð ,©u±U·\±\Á8ÉEÏLÉLÕCYÐ5YÕ/Zñ ,r   c                ó¨   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  '       d   TpTpWC3# R pTpWC3# r8   )rí   r&   rï   ræ   )r-   rò   rÝ   Úadaln_lora_B_T_3DÚ	emb_B_T_Ds   &&   r   r9   ÚTimestepEmbedding.forward  sd   € Ø�m‰m˜FÓ#ˆØ�o‰o˜cÓ"ˆØ�m‰m˜CÓ ˆà××ÐØ #ÐØˆIð
 Ð+Ð+ð !%ÐØˆIàÐ+Ð+r   )r&   rë   rí   rï   rì   ræ   )FNNNr;   rC   s   @@r   râ   râ   ÷   s    ù‡ € ÷rõ r÷,÷ ,ð ,r   râ   c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	Ú
PatchEmbedi  a  
PatchEmbed is a module for embedding patches from an input tensor by applying either 3D or 2D convolutional layers,
depending on the . This module can process inputs with temporal (video) and spatial (image) dimensions,
making it suitable for video and image processing tasks. It supports dividing the input into patches
and embedding each patch into a vector of size `out_channels`.

Parameters:
- spatial_patch_size (int): The size of each spatial patch.
- temporal_patch_size (int): The size of each temporal patch.
- in_channels (int): Number of input channels. Default: 3.
- out_channels (int): The dimension of the embedding vector for each patch. Default: 768.
- bias (bool): If True, adds a learnable bias to the output of the convolutional layers. Default: True.
c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )r   Úspatial_patch_sizeÚtemporal_patch_sizeÚin_channelsÚout_channelsr   )r   r   s   "€r   r   ÚPatchEmbed.__annotate__%  s9   ø€ ÷ _ñ _áð_ñ !ð_ñ ð	_ñ
 ñ_r   c                ó  <€ \         SV `  4        Wn        W n        \        P
                  ! \        R VVVR7      VP                  W1,          V,          V,          VRWVR7      4      V n        W1,          V,          V,          V n	        R# )z*b c (t r) (h m) (w n) -> b t h w (c r m n))ÚrÚmÚnFr   N)
r#   r$   rú   rû   r   Ú
Sequentialr   r'   ÚprojrÍ   )	r-   rú   rû   rü   rý   r!   r"   r.   r/   s	   &&&&&&&&€r   r$   ÚPatchEmbed.__init__%  sŒ   ø€ ô 	‰ÑÔØ"4ÔØ#6Ô ä—M’MÜØ<Ø%Ø$Ø$ô	ð ×ÑØÕ0Ð3EÕEÐH[Õ[Ð]iÐpuð  Eð ó ó

ˆŒ	ð Õ3Ð6HÕHÐK^Õ^ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r2   r4   )r   r   s   "€r   r   rþ   >  s#   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ñ r   c                ó4  € VP                  4       ^8X  g   Q hVP                  w   r#rEW@P                  ,          ^ 8X  d   WPP                  ,          ^ 8X  g   Q RWE3 RV P                   24       hW0P                  ,          ^ 8X  g   Q hV P	                  V4      pV# )a`  
Forward pass of the PatchEmbed module.

Parameters:
- x (torch.Tensor): The input tensor of shape (B, C, T, H, W) where
    B is the batch size,
    C is the number of channels,
    T is the temporal dimension,
    H is the height, and
    W is the width of the input.

Returns:
- torch.Tensor: The embedded patches as a tensor, with shape b t h w c.
zH,W z+ should be divisible by spatial_patch_size )rÍ   rR   rú   rû   r  )r-   r3   r¡   ÚTÚHÚWs   &&    r   r9   ÚPatchEmbed.forward>  s™   € ð �u‰u‹w˜!Œ|Ðˆ|ØŸ™‰ˆˆ1�à×'Ñ'Õ'¨1Ô,°×5LÑ5LÕ1LÐPQÔ1Qð	_à�1�&�ÐDÀT×E\ÑE\ÐD]Ð^ó	_ØQà×+Ñ+Õ+¨qÔ0Ð0Ð0Ø�I‰I�a‹LˆØˆr   )rÍ   r  rú   rû   )é   é   NNN©
r<   r=   r>   r?   rÀ   r$   r9   r@   rA   rB   rC   s   @@r   rø   rø     s%   ù‡ € ñ÷_õ _÷2÷ ð r   rø   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
Ú
FinalLayeriW  z
The final layer of video DiT.
c                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[RS[ /# )r   Úhidden_sizerú   rû   rý   ræ   Úadaln_lora_dimrç   )r   r   s   "€r   r   ÚFinalLayer.__annotate__\  sG   ø€ ÷ ñ áðñ  ðñ !ð	ñ
 ðñ ðñ ñr   c
                óf  <€ \         S
V `  4        V	P                  VR RR7      V n        V	P	                  WV,          V,          V,          R WxR7      V n        Wn        ^V n        WPn        W`n	        V'       dh   \        P                  ! \        P                  ! 4       V	P	                  WR WxR7      V	P	                  W`P                  V,          R WxR7      4      V n        R# \        P                  ! \        P                  ! 4       V	P	                  WP                  V,          R WxR7      4      V n        R# )Frf   )Úelementwise_affinerh   r   N)r#   r$   Ú	LayerNormÚ
layer_normr'   Úlinearr  Ún_adaln_chunksræ   r  r   r  rî   Úadaln_modulation)r-   r  rú   rû   rý   ræ   r  r!   r"   r.   r/   s   &&&&&&&&&&€r   r$   ÚFinalLayer.__init__\  s  ø€ ô 	‰ÑÔØ$×.Ñ.¨{ÈuÐZ^Ð.Ó_ˆŒØ ×'Ñ'ØÐ.@Õ@ÐCVÕVÐYeÕeÐlqð  {Að (ó 
ˆŒð 'ÔØˆÔØ,ÔØ,ÔßÜ$&§M¢MÜ—’“	Ø×!Ñ! +ÀEÐRXÐ!ÓfØ×!Ñ! .×2EÑ2EÈÕ2SÐZ_ÐhnÐ!Ó|ó%ˆDÖ!ô %'§M¢MÜ—’“	˜:×,Ñ,¨[×:MÑ:MÐP[Õ:[ÐbgÐpvÐ,ó  Eó%ˆDÖ!r   c                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          /# )r   Úx_B_T_H_W_Drõ   rô   ©r5   r6   r   )r   r   s   "€r   r   r  z  s:   ø€ ÷ ñ á—\‘\ðñ —<‘<ðñ $¡E§L¡LÕ1ñ	r   c                ó’  € V P                   '       dP   Vf   Q hV P                  V4      VRRR ^V P                  ,          13,          ,           P                  ^RR7      w  rEM$V P                  V4      P                  ^RR7      w  rE\	        VR4      \	        VR4      rvR R lpV! WP
                  Wv4      pV P                  V4      p	V	# )NrÏ   rÌ   úb t d -> b t 1 1 dc          
      ó¸   € V ^8„  d   QhR\         P                  R\        P                  R\         P                  R\         P                  R\         P                  /# )r   Ú_x_B_T_H_W_DÚ_norm_layerÚ_scale_B_T_1_1_DÚ_shift_B_T_1_1_Dr   )r5   r6   r   ÚModule)r   s   "r   r   Ú(FinalLayer.forward.<locals>.__annotate__Œ  sU   € ÷ 	Yñ 	YÜŸ,™,ð	YäŸ™ð	Yô $Ÿl™lð	Yô $Ÿl™lð		Yô
 �\‰\ñ	Yr   c                 ó<   € V! V 4      ^V,           ,          V,           # ©é   © ©r#  r$  r%  r&  s   &&&&r   Ú_fnÚFinalLayer.forward.<locals>._fnŒ  s    € ñ ˜|Ó,°Ð4DÕ0DÕEÐHXÕXÐXr   rO   )ræ   r  r  Úchunkr   r  r  )
r-   r  rõ   rô   Úshift_B_T_DÚscale_B_T_DÚshift_B_T_1_1_DÚscale_B_T_1_1_Dr.  Úx_B_T_H_W_Os
   &&&&      r   r9   ÚFinalLayer.forwardz  sÓ   € ð ××ÐØ$Ò0Ð0Ð0à×%Ñ% iÓ0Ð3DÀQÈÐKaÈQÐQU×QaÑQaÕMaÐKaÐEaÕ3bÕbß‰e�A˜2ˆeÓñ %ˆK˜ð (,×'<Ñ'<¸YÓ'G×'MÑ'MÈaÐUWÐ'MÓ'XÑ$ˆKä+4°[ÐBVÓ+WÔYbØÐ-óZ
˜õ	Yñ ˜+§¡¸ÓYˆØ—k‘k +Ó.ˆØÐr   )r  r  r  r  r  r  ræ   )Fé   NNNr8   r  rC   s   @@r   r  r  W  s#   ù‡ € ñ÷õ ÷<÷ ò r   r  c                   ód   a a€ ] tR tRt oRtR	V3R lV 3R llltRRR/ 3V3R lR lltRtVtV ;t	# )
ÚBlocki™  aG  
A transformer block that combines self-attention, cross-attention and MLP layers with AdaLN modulation.
Each component (self-attention, cross-attention, MLP) has its own layer normalization and AdaLN modulation.

Parameters:
    x_dim (int): Dimension of input features
    context_dim (int): Dimension of context features for cross-attention
    num_heads (int): Number of attention heads
    mlp_ratio (float): Multiplier for MLP hidden dimension. Default: 4.0
    use_adaln_lora (bool): Whether to use AdaLN-LoRA modulation. Default: False
    adaln_lora_dim (int): Hidden dimension for AdaLN-LoRA layers. Default: 256

The block applies the following sequence:
1. Self-attention with AdaLN modulation
2. Cross-attention with AdaLN modulation
3. MLP with AdaLN modulation

Each component uses skip connections and layer normalization.
Nc                ó>   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[RS[ /# )r   Úx_dimr^   Ú	num_headsÚ	mlp_ratioræ   r  )r   rb   rè   )r   r   s   "€r   r   ÚBlock.__annotate__®  sO   ø€ ÷ -Nñ -Náð-Nñ ð-Nñ ð	-Nñ
 ð-Nñ ð-Nñ ñ-Nr   c
                ó”  <€ \         S
V `  4        Wn        V	P                  VR RWxR7      V n        \        VRW1V,          WxV	R7      V n        V	P                  VR RWxR7      V n        \        WW1V,          WxV	R7      V n        V	P                  VR RWxR7      V n	        \        V\        W,          4      WxV	R7      V n        WPn        V P                  '       Ed   \        P                  ! \        P                   ! 4       V	P#                  WR WxR7      V	P#                  V^V,          R WxR7      4      V n        \        P                  ! \        P                   ! 4       V	P#                  WR WxR7      V	P#                  V^V,          R WxR7      4      V n        \        P                  ! \        P                   ! 4       V	P#                  WR WxR7      V	P#                  V^V,          R WxR7      4      V n        R# \        P                  ! \        P                   ! 4       V	P#                  V^V,          R WxR7      4      V n        \        P                  ! \        P                   ! 4       V	P#                  V^V,          R WxR7      4      V n        \        P                  ! \        P                   ! 4       V	P#                  V^V,          R WxR7      4      V n        R# )Frf   )r  rh   r!   r"   N)r!   r"   r.   r   )r#   r$   r;  r  Úlayer_norm_self_attnr[   Ú	self_attnÚlayer_norm_cross_attnÚ
cross_attnÚlayer_norm_mlpr   r   Úmlpræ   r   r  rî   r'   Úadaln_modulation_self_attnÚadaln_modulation_cross_attnÚadaln_modulation_mlp)r-   r;  r^   r<  r=  ræ   r  r!   r"   r.   r/   s   &&&&&&&&&&€r   r$   ÚBlock.__init__®  s‰  ø€ ô 	‰ÑÔØŒ
Ø$.×$8Ñ$8¸ÐSXÐ^bÐkqÐ$8Ó$ˆÔ!Ü" 5¨$°	ÀIÕ;MÐV\ð  wAô  BˆŒà%/×%9Ñ%9¸%ÐTYÐ_cÐlrÐ%9ó  &AˆÔ"Ü#Ø 	°IÕ+=ÀfÐfpô
ˆŒð )×2Ñ2°5ÈUÐX\ÐekÐ2ÓyˆÔÜ" 5¬#¨eÕ.?Ó*@ÈÐisÔtˆŒà,ÔØ××ÑÜ.0¯mªmÜ—’“	Ø×!Ñ! %¸eÈFÐ!Ó`Ø×!Ñ! .°!°eµ)À%ÐPVÐ!Ódó/ˆDÔ+ô
 02¯}ª}Ü—’“	Ø×!Ñ! %¸eÈFÐ!Ó`Ø×!Ñ! .°!°eµ)À%ÐPVÐ!Ódó0ˆDÔ,ô
 )+¯ªÜ—’“	Ø×!Ñ! %¸eÈFÐ!Ó`Ø×!Ñ! .°!°eµ)À%ÐPVÐ!Ódó)ˆDÖ%ô /1¯mªm¼B¿GºG»IÀz×GXÑGXÐY^Ð`aÐdiÕ`iÐpuð  EÐGXó  HSó  /TˆDÔ+Ü/1¯}ª}¼R¿WºW»YÈ
×HYÑHYÐZ_ÐabÐejÕajÐqvð  @FÐHYó  ITó  0UˆDÔ,Ü(*¯ª´b·g²g³iÀ×ARÑARÐSXÐZ[Ð^cÕZcÐjoÐx~ÐARó  BMó  )NˆDÖ%r   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[ P                  /# )	r   r  rõ   Úcrossattn_embÚrope_emb_L_1_1_Drô   Úextra_per_block_pos_embrH   r   rI   )r   r   s   "€r   r   r>  Ý  s”   ø€ ÷ xñ xá—\‘\ðxñ —<‘<ðxñ —|‘|ð	xñ
 #¡5§<¡<Õ0ðxñ $¡E§L¡LÕ1ðxñ "*©%¯,©,Õ!7ðxñ &¡d�^ðxñ 
�‰ñxr   c           
     ó”  a aaa#a$a%a&a'€ VP                   pVP                   o'Ve	   W,           pS P                  '       d‡   S P                  V4      V,           P                  ^RR7      w  ršpS P	                  V4      V,           P                  ^RR7      w  rÍpS P                  V4      V,           P                  ^RR7      w  pppMpS P                  V4      P                  ^RR7      w  ršpS P	                  V4      P                  ^RR7      w  rÍpS P                  V4      P                  ^RR7      w  ppp\        V	R4      p\        V
R4      p\        VR4      p\        VR4      p\        VR4      p\        VR4      p\        VR4      p\        VR4      p\        VR4      pVP                  w  po$o#o%pR o&S&! VS P                  VV4      p\        S P                  \        VP                  S'4      R4      R SVR7      RS$S#S%R7      p\        P                  ! VVP                  V4      VP                  V4      4      p/ 3R V#V$V%V&V'VVV 3R	 lllpV! VS P                  VVVR
7      p\        P                  ! VVP                  V4      VP                  V4      4      pS&! VS P                  VV4      P                  S'4      pVP                  R/ 4      p RV 9   d(   RVRV/p!V R,           F  p"V"! V!4      p!K  	  V!R,          pS P!                  V4      p\        P                  ! VVP                  V4      VP                  V4      4      pV# )NrÌ   r!  c                 ó<   € V! V 4      ^V,           ,          V,           # r*  r,  r-  s   &&&&r   r.  ÚBlock.forward.<locals>._fn  s   € Ù˜|Ó,°Ð4DÕ0DÕEÐHXÕXÐXr   úb t h w d -> b (t h w) drº   úb (t h w) d -> b t h w d©r‘   r�   Úwc                óÆ   € V ^8„  d   QhR\         P                  R\        R\         P                  R\         P                  R\        \        ,          R\         P                  /# )r   r#  rB  Ú_scale_cross_attn_B_T_1_1_DÚ_shift_cross_attn_B_T_1_1_DrH   r   )r5   r6   r   r   rJ   )r   s   "r   r   Ú#Block.forward.<locals>.__annotate__&  sX   € ÷ 	%ñ 	%ÜŸ,™,ð	%ä#+ð	%ô */¯©ð	%ô */¯©ð		%ô
 "*¬$¥ð	%ô �\‰\ñ	%r   c           	      ó’   <€ S
! WW#4      p\        SP                  \        VP                  S4      R 4      SSVR7      RSSS	R7      pV# )rQ  rº   rR  rS  )r   rC  Úto)r#  rB  rV  rW  rH   Ú_normalized_x_B_T_H_W_DÚ_result_B_T_H_W_Dr	  r  r
  r.  Úcompute_dtyperK  rL  r-   s   &&&&&  €€€€€€€€r   Ú_x_fnÚBlock.forward.<locals>._x_fn&  sj   ø€ ñ '*ØÐ5Pó'Ð#ô !*Ø—‘ÜÐ5×8Ñ8¸ÓGÐIcÓdØ!Ø-Ø(;ð	  ó ð +ØØØô!Ðð %Ð$r   r»   r„   Ú	mlp_patchr3   rH   rO   )r"   ræ   rF  r0  rG  rH  r   rR   r@  rA  rZ  r5   ÚaddcmulrB  rD  r§   rE  )(r-   r  rõ   rK  rL  rô   rM  rH   Úresidual_dtypeÚshift_self_attn_B_T_DÚscale_self_attn_B_T_DÚgate_self_attn_B_T_DÚshift_cross_attn_B_T_DÚscale_cross_attn_B_T_DÚgate_cross_attn_B_T_DÚshift_mlp_B_T_DÚscale_mlp_B_T_DÚgate_mlp_B_T_DÚshift_self_attn_B_T_1_1_DÚscale_self_attn_B_T_1_1_DÚgate_self_attn_B_T_1_1_DÚshift_cross_attn_B_T_1_1_DÚscale_cross_attn_B_T_1_1_DÚgate_cross_attn_B_T_1_1_DÚshift_mlp_B_T_1_1_DÚscale_mlp_B_T_1_1_DÚgate_mlp_B_T_1_1_DÚBÚDÚnormalized_x_B_T_H_W_DÚresult_B_T_H_W_Dr^  r„   Úargsr¯   r	  r  r
  r.  r]  s(   f&&ff&&&                           @@@@@r   r9   ÚBlock.forwardÝ  s¾  ÿ€ ð %×*Ñ*ˆØ!Ÿ™ˆØ"Ò.Ø%Õ?ˆKà××Ðà×/Ñ/°	Ó:Ð=NÕNß‰e�A˜2ˆeÓñ OÐ!Ð:Nð ×0Ñ0°Ó;Ð>OÕOß‰e�A˜2ˆeÓñ RÐ"Ð<Qð ×)Ñ)¨)Ó4Ð7HÕHß‰e�A˜2ˆeÓñ =ˆO˜_©nð RV×QpÑQpØóRç‰e�A˜2ˆeÓñ OÐ!Ð:Nð UY×TtÑTtØóUç‰e�A˜2ˆeÓñ RÐ"Ð<Qð @D×?XÑ?XÐYbÓ?c×?iÑ?iÐjkÐqsÐ?iÓ?tÑ<ˆO˜_¨nô %.Ð.CÐEYÓ$ZÐ!Ü$-Ð.CÐEYÓ$ZÐ!Ü#,Ð-AÐCWÓ#XÐ ä%.Ð/EÐG[Ó%\Ð"Ü%.Ð/EÐG[Ó%\Ð"Ü$-Ð.CÐEYÓ$ZÐ!ä'¨Ð9MÓNÐÜ'¨Ð9MÓNÐÜ& ~Ð7KÓLÐà#×)Ñ)‰ˆˆ1ˆa��Aò	Yñ "%ØØ×%Ñ%Ø%Ø%ó	"
Ðô %Ø�N‰NäÐ0×3Ñ3°MÓBÐD^Ó_ØØ)Ø$7ð ó ð 'ØØØô
Ðô —m’m KÐ1I×1LÑ1LÈ^Ó1\Ð^n×^qÑ^qð  sAó  _Bó  Cˆð 35÷	%÷ 	%ñ 	%ñ0 !ØØ×&Ñ&Ø&Ø&Ø 3ô
Ðô —m’m KÐ1J×1MÑ1MÈnÓ1]Ð_o×_rÑ_rð  tBó  `Có  Dˆá!$ØØ×ÑØØó	"
÷
 ‰"ˆ]Ó
ð 	ð &×)Ñ)¨)°RÓ8ˆØ˜'Ô!ØÐ/Ð1FÐH[Ð\ˆDØ  ×-Ð-�Ù˜T“{’ñ .à%)¨#¥YÐ"ØŸ8™8Ð$:Ó;ÐÜ—m’m KÐ1C×1FÑ1FÀ~Ó1VÐXh×XkÑXkÐlzÓX{Ó|ˆØÐr   )rG  rH  rF  rC  rB  rD  r@  rE  rA  ræ   r;  )ç      @Fr7  NNNr  rC   s   @@r   r9  r9  ™  s7   ù‡ € ñ÷(-Nõ -Nðh 48Ø48Ø:>Ø.0÷x÷ xò xr   r9  c                   ó°   a a€ ] tR tRt oRtRV3R lV 3R llltRV3R lR lltRV3R lR lltV3R	 lR
 ltRV3R lR llt	RV3R lR llt
RtVtV ;t# )ÚMiniTrainDITiX  a^  
A clean impl of DIT that can load and  reproduce the training results of the original DIT model in~(cosmos 1)
A general implementation of adaln-modulated VIT-like~(DiT) transformer for video processing.

Args:
    max_img_h (int): Maximum height of the input images.
    max_img_w (int): Maximum width of the input images.
    max_frames (int): Maximum number of frames in the video sequence.
    in_channels (int): Number of input channels (e.g., RGB channels for color images).
    out_channels (int): Number of output channels.
    patch_spatial (tuple): Spatial resolution of patches for input processing.
    patch_temporal (int): Temporal resolution of patches for input processing.
    concat_padding_mask (bool): If True, includes a mask channel in the input to handle padding.
    model_channels (int): Base number of channels used throughout the model.
    num_blocks (int): Number of transformer blocks.
    num_heads (int): Number of heads in the multi-head attention layers.
    mlp_ratio (float): Expansion ratio for MLP blocks.
    crossattn_emb_channels (int): Number of embedding channels for cross-attention.
    pos_emb_cls (str): Type of positional embeddings.
    pos_emb_learnable (bool): Whether positional embeddings are learnable.
    pos_emb_interpolation (str): Method for interpolating positional embeddings.
    min_fps (int): Minimum frames per second.
    max_fps (int): Maximum frames per second.
    use_adaln_lora (bool): Whether to use AdaLN-LoRA.
    adaln_lora_dim (int): Dimension for AdaLN-LoRA.
    rope_h_extrapolation_ratio (float): Height extrapolation ratio for RoPE.
    rope_w_extrapolation_ratio (float): Width extrapolation ratio for RoPE.
    rope_t_extrapolation_ratio (float): Temporal extrapolation ratio for RoPE.
    extra_per_block_abs_pos_emb (bool): Whether to use extra per-block absolute positional embeddings.
    extra_h_extrapolation_ratio (float): Height extrapolation ratio for extra embeddings.
    extra_w_extrapolation_ratio (float): Width extrapolation ratio for extra embeddings.
    extra_t_extrapolation_ratio (float): Temporal extrapolation ratio for extra embeddings.
c          :      óÆ   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[ RS[ RS[R	S[ R
S[ RS[ RS[RS[ RS[RS[RS[RS[ RS[ RS[RS[ RS[RS[RS[RS[RS[RS[RS[RS[RR/# )r   Ú	max_img_hÚ	max_img_wÚ
max_framesrü   rý   Úpatch_spatialÚpatch_temporalÚconcat_padding_maskÚmodel_channelsÚ
num_blocksr<  r=  Úcrossattn_emb_channelsÚpos_emb_clsÚpos_emb_learnableÚpos_emb_interpolationÚmin_fpsÚmax_fpsræ   r  Úrope_h_extrapolation_ratioÚrope_w_extrapolation_ratioÚrope_t_extrapolation_ratioÚextra_per_block_abs_pos_embÚextra_h_extrapolation_ratioÚextra_w_extrapolation_ratioÚextra_t_extrapolation_ratioÚrope_enable_fps_modulationr   N)r   rè   rb   Ústr)r   r   s   "€r   r   ÚMiniTrainDIT.__annotate__{  so  ø€ ÷ liñ liáðliñ ðliñ ð	liñ
 ðliñ ðliñ ðliñ ðliñ "ðliñ ðliñ ðliñ ðliñ ðliñ  !$ð!liñ$ ð%liñ&  ð'liñ(  #ð)liñ* ð+liñ, ð-liñ. ð/liñ0 ð1liñ2 %*ð3liñ4 %*ð5liñ6 %*ð7liñ8 &*ð9liñ: &+ð;liñ< &+ð=liñ> &+ð?liñ@ %)ðAliðJ 
ñKlir   c!                óØ  <€ \         S"V `  4        VV n        Wn        W n        W0n        W@n        WPn        W`n        Wpn	        W°n
        W n        W�n        W€n        Wàn        Wðn        VV n        VV n        VV n        VV n        VV n        VV n        VV n        VV n        VV n        VV n        VV n        V P7                  VVR 7       VV n        VV n        \<        P>                  ! \A        V	4      \C        W™VVVV R7      4      V n"        V'       d
   V^,           MTp\G        VVVV	VVV R7      V n$        \<        PJ                  ! \M        V
4       U!u. uF  p!\O        V	VVVVVVVV R7	      NK  	  up!4      V n(        \S        V P                  V P                  V P                  V P                  V P8                  V P:                  VVV R7	      V n*        V PW                  V	RVVR7      V n,        R# u up!i ))r!   r"   )ræ   r!   r"   r.   )rú   rû   rü   rý   r!   r"   r.   )	r;  r^   r<  r=  ræ   r  r!   r"   r.   )	r  rú   rû   rý   ræ   r  r!   r"   r.   rf   rg   N)-r#   r$   r"   r  r€  r�  rü   rý   r‚  rƒ  r<  r†  r…  r„  rˆ  r‰  rŠ  r‹  rŒ  r�  rŽ  r�  r�  r‘  r’  r“  r”  Úbuild_pos_embedræ   r  r   r  rÂ   râ   Ú
t_embedderrø   Ú
x_embedderÚ
ModuleListÚranger9  Úblocksr  Úfinal_layerrn   Út_embedding_norm)#r-   r  r€  r�  rü   rý   r‚  rƒ  r„  r…  r†  r<  r=  r‡  rˆ  r‰  rŠ  r‹  rŒ  ræ   r  r�  rŽ  r�  r�  r‘  r’  r“  r”  Úimage_modelr!   r"   r.   r¡   r/   s#   &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& €r   r$   ÚMiniTrainDIT.__init__{  s  ø€ ôL 	‰ÑÔØˆŒ
Ø"ŒØ"ŒØ$ŒØ&ÔØ(ÔØ*ÔØ,ÔØ"ŒØ$ŒØ,ÔØ#6Ô à&ÔØ!2ÔØ%:ˆÔ"ØˆŒØˆŒØ*DˆÔ'Ø*DˆÔ'Ø*DˆÔ'Ø+FˆÔ(Ø+FˆÔ(Ø+FˆÔ(Ø+FˆÔ(Ø*DˆÔ'à×Ñ F°%ÐÔ8Ø,ˆÔØ,ˆÔÜŸ-š-Ü�nÓ%Ü˜nÈ^ÐdjÐrwð  EOô  Qó
ˆŒ÷
 *=�k A–oÀ+ˆÜ$Ø,Ø .Ø#Ø'Ø °:ô
ˆŒô —m’mô ˜zÔ*óñ +�Aô Ø(Ø 6Ø'Ø'Ø#1Ø#1Ø!¨¸:÷ñ +ñó
ˆŒô &Ø×+Ñ+Ø#×1Ñ1Ø $× 3Ñ 3Ø×*Ñ*Ø×.Ñ.Ø×.Ñ.Ø °:ô
ˆÔð !+× 2Ñ 2°>ÀtÐTZÐbgÐ 2Ó hˆÖùò1s   ÅG'c                ó   <€ V ^8„  d   QhRR/# )r   r   Nr,  )r   r   s   "€r   r   r–  é  s   ø€ ÷ #ñ #¸$ñ #r   c                ó<  € V P                   R 8X  d   \        pM\        RV P                    24      h\        P                  ! RV P                    RV 24       \        V P                  V P                  V P                  ,          V P                  V P                  ,          V P                  V P                  ,          V P                  V P                  V P                  V P                  V P                  V P                   ,          V P"                  V P$                  V P&                  V P(                  VR7      pV! R/ VB V n        V P,                  '       dH   V P.                  VR&   V P0                  VR&   V P2                  VR&   WR&   W$R	&   \5        R/ VB V n        R
# R
# )Úrope3dzUnknown pos_emb_cls z#Building positional embedding with z class, impl )r…  Úlen_hÚlen_wÚlen_trŒ  r‹  Úis_learnableÚinterpolationr`   Úh_extrapolation_ratioÚw_extrapolation_ratioÚt_extrapolation_ratioÚenable_fps_modulationr!   rª  r«  r¬  r!   r"   Nr,  )rˆ  r	   Ú
ValueErrorrj   rk   rJ   r…  r  r‚  r€  r�  rƒ  rŒ  r‹  r‰  rŠ  r<  r�  rŽ  r�  r”  Úpos_embedderr�  r‘  r’  r“  r
   Úextra_pos_embedder)r-   r!   r"   Úcls_typeÚkwargss   &&&  r   r˜  ÚMiniTrainDIT.build_pos_embedé  su  € Ø×Ñ˜xÔ'Ü-‰HäÐ3°D×4DÑ4DÐ3EÐFÓGÐGä�ŠÐ;¸D×<LÑ<LÐ;MÈ]Ð[cÐZdÐeÔfÜØ×.Ñ.Ø—.‘. D×$6Ñ$6Õ6Ø—.‘. D×$6Ñ$6Õ6Ø—/‘/ T×%8Ñ%8Õ8Ø—L‘LØ—L‘LØ×/Ñ/Ø×4Ñ4Ø×(Ñ(¨D¯N©NÕ:Ø"&×"AÑ"AØ"&×"AÑ"AØ"&×"AÑ"AØ"&×"AÑ"AØô
ˆñ  %ñ 
Øñ
ˆÔð ×+×+Ð+Ø.2×.NÑ.NˆFÐ*Ñ+Ø.2×.NÑ.NˆFÐ*Ñ+Ø.2×.NÑ.NˆFÐ*Ñ+Ø%�8ÑØ#�7‰OÜ&9ñ 'Øñ'ˆDÖ#ñ ,r   c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[ P                  S[S[ P                  ,          S[S[ P                  ,          3,          /# )r   Úx_B_C_T_H_WÚfpsÚpadding_maskr   r•   )r   r   s   "€r   r   r–    sk   ø€ ÷ 30ñ 30á—\‘\ð30ñ ‘e—l‘lÕ#ð30ñ ™uŸ|™|Õ,ð	30ñ
 
‰u�|‰|™X¡e§l¡lÕ3±X¹e¿l¹lÕ5KÐKÕ	Lñ30r   c           
     ó–  € V P                   '       Ed   Vfe   \        P                  ! VP                  ^ ,          ^VP                  ^,          VP                  ^,          VP                  VP
                  R7      pMP\        P                  P                  V\        VP                  R	R 4      \        P                  P                  R7      p\        P                  ! WP                  ^4      P                  ^^VP                  ^,          ^^4      .^R7      pV P                  V4      pV P                   '       d*   V P#                  WBVP
                  VP                  R7      pMRpRV P$                  P'                  4       9   d    W@P)                  WBVP
                  R7      V3# W@P)                  WAP
                  R7      ,           pVRV3# )
a  
Prepares an embedded sequence tensor by applying positional embeddings and handling padding masks.

Args:
    x_B_C_T_H_W (torch.Tensor): video
    fps (Optional[torch.Tensor]): Frames per second tensor to be used for positional embedding when required.
                            If None, a default value (`self.base_fps`) will be used.
    padding_mask (Optional[torch.Tensor]): current it is not used

Returns:
    Tuple[torch.Tensor, Optional[torch.Tensor]]:
        - A tensor of shape (B, T, H, W, D) with the embedded sequence.
        - An optional positional embedding tensor, returned only if the positional embedding class
        (`self.pos_emb_cls`) includes 'rope'. Otherwise, None.

Notes:
    - If `self.concat_padding_mask` is True, a padding mask channel is concatenated to the input tensor.
    - The method of applying positional embeddings depends on the value of `self.pos_emb_cls`.
    - If 'rope' is in `self.pos_emb_cls` (case insensitive), the positional embeddings are generated using
        the `self.pos_embedder` with the shape [T, H, W].
    - If "fps_aware" is in `self.pos_emb_cls`, the positional embeddings are generated using the
    `self.pos_embedder` with the fps tensor.
    - Otherwise, the positional embeddings are generated without considering fps.
NrË   )r©  rÌ   )r¶  r!   r"   Úrope)r¶  r!   )r!   rN   )r„  r5   ÚzerosrR   r"   r!   r   Ú
functionalÚresizeÚlistÚInterpolationModeÚNEARESTrÙ   Ú	unsqueezeÚrepeatrš  r�  r°  rˆ  Úlowerr¯  )r-   rµ  r¶  r·  r  Úextra_pos_embs   &&&&  r   Úprepare_embedded_sequenceÚ&MiniTrainDIT.prepare_embedded_sequence  s¨  € ð< ×#×#Ñ#ØÒ#Ü$Ÿ{š{¨;×+<Ñ+<¸QÕ+?ÀÀK×DUÑDUÐVWÕDXÐZe×ZkÑZkÐlmÕZnð  wB÷  wHñ  wHð  Q\÷  Qcñ  Qcô   d‘ä)×4Ñ4×;Ñ;Ø ¤$ {×'8Ñ'8¸¸Ð'=Ó">Ìj×NjÑNj×NrÑNrð  <ó  �ô  Ÿ)š)Ø×4Ñ4°QÓ7×>Ñ>¸qÀ!À[×EVÑEVÐWXÕEYÐ[\Ð^_Ó`ÐaÐghôˆKð —o‘o kÓ2ˆà×+×+Ð+Ø ×3Ñ3°KÐQ\×QcÑQcÐkv×k|Ñk|Ð3Ó}‰Mà ˆMà�T×%Ñ%×+Ñ+Ó-Ô-Ø× 1Ñ 1°+È{×OaÑOaÐ 1Ó bÐdqÐqÐqØ!×$5Ñ$5°k×J\ÑJ\Ð$5Ó$]Õ]ˆà˜D -Ð/Ð/r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úx_B_T_H_W_Mr   r4   )r   r   s   "€r   r   r–  C  s#   ø€ ÷ ñ ¡e§l¡lð ±u·|±|ñ r   c                ób   € \        VR V P                  V P                  V P                  R7      pV# )z.B T H W (p1 p2 t C) -> B C (T t) (H p1) (W p2))Úp1Úp2r‘   )r   r‚  rƒ  )r-   rÇ  Úx_B_C_Tt_Hp_Wps   && r   Ú
unpatchifyÚMiniTrainDIT.unpatchifyC  s5   € Ü"ØØ<Ø×!Ñ!Ø×!Ñ!Ø×!Ñ!ô
ˆð Ðr   c                ó¼   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# ©r   r3   rÚ   r€   r¶  r·  r  )r   r   s   "€r   r   r–  M  s`   ø€ ÷ Fñ FÙ�<‰<ðFá—<‘<ðFñ —‘ðFñ ‘e—l‘lÕ#ð	Fñ
 ™uŸ|™|Õ,ñFr   c                ó2  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  VP                  R / 4      4      4      P                  ! WW4V3/ VB # r»   )
r™   Úpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELr§   Úexecute)r-   r3   rÚ   r€   r¶  r·  r²  s   &&&&&&,r   r9   ÚMiniTrainDIT.forwardM  sœ   € ô ×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4´U×5LÑ5L×5WÑ5W×5gÑ5gÐio×isÑisð  uJð  LNó  jOó  Pó
÷ ‰'ð	ð  ¨lñ	Fð ?Eñ	Fð 	Fr   c                ó¼   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# rÏ  r  )r   r   s   "€r   r   r–  [  sb   ø€ ÷ Hñ Há�<‰<ðHñ —<‘<ðHñ —‘ð	Hñ
 ‘e—l‘lÕ#ðHñ ™uŸ|™|Õ,ñHr   c           	     ó*  € \        VP                  4      p\        P                  P                  P                  WP                  V P                  V P                  34      pTpTp	Tp
 V P                  VVVR 7      w  r¼pV	P                  ^8X  d   V	P                  ^4      p	V P                  ^,          ! V P                  ^ ,          ! V	4      P                  VP                  4      4      w  rïV P                  V4      pVe<   VP                  VP                  8X  g!   Q VP                   RVP                   24       hVP                  R/ 4      pVP                  R/ 4      pRV9   d   VP!                  4       p/ VR&   RV9   d(   VR,           F  pV! RVRVRV/4      pVR,          pK  	  R	VP                  ^4      P                  ^ 4      R
VRVRV/pVP                  \"        P$                  8X  d   VP'                  4       p\)        V P*                  4       F  w  ppVVR&   V! VVV
3/ VB pK  	  V P-                  VP                  V
P                  4      WïR7      pV P/                  V4      RRRVR,          1RVR,          1RVR,          13,          pV# ))r¶  r·  Nz != rH   r„   Ú
post_inputÚmodel_patch_dataÚimgr3   rL  rô   rM  Úblock_index)rô   rÏ   éýÿÿÿrN   rO   )r½  rR   r™   ÚldmÚ
common_ditÚpad_to_patch_sizerƒ  r‚  rÄ  rÐ   rÀ  r™  rZ  r"   rŸ  r§   r¨   r5   Úfloat16rb   Ú	enumerater�  rž  rÌ  )r-   r3   rÚ   r€   r¶  r·  r²  Ú
orig_shaperµ  rÉ   rK  r  rL  Ú$extra_pos_emb_B_T_H_W_D_or_T_H_W_B_DÚt_embedding_B_T_Drô   rH   r„   r¯   r°   Úblock_kwargsrß  Úblockr5  rË  s   &&&&&&,                  r   rÔ  ÚMiniTrainDIT._forward[  sÀ  € ô ˜!Ÿ'™'“]ˆ
Ü�I‰I× Ñ ×2Ñ2°1×7JÑ7JÈD×L^ÑL^Ð`d×`rÑ`rÐ6sÓtˆØˆØ!ˆØˆð	ð OS×NlÑNlØØØ%ð Omó O
ÑKˆÐ'Kð ×Ñ Ô"Ø)×3Ñ3°AÓ6ˆMØ/3¯©¸qÖ/AÀ$Ç/Á/ÐRSÖBTÐUbÓBc×BfÑBfÐgr×gxÑgxÓByÓ/zÑ,ÐØ ×1Ñ1Ð2CÓDÐà/Ò;à×!Ñ!Ð%I×%OÑ%OÔOðVà×#Ñ#Ð$ DÐ)M×)SÑ)SÐ(TÐUóVØOð %Ÿj™jÐ)>ÀÓCÐØ%×)Ñ)¨)°RÓ8ˆØ˜7Ô"Ø"5×":Ñ":Ó"<ÐØ68ÐÐ 2Ñ3à˜7Ô"Ø  ×.Ð.�Ù˜U K°°kÐCXÐZmÐnÓo�Ø! %�j’ñ /ð
 Ð 0× :Ñ :¸1Ó =× GÑ GÈÓ JØÐ!2Ø%Ð'KØ!Ð#6ð	
ˆð ×Ñ¤§¡Ô-Ø%×+Ñ+Ó-ˆKä"+¨D¯K©KÖ"8ÑˆK˜Ø1<Ð Ñ.ÙØØ!Øñð ñ	ŠKñ #9ð ×&Ñ& {§~¡~°m×6IÑ6IÓ'JÐL]Ð&ó  DˆØŸ™¨Ó5°a¸¸O¸ZÈ½^¸OÈ_ÈjÐY[ÍnÈ_Ð^mÐ_iÐjlÕ_mÐ^mÐ6mÕnˆØÐr   )"r  r�  r„  r"   r‘  r�  r°  r“  r’  rž  rü   rŒ  r�  r  r€  r‹  r…  r†  r<  rý   r‚  rƒ  rˆ  rŠ  r‰  r¯  r”  r�  r�  rŽ  r™  rŸ  ræ   rš  )Tr  é
   é   r{  i   ÚsincosFÚcropr+  é   Fr7  ç      ð?rñ  rñ  Frñ  rñ  rñ  TNNNN)NN)r<   r=   r>   r?   rÀ   r$   r˜  rÄ  rÌ  r9   rÔ  r@   rA   rB   rC   s   @@r   r}  r}  X  sX   ù‡ € ñ ÷Dliõ li÷\#ò #÷J30ò 30÷jð ÷Fò F÷H÷ Hò Hr   r}  )$r5   r   Úeinopsr   Úeinops.layers.torchr   rj   Útypingr   r   r   rÒ   Úposition_embeddingr	   r
   Útorchvisionr   Úcomfy.patcher_extensionr™   Úcomfy.ldm.modules.attentionr   r   r   Úcomfy.ldm.common_ditÚ	comfy.opsÚcomfy.quant_opsr'  r   rY   r[   rÂ   râ   rø   r  r9  r}  r,  r   r   Ú<module>rü     sÚ   ðó Ý Ý Ý )Û ß ,Ñ ,Û ç KÝ "ã ß eÑ eÛ Û Û ô�b—i‘iô ð& IKð  ae÷ uôDSX�—	‘	ô SXôlf�—	‘	ô fô,,˜Ÿ	™	ô ,ô>>�—‘ô >ôB?�—‘ô ?ôD|ˆB�I‰Iô |ô~K�2—9‘9ö Kr   