+
    ƒ°ñjßk  ã                   ób  € ^ RI t ^ RIt^ RIHt ^ RIHu Ht ^ RIHtH	t	H
t
 ^ RIt^ RItRR ltR tRR ltRR ltRR ltRR lt ! R	 R
]P(                  4      t ! R R]P(                  4      t ! R R]P(                  4      t ! R R]P(                  4      t ! R R]P(                  4      tR# )é    N)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc           
     óº  € RV\         P                  ! ^ V ^\         P                  VP                  R7      V ,          ,          ,          p\         P                  ! VP                  4       VP                  4       4      pVP                  4       P                  ^RR7      P                  4       pVP                  4       P                  ^RR7      P                  4       pWV3# )zÂReturns (cos, sin) each with shape [seq_len, dim].

Frequencies are computed at dim//2 resolution then repeat_interleaved
to full dim, matching CogVideoX's interleaved (real, imag) pair format.
ç      ð?©ÚdtypeÚdevice©Údiméÿÿÿÿ)	ÚtorchÚarangeÚfloat32r
   ÚouterÚfloatÚcosÚrepeat_interleaveÚsin)r   ÚposÚthetaÚfreqsÚanglesr   r   s   &&&    Ú<C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\cogvideo\model.pyÚ_get_1d_rotary_pos_embedr      s¡   € ð �5œUŸ\š\¨!¨S°!¼5¿=¹=ÐQT×Q[ÑQ[Ô\Ð_bÕbÕcÕd€EÜ�[Š[˜Ÿ™› e§k¡k£mÓ4€FØ
�*‰*‹,×
(Ñ
(¨°Ð
(Ó
3×
9Ñ
9Ó
;€CØ
�*‰*‹,×
(Ñ
(¨°Ð
(Ó
3×
9Ñ
9Ó
;€CØˆ:Ðó    c                óè  € Vw  r#VR,          P                  V P                  4      pVR,          P                  V P                  4      pV P                  ! . V P                  RR ORN^N5!  P	                  R4      w  rE\
        P                  ! V) V.RR7      P                  ^4      pV P                  4       V,          VP                  4       V,          ,           P                  V P                  4      # )a  Apply CogVideoX rotary embedding to query or key tensor.

x: [B, heads, seq_len, head_dim]
freqs_cos_sin: (cos, sin) each [seq_len, head_dim//2]

Uses interleaved pair rotation (same as diffusers CogVideoX/Flux).
head_dim is reshaped to (-1, 2) pairs, rotated, then flattened back.
Nr   ©NNºNNNr   r   )
Útor
   ÚreshapeÚshapeÚunbindr   ÚstackÚflattenr   r	   )ÚxÚfreqs_cos_sinr   r   Úx_realÚx_imagÚ	x_rotateds   &&     r   Úapply_rotary_embr+      sÅ   € ð �H€CØ
ÐÕ
×
"Ñ
" 1§8¡8Ó
,€CØ
ÐÕ
×
"Ñ
" 1§8¡8Ó
,€Cð —Y’YÐ4 §¡¨¨ Ð4¨bÐ4°!Ó4×;Ñ;¸BÓ?�N€FÜ—’˜f˜W fÐ-°2Ô6×>Ñ>¸qÓA€Ià�G‰G‹I˜�O˜iŸo™oÓ/°#Õ5Õ5×9Ñ9¸!¿'¹'ÓBÐBr   c           
      ó¾  € V^,          p\         P                  ! \        P                  ! V4      ) \         P                  ! ^ V\         P
                  V P                  R7      ,          V,          4      pV R,          P                  4       VR,          ,          V,          p\         P                  ! \         P                  ! V4      \         P                  ! V4      .RR7      p	V'       d1   \         P                  ! V	RVR13,          V	RRV13,          .RR7      p	V^,          '       d6   \         P                  ! V	\         P                  ! V	R,          4      .RR7      p	V	# )é   ©ÚstartÚendr	   r
   r   Nr   ©r   Nr   ©r   :Né   N)r   ÚexpÚmathÚlogr   r   r
   r   Úcatr   r   Ú
zeros_like)
Ú	timestepsr   Úflip_sin_to_cosÚdownscale_freq_shiftÚscaleÚ
max_periodÚhalfr   ÚargsÚ	embeddings
   &&&&&&    r   Úget_timestep_embeddingrA   0   s  € Ø�!�8€DÜ�IŠI”t—x’x 
Ó+Ð+¬e¯lªlÀÈÔTY×TaÑTaÐjs×jzÑjzÔ.{Õ{ð  Cõ  Có  D€EØ�WÕ×#Ñ#Ó%¨¨d­Õ3°eÕ;€DÜ—	’	œ5Ÿ9š9 T›?¬E¯IªI°d«OÐ<À"ÔE€IßÜ—I’I˜y¨¨D©E¨Õ2°I¸aÀÀ$À¸hÕ4GÐHÈbÔQˆ	Ø
ˆQ‡w„wÜ—I’I˜y¬%×*:Ò*:¸9ÀUÕ;KÓ*LÐMÐSUÔVˆ	ØÐr   c                 óÆ  € \        V\        4      '       d   W3p\        P                  ! V^ ,          \        P                  VR7      V,          p\        P                  ! V^,          \        P                  VR7      V,          p\        P                  ! V\        P                  VR7      V,          p\        P
                  ! W‡VRR7      w  r‡p^V ^,          ,          p	V ^,          p
\        W—WeR7      p\        W¨R,          VR7      pVP                  w  rÞpVP                  ^4      P                  ^4      P                  RWïR4      p\        P                  ! WË.RR7      pV# )r   r   Úij)Úindexing©r
   r   )r   r   r   r   )Ú
isinstanceÚintr   r   r   ÚmeshgridÚ_get_2d_sincos_pos_embedÚ_get_1d_sincos_pos_embedr"   Ú	unsqueezeÚexpandr7   )Ú	embed_dimÚspatial_sizeÚtemporal_sizeÚspatial_interpolation_scaleÚtemporal_interpolation_scaler
   Úgrid_wÚgrid_hÚgrid_tÚembed_dim_spatialÚembed_dim_temporalÚpos_embed_spatialÚpos_embed_temporalÚTÚHÚWÚ	pos_embeds   &&&&&&           r   Úget_3d_sincos_pos_embedr]   <   s  € Ü�,¤×$Ò$Ø$Ð3ˆä�\Š\˜, q�/´·±ÀvÔNÐQlÕl€FÜ�\Š\˜, q�/´·±ÀvÔNÐQlÕl€FÜ�\Š\˜-¬u¯}©}ÀVÔLÐOkÕk€Fä"Ÿ^š^¨F¸FÈTÔRÑ€F�Fà˜Y¨!�^Õ,ÐØ" a�Ðä0Ð1BÈFÔbÐÜ1Ð2DÈWÅoÐ^dÔeÐà�l‰l�G€Aˆ!Ø+×5Ñ5°aÓ8×BÑBÀ1ÓE×LÑLÈRÐQRÐWYÓZÐÜ—	’	Ð-ÐAÀrÔJ€IàÐr   c                 ó  € VP                   w  rEpV ^,          p\        WqP                  R4      VR7      P                  WEWg4      p\        WrP                  R4      VR7      P                  WEWg4      p	\        P                  ! W‰.RR7      # )r-   rE   r   r   )r"   rJ   r!   r   r7   )
rM   rS   rR   r
   rY   rZ   r[   Úhalf_dimÚpos_hÚpos_ws
   &&&&      r   rI   rI   S   ss   € Ø�l‰l�G€Aˆ!Ø˜A�~€HÜ$ X¯~©~¸bÓ/AÈ&ÔQ×YÑYÐZ[Ð`aÓl€EÜ$ X¯~©~¸bÓ/AÈ&ÔQ×YÑYÐZ[Ð`aÓl€EÜ�9Š9�e�^¨Ô,Ð,r   c           
      óJ  € V ^,          p\         P                  ! \        P                  ! R4      ) \         P                  ! ^ V\         P
                  VR7      ,          V,          4      pVP                  4       P                  R4      R,          VR,          ,          p\         P                  ! \         P                  ! V4      \         P                  ! V4      .RR7      pV ^,          '       d6   \         P                  ! V\         P                  ! VR,          4      .RR7      pV# )r-   ç     ˆÃ@r.   Nr   r   r1   r2   )r   r4   r5   r6   r   r   r   r!   r7   r   r   r8   )rM   r   r
   r>   r   r?   r@   s   &&&    r   rJ   rJ   [   sÅ   € Ø˜�>€DÜ�IŠI”t—x’x Ó(Ð(¬5¯<ª<¸aÀTÔQV×Q^ÑQ^ÐgmÔ+nÕnÐquÕuÓv€EØ�9‰9‹;×Ñ˜rÓ" 7Õ+¨e°D­kÕ9€DÜ—	’	œ5Ÿ9š9 T›?¬E¯IªI°d«OÐ<À"ÔE€IØ�1‡}„}Ü—I’I˜y¬%×*:Ò*:¸9ÀUÕ;KÓ*LÐMÐSUÔVˆ	ØÐr   c                   óF   a a€ ] tR t^ft oRV 3R lltRR ltR tRtVtV ;t	# )ÚCogVideoXPatchEmbedc           
     óö  <€ \         SV `  4        Wn        W n        W@n        W€n        Wpn        W�n        W n        W°n	        WÀn
        WÐn        Wàn        Wðn        Vf   VP                  W4WVVVR7      V n        M.VP!                  W1,          V,          V,          VVVR7      V n        VP!                  WTVVR7      V n        V'       g	   V'       d+   TpV P%                  W‡V	4      pV P'                  RVVR7       R # R # )N)Úkernel_sizeÚstrideÚbiasr
   r	   ©r
   r	   Úpos_embedding)Ú
persistent)ÚsuperÚ__init__Ú
patch_sizeÚpatch_size_tr   Úsample_heightÚsample_widthÚsample_framesÚtemporal_compression_ratioÚmax_text_seq_lengthrP   rQ   Úuse_positional_embeddingsÚ!use_learned_positional_embeddingsÚConv2dÚprojÚLinearÚ	text_projÚ_get_positional_embeddingsÚregister_buffer)Úselfro   rp   Úin_channelsr   Útext_dimri   rr   rq   rs   rt   ru   rP   rQ   rv   rw   r
   r	   Ú
operationsrl   rk   Ú	__class__s   &&&&&&&&&&&&&&&&&&&  €r   rn   ÚCogVideoXPatchEmbed.__init__g   s  ø€ ô 	‰ÑÔØ$ŒØ(ÔØŒØ*ÔØ(ÔØ*ÔØ*DÔ'Ø#6Ô Ø+FÔ(Ø,HÔ)Ø)BÔ&Ø1RÔ.àÒØ"×)Ñ)¨+È
ÐlpÐyð  HMÐ)ó  NˆD�Ià"×)Ñ)¨+Õ*BÀZÕ*OÐR^Õ*^Ð`cÐlrÐzÐ)ó  AˆDŒIà#×*Ñ*¨8ÀÈuÐ*ÓUˆŒç$×(IØ:ˆJØ ×;Ñ;¸MÐYfÓgˆMØ× Ñ  °-ÈJÐ ÖWñ )Jr   c           	     óF  € WP                   ,          pW P                   ,          pV^,
          V P                  ,          ^,           pV P                  e   WpP                  ,          pWV,          V,          p\        V P                  We3VV P
                  V P                  VR7      p	V	P                  RV P                  4      p	V	P                  ^V P                  V,           V P                  RR7      p
V
P                  RV P                  R13,          P                  V	4       V
# )r3   NrE   F)Úrequires_gradr   r   )ro   rt   rp   r]   r   rP   rQ   r!   Ú	new_zerosru   ÚdataÚcopy_)r~   rq   rr   rs   r
   Úpost_patch_heightÚpost_patch_widthÚpost_time_compression_framesÚnum_patchesrk   Újoint_pos_embeddings   &&&&&      r   r|   Ú.CogVideoXPatchEmbed._get_positional_embeddingsˆ   s  € Ø)¯_©_Õ<ÐØ'¯?©?Õ:ÐØ(5¸Õ(9¸d×>]Ñ>]Õ']Ð`aÕ'aÐ$Ø×ÑÒ(Ø+G×K\ÑK\Õ+\Ð(Ø'Õ:Ð=YÕYˆä/Ø�H‰HØÐ1Ø(Ø×,Ñ,Ø×-Ñ-Øô
ˆð &×-Ñ-¨b°$·(±(Ó;ˆØ+×5Ñ5Øˆt×'Ñ'¨+Õ5°t·x±xÈuð 6ó 
Ðð 	× Ñ   D×$<Ñ$<Ñ$=Ð!=Õ>×DÑDÀ]ÔSØ"Ð"r   c           
     ó¸  € VP                   pV P                  VP                  V P                  P                  P                   4      4      P                  V4      pVP                  w  rErgpV P
                  P                  P                   p	V P                  fœ   VP                  RWgV4      pV P                  VP                  V	4      4      P                  V4      pVP                  ! WE.VP                  R,          O5!  pVP                  ^4      P                  ^^4      pVP                  ^^4      pM½V P                  p
V P                  pVP                  ^ ^^^^4      pVP                  WEV,          W·V
,          W¨V
,          W¦4      pVP                  ^ ^^^^^^^4      P                  ^^4      P                  ^^4      pV P                  VP                  V	4      4      P                  V4      p\        P                  ! W.^R7      P                  4       pV P                   '       g   V P"                  '       EdŸ   VP                  ^,          pVP                  ^,          pV P"                  '       dY   V P$                  RV P&                  V P&                  V,           13,          P                  VP(                  VP                   R7      pM½\+        V P,                  W€P                  ,          WpP                  ,          3WçV P                  ,          W€P                  ,          ,          ,          V P.                  V P0                  VP(                  R7      P                  ^WàP,                  4      P                  VP                   R7      p\        P2                  ! ^WÞ,           V P,                  VP(                  VP                   R7      pVVRVR 13&   VV,           pV# )N:r3   NNr   r   rj   rE   ©r	   r   )r	   r{   r    Úweightr"   ry   rp   r!   Úviewr%   Ú	transposero   Úpermuter   r7   Ú
contiguousrv   rw   rk   ru   r
   r]   r   rP   rQ   Úzeros)r~   Útext_embedsÚimage_embedsÚinput_dtypeÚ
batch_sizeÚ
num_framesÚchannelsÚheightÚwidthÚ
proj_dtypeÚpÚp_tÚembedsÚtext_seq_lengthÚnum_image_patchesÚ	image_posÚ	joint_poss   &&&              r   ÚforwardÚCogVideoXPatchEmbed.forwardŸ   s&  € Ø!×'Ñ'ˆØ—n‘n [§^¡^°D·N±N×4IÑ4I×4OÑ4OÓ%PÓQ×TÑTÐU`ÓaˆØ:F×:LÑ:LÑ7ˆ
 °%à—Y‘Y×%Ñ%×+Ñ+ˆ
Ø×ÑÒ$Ø'×/Ñ/°°HÀeÓLˆLØŸ9™9 \§_¡_°ZÓ%@ÓA×DÑDÀ[ÓQˆLØ'×,Ò,¨ZÐ]Àl×FXÑFXÐY[ÕF\Ó]ˆLØ'×/Ñ/°Ó2×<Ñ<¸QÀÓBˆLØ'×/Ñ/°°1Ó5‰Là—‘ˆAØ×#Ñ#ˆCØ'×/Ñ/°°1°a¸¸AÓ>ˆLØ'×/Ñ/Ø¨#Õ-¨s¸aµKÀÈQÅJÐPQóˆLð (×/Ñ/°°1°a¸¸A¸qÀ!ÀQÓG×OÑOÐPQÐSTÓU×]Ñ]Ð^_ÐabÓcˆLØŸ9™9 \§_¡_°ZÓ%@ÓA×DÑDÀ[ÓQˆLä—’˜KÐ6¸AÔ>×IÑIÓKˆà×)×)Ð)¨T×-S×-SÑ-SØ)×/Ñ/°Õ2ˆOØ ,× 2Ñ 2°1Õ 5Ðà×5×5Ð5Ø ×.Ñ.Ø�t×/Ñ/°×0HÑ0HÐK\Õ0\Ð\Ð\õç‘"˜FŸM™M°·±�"Ó>ñ ô 4Ø—H‘HØŸo™oÕ-¨v¿¹Õ/HÐIØ%°T·_±_Õ+DÈ×RaÑRaÕIaÕ*bÕcØ×4Ñ4Ø×5Ñ5Ø!Ÿ=™=ô÷ ‘'˜!Ð.·±Ó9¿"¹"À6Ç<Á<¸"Ó:Pð ô Ÿš A Õ'JÈDÏHÉHÐ]c×]jÑ]jÐrx×r~Ñr~ÔˆIØ-6ˆI�a˜Ñ)Ð)Ñ*Ø˜iÕ'ˆFàˆr   )r   ru   ro   rp   ry   rs   rq   rr   rP   rt   rQ   r{   rw   rv   )r-   Né   i€  é   TéZ   é<   é1   é   éâ   ç      þ?r   TTNNN©N)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rn   r|   r§   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r‚   Ú__classdict__s   @@r   re   re   f   s   ù‡ € ÷XôB#÷./ò /r   re   c                   ó<   a a€ ] tR t^Ñt oRV 3R lltR tRtVtV ;t# )ÚCogVideoXLayerNormZeroc	                óÌ   <€ \         S	V `  4        \        P                  ! 4       V n        VP                  V^V,          WVVR7      V n        VP                  W$W6VR7      V n        R# )é   ©ri   r
   r	   ©ÚepsÚelementwise_affiner
   r	   N©	rm   rn   ÚnnÚSiLUÚsilurz   ÚlinearÚ	LayerNormÚnorm)
r~   Útime_dimr   rÂ   rÁ   ri   r
   r	   r�   r‚   s
   &&&&&&&&&€r   rn   ÚCogVideoXLayerNormZero.__init__Ò   sV   ø€ ä‰ÑÔÜ—G’G“IˆŒ	Ø ×'Ñ'¨°!°cµ'ÀÐ[`Ð'ÓaˆŒØ×(Ñ(¨ÐJ\ÐsxÐ(ÓyˆŽ	r   c                ób  € V P                  V P                  V4      4      P                  ^^R7      w  rErgr‰V P                  V4      ^V,           R,          ,          VR,          ,           pV P                  V4      ^V,           R,          ,          VR,          ,           pWVR,          V	R,          3# )r¾   r   ©r   Nr   ©rÇ   rÆ   ÚchunkrÉ   )
r~   Úhidden_statesÚencoder_hidden_statesÚtembÚshiftr<   ÚgateÚ	enc_shiftÚ	enc_scaleÚenc_gates
   &&&&      r   r§   ÚCogVideoXLayerNormZero.forwardÙ   sœ   € Ø=A¿[¹[ÈÏÉÐSWËÓ=Y×=_Ñ=_Ð`aÐghÐ=_Ó=iÑ:ˆ�d yØŸ	™	 -Ó0°A¸µI¸zÕ3JÕJÈUÐS]ÕM^Õ^ˆØ $§	¡	Ð*?Ó @ÀAÈ	ÅMÐS]ÕC^Õ ^ÐajÐkuÕavÕ vÐØ°T¸*Õ5EÀxÐPZÕG[Ð[Ð[r   ©rÇ   rÉ   rÆ   )Tçñhãˆµøä>TNNN©	r²   r³   r´   rµ   rn   r§   r¶   r·   r¸   r¹   s   @@r   r¼   r¼   Ñ   s   ù‡ € ÷z÷\ò \r   r¼   c                   ó<   a a€ ] tR t^àt oRV 3R lltR tRtVtV ;t# )ÚCogVideoXAdaLayerNormc                óÊ   <€ \         SV `  4        \        P                  ! 4       V n        VP                  V^V,          WVR7      V n        VP                  W$W5VR7      V n        R# )r-   rj   rÀ   NrÃ   )	r~   rÊ   r   rÂ   rÁ   r
   r	   r�   r‚   s	   &&&&&&&&€r   rn   ÚCogVideoXAdaLayerNorm.__init__á   sS   ø€ ä‰ÑÔÜ—G’G“IˆŒ	Ø ×'Ñ'¨°!°cµ'À&Ð'ÓVˆŒØ×(Ñ(¨ÐJ\ÐsxÐ(ÓyˆŽ	r   c                óØ   € V P                  V P                  V4      4      pVP                  ^^R7      w  r4V P                  V4      ^V,           R,          ,          VR,          ,           pV# )r-   r   rÍ   rÎ   )r~   r&   rÒ   rÓ   r<   s   &&&  r   r§   ÚCogVideoXAdaLayerNorm.forwardè   sV   € Ø�{‰{˜4Ÿ9™9 T›?Ó+ˆØ—z‘z !¨�zÓ+‰ˆØ�I‰I�a‹L˜A �I zÕ2Õ2°U¸:Õ5FÕFˆØˆr   rÙ   )TrÚ   NNNrÛ   r¹   s   @@r   rÝ   rÝ   à   s   ù‡ € ÷z÷ò r   rÝ   c                   ó@   a a€ ] tR t^ït oRV 3R lltRR ltRtVtV ;t# )ÚCogVideoXBlockc           	     ól  <€ \         SV `  4        \        4       V n        Wn        W n        W0n        \        WAWXWšR 7      V n        V
P                  WRW‰R7      V n
        V
P                  WRW‰R7      V n        V
P                  WRW‰R7      V n        V
P                  VRRW‰R7      V n        V
P                  VRRW‰R7      V n        V
P                  WRW‰R7      V n        \        WAWXWšR 7      V n        T;'       g
    V^,          pV
P                  WWxV	R7      V n        V
P                  W±WxV	R7      V n        R# ))rÁ   r
   r	   r�   Tr¿   g�íµ ÷Æ°>rÀ   N)rm   rn   r   Úcomfy_attentionr   Ú	num_headsÚhead_dimr¼   Únorm1rz   ÚqÚkÚvrÈ   Únorm_qÚnorm_kÚattn_outÚnorm2Úff_projÚff_out)r~   r   ræ   rç   rÊ   rÁ   Úff_inner_dimÚff_biasr
   r	   r�   Ú	inner_dimr‚   s   &&&&&&&&&&& €r   rn   ÚCogVideoXBlock.__init__ð   s3  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØŒØ"ŒØ Œä+¨H¸sÐY^ÔvˆŒ
ð ×"Ñ" 3°$¸vÐ"ÓSˆŒØ×"Ñ" 3°$¸vÐ"ÓSˆŒØ×"Ñ" 3°$¸vÐ"ÓSˆŒØ ×*Ñ*¨8¸ÐRVÐ_eÐ*ÓsˆŒØ ×*Ñ*¨8¸ÐRVÐ_eÐ*ÓsˆŒØ"×)Ñ)¨#¸ÀfÐ)ÓZˆŒä+¨H¸sÐY^ÔvˆŒ
ð !×+Ð+ C¨!¥Gˆ	Ø!×(Ñ(¨¸gÐ\aÐ(ÓbˆŒØ ×'Ñ'¨	¸WÐ[`Ð'ÓaˆŽr   c           	     óN  € Vf   / pVP                  ^4      pV P                  WV4      w  rxrš\        P                  ! W‡.^R7      pVP                  w  rÍpV P
                  V P                  ppV P                  V4      P                  WÍVV4      pV P                  V4      P                  WÍVV4      pV P                  V4      pV P                  V4      P                  WÍVV4      pV P                  V4      P                  WÍVV4      pVe¿   VRVR 13,          P                  ^^4      pVRVR 13,          P                  ^^4      p\        VV4      p\        VV4      p\        P                  ! VRR V13,          VP                  ^^4      .^R7      p\        P                  ! VRR V13,          VP                  ^^4      .^R7      p??\        VP!                  WÍVV,          4      4      \        VP!                  WÍVV,          4      4      \        V4      ppp\#        VVVV P
                  V P$                  VR7      pV P'                  V4      pVP)                  WmV,
          .^R7      w  ppWV,          ,           pW*V,          ,           pV P+                  WV4      w  rxpp\        P                  ! W‡.^R7      pV P-                  \.        P0                  ! V P3                  V4      RR7      4      pVVVRVR 13,          ,          ,           pVVVRR V13,          ,          ,           pW3# )Nr   r   )ÚheadsÚpreferred_attentionÚtransformer_optionsÚtanh)Úapproximate)Úsizerè   r   r7   r"   ræ   rç   ré   r’   rê   rë   rì   rí   r“   r+   r   r!   r   rå   rî   Úsplitrï   rñ   ÚFÚgelurð   )r~   rÐ   rÑ   rÒ   Úimage_rotary_embrù   r£   Únorm_hiddenÚnorm_encoderÚgate_msaÚenc_gate_msaÚ	qkv_inputÚbÚsÚ_ÚnÚdré   rê   rë   Úq_imgÚk_imgrî   Úattn_encoderÚattn_hiddenÚgate_ffÚenc_gate_ffÚff_inputÚ	ff_outputs   &&&&&&                       r   r§   ÚCogVideoXBlock.forward
  s  € ØÒ&Ø"$ÐØ/×4Ñ4°QÓ7ˆð =A¿J¹JÀ}ÐmqÓ<rÑ9ˆ 8ô —I’I˜|Ð9¸qÔAˆ	Ø—/‘/‰ˆˆaØ�~‰~˜tŸ}™}ˆ1ˆà�F‰F�9Ó×"Ñ" 1¨¨AÓ.ˆØ�F‰F�9Ó×"Ñ" 1¨¨AÓ.ˆØ�F‰F�9Óˆà�K‰K˜‹N×Ñ  a¨Ó+ˆØ�K‰K˜‹N×Ñ  a¨Ó+ˆð Ò'Ø�a˜Ñ)Ð)Õ*×4Ñ4°Q¸Ó:ˆEØ�a˜Ñ)Ð)Õ*×4Ñ4°Q¸Ó:ˆEÜ$ UÐ,<Ó=ˆEÜ$ UÐ,<Ó=ˆEÜ—	’	˜1˜QÐ 0 Ð 0Ð0Õ1°5·?±?À1ÀaÓ3HÐIÈqÔQˆAÜ—	’	˜1˜QÐ 0 Ð 0Ð0Õ1°5·?±?À1ÀaÓ3HÐIÈqÔQˆAØ�uä*¨1¯9©9°Q¸1¸q½5Ó+AÓBÔD\Ð]^×]fÑ]fÐghÐmnÐqrÕmrÓ]sÓDtô  wOð  PQó  wRˆaˆ1ˆÜ&ØØØØ—.‘.Ø $× 4Ñ 4Ø 3ô
ˆð —=‘= Ó*ˆà$,§N¡N°OÈÕEXÐ3YÐ_` NÓ$aÑ!ˆ�kà%°;Õ(>Õ>ˆØ 5À|Õ8SÕ SÐð ;?¿*¹*À]ÐkoÓ:pÑ7ˆ 7¨Kô —9’9˜lÐ8¸aÔ@ˆØ—K‘K¤§¢ t§|¡|°HÓ'=È6Ô RÓSˆ	à%¨°)¸A¸Ñ?OÐ<OÕ2PÕ(PÕPˆØ 5¸ÀiÐPQÐScÐTcÐScÐPcÕFdÕ8dÕ dÐàÐ3Ð3r   )rî   rå   r   rñ   rð   rç   rê   rè   rï   rí   rì   ræ   ré   rë   )rÚ   NTNNN©NNrÛ   r¹   s   @@r   rã   rã   ï   s   ù‡ € ÷b÷494ô 94r   rã   c                   óP   a a€ ] tR tRt oRV 3R lltRR ltRR ltR tRtVt	V ;t
# )	ÚCogVideoXTransformer3DModeliF  c                ó–  <€ \         S V `  4        VV n        W,          pVV n        Wn        W n        W0n        W@n        VV n        VV n	        VV n
        VV n        \        R/ R VbRVbRVbRVbRV	bRVbRVbRVbRVbR	VbR
VbRVbRVbRV'       * bRVbRVbR\        P                  bRVb V n        VV n        WPn        W`n        VP'                  VVVVR7      V n        \*        P,                  ! 4       V n        VP'                  WwVVR7      V n        W€n        V'       dN   VP'                  WˆVVR7      V n        \*        P,                  ! 4       V n        VP'                  WˆVVR7      V n        MRV n        \*        P:                  ! \=        V
4       Uu. uF  p\?        VVVVRVVVR7      NK  	  up4      V n         VPC                  VRRVVR7      V n"        \G        VVRVVVR7      V n$        Vf   VV,          V,          pMVV,          V,          V,          pVP'                  VVVVR7      V n%        VV n&        VV n'        VV n(        R# u upi )ro   rp   r   r   r€   ri   rr   rq   rs   rt   ru   rP   rQ   rv   rw   r
   r	   r�   rj   NrÚ   )r   ræ   rç   rÊ   rÁ   r
   r	   r�   TrÀ   )rÊ   r   rÁ   r
   r	   r�   © ))rm   rn   r	   r   Únum_attention_headsÚattention_head_dimr   Úout_channelsro   rp   ru   Ú use_rotary_positional_embeddingsre   r   r   Úpatch_embedÚtime_proj_dimÚtime_proj_flipÚtime_proj_shiftrz   Útime_embedding_linear_1rÄ   rÅ   Útime_embedding_actÚtime_embedding_linear_2Úofs_proj_dimÚofs_embedding_linear_1Úofs_embedding_actÚofs_embedding_linear_2Ú
ModuleListÚrangerã   ÚblocksrÈ   Ú
norm_finalrÝ   Únorm_outÚproj_outrP   rQ   rt   )!r~   r  r  r   r  r:   Ú
freq_shiftÚtime_embed_dimÚofs_embed_dimÚtext_embed_dimÚ
num_layersÚdropoutÚattention_biasrr   rq   rs   ro   rp   rt   ru   rP   rQ   r  rw   Ú
patch_biasÚimage_modelr
   r	   r�   r   r  Ú
output_dimr‚   s!   &&&&&&&&&&&&&&&&&&&&&&&&&&&&&   €r   rn   Ú$CogVideoXTransformer3DModel.__init__G  sÕ  ø€ ô< 	‰ÑÔØˆŒ
Ø!Õ6ˆØˆŒØ#6Ô Ø"4ÔØ&ÔØ(ÔØ$ˆŒØ(ˆÔØ#6ˆÔ Ø0PˆÔ-ô /ò 
Ù!ð
á%ð
ñ $ð
ñ ð	
ñ
 $ð
ñ ð
ñ &ð
ñ (ð
ñ (ð
ñ (Bð
ñ !4ð
ñ )Dð
ñ *Fð
ð +KÕ&Jð
ñ /Pð
ñ  ð!
ô  "'§¢ð!
ñ  <Fð!
ˆÔð( !ˆÔØ-ÔØ)ÔØ'1×'8Ñ'8¸¸nÐU[ÐchÐ'8Ó'iˆÔ$Ü"$§'¢'£)ˆÔØ'1×'8Ñ'8¸Ð`fÐnsÐ'8Ó'tˆÔ$ð *ÔßØ*4×*;Ñ*;¸MÐagÐotÐ*;Ó*uˆDÔ'Ü%'§W¢W£YˆDÔ"Ø*4×*;Ñ*;¸MÐagÐotÐ*;Ó*uˆDÕ'à*.ˆDÔ'ô —m’mô ˜:Ô&ó
%
ñ '�ô ØØ-Ø+Ø'ØØ U°z÷ñ 'ñ
%
ó 
ˆŒð %×.Ñ.¨s¸ÐQUÐ^dÐlqÐ.ÓrˆŒô .Ø#¨°$Ø °:ô
ˆŒð
 ÒØ# jÕ0°<Õ?‰Jà# jÕ0°<Õ?À,ÕNˆJà"×)Ñ)¨#¨zÀ&ÐPUÐ)ÓVˆŒà+FˆÔ(Ø,HˆÔ)Ø*DˆÖ'ùò;
%
s   ÆIc                ó  € Vf   / p\         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  ! WW4V3/ VB # r±   )	ÚcomfyÚpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)r~   r&   ÚtimestepÚcontextÚofsrù   Úkwargss   &&&&&&,r   r§   Ú#CogVideoXTransformer3DModel.forward·  sŠ   € ØÒ&Ø"$ÐÜ×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4´U×5LÑ5L×5WÑ5W×5gÑ5gÐi|Ó}ó
÷ ‰'ð	ð ˜wÐ-@ñ	Lð EKñ	Lð 	Lr   c           
     ó¼  € Vf   / pVP                   w  rxršpV P                  e   V P                  M^p\        P                  P                  P                  WV P                  V P                  34      pVP                  ^ ^^^^4      pVP                   w  r}rŽp\        W P                  V P                  V P                  4      pVP                  VP                  R7      pV P                  V P                  V P!                  V4      4      4      pV P"                  e„   Ve€   \        W@P$                  V P                  V P                  4      pVP                  VP                  R7      pV P'                  V P)                  V P#                  V4      4      4      pVV,           pV P+                  W14      pVP                   ^,          pVRR V13,          pVRVR 13,          pR pV P,                  '       dg   WàP                  ,          pWðP                  ,          pV P                  f   TpMWÐP                  ,          pV P/                  VVVVP0                  R7      p\3        V P4                  4       F  w  ppV! VVVVVR7      w  ppK  	  V P7                  V4      pV P9                  VVR7      pV P;                  V4      pV P                  pV P                  pVf]   VP=                  W}VV,          VV,          RVV4      pVP                  ^ ^^^^^^4      P?                  ^^4      P?                  ^^4      pM‚VP=                  W}V,           ^,
          V,          VV,          VV,          RVVV4      pVP                  ^ ^^^^^^^4      P?                  ^^4      P?                  ^^4      P?                  ^^4      pVP                  ^ ^^^^4      RRR V	1R V
1R V13,          pV# )Nr�   r   rE   )rÐ   rÑ   rÒ   r   rù   )rÒ   r   ) r"   rp   r:  ÚldmÚ
common_ditÚpad_to_patch_sizero   r”   rA   r  r  r   r    r	   r#  r"  r!  r%  r$  r'  r&  r  r  Ú_get_rotary_embr
   Ú	enumerater*  r+  r,  r-  r!   r%   )r~   r&   rC  rD  rE  rù   rF  rš   rœ   ÚtÚhÚwr¡   r›   r�   rž   Út_embÚembÚofs_embrÐ   r£   rÑ   r   r‰   rŠ   Ú	post_timeÚiÚblockr    Úoutputs   &&&&&&,                       r   r>  Ú$CogVideoXTransformer3DModel._forwardÀ  s´  € ØÒ&Ø"$Ðà()¯©Ñ%ˆ
˜a Að $(×#4Ñ#4Ò#@ˆd×ÒÀaˆÜ�I‰I× Ñ ×2Ñ2°1¸D¿O¹OÈTÏ_É_Ð6]Ó^ˆð �I‰I�a˜˜A˜q !Ó$ˆØ:;¿'¹'Ñ7ˆ
 °%ô ' x×1CÑ1CÀT×EXÑEXÐZ^×ZnÑZnÓoˆØ—‘˜qŸw™w�Ó'ˆØ×*Ñ*¨4×+BÑ+BÀ4×C_ÑC_Ð`eÓCfÓ+gÓhˆà×&Ñ&Ò2°s²Ü,¨S×2CÑ2CÀT×EXÑEXÐZ^×ZnÑZnÓoˆGØ—j‘j q§w¡w�jÓ/ˆGØ×1Ñ1°$×2HÑ2HÈ×IdÑIdÐelÓImÓ2nÓoˆGØ˜•-ˆCð ×(Ñ(¨Ó4ˆà!Ÿ-™-¨Õ*ˆØ -¨aÐ1A°/Ð1AÐ.AÕ BÐØ% a¨Ñ)9Ð&9Õ:ˆð  ÐØ×0×0Ð0Ø &¯/©/Õ 9ÐØ$¯©Õ7ÐØ× Ñ Ò(Ø&‘	à&×*;Ñ*;Õ;�	Ø#×3Ñ3Ð4EÐGWÐYbÐkl×ksÑksÐ3ÓtÐô " $§+¡+Ö.‰HˆAˆuÙ38Ø+Ø&;ØØ!1Ø$7ô4Ñ0ˆMÒ0ñ /ð Ÿ™¨Ó6ˆð Ÿ™ m¸#˜Ó>ˆØŸ™ mÓ4ˆð �O‰OˆØ×ÑˆàŠ;Ø"×*Ñ*¨:À6ÈQÅ;ÐPUÐYZÕPZÐ\^Ð`aÐcdÓeˆFØ—^‘^ A q¨!¨Q°°1°aÓ8×@Ñ@ÀÀAÓF×NÑNÈqÐRSÓT‰Fà"×*Ñ*Ø¨#Õ-°Õ1°cÕ9¸6ÀQ½;ÈÐQRÍ
ÐTVÐX[Ð]^Ð`aóˆFð —^‘^ A q¨!¨Q°°1°a¸Ó;×CÑCÀAÀqÓI×QÑQÐRSÐUVÓW×_Ñ_Ð`aÐcdÓeˆFð —‘  1 a¨¨AÓ.¨q°!°R°a°R¸¸!¸¸R¸a¸RÐ/?Õ@ˆØˆr   c                óB  € V P                   pV^,          pV^,          ^,          pV^,          ^,          pV P                  eØ   V P                  P                  V P                  ,          p	V P                  P
                  V P                  ,          p
\        W‘4      p\        W¢4      p\        P                  ! W´\        P                  R7      p\        P                  ! WÄ\        P                  R7      p\        P                  ! W4\        P                  R7      pM¦\        P                  ! ^ V^,
          W\        P                  R7      V P                  ,          p\        P                  ! ^ V^,
          W$\        P                  R7      V P                  ,          p\        P                  ! W4\        P                  R7      p\        Wo4      p\        W}4      p\        WŽ4      pVw  ppVw  ppVw  ppVRV VRV ppVRV VRV ppVRV VRV ppVR,          P                  RWR4      pVR,          P                  RWR4      pVR,          P                  VRVR4      pVR,          P                  VRVR4      pVR,          P                  W1RR4      pVR,          P                  W1RR4      p\        P                  ! VVV.RR7      P                  W1,          V,          R4      p\        P                  ! VVV.RR7      P                  W1,          V,          R4      pVV3# )u?  Compute CogVideoX 3D rotary positional embeddings.

For CogVideoX 1.5 (patch_size_t != None): uses "slice" mode â€” grid positions
are integer arange computed at max_size, then sliced to actual size.
For CogVideoX 1.0 (patch_size_t == None): uses "linspace" mode with crop coords
scaled by spatial_interpolation_scale.
Nrj   r   )r   NNr   r   )Nr   Nr   r   )r  rp   r  rq   ro   rr   Úmaxr   r   r   ÚlinspacerP   r   rL   r7   r!   )r~   rO  rP  rN  r
   r
  Údim_tÚdim_hÚdim_wÚbase_hÚbase_wÚmax_hÚmax_wrS   rR   rT   Úfreqs_tÚfreqs_hÚfreqs_wÚt_cosÚt_sinÚh_cosÚh_sinÚw_cosÚw_sinr   r   s   &&&&&                      r   rL  Ú+CogVideoXTransformer3DModel._get_rotary_emb  s¢  € ð ×#Ñ#ˆØ�Q•ˆØ�Q•˜•
ˆØ�Q•˜•
ˆà×ÑÒ(ð ×%Ñ%×3Ñ3°t·±ÕFˆFØ×%Ñ%×2Ñ2°d·o±oÕEˆFÜ˜“NˆEÜ˜“NˆEä—\’\ %¼e¿m¹mÔLˆFÜ—\’\ %¼e¿m¹mÔLˆFÜ—\’\ !¼%¿-¹-ÔH‰Fô —^’^ A q¨1¥u¨aÄeÇmÁmÔTÐW[×WwÑWwÕwˆFÜ—^’^ A q¨1¥u¨aÄeÇmÁmÔTÐW[×WwÑWwÕwˆFÜ—\’\ !¼%¿-¹-ÔHˆFä*¨5Ó9ˆÜ*¨5Ó9ˆÜ*¨5Ó9ˆà‰ˆˆuØ‰ˆˆuØ‰ˆˆuð ˜R˜a�y %¨¨ )ˆuˆØ˜R˜a�y %¨¨ )ˆuˆØ˜R˜a�y %¨¨ )ˆuˆð Ð&Õ'×.Ñ.¨r°1¸Ó<ˆØÐ&Õ'×.Ñ.¨r°1¸Ó<ˆØÐ&Õ'×.Ñ.¨q°"°a¸Ó<ˆØÐ&Õ'×.Ñ.¨q°"°a¸Ó<ˆØÐ&Õ'×.Ñ.¨q°R¸Ó<ˆØÐ&Õ'×.Ñ.¨q°R¸Ó<ˆä�iŠi˜  uÐ-°2Ô6×>Ñ>¸q½uÀq½yÈ"ÓMˆÜ�iŠi˜  uÐ-°2Ô6×>Ñ>¸q½uÀq½yÈ"ÓMˆØ�SˆzÐr   )r  r*  r   r	   r   ru   r+  r,  r  r&  r%  r'  r$  r  r  ro   rp   r-  rP   rt   rQ   r"  r!  r#  r  r  r   r  )é   é@   r©   r©   Tr   i   Nrª   rm  g        Tr«   r¬   r­   r-   Nr®   r¯   r°   r   FFTNNNNr  )r²   r³   r´   rµ   rn   r§   r>  rL  r¶   r·   r¸   r¹   s   @@r   r  r  F  s$   ù‡ € ÷nEô`LôJ÷X5ò 5r   r  )rc   )Tr   r3   i'  )r   r   Nr±   )r5   r   Útorch.nnrÄ   Útorch.nn.functionalÚ
functionalrþ   Úcomfy.ldm.modules.attentionr   r   r   Úcomfy.patcher_extensionr:  Úcomfy.ldm.common_ditr   r+   rA   r]   rI   rJ   ÚModulere   r¼   rÝ   rã   r  r  r   r   Ú<module>rv     s›   ðó
 Û Ý ß Ð ç eÑ eÛ Û ô
òCô(	ôô.-ôôh˜"Ÿ)™)ô hôV\˜RŸY™Yô \ô˜BŸI™Iô ôT4�R—Y‘Yô T4ôn{ "§)¡)ö {r   