+
    ƒ°ñj�-  ã                   ó¼   € R t ^ RIt^ RIHt ^ RIHtHtHt ^ RIH	t	 ^ RI
HtHtHtHt ^ RIt^ RIt ! R R]P$                  4      t ! R R]4      t ! R	 R
]4      tR# )a�  
CausalWanModel: Wan 2.1 backbone with KV-cached causal self-attention for
autoregressive (frame-by-frame) video generation via Causal Forcing.

Weight-compatible with the standard WanModel -- same layer names, same shapes.
The difference is purely in the forward pass: this model processes one temporal
block at a time and maintains a KV cache across blocks.

Reference: https://github.com/thu-ml/Causal-Forcing
N)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention)Úapply_rope1)Úsinusoidal_embedding_1dÚrepeat_eÚWanModelÚWanAttentionBlockc                   óP   a a€ ] tR t^t oRtRRR/ 3V 3R lltR/ 3R ltRtVtV ;t	# )	ÚCausalWanSelfAttentionzBSelf-attention with KV cache support for autoregressive inference.Tç�íµ ÷Æ°>c                ó¶  <€ W,          ^ 8X  g   Q h\         S
V `  4        \        4       V n        Wn        W n        W,          V n        W@n        WPn        VP                  R4      pVP                  R4      pVP                  R4      p	VP                  WW‰R7      V n        VP                  WW‰R7      V n        VP                  WW‰R7      V n        VP                  WW‰R7      V n        V'       d   VP                  WRW‰R7      M\         P"                  ! 4       V n        V'       d   VP                  WRW‰R7      V n        R# \         P"                  ! 4       V n        R# )é    Ú
operationsÚdeviceÚdtype©r   r   T)ÚepsÚelementwise_affiner   r   N)ÚsuperÚ__init__r   Úcomfy_attentionÚdimÚ	num_headsÚhead_dimÚqk_normr   ÚgetÚLinearÚqÚkÚvÚoÚRMSNormÚnnÚIdentityÚnorm_qÚnorm_k)Úselfr   r   Úwindow_sizer   r   Úoperation_settingsÚopsr   r   Ú	__class__s   &&&&&&&   €Ú:C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\wan\ar_model.pyr   ÚCausalWanSelfAttention.__init__   s-  ø€ à� !Ô#Ð#Ð#Ü‰ÑÔÜ-Ó/ˆÔØŒØ"ŒØÕ(ˆŒØŒØŒà ×$Ñ$ \Ó2ˆØ#×'Ñ'¨Ó1ˆØ"×&Ñ& wÓ/ˆà—‘˜C¨V�ÓAˆŒØ—‘˜C¨V�ÓAˆŒØ—‘˜C¨V�ÓAˆŒØ—‘˜C¨V�ÓAˆŒßho�c—k‘k #À4ÐPV�kÔdÔuw÷  vAò  vAó  vCˆŒßho�c—k‘k #À4ÐPV�kÓdˆŽÔuw÷  vAò  vAó  vCˆŽó    Nc           	     ó~  € . VP                   R ,          OV P                  NV P                  N5w  rVrx\        V P	                  V P                  V4      4      P                  WVWx4      V4      p	\        V P                  V P                  V4      4      P                  WVWx4      V4      p
V P                  V4      P                  WVWx4      pVeZ   VR,          pWÆ,           pW£R,          RWÍ13&   W³R,          RWÍ13&   WÓR&   VR,          RRV13,          p
VR,          RRV13,          p\        V	P                  WVWx,          4      4      p	\        V
P                  VRWx,          4      4      p
\        VP                  VRWx,          4      4      p\        WšW°P                  V P                  VR7      pV P                  V4      pV# ):Né   NNÚendr   ºNNNr    ©ÚheadsÚpreferred_attentionÚtransformer_optionséÿÿÿÿ)Úshaper   r   r   r%   r   Úviewr&   r   r    r   Úreshaper   r   r!   )r'   ÚxÚfreqsÚkv_cacher6   ÚbÚsÚnÚdr   r   r    r1   Únew_ends   &&&&&         r,   ÚforwardÚCausalWanSelfAttention.forward4   s�  € Ø@�a—g‘g˜b•kÐ@ 4§>¡>Ð@°4·=±=Ñ@‰
ˆˆaä˜Ÿ™ D§F¡F¨1£IÓ.×3Ñ3°A¸!Ó?ÀÓGˆÜ˜Ÿ™ D§F¡F¨1£IÓ.×3Ñ3°A¸!Ó?ÀÓGˆØ�F‰F�1‹I�N‰N˜1 Ó&ˆàÒØ˜5•/ˆCØ•gˆGð -.�S�M˜!˜S˜[˜.Ñ)Ø,-�S�M˜!˜S˜[˜.Ñ)Ø%�U‰Oà˜•˜a  ' ˜kÕ*ˆAØ˜•˜a  ' ˜kÕ*ˆAä$ Q§V¡V¨A°!µ%Ó%8Ó9ˆÜ$ Q§Y¡Y¨q°"°aµeÓ%<Ó=ˆÜ$ Q§Y¡Y¨q°"°aµeÓ%<Ó=ˆÜØ�!Ÿ>™>Ø $× 4Ñ 4Ø 3ô
ˆð �F‰F�1‹IˆØˆr.   )r   r   r   r   r   r&   r%   r   r!   r   r   r    ©r7   r7   ©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   rC   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r+   Ú__classdict__s   @@r,   r   r      s)   ù‡ € ÙLà3;ÀTØ¨b÷Cð, *.À2÷ ô r.   r   c                   óV   a a€ ] tR t^Tt oRtR
RRR/ 3V 3R lltRRR/ 3R ltR	tVtV ;t	# )ÚCausalWanAttentionBlockzLTransformer block with KV-cached self-attention and cross-attention caching.TFr   c
                óX   <€ \         S
V `  WW4WVWxV	R 7	       \        W$WVVV	R 7      V n        R# ))r)   N)r   r   r   Ú	self_attn)r'   Úcross_attn_typer   Úffn_dimr   r(   r   Úcross_attn_normr   r)   r+   s   &&&&&&&&&&€r,   r   Ú CausalWanAttentionBlock.__init__W   s<   ø€ ô 	‰Ñ˜¨wØ$¨Ø,>ð 	ô 	@ô 0Ø˜K°#Ø1ô3ˆŽr.   i  Nc	                ó¦  € VP                   ^8  dZ   \        P                  P                  V P                  VP
                  VP                  R7      V,           P                  ^^R7      pMe\        P                  P                  V P                  VP
                  VP                  R7      P                  ^ 4      V,           P                  ^4      pVP                  4       pV P                  \        P                  ! \        V^ ,          V4      V P                  V4      ^\        V^,          V4      ,           4      W6VR7      p	\        P                  ! W\        V^,          V4      4      p?	VeÕ   VP!                  R4      '       d¾   \#        V P$                  P'                  V P$                  P)                  V P+                  V4      4      4      4      p
\-        V
\#        VR,          4      \#        VR,          4      V P.                  V P$                  P0                  VR7      pWP$                  P3                  V4      ,           pM‡WP%                  V P+                  V4      WEVR7      ,           pVe[   V P$                  P5                  V P$                  P7                  V4      4      VR&   V P$                  P9                  V4      VR&   R	VR&   V P;                  \        P                  ! \        V^,          V4      V P=                  V4      ^\        V^,          V4      ,           4      4      p	\        P                  ! W\        V^,          V4      4      pV# )
é   )r   r   ©r   )r=   r6   Úis_initr   r    r3   )Úcontext_img_lenr6   T)ÚndimÚcomfyÚmodel_managementÚcast_toÚ
modulationr   r   ÚchunkÚ	unsqueezeÚunbindÚ
contiguousrT   ÚtorchÚaddcmulr   Únorm1r   r   Ú
cross_attnr%   r   Únorm3r   r   r   r!   r&   r   r    ÚffnÚnorm2)r'   r;   Úer<   Úcontextr]   r=   Úcrossattn_cacher6   Úyr   Úx_cas   &&&&&&&&&   r,   rC   ÚCausalWanAttentionBlock.forwarda   sŸ  € à�6‰6�AŒ:Ü×'Ñ'×/Ñ/°·±ÀqÇwÁwÐWX×W_ÑW_Ð/Ó`ÐcdÕd×kÑkÐlmÐstÐkÓu‰Aä×'Ñ'×/Ñ/°·±ÀqÇwÁwÐWX×W_ÑW_Ð/Ó`×jÑjÐklÓmÐpqÕq×yÑyÐz{Ó|ˆAð �L‰L‹NˆØ�N‰NÜ�MŠMœ( 1 Q¥4¨Ó+¨T¯Z©Z¸«]¸AÄÈÈ1ÍÈqÓ@QÕ<QÓRØÐ:Mð ó Oˆô �MŠM˜!¤¨¨1­¨qÓ 1Ó2ˆØð Ò&¨?×+>Ñ+>¸y×+IÒ+IÜ(¨¯©×)?Ñ)?ÀÇÁ×@QÑ@QÐRV×R\ÑR\Ð]^ÓR_Ó@`Ó)aÓbˆAÜ&ØÔ+¨O¸CÕ,@ÓAÔC[Ð\kÐloÕ\pÓCqØ—n‘n¸$¿/¹/×:YÑ:Yð  pCôDˆDð —O‘O×%Ñ% dÓ+Õ+‰Aà—O‘O D§J¡J¨q£M°7ð  rE�Oó  Fõ  FˆAØÒ*Ø'+§¡×'=Ñ'=¸d¿o¹o×>OÑ>OÐPWÓ>XÓ'Y� Ñ$Ø'+§¡×'8Ñ'8¸Ó'A� Ñ$Ø-1� 	Ñ*ð �H‰H”U—]’]¤8¨A¨a­D°!Ó#4°d·j±jÀ³mÀQÌÐRSÐTUÕRVÐXYÓIZÕEZÓ[Ó\ˆÜ�MŠM˜!¤¨¨1­¨qÓ 1Ó2ˆØˆr.   )rT   rE   rF   rO   s   @@r,   rR   rR   T   s2   ù‡ € ÙVð &¨tÀUØ¨b÷3ð =@Ø¨tÈ÷ ô  r.   rR   c                   ó‚   a a€ ] tR t^„t oRtRV 3R lltRR ltR tR tR t	R t
]R	 4       tRR/ 3V 3R
 lltRtVtV ;t# )ÚCausalWanModelz¿
Wan 2.1 diffusion backbone with causal KV-cache support.

Same weight structure as WanModel -- loads identical state dicts.
Adds forward_block() for frame-by-frame autoregressive inference.
Nc                óJ   <€ \         SV `  WVWEWgW‰V
W¼VWïV\        VVVR 7       R# ))Ú
model_typeÚ
patch_sizeÚtext_lenÚin_dimr   rV   Úfreq_dimÚtext_dimÚout_dimr   Ú
num_layersr(   r   rW   r   Úimage_modelÚwan_attn_block_classr   r   r   N)r   r   rR   )r'   rw   rx   ry   rz   r   rV   r{   r|   r}   r   r~   r(   r   rW   r   r   r   r   r   r+   s   &&&&&&&&&&&&&&&&&&&&€r,   r   ÚCausalWanModel.__init__Œ   s:   ø€ ô( 	‰ÑØ!À8Ø¨GØ¸)Ø!ÀGØ+À+Ü!8Ø °:ð 	ö 	?r.   c                ó¸  € \         P                  P                  P                  WP                  4      pVP
                  w  r‰r«pV P                  VP                  4       4      P                  VP                  4      pVP
                  R,          pVP                  ^4      P                  ^^4      pV P                  \        V P                  VP                  4       4      P                  VP                  R7      4      pVP                  VP
                  ^ ,          RVP
                  R,          4      pV P!                  V4      P#                  ^^V P$                  34      pV P'                  V4      pRpVeM   V P(                  e?   V P)                  V4      p\*        P,                  ! VV.^R7      pVP
                  R	,          pV P/                  W«WÄVP0                  VP                  R7      p\3        V P4                  4       F"  w  ppV! WVVVVV,          VV,          R7      pK$  	  V P7                  W4      pV P9                  W4      pVRRRV
1RV1RV13,          # )
a   
Forward one temporal block for autoregressive inference.

Args:
    x: [B, C, block_frames, H, W] input latent for the current block
    timestep: [B, block_frames] per-frame timesteps
    context: [B, L, text_dim] raw text embeddings (pre-text_embedding)
    start_frame: temporal frame index for RoPE offset
    kv_caches: list of per-layer KV cache dicts
    crossattn_caches: list of per-layer cross-attention cache dicts
    clip_fea: optional CLIP features for I2V

Returns:
    flow_pred: [B, C_out, block_frames, H, W] flow prediction
:r0   NN)r   Nr[   )Út_startr   r   )rn   r<   ro   r]   r=   rp   r2   r7   éþÿÿÿ)r_   ÚldmÚ
common_ditÚpad_to_patch_sizerx   r8   Úpatch_embeddingÚfloatÚtor   ÚflattenÚ	transposeÚtime_embeddingr   r{   r:   Útime_projectionÚ	unflattenr   Útext_embeddingÚimg_embrg   ÚconcatÚrope_encoder   Ú	enumerateÚblocksÚheadÚ
unpatchify)r'   r;   Útimestepro   Ústart_frameÚ	kv_cachesÚcrossattn_cachesÚclip_feaÚbsÚcÚtÚhÚwÚ
grid_sizesrn   Úe0r]   Úcontext_clipr<   ÚiÚblocks   &&&&&&&&             r,   Úforward_blockÚCausalWanModel.forward_block©   sý  € ô" �I‰I× Ñ ×2Ñ2°1·o±oÓFˆØŸ™‰ˆˆq�Qà× Ñ  §¡£Ó+×.Ñ.¨q¯w©wÓ7ˆØ—W‘W˜R•[ˆ
Ø�I‰I�a‹L×"Ñ" 1 aÓ(ˆð ×ÑÜ# D§M¡M°8×3CÑ3CÓ3EÓF×IÑIÐPQ×PWÑPWÐIÓXóZˆà�I‰I�h—n‘n QÕ'¨¨Q¯W©W°R­[Ó9ˆØ×!Ñ! !Ó$×.Ñ.¨q°1°d·h±h°-Ó@ˆð ×%Ñ% gÓ.ˆàˆØÒ D§L¡LÒ$<ØŸ<™<¨Ó1ˆLÜ—l’l L°'Ð#:ÀÔBˆGØ&Ÿn™n¨RÕ0ˆOð × Ñ   qÀaÇhÁhÐVW×V]ÑV]Ð Ó^ˆô " $§+¡+Ö.‰HˆAˆuÙ�a U°GØ&5Ø(¨�|Ø&6°qÕ&9ô;ŠAñ /ð �I‰I�a‹Oˆð �O‰O˜AÓ*ˆØ��A�r˜�r˜2˜A˜2˜r ˜rÐ!Õ"Ð"r.   c                ó   € . p\        V P                  4       Fr  pVP                  R\        P                  ! WV P
                  V P                  W4R7      R\        P                  ! WV P
                  V P                  W4R7      R^ /4       Kt  	  V# )z&Create fresh KV caches for all layers.r   r   r    r1   )Úranger~   Úappendrg   Úzerosr   r   )r'   Ú
batch_sizeÚmax_seq_lenr   r   ÚcachesÚ_s   &&&&&  r,   Úinit_kv_cachesÚCausalWanModel.init_kv_cachesá   sq   € àˆÜ�t—‘Ö'ˆAØ�M‰MØ”U—[’[ ¸$¿.¹.È$Ï-É-Ð`fÔtØ”U—[’[ ¸$¿.¹.È$Ï-É-Ð`fÔtØ�qðö ñ (ð ˆr.   c                óh   € . p\        V P                  4       F  pVP                  RR/4       K  	  V# )z3Create fresh cross-attention caches for all layers.r\   F)rª   r~   r«   )r'   r­   r   r   r¯   r°   s   &&&&  r,   Úinit_crossattn_cachesÚ$CausalWanModel.init_crossattn_cachesì   s0   € àˆÜ�t—‘Ö'ˆAØ�M‰M˜9 eÐ,Ö-ñ (àˆr.   c                ó"   € V F  p^ VR&   K
  	  R# )z2Reset KV caches to empty (reuse allocated memory).r1   N© )r'   rš   Úcaches   && r,   Úreset_kv_cachesÚCausalWanModel.reset_kv_cachesó   s   € ãˆEØˆE�%‹Ló r.   c                ó"   € V F  pRVR&   K
  	  R# )zReset cross-attention caches.Fr\   Nr·   )r'   r›   r¸   s   && r,   Úreset_crossattn_cachesÚ%CausalWanModel.reset_crossattn_cachesø   s   € ã%ˆEØ$ˆE�)Óó &r.   c                ó<   € V P                   V P                  ,          # ©N)r   r   )r'   s   &r,   r   ÚCausalWanModel.head_dimý   s   € à�x‰x˜4Ÿ>™>Õ)Ð)r.   c           
     óB  <€ VP                  R 4      pVes   VP                  ^ ,          p	VP                  ^,          p
VP                  ^4      P                  Wš4      pV P	                  WVVR,          VR,          VR,          VR7      # \
        SV `  ! WV3RVRVRV/VB # )Úar_stater™   rš   r›   )r;   r˜   ro   r™   rš   r›   rœ   rœ   Útime_dim_concatr6   )r   r8   rd   Úexpandr§   r   rC   )r'   r;   r˜   ro   rœ   rÃ   r6   ÚkwargsrÂ   r�   Úblock_framesÚt_per_framer+   s   &&&&&&&,    €r,   rC   ÚCausalWanModel.forward  sÃ   ø€ Ø&×*Ñ*¨:Ó6ˆØÒØ—‘˜•ˆBØŸ7™7 1�:ˆLØ"×,Ñ,¨QÓ/×6Ñ6°rÓHˆKØ×%Ñ%Ø°7Ø$ ]Õ3Ø" ;Õ/Ø!)Ð*<Õ!=Ø!ð &ó ð ô ‰wŠ˜q¨Gñ R¸hð RØ/>ðRà3FðRàJPñRð 	Rr.   r·   )Út2v)é   r0   r0   i   é   i   i    é   i   rË   rË   é    rE   TTr   NNNNr¿   )rG   rH   rI   rJ   rK   r   r§   r±   r´   r¹   r¼   Úpropertyr   rC   rL   rM   rN   rO   s   @@r,   ru   ru   „   sS   ù‡ € ñ÷?ô:6#òp	òòò
%ð
 ñ*ó ð*ð 6:È4Ðeg÷ R÷ Rr.   ru   )rK   rg   Útorch.nnr#   Úcomfy.ldm.modules.attentionr   r   r   Úcomfy.ldm.flux.mathr   Úcomfy.ldm.wan.modelr   r   r   r	   Úcomfy.ldm.common_ditr_   Úcomfy.model_managementÚModuler   rR   ru   r·   r.   r,   Ú<module>rÖ      s\   ðñ	ó Ý ç eÑ eÝ +÷ó ó Û ô6˜RŸY™Yô 6ôr-Ð/ô -ô`MR�Xö MRr.   