+
    ƒ°ñjð  ã                   ó¾  € ^ RI HtHt ^ RIHt ^ RIt^ RIt^ RIt^ RIH	t	H
t
Ht ^ RIHt ^ RIt^ RIt^ RIHt ^ RIt^ RIt^ RIHtHt ^ RIt^ RIt^ RIt^ RIt^RIHtHt ]P<                  ! ]4      t R	 t! ! R
 R]"]4      t# ! R R]"]4      t$R5R R llt% ! R R]PL                  4      t' ! R R]PL                  4      t( ! R R]PL                  4      t) ! R R]PL                  4      t* ! R R]PL                  4      t+ ! R R]PL                  4      t, ! R R]PL                  4      t- ! R R]PL                  4      t.R  t/R! t0 ! R" R#4      t1R6R$ lt2 ! R% R&]PL                  4      t3^t4^	t5 ! R' R(]PL                  4      t6R) t7R/ 3R* lt8R+ t9]Pt                  ! ^R,7      R6R- l4       t;R. t<R/ t=R0 t> ! R1 R2]P"                  PL                  ]4      t? ! R3 R4]?4      t@R# )7é    )ÚABCÚabstractmethod)ÚEnumN)ÚDictÚOptionalÚTuple)Ú	rearrange)Únn)ÚAttentionTensorContainerÚComfyAttention)ÚSymmetricPatchifierÚlatent_to_pixel_coordsc                 ód   € \         P                  ! V 4      \         P                  ! V4      ,          # ©N)ÚnpÚlog)ÚxÚbases   &&Ú>C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\lightricks\model.pyÚ	_log_baser      s   € Ü�6Š6�!‹9”r—v’v˜d“|Õ#Ð#ó    c                   ó@   a € ] tR t^t o RtRtRt]RR l4       tRt	V t
R# )ÚLTXRopeTypeÚinterleavedÚsplitÚ	rope_typeNc                óf   € Vf   V P                   pV ! VP                  V P                  V4      4      # r   )ÚINTERLEAVEDÚgetÚKEY©ÚclsÚkwargsÚdefaults   &&&r   Ú	from_dictÚLTXRopeType.from_dict!   s*   € àŠ?Ø—o‘oˆGÙ�6—:‘:˜cŸg™g wÓ/Ó0Ð0r   © r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   ÚSPLITr    Úclassmethodr%   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r   r      s&   ø‡ € Ø€KØ€Eà
€Càó1ó ö1r   r   c                   ó@   a € ] tR t^(t o RtRtRt]RR l4       tRt	V t
R# )ÚLTXFrequenciesPrecisionÚfloat32Úfloat64Úfrequencies_precisionNc                óf   € Vf   V P                   pV ! VP                  V P                  V4      4      # r   )ÚFLOAT32r   r    r!   s   &&&r   r%   Ú!LTXFrequenciesPrecision.from_dict.   s*   € àŠ?Ø—k‘kˆGÙ�6—:‘:˜cŸg™g wÓ/Ó0Ð0r   r'   r   )r(   r)   r*   r+   r8   ÚFLOAT64r    r-   r%   r.   r/   r0   s   @r   r3   r3   (   s&   ø‡ € Ø€GØ€Gà
!€Càó1ó ö1r   r3   c                ót   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        R\        /# )é   Ú	timestepsÚembedding_dimÚflip_sin_to_cosÚdownscale_freq_shiftÚscaleÚ
max_period)ÚtorchÚTensorÚintÚboolÚfloat)Úformats   "r   Ú__annotate__rI   5   sH   € ÷ 1ñ 1Ü�|‰|ð1äð1ô ð1ô  ð	1ô
 ð1ô ñ1r   c                óþ  € \        V P                  4      ^8X  g   Q R4       hV^,          p\        P                  ! V4      ) \        P
                  ! ^ V\        P                  V P                  R7      ,          pWvV,
          ,          p\        P                  ! V4      pV R,          P                  4       VR,          ,          pWH,          p\        P                  ! \        P                  ! V4      \        P                  ! V4      .RR7      pV'       d1   \        P                  ! VRVR13,          VRRV13,          .RR7      pV^,          ^8X  d+   \        P                  P                  P                  VR	4      pV# )
aâ  
This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.

Args
    timesteps (torch.Tensor):
        a 1-D Tensor of N indices, one per batch element. These may be fractional.
    embedding_dim (int):
        the dimension of the output.
    flip_sin_to_cos (bool):
        Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False)
    downscale_freq_shift (float):
        Controls the delta between frequencies between dimensions
    scale (float):
        Scaling factor applied to the embeddings.
    max_period (int):
        Controls the maximum frequency of the embeddings
Returns
    torch.Tensor: an [N x dim] Tensor of positional embeddings.
zTimesteps should be a 1d-array)ÚstartÚendÚdtypeÚdeviceºNNNN©Údim)rO   N)NrO   éÿÿÿÿ)r   é   r   r   )ÚlenÚshapeÚmathr   rC   Úaranger4   rN   ÚexprG   ÚcatÚsinÚcosr
   Ú
functionalÚpad)	r=   r>   r?   r@   rA   rB   Úhalf_dimÚexponentÚembs	   &&&&&&   r   Úget_timestep_embeddingra   5   s(  € ô6 ˆy�‰Ó 1Ô$ÐFÐ&FÓFÐ$à Õ!€HÜ—’˜Ó$Ð$¤u§|¢|¸!ÀÔQV×Q^ÑQ^Ðgp×gwÑgwÔ'xÕx€HØÐ&:Õ:Õ;€Hä
�)Š)�HÓ
€CØ
�GÕ
×
"Ñ
"Ó
$ s¨7¥|Õ
3€Cð �+€Cô �)Š)”U—Y’Y˜s“^¤U§Y¢Y¨s£^Ð4¸"Ô
=€C÷ Ü�iŠi˜˜Q ¡	˜\Õ*¨C°°9°H°9°Õ,=Ð>ÀBÔGˆð �qÕ˜AÔÜ�h‰h×!Ñ!×%Ñ% c¨<Ó8ˆØ€Jr   c                   óL   a a€ ] tR t^it oRV3R lV 3R llltRR ltRtVtV ;t# )ÚTimestepEmbeddingc                óH   <€ V ^8„  d   QhRS[ RS[ RS[RS[ RS[S[,          /# )r<   Úin_channelsÚtime_embed_dimÚact_fnÚout_dimÚpost_act_fn)rE   Ústrr   )rH   r1   s   "€r   rI   ÚTimestepEmbedding.__annotate__j   sA   ø€ ÷ !!ñ !!áð!!ñ ð!!ñ ð	!!ñ
 ð!!ñ ™c•]ñ!!r   c                ó2  <€ \         SV `  4        V
P                  WWxV	R 7      V n        Ve   V
P                  WaRW‰R7      V n        MRV n        \
        P                  ! 4       V n        Ve   TpMTpV
P                  W+WxV	R 7      V n        Vf
   RV n	        R# R# )©rM   rN   NF©ÚbiasrM   rN   )
ÚsuperÚ__init__ÚLinearÚlinear_1Ú	cond_projr
   ÚSiLUÚactÚlinear_2Úpost_act)Úselfre   rf   rg   rh   ri   Úcond_proj_dimÚsample_proj_biasrM   rN   Ú
operationsÚtime_embed_dim_outÚ	__class__s   &&&&&&&&&&& €r   rq   ÚTimestepEmbedding.__init__j   s¦   ø€ ô 	‰ÑÔà"×)Ñ)¨+ÐGWÐmsÐ)ÓtˆŒàÒ$Ø'×.Ñ.¨}ÐPUÐ]bÐ.ÓrˆD�Nà!ˆDŒNä—7’7“9ˆŒàÒØ!(Ñà!/ÐØ"×)Ñ)ØÐ0@ÐV\ð *ó 
ˆŒð ÒØ ˆDŽMñ r   c                óü   € Ve   WP                  V4      ,           pV P                  V4      pV P                  e   V P                  V4      pV P                  V4      pV P                  e   V P	                  V4      pV# r   )rt   rs   rv   rw   rx   )ry   ÚsampleÚ	conditions   &&&r   ÚforwardÚTimestepEmbedding.forward�   sj   € ØÒ ØŸn™n¨YÓ7Õ7ˆFØ—‘˜vÓ&ˆà�8‰8ÒØ—X‘X˜fÓ%ˆFà—‘˜vÓ&ˆà�=‰=Ò$Ø—]‘] 6Ó*ˆFØˆr   )rv   rt   rs   rw   rx   )ÚsiluNNNTNNNr   ©	r(   r)   r*   r+   rq   rƒ   r.   r/   Ú__classcell__©r~   r1   s   @@r   rc   rc   i   s   ù‡ € ÷!!õ !!÷Jô r   rc   c                   óH   a a€ ] tR t^žt oRV3R lV 3R llltR tRtVtV ;t# )Ú	Timestepsc                ó2   <€ V ^8„  d   QhRS[ RS[RS[RS[ /# )r<   Únum_channelsr?   r@   rA   )rE   rF   rG   )rH   r1   s   "€r   rI   ÚTimesteps.__annotate__Ÿ   s+   ø€ ÷ ñ ¡Sð ¹4ð ÑW\ð Ñehñ r   c                óT   <€ \         SV `  4        Wn        W n        W0n        W@n        R # r   )rp   rq   rŒ   r?   r@   rA   )ry   rŒ   r?   r@   rA   r~   s   &&&&&€r   rq   ÚTimesteps.__init__Ÿ   s%   ø€ Ü‰ÑÔØ(ÔØ.ÔØ$8Ô!ØŽ
r   c                óv   € \        VV P                  V P                  V P                  V P                  R 7      pV# ))r?   r@   rA   )ra   rŒ   r?   r@   rA   )ry   r=   Út_embs   && r   rƒ   ÚTimesteps.forward¦   s8   € Ü&ØØ×ÑØ ×0Ñ0Ø!%×!:Ñ!:Ø—*‘*ô
ˆð ˆr   )r@   r?   rŒ   rA   )rS   r†   rˆ   s   @@r   rŠ   rŠ   ž   s   ù‡ € ÷õ ÷ò r   rŠ   c                   óL   a a€ ] tR t^±t oRtRV3R lV 3R llltR tRtVtV ;t	# )Ú)PixArtAlphaCombinedTimestepSizeEmbeddingszª
For PixArt-Alpha.

Reference:
https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L164C9-L168C29
c                ó    <€ V ^8„  d   QhRS[ /# )r<   Úuse_additional_conditions)rF   )rH   r1   s   "€r   rI   Ú6PixArtAlphaCombinedTimestepSizeEmbeddings.__annotate__¹   s   ø€ ÷ 
ñ 
ñ $(ñ	
r   c                ó|   <€ \         SV `  4        W n        \        R R^ R7      V n        \        R WWVR7      V n        R# )é   T)rŒ   r?   r@   )re   rf   rM   rN   r|   N)rp   rq   ÚoutdimrŠ   Ú	time_projrc   Útimestep_embedder)ry   r>   Úsize_emb_dimr–   rM   rN   r|   r~   s   &&&&&&&€r   rq   Ú2PixArtAlphaCombinedTimestepSizeEmbeddings.__init__¹   s:   ø€ ô 	‰ÑÔà"ŒÜ"°ÀTÐ`aÔbˆŒÜ!2Ø¨MÈvô"
ˆÖr   c                ój   € V P                  V4      pV P                  VP                  VR 7      4      pV# )©rM   )r›   rœ   Úto)ry   ÚtimestepÚ
resolutionÚaspect_ratioÚ
batch_sizeÚhidden_dtypeÚtimesteps_projÚtimesteps_embs   &&&&&&  r   rƒ   Ú1PixArtAlphaCombinedTimestepSizeEmbeddings.forwardÊ   s4   € ØŸ™¨Ó1ˆØ×.Ñ.¨~×/@Ñ/@À|Ð/@Ó/TÓUˆØÐr   )rš   r›   rœ   )FNNN©
r(   r)   r*   r+   Ú__doc__rq   rƒ   r.   r/   r‡   rˆ   s   @@r   r”   r”   ±   s   ù‡ € ñ÷
õ 
÷"ò r   r”   c                   ó\   a a€ ] tR t^Ðt oRtRV3R lV 3R llltRV3R lR lltRtVtV ;t	# )	ÚAdaLayerNormSinglea4  
Norm layer adaptive layer norm single (adaLN-single).

As proposed in PixArt-Alpha (see: https://arxiv.org/abs/2310.00426; Section 2.3).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    use_additional_conditions (`bool`): To use additional conditions for normalization or not.
c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r<   r>   Úembedding_coefficientr–   )rE   rF   )rH   r1   s   "€r   rI   ÚAdaLayerNormSingle.__annotate__Û   s*   ø€ ÷ Eñ EÙ ðEÙ9<ðEÙ]añEr   c           	     óÒ   <€ \         SV `  4        \        VV^,          VVVVR7      V n        \        P
                  ! 4       V n        VP                  WV,          RWER7      V n        R# )é   )r�   r–   rM   rN   r|   Trn   N)	rp   rq   r”   r`   r
   ru   r…   rr   Úlinear)ry   r>   r¯   r–   rM   rN   r|   r~   s   &&&&&&&€r   rq   ÚAdaLayerNormSingle.__init__Û   se   ø€ ô 	‰ÑÔä<ØØ&¨!Õ+Ø&?ØØØ!ô
ˆŒô —G’G“IˆŒ	Ø ×'Ñ'¨È}Õ7\ÐcgÐotÐ'ó  EˆŽr   c                ó@  <€ V ^8„  d   QhRS[ P                  RS[S[S[S[ P                  3,          ,          RS[S[,          RS[S[ P                  ,          RS[S[ P                  S[ P                  S[ P                  S[ P                  S[ P                  3,          /# )r<   r¢   Úadded_cond_kwargsr¥   r¦   Úreturn)rC   rD   r   r   rj   rE   rM   r   )rH   r1   s   "€r   rI   r°   ì   sŒ   ø€ ÷ 
Lñ 
Lá—,‘,ð
Lñ $¡D©©e¯l©lÐ):Õ$;Õ<ð
Lñ ™S•Mð	
Lñ
 ™uŸ{™{Õ+ð
Lñ 
‰u�|‰|™UŸ\™\©5¯<©<¹¿¹ÁuÇ|Á|ÐSÕ	Tñ
Lr   c                ó˜   € T;'       g    R RRR/pV P                   ! V3/ VBRVRV/B pV P                  V P                  V4      4      V3# )r£   Nr¤   r¥   r¦   ©r`   r³   r…   )ry   r¢   r¶   r¥   r¦   Úembedded_timesteps   &&&&& r   rƒ   ÚAdaLayerNormSingle.forwardì   s[   € ð .×[Ð[°,ÀÀnÐVZÐ1[ÐØ ŸHšH XÑuÐ1BÑuÈzÐuÐhtÒuÐØ�{‰{˜4Ÿ9™9Ð%6Ó7Ó8Ð:KÐKÐKr   r¹   )é   FNNN©NNNrª   rˆ   s   @@r   r­   r­   Ð   s(   ù‡ € ñ÷Eõ E÷"
L÷ 
Lò 
Lr   r­   c                   ó@   a a€ ] tR t^ùt oRtRV 3R lltR tRtVtV ;t	# )ÚPixArtAlphaTextProjectionz¾
Projects caption embeddings. Also handles dropout for classifier-free guidance.

Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/nets/PixArt_blocks.py
c                ó<  <€ \         SV `  4        Vf   TpVP                  WRWVR7      V n        VR8X  d   \        P
                  ! RR7      V n        M0VR8X  d   \        P                  ! 4       V n        M\        RV 24      hVP                  W#RWVR7      V n	        R # )NT)Úin_featuresÚout_featuresro   rM   rN   Ú	gelu_tanhÚtanh©Úapproximater…   zUnknown activation function: )
rp   rq   rr   rs   r
   ÚGELUÚact_1ru   Ú
ValueErrorrw   )	ry   rÁ   Úhidden_sizerÂ   rg   rM   rN   r|   r~   s	   &&&&&&&&€r   rq   Ú"PixArtAlphaTextProjection.__init__   s›   ø€ ô 	‰ÑÔØÒØ&ˆLØ"×)Ñ)Ø#ÀDÐPUð *ó 
ˆŒð �[Ô ÜŸš¨VÔ4ˆD�JØ�vÔÜŸš›ˆD�JäÐ<¸V¸HÐEÓFÐFØ"×)Ñ)Ø#ÀTÐQVð *ó 
ˆŽr   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r   )rs   rÈ   rw   )ry   ÚcaptionÚhidden_statess   && r   rƒ   Ú!PixArtAlphaTextProjection.forward  s2   € ØŸ™ gÓ.ˆØŸ
™
 =Ó1ˆØŸ™ mÓ4ˆØÐr   )rÈ   rs   rw   )NrÃ   NNNrª   rˆ   s   @@r   r¿   r¿   ù   s   ù‡ € ñ÷
÷&ò r   r¿   c                   ó@   a a€ ] tR tRt oRtRV 3R lltR tRtVtV ;t	# )ÚNormSingleLinearTextProjectioni  zLText projection for 20B models - single linear with RMSNorm (no activation).c                óÜ   <€ \         SV `  4        Vf   \        P                  P                  pVP                  VRRR7      V n        VP                  WRW4R7      V n        W n	        Wn
        R # )Nç�íµ ÷Æ°>F)ÚepsÚelementwise_affineTrn   )rp   rq   ÚcomfyÚopsÚdisable_weight_initÚRMSNormÚin_normrr   rs   rÊ   rÁ   )ry   rÁ   rÊ   rM   rN   r|   r~   s   &&&&&&€r   rq   Ú'NormSingleLinearTextProjection.__init__  sq   ø€ ô 	‰ÑÔØÒÜŸ™×6Ñ6ˆJØ!×)Ñ)Ø˜T°eð *ó 
ˆŒð #×)Ñ)Ø¨4°uð *ó 
ˆŒð 'ÔØ&Ör   c                óš   € V P                  V4      pWP                  V P                  ,          R ,          ,          pV P                  V4      # )g      à?)rÚ   rÊ   rÁ   rs   )ry   rÍ   s   &&r   rƒ   Ú&NormSingleLinearTextProjection.forward,  s=   € Ø—,‘,˜wÓ'ˆØ×-Ñ-°×0@Ñ0@Õ@ÀSÕHÕHˆØ�}‰}˜WÓ%Ð%r   )rÊ   rÁ   rÚ   rs   r½   rª   rˆ   s   @@r   rÑ   rÑ     s   ù‡ € ÙV÷'÷&ò &r   rÑ   c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚGELU_approxi2  c                óV   <€ \         SV `  4        VP                  WW4VR 7      V n        R# )rn   N)rp   rq   rr   Úproj)ry   Údim_inÚdim_outro   rM   rN   r|   r~   s   &&&&&&&€r   rq   ÚGELU_approx.__init__3  s(   ø€ Ü‰ÑÔØ×%Ñ% f¸DÐV\Ð%Ó]ˆŽ	r   c                óv   € \         P                  P                  P                  V P	                  V4      R R7      # )rÄ   rÅ   )rC   r
   r\   Úgelurá   ©ry   r   s   &&r   rƒ   ÚGELU_approx.forward7  s*   € Ü�x‰x×"Ñ"×'Ñ'¨¯	©	°!«À&Ð'ÓIÐIr   )rá   )TNNNr†   rˆ   s   @@r   rß   rß   2  s   ù‡ € ÷^÷Jò Jr   rß   c                   ó<   a a€ ] tR tRt oRV 3R lltR tRtVtV ;t# )ÚFeedForwardi;  c
                óè   <€ \         SV `  4        \        W,          4      p
\        WWgW‰R 7      p\        P
                  ! V\        P                  ! V4      V	P                  W¢WgVR7      4      V n        R# ))ro   rM   rN   r|   rn   N)	rp   rq   rE   rß   r
   Ú
SequentialÚDropoutrr   Únet)ry   rQ   rã   ÚmultÚgluÚdropoutÚff_biasrM   rN   r|   Ú	inner_dimÚ
project_inr~   s   &&&&&&&&&&  €r   rq   ÚFeedForward.__init__<  s\   ø€ Ü‰ÑÔÜ˜�
“Oˆ	Ü  °gÐSYÔqˆ
ä—=’=ØœŸ
š
 7Ó+¨Z×->Ñ->¸yÐX_Ðu{Ð->Ó-|ó
ˆŽr   c                ó  € \         P                  P                  '       d   V P                  V4      # \         P                  P                  V P                  ^,          V P                  ^ ,          P                  V4      R4      # )r<   rÃ   )rÖ   Úmodel_managementÚin_trainingrî   r×   Úlinear_input_actrá   rç   s   &&r   rƒ   ÚFeedForward.forwardE  sX   € ô ×!Ñ!×-×-Ð-Ø—8‘8˜A“;ÐÜ�y‰y×)Ñ)¨$¯(©(°1­+°t·x±xÀµ{×7GÑ7GÈÓ7JÈKÓXÐXr   )rî   )é   Fç        TNNNr†   rˆ   s   @@r   rê   rê   ;  s   ù‡ € ÷
÷Yò Yr   rê   c                 óÔ  € Vw  r#V P                   pV P                  V P                   ^ ,          V P                   ^,          VP                   ^,          R4      p \        P                  P                  '       Ed   V'       dE   V P                  ! . V P                   RR O^NRN5!  P                  RR4      P                  R4      pM&V P                  ! . V P                   RR ORN^N^N5!  pVP                  VP                  4      pVR,          VR,          ,          VR,          VR,          ,          ,           pV'       d   VP                  RR4      pVP                  V P                   4      P                  V 4      pM[V'       d+   \        P                  P                  P                  W4      pM)\        P                  P                  P                  W4      pVP                  V4      # )r   NrR   éþÿÿÿ©.r   ©.rS   )rU   ÚreshaperÖ   r÷   rø   ÚmovedimÚ	unsqueezer¡   rM   Útype_asÚ	quant_opsÚckÚapply_rope_split_half1Úapply_rope1)Úinput_tensorÚ	freqs_cisÚrotation_matrixÚsplit_peÚoriginal_shapeÚtÚoutputs   &&     r   Úapply_rotary_embr  L  s�  € Ø )Ñ€OØ!×'Ñ'€NØ×'Ñ'Ø×Ñ˜1Õ˜|×1Ñ1°!Õ4°o×6KÑ6KÈAÕ6NÐPRó€Lô ×Ñ×)×)Ñ)ßØ×$Ò$ÐE l×&8Ñ&8¸¸"Ð&=ÐE¸qÐEÀ"ÓE×MÑMÈbÐRTÓU×_Ñ_Ð`bÓc‰Aà×$Ò$ÐH l×&8Ñ&8¸¸"Ð&=ÐH¸rÐHÀ1ÐHÀaÓHˆAØ�D‰D�×&Ñ&Ó'ˆØ  Õ(¨1¨V­9Õ4°ÀvÕ7NÐQRÐSYÕQZÕ7ZÕZˆßØ—^‘^ B¨Ó+ˆFØ—‘ × 2Ñ 2Ó3×;Ñ;¸LÓI‰ß	Ü—‘×#Ñ#×:Ñ:¸<ÓY‰ä—‘×#Ñ#×/Ñ/°ÓNˆØ�>‰>˜.Ó)Ð)r   c                 óÈ  € \         P                  P                  '       d   \        W4      \        W4      3# Vw  r4V P                  pVP                  pV P                  V P                  ^ ,          V P                  ^,          VP                  ^,          R4      p VP                  VP                  ^ ,          VP                  ^,          VP                  ^,          R4      pV'       d.   \         P                  P                  P                  WV4      w  rM,\         P                  P                  P                  WV4      w  rV P                  V4      VP                  V4      3# )r   rR   )
rÖ   r÷   rø   r  rU   r  r  r  Úapply_rope_split_halfÚ
apply_rope)ÚqÚkr
  r  r  Úq_shapeÚk_shapes   &&&    r   Úapply_rotary_emb_qkr  c  s   € Ü×Ñ×)×)Ð)Ü Ó-Ô/?ÀÓ/MÐMÐMà )Ñ€OØ�g‰g€GØ�g‰g€GØ	�	‰	�!—'‘'˜!•*˜aŸg™g a�j¨/×*?Ñ*?ÀÕ*BÀBÓG€AØ	�	‰	�!—'‘'˜!•*˜aŸg™g a�j¨/×*?Ñ*?ÀÕ*BÀBÓG€AßÜ�‰×!Ñ!×7Ñ7¸¸oÓN‰ˆˆ1ä�‰×!Ñ!×,Ñ,¨Q°?ÓC‰ˆØ�9‰9�WÓ˜qŸy™y¨Ó1Ð1Ð1r   c                   ó.   a € ] tR tRt o RtRtR tRtV tR# )ÚGuideAttentionMaskis  z½Holds the two per-group masks for LTXV guide self-attention.
_attention_with_guide_mask splits queries into noisy and tracked-guide
groups, so the largest mask is (1, 1, tracked_count, T).
c                ó^  € VP                   pVP                  p\        P                  ! V4      pV^ 8„  p\        P                  ! WGP
                  4      p	\        P                  ! WH,          P                  VP                  R7      4      W˜&   W n	        W0n
        \        P                  ! ^^^V3WVR7      V n        V	P                  ^^^R4      V P                  RRRW"V,           13&   \        P                  ! ^^W13WVR7      V n        V	P                  ^^R^4      V P                  RRRRV13&   R# )r   )Úmin©rN   rM   rO   NrR   )rN   rM   rC   ÚfinfoÚ	full_liker  r   ÚclampÚtinyÚguide_startÚtracked_countÚzerosÚ
noisy_maskÚviewÚtracked_mask)
ry   Útotal_tokensr"  r#  Útracked_weightsrN   rM   r  ÚposÚlog_ws
   &&&&&     r   rq   ÚGuideAttentionMask.__init__z  s  € Ø ×'Ñ'ˆØ×%Ñ%ˆÜ—’˜EÓ"ˆà Ñ!ˆÜ—’ ·±Ó;ˆÜ—Y’Y˜Õ3×9Ñ9¸e¿j¹jÐ9ÓIÓJˆ‰
à&ÔØ*ÔäŸ+š+ q¨!¨Q°Ð&=ÀfÔZˆŒØLQÏJÉJÐWXÐZ[Ð]^Ð`bÓLcˆ�‰˜˜1˜a ¸=Õ-HÐ!HÐHÑIä!ŸKšK¨¨A¨}Ð(KÐTZÔhˆÔØ38·:±:¸aÀÀBÈÓ3Jˆ×Ñ˜!˜Q  < K <Ð/Ó0r   )r"  r%  r#  r'  N)r"  r#  r%  r'  )	r(   r)   r*   r+   r«   Ú	__slots__rq   r.   r/   r0   s   @r   r  r  s  s   ø‡ € ñð O€I÷Kð Kr   r  c                óB  € VP                   pW„P                  ,           p	\        P                  ! V 4      p
V^ 8”  dt   \        P
                  P                  P                  P                  \        V RRV1R3,          4      \        V4      \        V4      W4P                  WVVRR7	      V
RRV1R3&   \        P
                  P                  P                  P                  \        V RW‰1R3,          4      \        V4      \        V4      W4P                  WVVRR7	      V
RW‰1R3&   W�P                  ^,          8  di   \        P
                  P                  P                  P                  \        V RV	R1R3,          4      \        V4      \        V4      VWVVR7      V
RV	R1R3&   V
# )z¦Apply the guide mask by partitioning Q into noisy and tracked-guide
groups, so each group needs only its own sub-mask. Avoids materializing
the (1,1,T,T) dense mask.
rO   NF)ÚmaskÚattn_precisionÚtransformer_optionsÚpreferred_attentionÚlow_precision_attention©r0  r1  r2  )r"  r#  rC   Ú
empty_likerÖ   ÚldmÚmodulesÚ	attentionÚoptimized_attentionr   r%  r'  rU   )r  r  ÚvÚheadsÚ
guide_maskr0  r1  r2  r"  Útracked_endÚouts   &&&&&&&&   r   Ú_attention_with_guide_maskr?  �  s¨  € ð
 ×(Ñ(€KØ× 8Ñ 8Õ8€Kä
×
Ò
˜1Ó
€Cà�Q„Ü"'§)¡)×"3Ñ"3×"=Ñ"=×"QÑ"QÜ$ Q q¨,¨;¨,¸Ð'9Õ%:Ó;Ô=UÐVWÓ=XÔZrÐstÓZuÐw|÷  DYñ  DYØ)Ðh{Ø$)ð #Ró #
ˆˆAˆ|�ˆ|˜QÐÑô
 */¯©×):Ñ):×)DÑ)D×)XÑ)XÜ   1 kÐ&=¸qÐ#@Õ!AÓBÔD\Ð]^ÓD_ÔayÐz{Óa|ð  D÷  Kbñ  KbØ%ÐdwØ %ð *Yó *€Cˆˆ;Ð" AÐ%Ñ&ð
 —W‘W˜Q•ZÔÜ"'§)¡)×"3Ñ"3×"=Ñ"=×"QÑ"QÜ$ Q q¨+©,¸Ð'9Õ%:Ó;Ô=UÐVWÓ=XÔZrÐstÓZuÐw|Ø)Ðh{ð #Ró #
ˆˆAˆ{‰|˜QÐÑð €Jr   c                   óJ   a a€ ] tR tRt oRV 3R lltRRRR/ 3R ltRtVtV ;t# )ÚCrossAttentioniª  Nc           	     óT  <€ \         SV `  4        \        4       V n        WC,          pVf   TMTpW`n        W0n        W@n        V
P                  VRW‰R7      V n        V
P                  VRW‰R7      V n	        V
P                  WRW‰R7      V n        V
P                  W+RW‰R7      V n        V
P                  W+RW‰R7      V n        V'       d   V
P                  WRW‰R7      V n        MR V n        \        P                   ! V
P                  W±W‰R7      \        P"                  ! V4      4      V n        R # )Ngñhãˆµøä>)rÔ   rM   rN   Trn   rm   )rp   rq   r   Úcomfy_attentionr0  r;  Údim_headrÙ   Úq_normÚk_normrr   Úto_qÚto_kÚto_vÚto_gate_logitsr
   rì   rí   Úto_out)ry   Ú	query_dimÚcontext_dimr;  rD  rñ   r0  Úapply_gated_attentionrM   rN   r|   ró   r~   s   &&&&&&&&&&& €r   rq   ÚCrossAttention.__init__«  s  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØÕ$ˆ	Ø#.Ò#6‘i¸KˆØ,ÔàŒ
Ø Œà ×(Ñ(¨¸ÀEÐ(ÓYˆŒØ ×(Ñ(¨¸ÀEÐ(ÓYˆŒà×%Ñ% iÀÈUÐ%ÓbˆŒ	Ø×%Ñ% kÀ4ÈuÐ%ÓdˆŒ	Ø×%Ñ% kÀ4ÈuÐ%ÓdˆŒ	÷ !Ø",×"3Ñ"3°IÈ4ÐW\Ð"3Ó"lˆDÕà"&ˆDÔä—m’mØ×Ñ˜i¸%ÐÓOÔQS×Q[ÒQ[Ð\cÓQdó
ˆŽr   c                óª  € VR J pV P                  V4      pVf   TMTpV P                  V4      p	V P                  V4      p
V'       d   VP                  RR4      '       d   T
pEM)V P	                  V4      pV P                  V	4      p	VeK   Vf+   VP                  V	P                  8X  d   \        W‰V4      w  r‰M\        W„4      p\        Y•f   TMT4      p	\        V\        4      '       d/   \        W‰W P                  W0P                  W`P                  R7      pMu\        V4      \        V	4      \        V
4      r©p\         P"                  P$                  P&                  P)                  W‰W P                  W0P                  W`P                  R7      pV P*                  eª   V P+                  V4      pVP                  w  rÞpVP-                  WÞV P                  V P.                  4      pR\0        P2                  ! V4      ,          pVVP5                  R4      ,          pVP-                  WÞV P                  V P.                  ,          4      pV P7                  V4      # )NÚstg_skip_self_attnFr4  )r/  r0  r1  r2  ç       @rR   )rG  rH  rI  r   rE  rF  rU   r  r  Ú
isinstancer  r?  r;  r0  rC  r   rÖ   r6  r7  r8  r9  rJ  r&  rD  rC   Úsigmoidr  rK  )ry   r   Úcontextr/  ÚpeÚk_per1  Ú	self_attnr  r  r:  r>  Úgate_logitsÚbr  Ú_Úgatess   &&&&&&&          r   rƒ   ÚCrossAttention.forwardÒ  s  € Ø˜t�Oˆ	Ø�I‰I�a‹LˆØ’‘!¨GˆØ�I‰I�gÓˆØ�I‰I�gÓˆ÷ Ð,×0Ñ0Ð1EÀu×MÒMØŠCà—‘˜A“ˆAØ—‘˜A“ˆAð Š~Ø’< A§G¡G¨q¯w©wÔ$6Ü.¨q°RÓ8‘D�A�qä(¨Ó/�AÜ(¨²,©BÀDÓI�Aä˜$Ô 2×3Ò3Ü0°°q¿*¹*Àd×[nÑ[nð  EX÷  nBñ  nBô  C‘ä2°1Ó5Ô7OÐPQÓ7RÔTlÐmnÓTo�a�Ü—i‘i×'Ñ'×1Ñ1×EÑEÀaÈAÏzÉzÐ`d÷  vIñ  vIð  _r÷  H\ñ  H\ÐEó  ]�ð ×ÑÒ*Ø×-Ñ-¨aÓ0ˆKØ—i‘i‰GˆA�!Ø—(‘(˜1 §¡¨T¯]©]Ó;ˆCØœ%Ÿ-š-¨Ó4Õ4ˆEØ˜Ÿ™¨Ó+Õ+ˆCØ—(‘(˜1 §¡¨d¯m©mÕ!;Ó<ˆCà�{‰{˜3ÓÐr   )r0  rC  rD  r;  rF  rE  rJ  rH  rK  rG  rI  )	Né   é@   rü   NFNNNr†   rˆ   s   @@r   rA  rA  ª  s#   ù‡ € ÷%
ðN "&¨D°TÀÐZ\÷ & ô & r   rA  c                   óN   a a€ ] tR tRt oRV 3R lltRRRR/ RR3R ltRtVtV ;t# )ÚBasicTransformerBlockiþ  Nc                óî  <€ \         SV `  4        WPn        W`n        \	        VVVR V P                  VV	V
R7      V n        \        WRWxWšR7      V n        \	        VVVVV P                  VV	V
R7      V n        V'       d   \        M\        p\        P                  ! \        P                  ! W±W˜R7      4      V n        V'       d5   \        P                  ! \        P                  ! ^WVR7      4      V n        R # R # )N)rL  r;  rD  rM  r0  rM   rN   r|   T)rã   rð   rò   rM   rN   r|   )rL  rM  r;  rD  r0  rM   rN   r|   r  )rp   rq   r0  Úcross_attention_adalnrA  Úattn1rê   ÚffÚattn2ÚADALN_CROSS_ATTN_PARAMS_COUNTÚADALN_BASE_PARAMS_COUNTr
   Ú	ParameterrC   ÚemptyÚscale_shift_tableÚprompt_scale_shift_table)ry   rQ   Ún_headsÚd_headrM  r0  rc  rò   rM   rN   r|   Únum_ada_paramsr~   s   &&&&&&&&&&& €r   rq   ÚBasicTransformerBlock.__init__ÿ  s×   ø€ ô 	‰ÑÔà,ÔØ%:Ô"Ü#ØØØØØ×.Ñ.ØØØ!ô	
ˆŒ
ô ˜c°DÀ'Ð_eÔ}ˆŒä#ØØ#ØØØ×.Ñ.ØØØ!ô	
ˆŒ
÷ ;PÕ6ÔUlˆÜ!#§¢¬e¯kªk¸.ÐV\Ô.jÓ!kˆÔç Ü,.¯LªL¼¿ºÀQÈÐbgÔ9hÓ,iˆDÖ)ñ !r   c	                ó�  € V P                   R,          P                  VP                  VP                  R7      VP	                  VP
                  ^ ,          VP
                  ^,          V P                   P
                  ^ ,          R4      R,          ,           P                  ^R7      w  ršr¼rÞ\        P                  P                  '       d@   \        P                  P                  P                  V4      ^V
,           ,          V	,           pM*\        P                  P                  P                  WV	4      pWP!                  WõWvR7      V,          ,          pV P"                  '       dØ   V P                   R,          P                  VP                  VP                  R7      VP	                  VP
                  ^ ,          VP
                  ^,          V P                   P
                  ^ ,          R4      R	,          ,           P                  ^R7      w  pppV\%        WV P&                  VVVV P(                  WƒV4
      ,          pMWP'                  WW6R7      ,          p\        P                  P                  P                  V4      p\*        P,                  ! VVV4      P/                  V4      pVP1                  V P3                  V4      V4       V# )
Nr  rP   )rV  r/  r1  ©rU  r/  r1  )NNºNr¼   NrR   )rO   rO   rs  rO   )NNºr¼   é	   N)rO   rO   rt  rO   )rk  r¡   rN   rM   r  rU   ÚunbindrÖ   r÷   rø   r6  Ú
common_ditÚrms_normr  r  Ú	rms_adalnrd  rc  Úapply_cross_attention_adalnrf  rl  rC   ÚaddcmulÚadd_Úaddcmul_re  )ry   r   rU  Úattention_maskr¢   rV  r1  Úself_attention_maskÚprompt_timestepÚ	shift_msaÚ	scale_msaÚgate_msaÚ	shift_mlpÚ	scale_mlpÚgate_mlpÚnorm_xÚshift_q_mcaÚscale_q_mcaÚgate_mcaÚys   &&&&&&&&&           r   rƒ   ÚBasicTransformerBlock.forward#  s:  € ØJN×J`ÑJ`ÐaoÕJp×JsÑJsÐ{|÷  |Dñ  |Dð  LM÷  LSñ  LSÐJsó  KTð  W_÷  Wgñ  Wgð  hi÷  hoñ  hoð  pqõ  hrð  t|÷  tBñ  tBð  CDõ  tEð  GK÷  G]ñ  G]÷  Gcñ  Gcð  deõ  Gfð  hjó  Wkð  lwõ  Wxõ  Kx÷  J@ñ  J@ð  EFð  J@ó  JGÑFˆ	˜h°9ä×!Ñ!×-×-Ð-Ü—Y‘Y×)Ñ)×2Ñ2°1Ó5¸¸Y½ÕGÈ)ÕS‰Fä—_‘_×'Ñ'×1Ñ1°!À	ÓJˆFà	�Z‰Z˜Ð,?ˆZÓiÐltÕtÕtˆà×%×%Ð%Ø26×2HÑ2HÈÕ2Y×2\Ñ2\Ðde×dlÑdlÐtu×t{Ñt{Ð2\Ó2|ð  @H÷  @Pñ  @Pð  QR÷  QXñ  QXð  YZõ  Q[ð  ]e÷  ]kñ  ]kð  lmõ  ]nð  pt÷  pFñ  pF÷  pLñ  pLð  MNõ  pOð  QSó  @Tð  Uaõ  @bõ  3b÷  2jñ  2jð  opð  2jó  2qÑ.ˆK˜ hØÔ,Ø˜DŸJ™J¨°[À(Ø×-Ñ-¨ÐPcóõ ‰Að
 —‘˜A°^�ÓmÕmˆAä�I‰I× Ñ ×)Ñ)¨!Ó,ˆÜ�MŠM˜!˜Q 	Ó*×/Ñ/°	Ó:ˆØ	�
‰
�4—7‘7˜1“:˜xÔ(àˆr   )rd  rf  r0  rc  re  rl  rk  )NNFTNNNr†   rˆ   s   @@r   ra  ra  þ  s2   ù‡ € ÷"jðH "&°dÀTÈdÐhjð  AEð  W[÷ ô r   ra  c                ó  € V f   R# VP                  4       ^8”  d,   VP                  ^RR7      P                  P                  4       MVP                  4       pV ! VRRRR/VVR7      w  rVVP	                  V^VP
                  R,          4      # )zÊCompute a single global prompt timestep for cross-attention ADaLN.

Uses the max across tokens (matching JAX max_per_segment) and broadcasts
over text tokens.  Returns None when *adaln_module* is None.
NT)rQ   Úkeepdimr£   r¤   ©r¥   r¦   rR   )rQ   ÚmaxÚvaluesÚflattenr&  rU   )Úadaln_moduleÚtimestep_scaledr¥   r¦   Úts_inputÚ	prompt_tsr[  s   &&&&   r   Úcompute_prompt_timestepr—  <  s–   € ð ÒÙð ×ÑÓ  1Ô$ð 	×Ñ ¨4ÐÓ0×7Ñ7×?Ñ?ÔAà×$Ñ$Ó&ð ñ
  ØØ	�t˜^¨TÐ2ØØ!ô	�L€Ið �>‰>˜* a¨¯©¸Õ)<Ó=Ð=r   c
                ó\  € V P                   ^ ,          p
VR,          P                  V P                  V P                  R7      VP	                  W§P                   ^,          ^R4      ,           P                  ^R7      w  r¼\        P                  P                  '       d@   \        P                  P                  P                  V 4      ^V,           ,          V,           pM*\        P                  P                  P                  WV4      pV^V,           ,          V,           pV! WÞW‰R7      V,          # )zÔApply cross-attention with ADaLN modulation (shift/scale/gate on Q and KV).

Q params (q_shift, q_scale, q_gate) are pre-extracted by the caller so
that both regular tensors and CompressedTimestep are supported.
r  rP   rr  ©NNrR   )rU   r¡   rN   rM   r  rv  rÖ   r÷   rø   r6  rw  rx  r  r  ry  )r   rU  ÚattnÚq_shiftÚq_scaleÚq_gaterl  r€  r~  r1  r¥   Úshift_kvÚscale_kvÚ
attn_inputÚencoder_hidden_statess   &&&&&&&&&&     r   rz  rz  R  sï   € ð —‘˜•€Jà  Õ,×/Ñ/°q·x±xÀqÇwÁwÐ/ÓOØ
×
!Ñ
! *×.CÑ.CÀAÕ.FÈÈ2Ó
Nõ	Oç�f�€fƒmñ €Hô ×Ñ×)×)Ð)Ü—Y‘Y×)Ñ)×2Ñ2°1Ó5¸¸W½ÕEÈÕO‰
ä—_‘_×'Ñ'×1Ñ1°!¸gÓFˆ
Ø# q¨8¥|Õ4°xÕ?ÐÙ�
ÀÔxð  |Bõ  Bð  Br   c                 ó  € V P                   ^,          pV\        V4      8X  g   Q RV R\        V4       R24       h\        P                  ! \	        V4       Uu. uF  q0RV3,          W,          ,          NK  	  upRR7      pV# u upi )rS   zNumber of position dimensions (z) must match max_pos length (Ú)rO   )ÚaxisrR   )rU   rT   rC   ÚstackÚrange)Úindices_gridÚmax_posÚ
n_pos_dimsÚiÚfractional_positionss   &&   r   Úget_fractional_positionsr¬  h  sš   € Ø×#Ñ# AÕ&€JØœ˜W›Ô%ð  BÐ)HÈÈÐTqÔruÐv}Ór~Ðqð  @Að  (Bó  BÐ%Ü Ÿ;š;Ü27¸
Ô2CÓDÑ2C¨Q�a˜�dÕ	˜g�j×	(Ð	(Ñ2CÑDØôÐð  Ðùò 	Es   Á B)Úmaxsizec           
      óN  € T p^pTp^V,          p\         P                  ! V\         P                  ! \        WT4      \        Wd4      W',          \         P                  R7      4      p\
        P                  ! V\        P                  ,          ^,          \
        P                  R7      # )rS   r    )
r   ÚpowerÚlinspacer   r5   rC   ÚtensorrV   Úpir4   )	Úpositional_embedding_thetaÚ"positional_embedding_max_pos_countró   r[  ÚthetarK   rL   Ún_elemÚpow_indicess	   &&&&     r   Úgenerate_freq_grid_npr¸  r  s{   € à&€EØ€EØ
€CàÐ3Õ3€FÜ—(’(ØÜ
�ŠÜ�eÓ#Ü�cÓ!ØÕÜ—*‘*ô		
ó€Kô �<Š<˜¤d§g¡gÕ-°Õ1¼¿¹ÔGÐGr   c           	      ó\  € T p^pTp^V,          pV\         P                  ! \        P                  ! WT4      \        P                  ! Wd4      W',          V\         P                  R7      ,          pVP                  \         P                  R7      pV\        P                  ,          ^,          pV# )rS   r  r    )rC   r°  rV   r   r4   r¡   r²  )	r³  r´  ró   rN   rµ  rK   rL   r¶  Úindicess	   &&&&     r   Úgenerate_freq_grid_pytorchr»  „  sˆ   € Ø&€EØ€EØ
€CØÐ3Õ3€FàÜ�ŠÜ�HŠH�UÓ"Ü�HŠH�SÓ ØÕØÜ—-‘-ô	
õ€Gð �j‰jœuŸ}™}ˆjÓ-€GàœŸ™Õ !Õ#€Gà€Nr   c                 óÜ  € V'       dV   \        VP                  4      ^8X  d   VP                  R,          ^8X  g   Q hVR,          VR,          rTWE,           R,          pM#\        VP                  4      ^8X  d
   VR,          p\        W4      pV P                  VP                  R7      p WP                  R4      ^,          ^,
          ,          P                  RR4      P                  ^4      pV# )rû   rR  )rN   rR   rÿ   r   rþ   )rT   rU   r¬  r¡   rN   r  Ú	transposer’  )rº  r§  r¨  Úuse_middle_indices_gridÚindices_grid_startÚindices_grid_endr«  Úfreqss   &&&&    r   Úgenerate_freqsrÂ  ™  sÏ   € ßÜ�<×%Ñ%Ó&¨!Ô+°×0BÑ0BÀ2Õ0FÈÔ0JÐKÐJØ/;¸FÕ/CÀ\ÐRXÕEYÐ,Ø*Õ=ÀÕD‰Ü	ˆ\×ÑÓ	  AÔ	%Ø# FÕ+ˆô 4°LÓJÐØ�j‰jÐ 4× ;Ñ ;ˆjÓ<€Gð 
×2Ñ2°2Ó6¸Õ:¸QÕ>Õ	?ß	‰�2�rÓ	ß	‰�‹ð 
ð
 €Lr   c                 ó¤  € V P                  4       P                  V4      pV P                  4       P                  V4      pV'       dŠ   V'       d   TMV^,          p\        P                  ! VRRRV13,          4      p\        P
                  ! VRRRV13,          4      p	\        P                  ! W…3RR7      p\        P                  ! W–3RR7      pVP                  w  r«pVP                  W«W<V,          4      pVP                  W«W<V,          4      p\        P                  ! WV) We3RR7      pVP                  ! . VP                  RR O^N^N5!  V3# )r<   rO   NrP   rR   )
r[   r¡   rZ   rC   Ú	ones_likeÚ
zeros_likerY   rU   r  r¥  )rÁ  Úpad_sizeÚ
split_modeÚnum_attention_headsÚ	out_dtypeÚcos_freqÚsin_freqÚmatrix_pad_sizeÚcos_paddingÚsin_paddingÚBÚTÚhalf_HDr  s   &&&&&         r   Úfreqs_cis_matrixrÒ  ¬  s-  € Ø�y‰y‹{�~‰~˜iÓ(€HØ�y‰y‹{�~‰~˜iÓ(€Hßß&0™(°hÀ!µmˆÜ—o’o h¨q°!Ð5E°oÐ5EÐ/EÕ&FÓGˆÜ×&Ò& x°°1Ð6F°Ð6FÐ0FÕ'GÓHˆÜ—9’9˜kÐ4¸"Ô=ˆÜ—9’9˜kÐ4¸"Ô=ˆà—N‘N�M€Aˆ'Ø×Ñ Ð&9ÐFYÕ;YÓZ€HØ×Ñ Ð&9ÐFYÕ;YÓZ€HÜ—k’kØ	�9˜hÐ1°rô€Oð ×"Ò"ÐE O×$9Ñ$9¸#¸2Ð$>ÐEÀÐEÀ1ÓEÀzÐQÐQr   c                   ó.  a a€ ] tR tRt oRtR. RORRRRRRRRRRRRR3V3R lV 3R	 llltR
 t]R 4       t]R 4       t	]R 4       t
]R 4       tR t]RR l4       t]R 4       tR tRR ltR. ROR^ 3R ltR tR t^/ RR3R lt^/ RR3R ltRtVtV ;t# )ÚLTXBaseModeli¾  zÔ
Abstract base class for LTX models (Lightricks Transformer models).

This class defines the common interface and shared functionality for all LTX models,
including LTXV (video) and LTXAV (audio-video) variants.
ç     ˆÃ@Fç     @�@TNc                óV   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[ RS[RS[R	S[R
S[/
# )r<   re   Úcross_attention_dimÚattention_head_dimrÈ  Úcaption_channelsÚ
num_layersr³  Úpositional_embedding_max_posÚcausal_temporal_positioningÚvae_scale_factors)rE   rG   ÚlistrF   Útuple)rH   r1   s   "€r   rI   ÚLTXBaseModel.__annotate__Æ  s{   ø€ ÷ C4ñ C4áðC4ñ !ðC4ñ  ð	C4ñ
 !ðC4ñ ðC4ñ ðC4ñ %*ðC4ñ '+ðC4ñ &*ðC4ñ !ñC4r   c                ó�  <€ \         SV `  4        R V n        W n        W°n        VV n        Wn        W n        W0n        W@n	        WPn
        W`n        Wpn        W€n        \        P                  V4      V n        \"        P                  V4      \"        P$                  8X  d   \&        M\(        V n        W�n        VV n        WÀn        WÐn        Wàn        Wðn        VV n        VV n        VV n        WC,          V n        Wn         V PC                  VV4       V PD                  ! VV3/ VB  V PF                  ! VV3/ VB  V PI                  VV4       R # r   )%rp   rq   Ú	generatorrÞ  r¾  rM   re   rØ  rÙ  rÈ  rÚ  rÛ  r³  rÜ  r   r%   Úsplit_positional_embeddingr3   r:   r¸  r»  Úfreq_grid_generatorrÝ  r|   Útimestep_scale_multiplierÚcaption_proj_before_connectorrc  Úcaption_projection_first_linearrò   Úuse_prompt_adaln_singleÚuse_keyframes_abs_pos_embeddingró   Úout_channelsÚ_init_common_componentsÚ_init_model_componentsÚ_init_transformer_blocksÚ_init_output_components)ry   re   rØ  rÙ  rÈ  rÚ  rÛ  r³  rÜ  rÝ  rÞ  r¾  ræ  rç  rc  rè  rò   ré  rê  rM   rN   r|   r#   r~   s   &&&&&&&&&&&&&&&&&&&&&&,€r   rq   ÚLTXBaseModel.__init__Æ  s7  ø€ ô2 	‰ÑÔØˆŒØ!2ÔØ'>Ô$ØˆŒ
Ø&ÔØ#6Ô Ø"4ÔØ#6Ô Ø 0ÔØ$ŒØ*DÔ'Ø,HÔ)Ü*5×*?Ñ*?ÀÓ*GˆÔ'ä%<×%FÑ%FÀvÓ%NÔRi×RqÑRqÔ%qÕ!Ü+ð 	Ô ð ,GÔ(Ø$ˆŒØ)BÔ&Ø-JÔ*Ø%:Ô"Ø/NÔ,ØˆŒØ'>ˆÔ$Ø/NˆÔ,ð -ÕAˆŒØ'Ôð 	×$Ñ$ V¨UÔ3ð 	×#Ò# F¨EÑ<°VÒ<ð 	×%Ò% f¨eÑ>°vÒ>ð 	×$Ñ$ V¨UÖ3r   c           	     ó†  € V P                   P                  V P                  V P                  RW!R7      V n        V P
                  '       d>   \        P                  ! \        P                  ! ^V P                  W!R7      4      V n
        MRV n
        V P                  '       d   \        M\        p\        V P                  VRW!V P                   R7      V n        V P                  '       d=   V P                   '       d+   \        V P                  ^RW!V P                   R7      V n        MRV n        V P$                  '       dR   V P&                  '       d6   \)        V P*                  V P                  VVV P                   R7      V n        R# R V n        R# \/        V P*                  V P                  VVV P                   R7      V n        R# )	zÙInitialize components common to all LTX models
- patchify_proj: Linear projection for patchifying input
- adaln_single: AdaLN layer for timestep embedding
- caption_projection: Linear projection for caption embedding
Trn   rm   NF)r¯   r–   rM   rN   r|   )rÁ   rÊ   rM   rN   r|   c                 ó   € V # r   r'   )Úas   &r   Ú<lambda>Ú6LTXBaseModel._init_common_components.<locals>.<lambda>0  s   € ±Ar   )r|   rr   re   ró   Úpatchify_projrê  r
   ri  rC   r$  Úkeyframes_abs_pos_embeddingrc  rg  rh  r­   Úadaln_singleré  Úprompt_adaln_singlerç  rè  rÑ   rÚ  Úcaption_projectionr¿   )ry   rN   rM   r¯   s   &&& r   rì  Ú$LTXBaseModel._init_common_components  s}  € ð "Ÿ_™_×3Ñ3Ø×Ñ˜dŸn™n°4¸uð 4ó 
ˆÔð ×/×/Ð/Ü/1¯|ª|¼E¿KºKÈÈ4Ï>É>ÐafÔ<vÓ/wˆDÕ,à/3ˆDÔ,àAE×A[×A[ÐA[Õ =ÔaxÐÜ.Ø�N‰NÐ2GÐchÐpuð  RV÷  Rañ  Raô
ˆÔð ×%×%Ð%¨$×*F×*FÐ*FÜ'9Ø—‘°aÐSXÐ`eð  BF÷  BQñ  BQô(ˆDÕ$ð (,ˆDÔ$à×-×-Ð-Ø×3×3Ð3Ü*HØ $× 5Ñ 5Ø $§¡ØØ!Ø#Ÿ™ô+�Ö'ñ +6�Ö'ä&?Ø ×1Ñ1Ø ŸN™NØØØŸ?™?ô'ˆDÖ#r   c                ó   € R# )zHInitialize model-specific components. Must be implemented by subclasses.Nr'   ©ry   rN   rM   r#   s   &&&,r   rí  Ú#LTXBaseModel._init_model_components:  ó   € ñ 	r   c                ó   € R# )zAInitialize transformer blocks. Must be implemented by subclasses.Nr'   rý  s   &&&,r   rî  Ú%LTXBaseModel._init_transformer_blocks?  rÿ  r   c                ó   € R# )z@Initialize output components. Must be implemented by subclasses.Nr'   ©ry   rN   rM   s   &&&r   rï  Ú$LTXBaseModel._init_output_componentsD  rÿ  r   c                ó   € R# )z6Process input data. Must be implemented by subclasses.Nr'   )ry   r   Úkeyframe_idxsÚdenoise_maskr#   s   &&&&,r   Ú_process_inputÚLTXBaseModel._process_inputI  rÿ  r   c                ó   € R# )z¿Build self-attention mask for per-guide attention attenuation.

Base implementation returns None (no attenuation). Subclasses that
support guide-based attention control should override this.
Nr'   )ry   r   r1  Úmerged_argss   &&&&r   Ú _build_guide_self_attention_maskÚ-LTXBaseModel._build_guide_self_attention_maskN  s   € ñ r   c                ó   € R# )z>Process transformer blocks. Must be implemented by subclasses.Nr'   )ry   r   rU  r~  r¢   rV  r  r#   s   &&&&&&&,r   Ú_process_transformer_blocksÚ(LTXBaseModel._process_transformer_blocksV  rÿ  r   c                ó   € R# )z7Process output data. Must be implemented by subclasses.Nr'   )ry   r   rº   r  r#   s   &&&&,r   Ú_process_outputÚLTXBaseModel._process_output[  rÿ  r   c                ó€  € VP                  RR4      pVe   VRV3,          pWP                  ,          pV P                  VP                  4       RRRR/VVR7      w  rVP	                  VRVP
                  R,          4      pVP	                  VRVP
                  R,          4      p\        V P                  WbV4      pWV3# )zPrepare timestep embeddings.Ú	grid_maskNrO   r£   r¤   r�  rR   )r   ræ  rø  r’  r&  rU   r—  rù  )	ry   r¢   r¥   r¦   r#   r  r”  rº   r€  s	   &&&&,    r   Ú_prepare_timestepÚLTXBaseModel._prepare_timestep`  sÍ   € à—J‘J˜{¨DÓ1ˆ	ØÒ Ø  9 Õ-ˆHà"×%CÑ%CÕCˆØ&*×&7Ñ&7Ø×#Ñ#Ó%Ø˜4 °Ð6Ø!Ø%ð	 '8ó '
Ñ#ˆð —=‘= ¨R°·±ÀÕ1CÓDˆØ-×2Ñ2°:¸rÐCT×CZÑCZÐ[]ÕC^Ó_Ðä1Ø×$Ñ$ oÀ<ó
ˆð ¨OÐ;Ð;r   c                ó’   € V P                   RJ d   V P                  V4      pVP                  VRVP                  R,          4      pW3# )z'Prepare context for transformer blocks.FrR   )rç  rú  r&  rU   )ry   rU  r¥   r   r~  s   &&&&&r   Ú_prepare_contextÚLTXBaseModel._prepare_contextx  sC   € à×-Ñ-°Ó6Ø×-Ñ-¨gÓ6ˆGà—,‘,˜z¨2¨q¯w©w°r­{Ó;ˆØÐ&Ð&r   c                ój  € V P                   \        P                  8H  pV P                  WAP                  ^,          W!P
                  4      p	\        W‘WV4      p
V'       d&   V^,          pV
P                  R,          pW¼,
          pM"^VP                  ^,          ,          pW.,          p\        V
VVVV4      # )rS   rR   )rä  r   r,   rå  rU   rN   rÂ  rÒ  )ry   r§  rQ   rÉ  rµ  r¨  r¾  rÈ  rÇ  rº  rÁ  Úexpected_freqsÚcurrent_freqsrÆ  r¶  s   &&&&&&&&       r   Ú_precompute_freqs_cisÚ"LTXBaseModel._precompute_freqs_cis€  s¤   € ð ×4Ñ4¼×8IÑ8IÑIˆ
Ø×*Ñ*¨5×2DÑ2DÀQÕ2GÈ×NaÑNaÓbˆÜ˜w°gÓWˆçØ  A�XˆNØ!ŸK™K¨�OˆMØ%Õ5‰Hð ˜×+Ñ+¨AÕ.Õ.ˆFØ•|ˆHÜØØØØØó
ð 	
r   c           	     óö   € VP                  \        P                  4      pVR,          RV,          ,          VR&   V P                  VV P                  VV P
                  V P                  V P                  R7      pV# )zPrepare positional embeddings.ç      ð?)rQ   rÉ  r¨  r¾  rÈ  ©rO   r   )r¡   rC   r4   r  ró   rÜ  r¾  rÈ  )ry   Úpixel_coordsÚ
frame_rateÚx_dtypeÚfractional_coordsrV  s   &&&&  r   Ú_prepare_positional_embeddingsÚ+LTXBaseModel._prepare_positional_embeddingsž  ss   € à(ŸO™O¬E¯M©MÓ:ÐØ"3°DÕ"9¸SÀ:Õ=MÕ"NÐ˜$ÑØ×'Ñ'ØØ—‘ØØ×5Ñ5Ø$(×$@Ñ$@Ø $× 8Ñ 8ð (ó 
ˆð ˆ	r   c                ó*  € Ve�   \         P                  ! V4      '       gs   V^,
          P                  V4      P                  VP                  ^ ,          ^RVP                  R,          34      \         P
                  ! V4      P                  ,          pV# )zPrepare attention mask.rR   )rC   Úis_floating_pointr¡   r  rU   r  r�  )ry   r~  r%  s   &&&r   Ú_prepare_attention_maskÚ$LTXBaseModel._prepare_attention_mask¬  sy   € àÒ%¬e×.EÒ.EÀn×.UÒ.UØ,¨qÕ0×4Ñ4°WÓ=×EÑEØ×%Ñ% aÕ(¨!¨R°×1EÑ1EÀbÕ1IÐJóä—’˜GÓ$×(Ñ(õ)ˆNð Ðr   c	           	     ó  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  ! WW4WVV3RV/V	B # )a¸  
Forward pass for LTX models.

Args:
    x: Input tensor
    timestep: Timestep tensor
    context: Context tensor (e.g., text embeddings)
    attention_mask: Attention mask tensor
    frame_rate: Frame rate for temporal processing
    transformer_options: Additional options for transformer blocks
    keyframe_idxs: Keyframe indices for temporal processing
    **kwargs: Additional keyword arguments

Returns:
    Processed output tensor
r  )	rÖ   Úpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)
ry   r   r¢   rU  r~  r$  r1  r  r  r#   s
   &&&&&&&&&,r   rƒ   ÚLTXBaseModel.forward´  s’   € ô& ×&Ñ&×6Ñ6×IÑIØ�M‰MØÜ×#Ñ#×4Ñ4Ü×'Ñ'×2Ñ2×BÑBÐDWóó
÷ ‰'ðð ˜w¸
ÐYfñMð vBðMð FLñMð 	Mr   c	                óL  € \        V\        4      '       d/   V^ ,          P                  p
V^ ,          P                  ^ ,          pMVP                  p
VP                  ^ ,          p/ VCV	CpV P                  ! WV3/ VB w  rpVP                  V4       V P                  ! W+V
3/ VB w  r/pVVR&   V P                  W;W4      w  r4V P                  WJ4      pV P                  WÕV
4      pV P                  WV4      pV P                  ! WWBV3RVRV/VB pV P                  ! WV3/ VB pV# )aÁ  
Internal forward pass for LTX models.

Args:
    x: Input tensor
    timestep: Timestep tensor
    context: Context tensor (e.g., text embeddings)
    attention_mask: Attention mask tensor
    frame_rate: Frame rate for temporal processing
    transformer_options: Additional options for transformer blocks
    keyframe_idxs: Keyframe indices for temporal processing
    **kwargs: Additional keyword arguments

Returns:
    Processed output tensor
r€  r1  r  )rS  rß  rM   rU   r  Úupdater  r  r+  r'  r  r  r  )ry   r   r¢   rU  r~  r$  r1  r  r  r#   Úinput_dtyper¥   r  r#  Úadditional_argsrº   r€  rV  r  s   &&&&&&&&&,         r   r1  ÚLTXBaseModel._forwardÏ  sP  € ô& �aœ×ÒØ˜A�$Ÿ*™*ˆKØ˜1�Ÿ™ A�‰JàŸ'™'ˆKØŸ™ �ˆJà7Ð,Ð7°Ð7ˆØ+/×+>Ò+>¸qÐQ]Ñ+mÐalÑ+mÑ(ˆ˜Ø×Ñ˜?Ô+ð 8<×7MÒ7MÈhÐdoÑ7Ðs~Ñ7Ñ4ˆ _Ø)8ˆÐ%Ñ&Ø"&×"7Ñ"7¸ÈQÓ"_Ñˆð ×5Ñ5°nÓRˆØ×0Ñ0°È;ÓWˆð #×CÑCØ Kó
Ðð
 ×,Ò,Ø˜°"ñ
à 3ð
ð !4ð
ð ñ	
ˆð × Ò  °}ÑTÈÑTˆØˆr   )rø  rÙ  rÚ  rç  rú  rè  rÝ  rc  rØ  rM   rò   rå  rã  re   ró   r÷  rÈ  rÛ  r|   rë  rö  rÜ  r³  rù  rä  ræ  rê  r¾  ré  rÞ  ©é   é   r>  ©r^  é    r@  r   )r(   r)   r*   r+   r«   rq   rì  r   rí  rî  rï  r  r  r  r  r  r  r  r'  r+  rƒ   r1  r.   r/   r‡   rˆ   s   @@r   rÔ  rÔ  ¾  s(  ù‡ € ñð -4Ú-=Ø,1Ø#.Ø %Ø$*Ø&+Ø#Ø(,ØØ $Ø(-ØØØ÷-C4õ C4òJ-ð^ ñó ðð ñó ðð ñó ðð ñó ðòð óó ðð ñó ðò<ô0'ð Ú Ø %Øô
ò<òð @BÐWYÐimð  }AôMð8 @BÐWYÐimð  }A÷6ô 6r   rÔ  c                   ó´   a a€ ] tR tRt oRt^€R^@^ R^R. RORRRRRRRRR3V 3R	 lltR
 tR tR tR t	R t
R tR tR t]R 4       t/ R3R ltR tRtVtV ;t# )Ú	LTXVModeli  z LTXV model for video generation.r>  i   rÕ  FrÖ  Nc                óŽ   <€ \         SV `  ! R/ R VbRVbRVbRVbRVbRVbRVbRVbRV	bR	V
bR
VbRVbRVbRVbRVbRVbRVbVB  R# )re   rØ  rÙ  rÈ  rÚ  rÛ  r³  rÜ  rÝ  rÞ  r¾  ræ  rç  rc  rM   rN   r|   Nr'   )rp   rq   )ry   re   rØ  rÙ  rÈ  rÚ  rÛ  r³  rÜ  rÝ  rÞ  r¾  ræ  rç  rc  rM   rN   r|   r#   r~   s   &&&&&&&&&&&&&&&&&&,€r   rq   ÚLTXVModel.__init__  sÀ   ø€ ô* 	‰Òò 	
Ù#ð	
á 3ð	
ñ  2ð	
ñ !4ð		
ñ
 .ð	
ñ "ð	
ñ (Bð	
ñ *Fð	
ñ )Dð	
ñ 0ð	
ñ %<ð	
ñ '@ð	
ñ +Hð	
ñ #8ð	
ñ ð	
ñ  ð!	
ñ" "Øô%	
r   c                ó   € R# )z$Initialize LTXV-specific components.Nr'   rý  s   &&&,r   rí  Ú LTXVModel._init_model_components5  s   € ár   c                ó<  € \         P                  ! \        V P                  4       Uu. uF]  p\	        V P
                  V P                  V P                  V P                  V P                  V P                  VVV P                  R7	      NK_  	  up4      V n        R# u upi )z'Initialize transformer blocks for LTXV.)rM  rc  rò   rM   rN   r|   N)r
   Ú
ModuleListr¦  rÛ  ra  ró   rÈ  rÙ  rØ  rc  rò   r|   Útransformer_blocks)ry   rN   rM   r#   r[  s   &&&, r   rî  Ú"LTXVModel._init_transformer_blocks9  s†   € ä"$§-¢-ô ˜tŸ™Ô/óñ 0�Aô &Ø—N‘NØ×,Ñ,Ø×+Ñ+Ø $× 8Ñ 8Ø*.×*DÑ*DØ ŸL™LØØ!Ø#Ÿ™÷
ñ 0ñó#
ˆÖùòs   ¨A#Bc           	     ól  € \         P                  ! \        P                  ! ^V P                  W!R7      4      V n        V P                  P                  V P                  RRW!R7      V n        V P                  P                  V P                  V P                  W!R7      V n        \        ^RR7      V n        R# )z&Initialize output components for LTXV.rm   FrÓ   )rÕ   rÔ   rM   rN   T)Ú	start_endN)r
   ri  rC   rj  ró   rk  r|   Ú	LayerNormÚnorm_outrr   rë  Úproj_outr   Ú
patchifierr  s   &&&r   rï  Ú!LTXVModel._init_output_componentsL  s…   € ä!#§¢¬e¯kªk¸!¸T¿^¹^ÐSXÔ.hÓ!iˆÔØŸ™×1Ñ1Ø�N‰N¨u¸$Àeð 2ó 
ˆŒð Ÿ™×.Ñ.¨t¯~©~¸t×?PÑ?PÐX]Ð.ÓmˆŒÜ-¨a¸4Ô@ˆŽr   c                óÔ  € R\        VP                  4      /pV P                  P                  V4      w  r\	        VV P
                  V P                  R7      pRpVEe:   VP                  ^,          ^ 8”  Ed!   V P                  VR,          4      p	VP                  ^,          V	,          ^ 8w  d$   \        RVP                  ^,           RV	 R24      hVP                  R\        VP                  4      /4       V P                  P                  V4      ^ ,          p\        P                  ! V^ 8  RR7      ^ ,          ( pVP                  R	V/4       VR
VR
3,          pVR
R
VR3,          pW‚P                  ^,          ) R p
VP                  RR4      pV'       dª   \        R V 4       4      pV\        V
4      8w  d   \        RV R\        V
4       R24      h. p^ pV F^  pVR,          pW®VV,            p\        VP                  4       P!                  4       4      pVP#                  / VCRV/C4       VV,          pK`  	  WÕR&   VRV
R
3,          pVP                  ^,          ^ 8”  d   W'R
R
VP                  ^,          ) R1R
3&   VP                  ^,          VR&   V P%                  V4      pV P'                  VVVR,          VVP                  R^ 4      VP                  RR4      R7      pWV3# )zProcess input for LTXV.Ú
orig_shape)Úlatent_coordsÚscale_factorsÚ
causal_fixNzkeyframe_idxs holds z( tokens, which is not a whole number of a  -token latent frames. The appended frames were recorded against a different spatial resolution than the latent being sampled, so their positions would land on the wrong tokens. Crop the guides and separate the generated keyframes before upscaling the latent.Úorig_patchified_shaperP   r  rO   .Úguide_attention_entriesc              3   ó2   "  € T F  qR ,          x € K  	  R# 5i)Úpre_filter_countNr'   ©Ú.0Úes   & r   Ú	<genexpr>Ú+LTXVModel._process_input.<locals>.<genexpr>x  s   é € ÐM¹}¸!Ð"4× 5Ò 5»}ùs   ‚zguide pre_filter_counts (z ) != keyframe grid mask length (r£  rZ  Úsurviving_countÚresolved_guide_entriesÚnum_guide_tokensÚgenerated_keyframes)rS  r  rb  rc  rR   )rß  rU   rP  Úpatchifyr   rÞ  rÝ  Útokens_per_latent_framerÉ   r8  rC   Úanyr   ÚsumrT   rE   ÚitemÚappendrö  Ú!apply_keyframes_abs_pos_embedding)ry   r   r  r  r#   r:  rT  r#  r  Útokens_per_frameÚkf_grid_maskÚguide_entriesÚ	total_pfcÚresolved_entriesÚoffsetÚentryÚpfcÚ
entry_maskÚ	survivings   &&&&,              r   r  ÚLTXVModel._process_inputU  s  € à'¬¨a¯g©g«Ð7ˆØŸ?™?×3Ñ3°AÓ6ÑˆÜ-Ø'Ø×0Ñ0Ø×7Ñ7ô
ˆð ˆ	ØÓ$¨×)<Ñ)<¸QÕ)?À!Õ)CØ#×;Ñ;¸OÈLÕ<YÓZÐØ×"Ñ" 1Õ%Ð(8Õ8¸AÔ=Ü Ø*¨=×+>Ñ+>¸qÕ+AÐ*BÐBjØ'Ð(ð ),ð,óð ð ×"Ñ"Ð%<¼dÀ1Ç7Á7»mÐ#LÔMØŸ?™?×3Ñ3°LÓAÀ!ÕDˆLÜŸš <°!Ñ#3¸Ô<¸QÕ?Ð?ˆIØ×"Ñ" K°Ð#;Ô<Ø�!�Y �/Õ"ˆAØ'¨¨1¨i¸Ð(<Õ=ˆLà$×&9Ñ&9¸!Õ&<Ð%<Ð%=Ð>ˆLð
 #ŸJ™JÐ'@À$ÓGˆMßÜÑM¹}ÓMÓM�	Ø¤ LÓ 1Ô1Ü$Ø3°I°;ð ?6Ü69¸,Ó6GÐ5HÈðKóð ð $&Ð Ø�Û*�EØÐ 2Õ3�CØ!-°V¸cµ\Ð!B�JÜ # J§N¡NÓ$4×$9Ñ$9Ó$;Ó <�IØ$×+Ñ+ð -Øð-à)¨9ñ-ô ð ˜c•M’Fñ +ð =MÐ 8Ñ9à)¨#¨|¸QÐ*>Õ?ˆMà×"Ñ" 1Õ%¨Ô)ØBO˜Q  M×$7Ñ$7¸Õ$:Ð#:Ñ#;¸QÐ>Ñ?ð 3@×2EÑ2EÀaÕ2HˆOÐ.Ñ/à×Ñ˜qÓ!ˆØ×2Ñ2ØØØ& |Õ4ØØ,×0Ñ0Ð1CÀQÓGØ &§
¡
Ð+@À$Ó Gð 3ó 
ˆð  Ð/Ð/r   c                ó”   € V P                   P                  pV^,          V^,          ,          V^,          V^,          ,          ,          # )z?Token count of a single latent frame at the given latent shape.)rP  Ú
patch_size)ry   rS  rw  s   && r   re  Ú!LTXVModel.tokens_per_latent_framež  s6   € à—_‘_×/Ñ/ˆ
Ø˜1• ¨A¥Õ.°:¸aµ=ÀJÈqÅMÕ3QÕRÐRr   c                óê  € VR,          pVP                   ^8X  d
   VR,          pV^ 8H  pV^ 8”  d   RVRV) R13&   Ve¶   V P                  V4      pVR,          V8w  d   \        RVR,           RV R24      hVR,          V,          p	VR	,          V,          p
\        P                  ! V^,          V,          \        P
                  VP                  R
7      pRW¹Wš,           % Ve	   W³,          pW{,          pV# )z©Per-token mask selecting the latents that encode a single standalone pixel frame.

Returns a (batch, tokens) boolean mask over the already grid-filtered token sequence.
rO   FNrk  z)The generated keyframes were recorded at z- tokens per latent frame but this latent has z?. Separate the generated keyframes before upscaling the latent.Úfirst_latent_frameÚnum_keyframesrm   Tr"  rÿ   )Úndimre  rÉ   rC   r$  rF   rN   )ry   r#  rS  r  rb  rc  Útemporal_startr/  rk  Úfirst_tokenÚnum_slot_tokensÚslotss   &&&&&&      r   Úkeyframes_abs_pos_maskÚ LTXVModel.keyframes_abs_pos_mask£  s  € ð
 & dÕ+ˆØ×Ñ !Ô#Ø+¨FÕ3ˆNØ Ñ"ˆØ˜aÔØ*/ˆD�Ð%Ð%Ñ&Ð&Ñ'àÒ*à#×;Ñ;¸JÓGÐØ"Ð#5Õ6Ð:JÔJÜ Ø?Ð@SÐTfÕ@gÐ?hð i<Ø<LÐ;Mð N3ð3óð ð
 .Ð.BÕCÐFVÕVˆKØ1°/ÕBÐEUÕUˆOÜ—K’K 
¨1¥Ð0@Õ @ÌÏ
É
Ð[_×[fÑ[fÔgˆEØ?CˆE˜kÕ;Ð<ØÒ$ØÕ(�Ø•<ˆDàˆr   c                ó  € V P                   f   V# V P                  W#WEV4      pV P                   P                  VP                  VP                  R7      pWP                  R4      P                  VP                  4      V,          ,           # )z}Add the learned keyframe marker to the single-pixel-frame tokens.

A no-op for every checkpoint built without the parameter.
r  rR   )r÷  r�  r¡   rN   rM   r  )	ry   r   r#  rS  r  rb  rc  r/  Ú	embeddings	   &&&&&&&  r   rj  Ú+LTXVModel.apply_keyframes_abs_pos_embeddingÂ  ss   € ð
 ×+Ñ+Ò3ØˆHà×*Ñ*¨<ÀYÐbuÓvˆØ×4Ñ4×7Ñ7¸q¿x¹xÈqÏwÉwÐ7ÓWˆ	Ø—>‘> "Ó%×(Ñ(¨¯©Ó1°IÕ=Õ=Ð=r   c                ó’  € \        V\        4      '       dB   V^ ,          P                  ^,          pV^ ,          P                  pV^ ,          P                  pM+VP                  ^,          pVP                  pVP                  pVP                  R^ 4      pV^ 8X  d   R# VP                  RR4      pV'       g   R# \        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      p	V	'       g   R# WG,
          p
. p^ pV EFt  pVR,          pV^ 8X  d   K  VR,          pVP                  R	4      pVP                  R
4      pVeê   Veæ   Vw  pppV P                  VP                  WVR7      VVV4      pVP                  ^ ,          ^8”  dr   V^ ,          p\        ^VP                  ^ ,          4       F>  p\        P                  ! VVV,          4      '       d   K)  \        P                  R4        M	  VR,          p\        VP                  ^,          V4      pVRRV13,          V,          pM\        P                  ! ^V3WõVR7      pVP!                  V4       VVP                  ^,          ,          pEKw  	  V'       g   R# \        P"                  ! V^R7      pVR8H  P%                  4       '       d   R# \'        WJVV4      # )up  Build self-attention mask for per-guide attention attenuation.

Reads resolved_guide_entries from merged_args (computed in _process_input)
to build a log-space additive bias mask that attenuates noisy â†” guide
attention for each guide reference independently.

Returns None if no attenuation is needed (all strengths == 1.0 and no
spatial masks, or no guide tokens).
rb  Nra  c              3   óp   "  € T F,  pVR ,          R8g  ;'       g    VP                  R4      RJx € K.  	  R# 5i)Ústrengthr!  Ú
pixel_maskN©r   r[  s   & r   r^  Ú=LTXVModel._build_guide_self_attention_mask.<locals>.<genexpr>ë  s9   é € ð 
á%�ð ˆj�M˜SÑ ×CÐC A§E¡E¨,Ó$7¸tÐ$CÔCÛ%ùs   ‚6š6TFr`  rˆ  r‰  Úlatent_shaper  zCpixel_mask differs across batch elements; using first element only.ºNrS   NrO   rP   r!  )rS  rß  rU   rN   rM   r   rf  Ú_downsample_mask_to_latentr¡   r¦  rC   ÚequalÚloggerÚwarningr  Úfullri  rY   Úallr  )ry   r   r1  r  r(  rN   rM   rb  ro  Ú
needs_maskr"  Úall_weightsÚtotal_trackedrq  rt  rˆ  r‰  rŒ  Úf_latÚh_latÚw_latÚ	per_tokenÚrefÚbiÚ	n_weightsÚweightsr)  s   &&&&                       r   r  Ú*LTXVModel._build_guide_self_attention_maskÎ  su  € ô �aœ×Òà˜Q�4Ÿ:™: a�=ˆLØ�q•T—[‘[ˆFØ�a•D—J‘J‰EàŸ7™7 1�:ˆLØ—X‘XˆFØ—G‘GˆEà&Ÿ?™?Ð+=¸qÓAÐØ˜qÔ Ùà&Ÿ?™?Ð+CÀTÓJÐßÙ÷ “Sñ 
á%ó
—S—S’Sñ 
á%ó
ó 
ˆ
÷ Ùð #Õ5ˆØˆØˆä%ˆEØÐ/Õ0ˆIØ˜AŒ~Ùà˜ZÕ(ˆHØŸ™ <Ó0ˆJØ Ÿ9™9 ^Ó4ˆLàÒ%¨,Ò*BØ&2Ñ#��u˜eØ ×;Ñ;Ø—M‘M¨�MÓ=Ø˜5 %ó�	ð —?‘? 1Õ%¨Ô)Ø# A�,�CÜ# A y§¡°qÕ'9Ö:˜Ü$Ÿ{š{¨3°	¸"µ×>Ô>Ü"ŸN™Nð!<ôñ "ñ ;ð !*¨"¥�Iô   	§¡°Õ 2°IÓ>�	Ø# A z¨	 z MÕ2°XÕ=‘äŸ*š*Ø˜	�N HÀ5ô�ð ×Ñ˜wÔ'Ø˜WŸ]™]¨1Õ-Õ-‹MñM &÷P Ùô  Ÿ)š) K°QÔ7ˆð ˜sÑ"×'Ñ'×)Ò)Ùä! ,¸]ÈOÓ\Ð\r   c                óè  € V P                   ^ ,          pV P                   ^,          p\        P                  P                  P	                  \        V R4      W#3RR7      p\        VRVR7      pVR,          pV^8”  dÎ   V^8”  dÇ   V^,
          pV^,
          p	W‰,          p
V
^8  dO   \        VR,          R4      p\        P                  P                  P	                  W¹RR7      p\        VR	WBVR
7      pM=Wš,          p\        VRR^^V,           1RR3,          RV
R7      pVP                  ^R7      p\        P                  ! W}.^R7      pMV^8”  d   VP                  RRVRR4      pMTp\        VR4      # )aˆ  Downsample a pixel-space mask to per-token latent weights.

Args:
    mask: (B, 1, F_pix, H_pix, W_pix) pixel-space mask with values in [0, 1].
    f_lat: Number of latent frames (pre-dilation original count).
    h_lat: Latent height (pre-dilation original height).
    w_lat: Latent width (pre-dilation original width).

Returns:
    (B, F_lat * H_lat * W_lat) flattened per-token weights.
zb 1 f h w -> (b f) 1 h wÚarea)ÚsizeÚmodez(b f) 1 h w -> b 1 f h w)rZ  rO   zb 1 f h w -> (b h w) 1 fÚnearestz(b h w) 1 f -> b 1 f h w)rZ  ÚhÚwzb 1 (f t) h w -> b 1 f t h w)r  rP   zb 1 f h w -> b (f h w))rO   rO   r�  rO   rO   )rO   rO   :rS   NNrO   rO   rR   )	rU   rC   r
   r\   Úinterpolater	   ÚmeanrY   Úexpand)r/  r—  r˜  r™  rZ  Úf_pixÚspatial_downÚfirst_frameÚremaining_pixÚremaining_latr  Ú	rest_flatÚrest_upÚrestÚusableÚlatent_masks   &&&&            r   rŽ  Ú$LTXVModel._downsample_mask_to_latent,  s  € ð �J‰J�q�MˆØ—
‘
˜1•ˆô —x‘x×*Ñ*×6Ñ6Ü�dÐ6Ó7Ø�Øð 7ó 
ˆô
 ! Ð/IÈQÔOˆð # >Õ2ˆØ�1Œ9˜ œØ! A�IˆMØ! A�IˆMØÕ.ˆAØ�1Œuô &Ø  Õ0Ø.ó�	ô  Ÿ(™(×-Ñ-×9Ñ9Ø¸	ð :ó �ô !ØÐ7Ø Eô‘ð 'Õ*�Ü Ø   A q¨¨V­ |°Q¸Ð!9Õ:Ø2Øô�ð
 —y‘y Q�yÓ'�ÜŸ)š) [Ð$7¸QÔ?‰KØ�QŒYð &×,Ñ,¨R°°U¸BÀÓC‰Kà%ˆKä˜Ð&>Ó?Ð?r   c                óN  a€ VP                  R/ 4      p	V	P                  R/ 4      p
VP                  RR4      p\        V P                  4       FT  w  poRV3V
9   d6   V3R lpV
RV3,          ! RVRVR	VR
VRVRVRVRV/RV/4      pVR,          pKD  S! VVVVVVVVR7      pKV  	  V# )z$Process transformer blocks for LTXV.Úpatches_replaceÚditr€  NÚdouble_blockc                 óÂ   <€ / pS! V R ,          V R,          V R,          V R,          V R,          V R,          V P                  R4      V P                  R4      R7      VR &   V# )	ÚimgÚtxtr~  ÚvecrV  r1  r  r€  ©rU  r~  r¢   rV  r1  r  r€  rŠ  )Úargsr>  Úblocks   & €r   Ú
block_wrapÚ9LTXVModel._process_transformer_blocks.<locals>.block_wrapu  s²   ø€ Ø�CÙ!& t¨E¥{¸DÀ½KÐX\Ð]mÕXnÐy}ð  Dõ  zEð  JNð  OSõ  JTð  jnð  oDõ  jEð  [_÷  [cñ  [cð  dyó  [zð  LP÷  LTñ  LTð  Ufó  Lgô  "h�C˜‘JØ�Jr   rº  r»  r~  r¼  rV  r1  r  Úoriginal_blockr½  )r   Ú	enumeraterI  )ry   r   rU  r~  r¢   rV  r1  r  r#   r¶  Úblocks_replacer€  rª  rÀ  r>  r¿  s   &&&&&&&&,      @r   r  Ú%LTXVModel._process_transformer_blocksl  s  ø€ à-×1Ñ1Ð2CÀRÓHˆØ(×,Ñ,¨U°BÓ7ˆØ Ÿ*™*Ð%6¸Ó=ˆä! $×"9Ñ"9Ö:‰HˆAˆuØ Ð" nÔ4õð
 % n°aÐ%8Ö9¸5À!ÀUÈGÐUeÐguÐw|ð  Gð  IMð  OQð  Shð  j}ð  Tð  Við  k|ð  ~Mð  ;Nð  Qað  cmð  Pnó  o�Ø˜•J’áØØ#Ø#1Ø%ØØ(;Ø(;Ø$3ô	’ñ ;ð, ˆr   c           
     óÔ  € V P                   R,          P                  VP                  VP                  R7      VR	,          ,           pVR
,          VR,          rvV P	                  V4      pV^V,           ,          V,           pV P                  V4      pVe`   VP                  ^,          ^ 8”  dH   VR,          pVR,          p	\        P                  ! W‘P                  VP                  R7      p
WRVR3&   T
pVR,          pV P                  P                  VV^,          V^,          V^,          V^,          \        P                  ! V P                  P                  4      ,          R7      pV# )zProcess output for LTXV.r  rO   r  rW  rm   rS  )ÚlatentsÚoutput_heightÚoutput_widthÚoutput_num_framesrë  r™  )rO   rO   N)rO   rO   r   )rO   rO   rS   )rk  r¡   rN   rM   rN  rO  rU   rC   r$  rP  Ú
unpatchifyrV   Úprodrw  )ry   r   rº   r  r#   Úscale_shift_valuesÚshiftrA   r  rW  Úfull_xrS  s   &&&&,       r   r  ÚLTXVModel._process_outputŠ  s8  € ð ×"Ñ" :Õ.×1Ñ1¸¿¹ÈÏÉÐ1ÓQÐTeÐfpÕTqÕqð 	ð *¨'Õ2Ð4FÀwÕ4Oˆuà�M‰M˜!ÓˆØ��U•�O˜eÕ#ˆØ�M‰M˜!ÓˆàÒ$¨×)<Ñ)<¸QÕ)?À!Ô)CØ˜{Õ+ˆIØ$*Ð+BÕ$CÐ!Ü—[’[Ð!6¿g¹gÈaÏhÉhÔWˆFØ&'�1�i �?Ñ#ØˆAà˜LÕ)ˆ
Ø�O‰O×&Ñ&ØØ$ Q�-Ø# A�Ø(¨�mØ# A�¬$¯)ª)°D·O±O×4NÑ4NÓ*OÕOð 'ó 
ˆð ˆr   )rN  rP  rO  rk  rI  r<  r?  )r(   r)   r*   r+   r«   rq   rí  rî  rï  r  re  r�  rj  r  ÚstaticmethodrŽ  r  r  r.   r/   r‡   rˆ   s   @@r   rB  rB    s¦   ù‡ € Ù*ð Ø ØØØØØ#*Ú%5Ø$)Ø%Ø %Ø"(Ø&+Ø#ØØØ÷%(
òTò
ò&AòG0òRSò
ò>
>ò\]ð| ñ=@ó ð=@ð~ ikð  AEô ÷<ò r   rB  )FrS   rS   i'  r   )AÚabcr   r   Úenumr   Ú	functoolsÚloggingrV   Útypingr   r   r   Úeinopsr	   Únumpyr   rC   r
   Úcomfy.patcher_extensionrÖ   Úcomfy.ldm.modules.attentionr   r   Úcomfy.ldm.common_ditÚcomfy.model_managementÚ	comfy.opsÚcomfy.quant_opsÚsymmetric_patchifierr   r   Ú	getLoggerr(   r�  r   rj   r   r3   ra   ÚModulerc   rŠ   r”   r­   r¿   rÑ   rß   rê   r  r  r  r?  rA  rh  rg  ra  r—  rz  r¬  Ú	lru_cacher¸  r»  rÂ  rÒ  rÔ  rB  r'   r   r   Ú<module>rã     sº  ðß #Ý Û Û Û ß (Ñ (å Û Û Ý Û Û "ß PÛ Û Û Û ç Mà	×	Ò	˜8Ó	$€ò$ô
1�#�tô 
1ô
1˜c 4ô 
1÷1ôh2˜Ÿ	™	ô 2ôj�—	‘	ô ô&°·	±	ô ô>&L˜Ÿ™ô &LôR §	¡	ô ôB& R§Y¡Yô &ô0J�"—)‘)ô JôY�"—)‘)ô Yò"*ò.2÷ Kñ Kô4ô:N �R—Y‘Yô N ðb Ð Ø !Ð ô<˜BŸI™Iô <ò|>ð2 ¨RôBò, ð ×Ò˜QÔóHó  ðHò"ò*ò&Rô$G�5—8‘8—?‘? Cô GôT
^�ö ^r   