+
    ƒ°ñj40  ã                   ó¢   € ^ RI t ^ RIHt ^ RIt^ RIt^ RIHtHtH	t	H
t
 ^ RIHt  ! R R]P                  4      t ! R R]P                  4      tR# )	é    N)ÚOptional)ÚCrossAttentionÚFeedForwardÚfreqs_cis_matrixÚgenerate_freq_grid_np)Únnc                   óP   a a€ ] tR t^t oRtRV 3R lltRV3R lR lltRtVtV ;t	# )ÚBasicTransformerBlock1Da±  
A basic Transformer block.

Parameters:

    dim (`int`): The number of channels in the input and output.
    num_attention_heads (`int`): The number of heads to use for multi-head attention.
    attention_head_dim (`int`): The number of channels in each head.
    dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
    activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
    attention_bias (:
        obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.
    upcast_attention (`bool`, *optional*):
        Whether to upcast the attention computation to float32. This is useful for mixed precision training.
    norm_elementwise_affine (`bool`, *optional*, defaults to `True`):
        Whether to use learnable elementwise affine parameters for normalization.
    standardization_norm (`str`, *optional*, defaults to `"layer_norm"`): The type of pre-normalization to use. Can be `"layer_norm"` or `"rms_norm"`.
    norm_eps (`float`, *optional*, defaults to 1e-5): Epsilon value for normalization layers.
    qk_norm (`str`, *optional*, defaults to None):
        Set to 'layer_norm' or `rms_norm` to perform query and key normalization.
    final_dropout (`bool` *optional*, defaults to False):
        Whether to apply a final dropout after the last feed-forward layer.
    ff_inner_dim (`int`, *optional*): Dimension of the inner feed-forward layer. If not provided, defaults to `dim * 4`.
    ff_bias (`bool`, *optional*, defaults to `True`): Whether to use bias in the feed-forward layer.
    attention_out_bias (`bool`, *optional*, defaults to `True`): Whether to use bias in the attention output layer.
    use_rope (`bool`, *optional*, defaults to `False`): Whether to use Rotary Position Embeddings (RoPE).
    ffn_dim_mult (`int`, *optional*, defaults to 4): Multiplier for the inner dimension of the feed-forward layer.
c                ó‚   <€ \         SV `  4        \        VVVR VVV	V
R7      V n        \	        VVRVVV	V
R7      V n        R # )N)Ú	query_dimÚheadsÚdim_headÚcontext_dimÚapply_gated_attentionÚdtypeÚdeviceÚ
operationsT)Údim_outÚgluÚff_biasr   r   r   )ÚsuperÚ__init__r   Úattn1r   Úff)ÚselfÚdimÚn_headsÚd_headr   Úattn_precisionr   r   r   r   r   Ú	__class__s   &&&&&&&&&&&€ÚMC:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\lightricks\embeddings_connector.pyr   Ú BasicTransformerBlock1D.__init__-   sY   ø€ ô 	‰ÑÔô $ØØØØØ"7ØØØ!ô	
ˆŒ
ô ØØØØØØØ!ô
ˆŽó    c                ó4   <€ V ^8„  d   QhRS[ P                  /# )é   Úreturn)ÚtorchÚFloatTensor)ÚformatÚ__classdict__s   "€r!   Ú__annotate__Ú$BasicTransformerBlock1D.__annotate__T   s   ø€ ÷ ñ Áe×FWÑFWñ r#   c                ó¼  € \         P                  P                  P                  V4      pVP	                  ^4      pV P                  WBVR7      pWQ,           pVP                  ^8X  d   VP	                  ^4      p\         P                  P                  P                  V4      pV P                  V4      pWa,           pVP                  ^8X  d   VP	                  ^4      pV# )é   )ÚmaskÚpe)ÚcomfyÚldmÚ
common_ditÚrms_normÚsqueezer   Úndimr   )r   Úhidden_statesÚattention_maskr0   Únorm_hidden_statesÚattn_outputÚ	ff_outputs   &&&&   r!   ÚforwardÚBasicTransformerBlock1D.forwardT   sÅ   € ô
 #ŸY™Y×1Ñ1×:Ñ:¸=ÓIÐà/×7Ñ7¸Ó:Ðð —j‘jÐ!3ÈR�jÓPˆà#Õ3ˆØ×Ñ Ô"Ø)×1Ñ1°!Ó4ˆMô #ŸY™Y×1Ñ1×:Ñ:¸=ÓIÐð —G‘GÐ.Ó/ˆ	à!Õ1ˆØ×Ñ Ô"Ø)×1Ñ1°!Ó4ˆMàÐr#   )r   r   )NNFTNNN)NN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r<   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r    r*   s   @@r!   r
   r
      s   ù‡ € ñ÷:%
÷N÷ ò r#   r
   c                   óš   a a€ ] tR t^qt oRt^€R^€^^RR.R^€RRRRRRR3V3R lV 3R llltR	 tR
 tR tRR lt	RV3R lR llt
RtVtV ;t# )ÚEmbeddings1DConnectorTi   g     ˆÃ@i   FNc                ó0   <€ V ^8„  d   QhRS[ S[,          /# )r%   Únum_learnable_registers)r   Úint)r)   r*   s   "€r!   r+   Ú"Embeddings1DConnector.__annotate__t   s   ø€ ÷ 6ñ 6ñ "*©#¥ñ6r#   c                ó  <€ \         SV `  4        WÀn        Wn        W@n        WC,          V n        W€n        W`n        Wpn        Wðn	        VV n
        \        P                  ! \        V4       Uu. uF!  p\        V P
                  VVVV
VVVVR 7	      NK#  	  up4      V n        WC,          pW�n        V P                   '       d?   \        P"                  ! \$        P&                  ! V P                   VWÍR7      4      V n        R# R# u upi ))r   r   r   r   r   r   ©r   r   N)r   r   r   Úout_channelsÚnum_attention_headsÚ	inner_dimÚcausal_temporal_positioningÚpositional_embedding_thetaÚpositional_embedding_max_posÚ
split_ropeÚdouble_precision_roper   Ú
ModuleListÚranger
   Útransformer_1d_blocksrJ   Ú	Parameterr'   ÚemptyÚlearnable_registers)r   Úin_channelsÚcross_attention_dimÚattention_head_dimrP   Ú
num_layersrS   rT   rR   rJ   r   Úconnector_ff_biasr   r   r   rU   rV   ÚkwargsÚ_rQ   r    s   &&&&&&&&&&&&&&&&&,  €r!   r   ÚEmbeddings1DConnector.__init__t   sõ   ø€ ô( 	‰ÑÔØŒ
Ø'ÔØ#6Ô Ø,ÕAˆŒØ+FÔ(Ø*DÔ'Ø,HÔ)Ø$ŒØ%:ˆÔ"Ü%'§]¢]ô ˜zÔ*óñ +�Aô (Ø—N‘NØ'Ø&Ø 3Ø*?Ø-ØØ!Ø)÷
ñ +ñó&
ˆÔ"ð" (Õ<ˆ	Ø'>Ô$Ø×'×'Ð'Ü')§|¢|Ü—’Ø×0Ñ0°)À5ôó(ˆDÖ$ñ (ùò%s   Á+'C>c                ó´   € \         P                  ! \        ^4       Uu. uF&  pVRV3,          V P                  V,          ,          NK(  	  upRR7      pV# u upi )r.   ºNNN©r   éÿÿÿÿ)r'   ÚstackrX   rT   )r   Úindices_gridÚiÚfractional_positionss   &&  r!   Úget_fractional_positionsÚ.Embeddings1DConnector.get_fractional_positions¬   s`   € Ü$Ÿ{š{ô ˜qœóá!�Að ˜Q ˜TÕ" T×%FÑ%FÀqÕ%I×IÐIÙ!ñð ô 
Ðð $Ð#ùòs   ž,Ac                ó¨  € VP                   pV\        P                  \        P                  39   d   \        P                  MTpV P                  V4      pV P                  '       d3   \        V P                  VP                  ^,          V P                  4      MV P                  W$VP                  4      P                  VP                  R7      pVR8X  d9   WeP                  R4      ,          P                  RR4      P!                  ^4      pV# WeP                  R4      ^,          ^,
          ,          P                  RR4      P!                  ^4      pV# )r.   ©r   Úexp_2rh   éþÿÿÿ)r   r'   Úbfloat16Úfloat16Úfloat32rm   rV   r   rS   ÚshaperQ   Úgenerate_freq_gridr   ÚtoÚ	unsqueezeÚ	transposeÚflatten)r   rj   ÚspacingÚsource_dtyper   rl   ÚindicesÚfreqss   &&&     r!   Úprecompute_freqsÚ&Embeddings1DConnector.precompute_freqs¶   s)  € Ø#×)Ñ)ˆð ¤§¡´·±Ð>Ô>ô �MŠMàð 	ð  $×<Ñ<¸\ÓJÐð ×)×)Ð)ô "Ø×/Ñ/Ø×"Ñ" 1Õ%Ø—‘ôð ×(Ñ(¨Ð9M×9TÑ9TÓUß
‰"Ð(×/Ñ/ˆ"Ó
0ð 	ð �gÔà×9Ñ9¸"Ó=Õ=ß‘˜2˜rÓ"ß‘˜“ð ð ˆð	 ×:Ñ:¸2Ó>ÀÕBÀQÕFÕGß‘˜2˜rÓ"ß‘˜“ð ð
 ˆr#   c           	     ó   € V P                   pV P                  p^p^V,          p^pTp	VR8X  dP   V\        P                  ! ^ WGR\        P                  R7      WG,
          ,          ,          p
V
P                  W#R7      p
M²VR8X  dB   RV\        P                  ! ^ WGVR7      V,          ,          ,          p
V
P                  VR7      p
MjVR	8X  d!   \        P                  ! W‰WG,          W2R7      p
MCVR
8X  d=   \        P                  ! V^,          V	^,          WG,          W2R7      P                  4       p
X
\        P                  ,          ^,          p
V
# )r.   ÚexpÚcpu)r   r   rN   rq   g      ð?rp   )r   ÚlinearÚsqrt)
rQ   rS   r'   Úarangeru   rx   Úlinspacer†   ÚmathÚpi)r   r|   r   r   r   ÚthetaÚ
n_pos_dimsÚn_elemÚstartÚendr~   s   &&&&       r!   rw   Ú(Embeddings1DConnector.generate_freq_grid×   s  € Ø�n‰nˆØ×/Ñ/ˆØˆ
Ø�Z•ˆØˆØˆà�eÔØ¤§¢¨Q°ÀEÔQV×Q^ÑQ^Ô _ÐcfÕcoÕ pÕqˆGØ—j‘j u�jÓ<‰GØ˜ÔØ˜E¤e§l¢l°1°cÈ&Ô&QÐTWÕ&WÕXÕXˆGØ—j‘j u�jÓ-‰GØ˜Ô Ü—n’nØ˜C�M°&ô‰Gð ˜ÔÜ—n’nØ�q•˜#˜q�& #¥-¸ôç‰d‹fð ð œDŸG™GÕ# aÕ'ˆàˆr#   c                ó  € V P                   p^pV P                  W4      pV P                  '       d&   V^,          pVP                  R,          pWx,
          p	MWE,          p	\	        VV	V P                  V P
                  V4      # )r%   rh   )rQ   r€   rU   rv   r   rP   )
r   rj   r|   Ú	out_dtyper   r�   r   Úexpected_freqsÚcurrent_freqsÚpad_sizes
   &&&&      r!   Úprecompute_freqs_cisÚ*Embeddings1DConnector.precompute_freqs_cisò   st   € Ø�n‰nˆØˆØ×%Ñ% lÓ<ˆØ�?�?ˆ?Ø  A�XˆNØ!ŸK™K¨�OˆMØ%Õ5‰Hà•|ˆHÜØØØ�O‰OØ×$Ñ$Øó
ð 	
r#   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# )r%   r7   r8   )r'   ÚTensorr   )r)   r*   s   "€r!   r+   rL     s,   ø€ ÷ 5-ñ 5-á—|‘|ð5-ñ !¡§¡Õ.ñ5-r#   c                óÎ  € V P                   '       Ed   \        P                  ! \        RVP                  ^,          4      V P                   ,          4      p\
        P                  ! V P                  P                  V4      V^34      p\
        P                  ! WVP                  ^,          R P                  ^ 4      P                  VP                  ^ ,          ^^4      3^R7      pVeC   \
        P                  ! ^^^VP                  ^,          .VP                  VP                  R7      p\
        P                  ! VP                  ^,          \
        P                   VP                  R7      pVR,          pV P#                  WQP                  R7      p\%        V P&                  4       F  w  rxV! WVR7      pK  	  \(        P*                  P,                  P/                  V4      pW3# )ag  
The [`Transformer2DModel`] forward method.

Args:
    hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, channel, height, width)` if continuous):
        Input `hidden_states`.
    indices_grid (`torch.LongTensor` of shape `(batch size, 3, num latent pixels)`):
    attention_mask ( `torch.Tensor`, *optional*):
        An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask
        is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large
        negative values to the attention scores corresponding to "discard" tokens.
Returns:
    If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a
    `tuple` where the first element is the sample tensor.
i   Nrg   rN   )r’   )r8   r0   )NNrf   )rJ   r‰   ÚceilÚmaxrv   r'   Útiler\   rx   Úcatry   ÚrepeatÚzerosr   r   r‡   ru   r–   Ú	enumeraterY   r1   r2   r3   r4   )	r   r7   r8   Únum_registers_duplicationsr\   rj   Ú	freqs_cisÚ	block_idxÚblocks	   &&&      r!   r<   ÚEmbeddings1DConnector.forward  sÑ  € ð, ×'×'Ñ'Ü)-¯ªÜ�D˜-×-Ñ-¨aÕ0Ó1°D×4PÑ4PÕPó*Ð&ô #(§*¢*Ø×(Ñ(×+Ñ+¨MÓ:Ð=WÐYZÐ<[ó#Ðô "ŸIšI }È-×J]ÑJ]Ð^_ÕJ`ÐJaÐ6b×6lÑ6lÐmnÓ6o×6vÑ6vð  xE÷  xKñ  xKð  LMõ  xNð  PQð  STó  7Uð  'Vð  \]ô  ^ˆMàÒ)Ü!&§¢¨a°°A°}×7JÑ7JÈ1Õ7MÐ-NÐVd×VjÑVjð  tB÷  tIñ  tIô  "J�ä—|’|Ø×Ñ Õ"¬%¯-©-À×@TÑ@Tô
ˆð $ MÕ2ˆØ×-Ñ-¨l×FYÑFYÐ-ÓZˆ	ô !*¨$×*DÑ*DÖ EÑˆIÙ!ØÀôŠMñ !Fô Ÿ	™	×,Ñ,×5Ñ5°mÓDˆàÐ,Ð,r#   )rR   rV   r   rQ   r\   rP   rJ   rO   rT   rS   rU   rY   )rƒ   N)N)r>   r?   r@   rA   Ú _supports_gradient_checkpointingr   rm   r€   rw   r–   r<   rC   rD   rE   rF   s   @@r!   rH   rH   q   sn   ù‡ € Ø'+Ð$ð Ø ØØØØ#*Ø&* VØ$)Ø14Ø#ØØØØØØ#÷#6õ 6òp$òòBô6
÷$5-÷ 5-ò 5-r#   rH   )r‰   Útypingr   Úcomfy.ldm.common_ditr1   r'   Úcomfy.ldm.lightricks.modelr   r   r   r   r   ÚModuler
   rH   © r#   r!   Ú<module>r­      sE   ðÛ Ý ã Û ÷ó õ ô_˜bŸi™iô _ôDH-˜BŸI™Iö H-r#   