+
    ƒ°ñjç  ã                   ó�   € ^ RI Ht ^ RIt^ RIHtHtHt ^ RIHt ^ RI	Hu H
t R tRR/R R llt ! R	 R
]P                  4      tR# )é    )ÚOptionalN)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc                 ól   € V ^VP                  ^4      ,           ,          VP                  ^4      ,           # )é   )Ú	unsqueeze)ÚxÚshiftÚscales   &&&ÚEC:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\genmo\joint_model\utils.pyÚmodulater      s)   € Ø��E—O‘O AÓ&Õ&Õ'¨%¯/©/¸!Ó*<Õ<Ð<ó    ÚkeepdimFc                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )é   r
   ÚmaskÚreturn)ÚtorchÚTensor)Úformats   "r   Ú__annotate__r      s-   € ÷ ñ ”5—<‘<ð ¤u§|¡|ð Ì%Ï,É,ñ r   c               ól  € V P                  ^4      VP                  ^4      8X  g   Q hV P                  ^ 4      VP                  ^ 4      8X  g   Q hVR,          P                  V P                  R7      pWP                  ^RR7      P	                  ^R7      ,          pW,          P                  ^VR7      pV# )zö
Pool tokens in x using mask.

NOTE: We assume x does not require gradients.

Args:
    x: (B, L, D) tensor of tokens.
    mask: (B, L) boolean tensor indicating which tokens are not padding.

Returns:
    pooled: (B, D) tensor of pooled tokens.
)ÚdtypeT)Údimr   )Úmin)ºNNNr   N)ÚsizeÚtor   ÚsumÚclamp)r
   r   r   Úpooleds   &&$ r   Úpool_tokensr#      s˜   € ð �6‰6�!‹9˜Ÿ	™	 !›Ô$Ð$Ð$Ø�6‰6�!‹9˜Ÿ	™	 !›Ô$Ð$Ð$Ø�
Õ×Ñ Q§W¡WÐÓ-€DØ—(‘(˜q¨$�(Ó/×5Ñ5¸!Ð5Ó<Õ<€DØ�h�^‰^ ¨7ˆ^Ó3€FØ€Mr   c                   óH   a a€ ] tR t^%t oRV3R lV 3R llltR tRtVtV ;t# )ÚAttentionPoolc          	      óV   <€ V ^8„  d   QhRS[ RS[ RS[ RS[S[P                  ,          /# )r   Ú	embed_dimÚ	num_headsÚ
output_dimÚdevice)Úintr   r   r*   )r   Ú__classdict__s   "€r   r   ÚAttentionPool.__annotate__&   sB   ø€ ÷ hñ háðhñ ðhñ ð	hñ
 ™Ÿ™Õ&ñhr   c                ó  <€ \         SV `  4        \        4       V n        W n        VP                  V^V,          WER7      V n        VP                  WWER7      V n        TP                  Y;'       g    TWER7      V n        R# )zö
Args:
    spatial_dim (int): Number of tokens in sequence length.
    embed_dim (int): Dimensionality of input tokens.
    num_heads (int): Number of attention heads.
    output_dim (int): Dimensionality of output tokens. Defaults to embed_dim.
)r*   r   N)	ÚsuperÚ__init__r   Úcomfy_attentionr(   ÚLinearÚto_kvÚto_qÚto_out)Úselfr'   r(   r)   r*   r   Ú
operationsÚ	__class__s   &&&&&&&€r   r0   ÚAttentionPool.__init__&   st   ø€ ô  	‰ÑÔÜ-Ó/ˆÔØ"ŒØ×&Ñ& y°!°iµ-ÈÐ&Ó\ˆŒ
Ø×%Ñ% iÀ6Ð%ÓWˆŒ	Ø ×'Ñ'¨	×3JÐ3JÀÐSYÐ'ÓgˆŽr   c                ó   € VP                  ^4      pVR,          P                  4       p\        P                  ! VRRR7      p\	        WRR7      p\
        P                  ! WQ.^R7      pV P                  V4      pV P                  VR,          4      pW0P                  ,          pVP                  ^^V P                  V34      pVP                  ^^4      pVP                  ^4      w  ršVP                  ^V P                  V34      pVP                  ^4      p?\        V4      \        V	4      \        V
4      r©p\        WyW P                  VRRV P                   R7      pVP#                  ^4      P%                  ^^4      pV P'                  V4      pV# )	a  
Args:
    x (torch.Tensor): (B, L, D) tensor of input tokens.
    mask (torch.Tensor): (B, L) boolean tensor indicating which tokens are not padding.

NOTE: We assume x does not require gradients.

Returns:
    x (torch.Tensor): (B, D) tensor of pooled tokens.
T)Úvalue)r   )r   )r   Úskip_reshapeÚskip_output_reshapeÚpreferred_attention)r   NNr   )r   r   )r   r   )r   ÚboolÚFÚpadr#   r   Úcatr3   r4   r(   Ú	unflattenÚ	transposeÚunbindr	   r   r   r1   ÚsqueezeÚflattenr5   )r6   r
   r   ÚDÚ	attn_maskÚx_poolÚkvÚqÚhead_dimÚkÚvs   &&&        r   ÚforwardÚAttentionPool.forward=   sj  € ð �F‰F�1‹Iˆð Ð)Õ*×/Ñ/Ó1ˆ	Ü—E’E˜) V°4Ô8ˆ	ô ˜Q¨dÔ3ˆô �IŠI�v�k qÔ)ˆð �Z‰Z˜‹]ˆØ�I‰I�a˜•gÓˆð Ÿ™Õ&ˆØ�\‰\˜!˜a §¡°Ð:Ó;ˆØ�\‰\˜!˜QÓˆØ�y‰y˜‹|‰ˆØ�K‰K˜˜DŸN™N¨HÐ5Ó6ˆØ�K‰K˜‹Nˆð Ü*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆÜ  a¯©¸iÐVZÐptð  KO÷  K_ñ  K_ô  `ˆð �I‰I�a‹L× Ñ   AÓ&ˆØ�K‰K˜‹NˆØˆr   )r1   r(   r3   r5   r4   )NNNN)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r0   rP   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r8   r,   s   @@r   r%   r%   %   s   ù‡ € ÷hõ h÷.+ò +r   r%   )Útypingr   r   Úcomfy.ldm.modules.attentionr   r   r   Útorch.nnÚnnÚtorch.nn.functionalÚ
functionalr@   r   r#   ÚModuler%   © r   r   Ú<module>ra      s?   ðõ ã ß eÑ eÝ ß Ð ò=ðÀ÷ ô*C�B—I‘Iö Cr   