+
    ƒ°ñjf)  ã                   ó¼   € ^ RI t ^ RIHt ^ RIHtHtHt ^ RIHtH	t	H
t
 RR R lltR tRR R llt ! R	 R
]P                  4      t ! R R]P                  4      tR# )é    N)ÚTupleÚUnionÚOptional)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc                ó¤   € V ^8„  d   QhR\         \        P                  \        \        P                  ,          3,          R\        P                  /# )é   Ú	freqs_cisÚx)r   ÚtorchÚTensorr   )Úformats   "Ú?C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\hydit\attn_layers.pyÚ__annotate__r      s8   € ÷ '&ñ '&¤U¬5¯<©<¼¼u¿|¹|Õ9LÐ+LÕ%Mð '&ÔRW×R^ÑR^ñ '&ó    c                ó0  € VP                   p^ ^u;8:  d	   V8  g   Q h Q h\        V \        4      '       Ed‚   V'       dª   V ^ ,          P                  VP                  R,          VP                  R,          38X  g)   Q RV ^ ,          P                   RVP                   24       h\	        VP                  4       UUu. uF#  w  rEWC^,
          8X  g   WC^,
          8X  d   TM^NK%  	  pppM¢V ^ ,          P                  VP                  ^,          VP                  R,          38X  g)   Q RV ^ ,          P                   RVP                   24       h\	        VP                  4       UUu. uF  w  rEV^8X  g   WC^,
          8X  d   TM^NK  	  pppV ^ ,          P
                  ! V!  V ^,          P
                  ! V!  3# V'       dœ   V P                  VP                  R,          VP                  R,          38X  g"   Q RV P                   RVP                   24       h\	        VP                  4       UUu. uF#  w  rEWC^,
          8X  g   WC^,
          8X  d   TM^NK%  	  pppM”V P                  VP                  ^,          VP                  R,          38X  g"   Q RV P                   RVP                   24       h\	        VP                  4       UUu. uF  w  rEV^8X  g   WC^,
          8X  d   TM^NK  	  pppV P
                  ! V!  # u uppi u uppi u uppi u uppi )aÍ  
Reshape frequency tensor for broadcasting it with another tensor.

This function reshapes the frequency tensor to have the same shape as the target tensor 'x'
for the purpose of broadcasting the frequency tensor during element-wise operations.

Args:
    freqs_cis (Union[torch.Tensor, Tuple[torch.Tensor]]): Frequency tensor to be reshaped.
    x (torch.Tensor): Target tensor for broadcasting compatibility.
    head_first (bool): head dimension first (except batch dim) or not.

Returns:
    torch.Tensor: Reshaped frequency tensor.

Raises:
    AssertionError: If the frequency tensor doesn't match the expected shape.
    AssertionError: If the target tensor 'x' doesn't have the expected number of dimensions.
zfreqs_cis shape z does not match x shape éþÿÿÿéÿÿÿÿ)ÚndimÚ
isinstanceÚtupleÚshapeÚ	enumerateÚview)r   r   Ú
head_firstr   ÚiÚdr   s   &&&    r   Úreshape_for_broadcastr      s  € ð& �6‰6€DØ�Ž=�DŒ=Ð‰=Ðˆ=ä�)œU×#Ó#çØ˜Q•<×%Ñ%¨!¯'©'°"­+°q·w±w¸rµ{Ð)CÔCð  NÐGWÐXaÐbcÕXd×XjÑXjÐWkð  lDð  EF÷  ELñ  ELð  DMð  FNó  NÐCÜLUÐVW×V]ÑV]ÔL^Ô_ÑL^ÁDÀA˜! a�xœ-¨1°qµ¬=‘Q¸aÒ?ÑL^ˆEÑ_ˆEà˜Q•<×%Ñ%¨!¯'©'°!­*°a·g±g¸bµkÐ)BÔBð  MÐFVÐW`ÐabÕWc×WiÑWiÐVjð  kCð  DE÷  DKñ  DKð  CLð  EMó  MÐBÜENÈqÏwÉwÔEWÔXÑEW¹T¸Q˜!˜qœ& A°­¤M‘Q°qÒ8ÑEWˆEÑXØ˜�|× Ò  %Ñ(¨)°A­,×*;Ò*;¸UÑ*CÐCÐC÷ Ø—?‘? q§w¡w¨r¥{°A·G±G¸BµKÐ&@Ô@ð  HÐDTÐU^×UdÑUdÐTeÐe}Ð~÷  Fñ  Fð  ~Gð  CHó  HÐ@ÜLUÐVW×V]ÑV]ÔL^Ô_ÑL^ÁDÀA˜! a�xœ-¨1°qµ¬=‘Q¸aÒ?ÑL^ˆEÑ_ˆEà—?‘? q§w¡w¨q¥z°1·7±7¸2µ;Ð&?Ô?ð  GÐCSÐT]×TcÑTcÐSdÐd|Ð}~÷  ~Eñ  ~Eð  }Fð  BGó  GÐ?ÜENÈqÏwÉwÔEWÔXÑEW¹T¸Q˜!˜qœ& A°­¤M‘Q°qÒ8ÑEWˆEÑXØ�~Š~˜uÑ%Ð%ùó `ùó Yùó `ùó Ys   Â<)L Å%#LÈ/)LË
#Lc                 óÀ   € V P                   ! . V P                  R R ORN^N5!  P                  R4      w  r\        P                  ! V) V.RR7      P                  ^4      # )N©Údimr   )Úreshaper   Úunbindr   ÚstackÚflatten)r   Úx_realÚx_imags   &  r   Úrotate_halfr)   1   sW   € Ø—Y’YÐ4 §¡¨¨ Ð4¨bÐ4°!Ó4×;Ñ;¸BÓ?�N€FÜ�;Š;˜˜ Ð(¨bÔ1×9Ñ9¸!Ó<Ð<r   c                ó<  € V ^8„  d   QhR\         P                  R\        \         P                  ,          R\        \         P                  \        \         P                  ,          3,          R\
        R\        \         P                  \         P                  3,          /# )r
   ÚxqÚxkr   r   Úreturn)r   r   r   r   r   Úbool)r   s   "r   r   r   6   sp   € ÷ &ñ &Ü�L‰Lð&ä”U—\‘\Õ"ð&ô œŸ™¤u¬U¯\©\Õ':Ð:Õ;ð&ô ð	&ô
 Œ5�<‰<œŸ™Ð%Õ&ñ&r   c                ó0  € Rp\        V\        4      '       dT   \        W V4      w  rVW,          \        V 4      V,          ,           pVe    W,          \        V4      V,          ,           pWt3# \        P
                  ! V P                  4       P                  ! . V P                  RR ORN^N5!  4      p\        W(V4      P                  V P                  4      p\        P                  ! W‚,          4      P                  ^4      P                  V 4      pVe�   \        P
                  ! VP                  4       P                  ! . VP                  RR ORN^N5!  4      p	\        P                  ! W’,          4      P                  ^4      P                  V4      pWt3# )an  
Apply rotary embeddings to input tensors using the given frequency tensor.

This function applies rotary embeddings to the given query 'xq' and key 'xk' tensors using the provided
frequency tensor 'freqs_cis'. The input tensors are reshaped as complex numbers, and the frequency tensor
is reshaped for broadcasting compatibility. The resulting tensors contain rotary embeddings and are
returned as real tensors.

Args:
    xq (torch.Tensor): Query tensor to apply rotary embeddings. [B, S, H, D]
    xk (torch.Tensor): Key tensor to apply rotary embeddings.   [B, S, H, D]
    freqs_cis (Union[torch.Tensor, Tuple[torch.Tensor]]): Precomputed frequency tensor for complex exponentials.
    head_first (bool): head dimension first (except batch dim) or not.

Returns:
    Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings.

Nr   )r   r   r   r)   r   Úview_as_complexÚfloatr#   r   ÚtoÚdeviceÚview_as_realr&   Útype_as)
r+   r,   r   r   Úxk_outÚcosÚsinÚxq_outÚxq_Úxk_s
   &&&&      r   Úapply_rotary_embr<   6   sG  € ð0 €FÜ�)œU×#Ò#Ü(¨¸
ÓC‰ˆØ•(œ[¨›_¨sÕ2Õ2ˆØŠ>Ø•h¤¨R£°3Õ!6Õ6ˆFð ˆ>Ðô ×#Ò# B§H¡H£J×$6Ò$6Ð$M¸¿¹ÀÀ"¸Ð$MÀrÐ$MÈ1Ó$MÓNˆÜ)¨)¸*ÓE×HÑHÈÏÉÓSˆ	Ü×#Ò# C¥OÓ4×<Ñ<¸QÓ?×GÑGÈÓKˆØŠ>Ü×'Ò'¨¯©«
×(:Ò(:Ð(Q¸B¿H¹HÀSÀb¸MÐ(QÈ2Ð(QÈqÓ(QÓRˆCÜ×'Ò'¨­Ó8×@Ñ@ÀÓC×KÑKÈBÓOˆFàˆ>Ðr   c                   óD   a a€ ] tR t^`t oRtRV 3R lltRR ltRtVtV ;t	# )ÚCrossAttentionz
Use QK Normalization.
c                ó˜  <€ R V	RV
/p\         SV `  4        W€n        \        4       V n        Wn        W n        W0n        V P
                  V,          ^ 8X  g   Q R4       hV P
                  V,          V n        V P                  ^,          ^ 8X  d   V P                  ^€8:  g   Q R4       hV P                  R	,          V n	        VP                  ! W3RV/VB V n        VP                  ! V^V,          3RV/VB V n        V'       d    VP                  V P                  RRW©R7      M\        P                  ! 4       V n        V'       d    VP                  V P                  RRW©R7      M\        P                  ! 4       V n        \        P$                  ! V4      V n        VP                  ! W3RV/VB V n        \        P$                  ! V4      V n        R# )
r3   Údtypez(self.qdim must be divisible by num_headsú/Only support head_dim <= 128 and divisible by 8ÚbiasTç�íµ ÷Æ°>©Úelementwise_affineÚepsr@   r3   Nç      à¿)ÚsuperÚ__init__Úattn_precisionr   Úcomfy_attentionÚqdimÚkdimÚ	num_headsÚhead_dimÚscaleÚLinearÚq_projÚkv_projÚ	LayerNormÚnnÚIdentityÚq_normÚk_normÚDropoutÚ	attn_dropÚout_projÚ	proj_drop)ÚselfrL   rM   rN   Úqkv_biasÚqk_normrZ   r\   rJ   r3   r@   Ú
operationsÚfactory_kwargsÚ	__class__s   &&&&&&&&&&&& €r   rI   ÚCrossAttention.__init__d   s£  ø€ ð # F¨G°UÐ;ˆÜ‰ÑÔØ,ÔÜ-Ó/ˆÔØŒ	ØŒ	Ø"ŒØ�y‰y˜9Õ$¨Ô)ÐUÐ+UÓUÐ)ØŸ	™	 YÕ.ˆŒØ�}‰}˜qÕ  AÔ%¨$¯-©-¸3Ô*>ÐqÐ@qÓqÐ>Ø—]‘] dÕ*ˆŒ
à ×'Ò'¨ÑT¸ÐTÀ^ÑTˆŒØ!×(Ò(¨¨q°4­xÑY¸hÐYÈ.ÑYˆŒ÷ }D�j×*Ñ*¨4¯=©=ÈTÐW[ÐchÐ*Ôxô  JL÷  JUò  JUó  JWˆŒ÷ }D�j×*Ñ*¨4¯=©=ÈTÐW[ÐchÐ*Ôxô  JL÷  JUò  JUó  JWˆŒÜŸš IÓ.ˆŒØ"×)Ò)¨$ÑV¸8ÐVÀ~ÑVˆŒÜŸš IÓ.ˆŽr   c           
     óÔ  € VP                   w  rEpVP                   w  rxpV P                  V4      P                  WEV P                  V P                  4      p	V P                  V4      P                  WH^V P                  V P                  4      p
V
P                  ^R7      w  r¼V P                  V	4      p	V P                  V4      pVeO   \        V	RV4      w  r×VP                   V	P                   8X  g"   Q RVP                    RV	P                    24       hTp	?V	P                  RR4      P                  4       p	VP                  RR4      P                  4       pVP                  RR4      P                  4       p\        V	4      \        V4      \        V4      rËp	?
\        W›WÀP                  RV P                  V P                  R7      pV P!                  V4      pV P#                  V4      pV3pV# )	zí
Parameters
----------
x: torch.Tensor
    (batch, seqlen1, hidden_dim) (where hidden_dim = num heads * head dim)
y: torch.Tensor
    (batch, seqlen2, hidden_dim2)
freqs_cis_img: torch.Tensor
    (batch, hidden_dim // 2), RoPE for image
r!   Núqq: ú, q: T©Úskip_reshaperJ   Úpreferred_attentionr   éýÿÿÿ)r   rR   r   rN   rO   rS   r$   rW   rX   r<   Ú	transposeÚ
contiguousr   r   rJ   rK   r[   r\   )r]   r   ÚyÚfreqs_cis_imgÚbÚs1ÚcÚ_Ús2ÚqÚkvÚkÚvÚqqÚcontextÚoutÚ	out_tuples   &&&&             r   ÚforwardÚCrossAttention.forward‡   s«  € ð —7‘7‰ˆˆqØ—7‘7‰ˆˆqà�K‰K˜‹N×Ñ  t§~¡~°t·}±}ÓEˆØ�\‰\˜!‹_×!Ñ! !¨¨D¯N©N¸D¿M¹MÓJˆØ�y‰y˜QˆyÓ‰ˆØ�K‰K˜‹NˆØ�K‰K˜‹Nˆð Ò$Ü$ Q¨¨mÓ<‰EˆBØ—8‘8˜qŸw™wÔ&ÐG¨$¨r¯x©x¨j¸¸a¿g¹g¸YÐ(GÓGÐ&ØˆAØà�K‰K˜˜BÓ×*Ñ*Ó,ˆØ�K‰K˜˜BÓ×*Ñ*Ó,ˆØ�K‰K˜˜BÓ×*Ñ*Ó,ˆä*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆØÜ% a¨A¯~©~ÈDÐae×atÑatð  KO÷  K_ñ  K_ô  `ˆà�m‰m˜GÓ$ˆØ�n‰n˜SÓ!ˆà�Fˆ	àÐr   )rZ   rJ   rK   rO   rX   rM   rS   rN   r[   r\   rW   rR   rL   rP   ©TFç        r   NNNN©N©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rI   r|   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rb   Ú__classdict__s   @@r   r>   r>   `   s   ù‡ € ñ÷!/÷F(ô (r   r>   c                   óD   a a€ ] tR t^²t oRtRV 3R lltRR ltRtVtV ;t	# )Ú	Attentionz:
We rename some layer names to align with flash attention
c                óD  <€ \         SV `  4        Wpn        \        4       V n        Wn        W n        V P
                  V,          ^ 8X  g   Q R4       hV P
                  V,          V n        V P                  ^,          ^ 8X  d   V P                  ^€8:  g   Q R4       hV P                  R	,          V n        V
P                  W^,          W8V	R7      V n
        V'       d    V
P                  V P                  RRW‰R7      M\        P                  ! 4       V n        V'       d    V
P                  V P                  RRW‰R7      M\        P                  ! 4       V n        \        P                   ! V4      V n        V
P                  WW‰R7      V n        \        P                   ! V4      V n        R# )
r   z$dim should be divisible by num_headsrA   )rB   r@   r3   TrC   rD   )r@   r3   NrG   )rH   rI   rJ   r   rK   r"   rN   rO   rP   rQ   ÚWqkvrT   rU   rV   rW   rX   rY   rZ   r[   r\   )r]   r"   rN   r^   r_   rZ   r\   rJ   r@   r3   r`   rb   s   &&&&&&&&&&&€r   rI   ÚAttention.__init__¶   sk  ø€ Ü‰ÑÔØ,ÔÜ-Ó/ˆÔØŒØ"ŒØ�x‰x˜)Õ# qÔ(ÐPÐ*PÓPÐ(ØŸ™ IÕ-ˆŒà�}‰}˜qÕ  AÔ%¨$¯-©-¸3Ô*>ÐqÐ@qÓqÐ>Ø—]‘] dÕ*ˆŒ
ð ×%Ñ% c°­7¸ÐW]Ð%Ó^ˆŒ	÷ }D�j×*Ñ*¨4¯=©=ÈTÐW[ÐchÐ*Ôxô  JL÷  JUò  JUó  JWˆŒ÷ }D�j×*Ñ*¨4¯=©=ÈTÐW[ÐchÐ*Ôxô  JL÷  JUò  JUó  JWˆŒÜŸš IÓ.ˆŒØ"×)Ñ)¨#¸%Ð)ÓOˆŒÜŸš IÓ.ˆŽr   c           
     ó  € VP                   w  r4pV P                  V4      P                  W4^V P                  V P                  4      P                  ^^ ^^^4      pVP                  ^ 4      w  rxp	V P                  V4      pV P                  V4      pVe†   \        WxVRR7      w  r«V
P                   VP                   8X  d   VP                   VP                   8X  g<   Q RV
P                    RVP                    RVP                    RVP                    24       hY«r‡?
?\        V4      \        V4      \        V	4      r˜p?\        WxW�P                  RV P                  V P                  R7      pV P                  V4      pV P                  V4      pV3pV# )é   T)r   re   rf   z, kk: z, k: rg   )r   r�   r#   rN   rO   Úpermuter$   rW   rX   r<   r   r   rJ   rK   r[   r\   )r]   r   rn   ÚBÚNÚCÚqkvrt   rv   rw   rx   Úkkr{   s   &&&          r   r|   ÚAttention.forwardË   sp  € Ø—'‘'‰ˆˆaØ�i‰i˜‹l×"Ñ" 1¨¨D¯N©N¸D¿M¹MÓJ×RÑRÐSTÐVWÐYZÐ\]Ð_`ÓaˆØ—*‘*˜Q“-‰ˆˆaØ�K‰K˜‹NˆØ�K‰K˜‹Nˆð Ò$Ü% a¨MÀdÔK‰FˆBØ—8‘8˜qŸw™wÔ&¨2¯8©8°q·w±wÔ+>ð NØ�r—x‘x�j  a§g¡g Y¨f°R·X±X°J¸eÀAÇGÁGÀ9ÐMóNÐ>àˆqØ�Bä*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆØÜ  a¯©ÀdÐ[_×[nÑ[nð  EI÷  EYñ  EYô  ZˆØ�M‰M˜!ÓˆØ�N‰N˜1Óˆà�Dˆ	àÐr   )r�   rZ   rJ   rK   r"   rO   rX   rN   r[   r\   rW   rP   r~   r€   r�   rŠ   s   @@r   r�   r�   ²   s   ù‡ € ñ÷/÷*ô r   r�   )F)r   Útorch.nnrU   Útypingr   r   r   Úcomfy.ldm.modules.attentionr   r   r   r   r)   r<   ÚModuler>   r�   © r   r   Ú<module>rŸ      sK   ðÛ Ý ß )Ñ )ß eÑ e÷'&òT=÷
&ôTO�R—Y‘Yô Oôd0�—	‘	ö 0r   