+
    ƒ°ñjj  ã                   ó°   € ^ RI Ht ^ RIt^ RIHt ^ RIHt  ! R R]P                  4      t ! R R]P                  4      t	 ! R R	]P                  4      t
R# )
é    )ÚOptionalN)Úoptimized_attentionc                   óX   a a€ ] tR t^t o]P
                  RRR3V 3R lltR tRtVt	V ;t
# )ÚMLPNc	                óè   <aaaa	€ \         S
V `  4        V.V.V^,
          ,          ,           V.,           o	\        P                  ! VV	VV3R l\	        V4       4       4      V n        V! 4       V n        R# )é   c              3   óx   <"  € T F/  pSP                  SV,          SV^,           ,          SSR7      x € K1  	  R# 5i)r   )ÚdeviceÚdtypeN©ÚLinear)Ú.0Úir
   Údimsr   Ú
operationss   & €€€€ÚJC:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\sam3d_body\model\transformer.pyÚ	<genexpr>ÚMLP.__init__.<locals>.<genexpr>   s;   øé € ð $
á&�ð ×Ñ˜d 1�g t¨A°­E¥{¸6ÈÐ×OÐOÛ&ùs   ƒ7:N)ÚsuperÚ__init__ÚnnÚ
ModuleListÚrangeÚlayersÚact)ÚselfÚ	input_dimÚ
hidden_dimÚ
output_dimÚ
num_layersÚ	act_layerr
   r   r   r   Ú	__class__s   &&&&&&fff@€r   r   ÚMLP.__init__	   sY   ü€ Ü‰ÑÔØˆ{˜j˜\¨Z¸!­^Õ<Õ<À
¸|ÕKˆÜ—m’m÷ $
ä˜:Ô&ó$
ó 
ˆŒñ “;ˆŽó    c                óÂ   € \        V P                  4       FE  w  r#V\        V P                  4      ^,
          8  d   V P                  V! V4      4      MV! V4      pKG  	  V# ©r   )Ú	enumerater   Úlenr   )r   Úxr   Úlayers   &&  r   ÚforwardÚMLP.forward   sI   € Ü! $§+¡+Ö.‰HˆAØ&'¬#¨d¯k©kÓ*:¸QÕ*>Ô&>�—‘™˜q›Ô"ÁEÈ!ÃHŠAñ /àˆr$   )r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   ÚReLUr   r+   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r"   Ú__classdict__s   @@r   r   r      s*   ù‡ € ØPR×PWÑPWÐ`dÐlpð  ~B÷ ÷ò r$   r   c                   ó^   a a€ ] tR t^t oRV 3R lltV3R lR ltRV3R lR lltRtVtV ;t	# )	Ú	Attentionc                ó’  <aa	a
€ \         SV `  4        T;'       g    TV n        T;'       g    TV n        T;'       g    TV n        Wn        W n        W,          V n        VV	V
3R  lpV! V P                  W4      V n        V! V P                  W4      V n	        V! V P                  W4      V n
        V! WP                  V4      V n        R# )c                 ó.   <€ SP                  WVSSR 7      # ))Úbiasr
   r   r   )r   ÚoÚbr
   r   r   s   &&&€€€r   Ú<lambda>Ú$Attention.__init__.<locals>.<lambda>"   s   ø€ ˜j×/Ñ/°¸1ÀVÐSXÐ/ÔYr$   N)r   r   Ú
query_dimsÚkey_dimsÚ
value_dimsÚ
embed_dimsÚ	num_headsÚ	head_dimsÚq_projÚk_projÚv_projÚproj)r   rC   rD   r@   rA   rB   Úqkv_biasÚ	proj_biasr
   r   r   Úlinr"   s   &&&&&&&&fff €r   r   ÚAttention.__init__   sœ   û€ ä‰ÑÔØ$×2Ð2¨
ˆŒØ ×.Ð. JˆŒØ$×2Ð2¨
ˆŒØ$ŒØ"ŒØ#Õ0ˆŒæYˆÙ˜$Ÿ/™/¨:Ó@ˆŒÙ˜$Ÿ-™-¨:Ó@ˆŒÙ˜$Ÿ/™/¨:Ó@ˆŒÙ˜*§o¡o°yÓAˆŽ	r$   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )é   r)   Úreturn)ÚtorchÚTensor)Úformatr6   s   "€r   Ú__annotate__ÚAttention.__annotate__(   s'   ø€ ÷ Oñ O™Ÿ™ð O©¯©ñ Or$   c                óŽ   € VP                   w  r#pVP                  W#V P                  V P                  4      P	                  ^^4      # r&   )ÚshapeÚreshaperD   rE   Ú	transpose)r   r)   r=   ÚnÚ_s   &&   r   Ú_splitÚAttention._split(   s7   € Ø—'‘'‰ˆˆaØ�y‰y˜˜tŸ~™~¨t¯~©~Ó>×HÑHÈÈAÓNÐNr$   c                óD   <€ V ^8„  d   QhRS[ S[P                  ,          /# )rO   Ú	attn_mask)r   rQ   rR   )rS   r6   s   "€r   rT   rU   ,   s   ø€ ÷ ñ ©(±5·<±<Õ*@ñ r$   c           
     ó  € V P                  V P                  V4      4      V P                  V P                  V4      4      V P                  V P                  V4      4      r2p\	        WW0P
                  VR RR7      pV P                  V4      # )TF)ÚmaskÚskip_reshapeÚlow_precision_attention)r\   rF   rG   rH   r   rD   rI   )r   ÚqÚkÚvr_   r)   s   &&&&& r   r+   ÚAttention.forward,   sk   € Ø—+‘+˜dŸk™k¨!›nÓ-¨t¯{©{¸4¿;¹;Àq»>Ó/JÈDÏKÉKÐX\×XcÑXcÐdeÓXfÓLgˆaˆÜ  a¯©¸iÐVZÐtyÔzˆØ�y‰y˜‹|Ðr$   )
rC   rE   rG   rA   rD   rI   rF   r@   rH   rB   )NNNTTNNN)N)
r-   r.   r/   r0   r   r\   r+   r2   r3   r4   r5   s   @@r   r8   r8      s&   ù‡ € ÷B÷ Oð O÷÷ ò r$   r8   c                   ó@   a a€ ] tR t^1t oRV 3R lltRR ltRtVtV ;t# )ÚTransformerDecoderLayerc                óÒ  <aa	a
€ \         SV `  4        W`n        Wpn        VV	V
3R  lpW4,          p\	        WÃSS	S
R7      pV'       d   V! V4      V! V4      uV n        V n        V! V4      V n        \        RRVRVRV/VB V n	        V! V4      V! V4      uV n
        V n        \        RRVRVRV/VB V n        V! V4      V n        \        WV^\        P                   SS	S
R7      V n        R# )c                 ó.   <€ SP                  V R SSR7      # )g�íµ ÷Æ°>)Úepsr
   r   )Ú	LayerNorm)Údr
   r   r   s   &€€€r   r>   Ú2TransformerDecoderLayer.__init__.<locals>.<lambda>8   s   ø€ �z×+Ñ+¨A°4ÀÈeÐ+ÔTr$   )rC   rD   r
   r   r   r@   rA   rB   )r    r!   r
   r   r   N© )r   r   Ú	repeat_peÚskip_first_peÚdictÚln_pe_1Úln_pe_2Úln1r8   Ú	self_attnÚln2_1Úln2_2Ú
cross_attnÚln3r   r   ÚGELUÚffn)r   Ú
token_dimsÚcontext_dimsrD   rE   Úmlp_dimsrq   rr   r
   r   r   ÚlnÚattn_dimÚattn_kwargsr"   s   &&&&&&&&fff   €r   r   Ú TransformerDecoderLayer.__init__2   sì   û€ ä‰ÑÔØ"ŒØ*ÔæTˆØÕ(ˆÜ hÈFÐZ_ÐlvÔwˆçÙ)+¨J«¹¸LÓ9IÐ&ˆDŒL˜$œ,á�j“>ˆŒÜ"Ñt¨jÐtÀ:ÐtÐZdÐtÐhsÑtˆŒá!# J£±°LÓ1AÐˆŒ
�D”JÜ#Ñy¨zÐyÀLÐyÐ]iÐyÐmxÑyˆŒá�j“>ˆŒÜ�z¨ZÀAÔQS×QXÑQXÐagÐotð  BLô  MˆŽr$   c                ór  € V P                   '       d'   Ve#   V P                  V4      pV P                  V4      pV P                   '       dB   V P                  '       g0   Ve,   V P	                  V4      V,           ;rgV P	                  V4      pMV P	                  V4      ;p;rxRp	Ve?   VR,          VR,          ,          p	V	P                  ^^R7      P                  ^4       V	^ 8„  p	WP                  WgW‰R7      ,           pV P                   '       dG   VeC   V P                  V4      V,           pV P                  V4      V,           pV P                  V4      pM#V P                  V4      pV P                  V4      ;rxWP                  WgV4      ,           pWP                  V P                  V4      4      ,           pW3# )zKx: [B, N_tokens, C], context: [B, N_ctx, C], x_mask: [B, N_tokens] or None.N)Údim1Údim2)r_   )ºNNNrˆ   N)rˆ   Nrˆ   )rq   rt   ru   rr   rv   ÚdiagonalÚfill_rw   rx   ry   rz   r}   r{   )
r   r)   ÚcontextÚx_peÚ
context_peÚx_maskrd   re   rf   r_   s
   &&&&&&    r   r+   ÚTransformerDecoderLayer.forwardH   sp  € ð �>�>ˆ>˜jÒ4Ø—<‘< Ó%ˆDØŸ™ jÓ1ˆJð �>�>ˆ> $×"4×"4Ð"4¸Ò9IØ—H‘H˜Q“K $Õ&Ð&ˆAØ—‘˜“‰AàŸ™ ›Ð#ˆAÐ#�àˆ	ØÒØ˜zÕ*¨V°JÕ-?Õ?ˆIØ×Ñ A¨AÐÓ.×4Ñ4°QÔ7Ø! A™ˆIØ—‘˜q Q�Ó<Õ<ˆð �>�>ˆ>˜jÒ4Ø—
‘
˜1“ Õ$ˆAØ—
‘
˜7Ó# jÕ0ˆAØ—
‘
˜7Ó#‰Aà—
‘
˜1“ˆAØ—J‘J˜wÓ'Ð'ˆAØ—‘  aÓ(Õ(ˆà—‘˜Ÿ™ !›Ó%Õ%ˆØˆzÐr$   )rz   r}   rv   rx   ry   r{   rt   ru   rq   rw   rr   )é   é@   i   FFNNN)NNN)	r-   r.   r/   r0   r   r+   r2   r3   r4   r5   s   @@r   ri   ri   1   s   ù‡ € ÷M÷, ô  r$   ri   )Útypingr   rQ   Útorch.nnr   Úcomfy.ldm.modules.attentionr   ÚModuler   r8   ri   rp   r$   r   Ú<module>r–      sA   ðÝ ã Ý Ý ;ôˆ"�)‰)ô ô�—	‘	ô ô47˜bŸi™iö 7r$   