+
    ƒ°ñj4  ã                   óP  € ^ RI t ^ RIHt ^ RIt ! R R] P
                  P                  4      t ! R R] P
                  P                  4      t ! R R] P
                  P                  4      t	 ! R	 R
] P
                  P                  4      t
 ! R R] P
                  P                  4      t ! R R] P
                  P                  4      t ! R R] P
                  P                  4      t ! R R] P
                  P                  4      t ! R R] P
                  P                  4      tR# )é    N)Úoptimized_attention_for_devicec                   ó<   a a€ ] tR t^t oV 3R ltRR ltRtVtV ;t# )ÚBertAttentionc                óÆ   <€ \         SV `  4        W n        VP                  WR W4R7      V n        VP                  WR W4R7      V n        VP                  WR W4R7      V n        R# )T)ÚbiasÚdtypeÚdeviceN)ÚsuperÚ__init__ÚheadsÚLinearÚqueryÚkeyÚvalue)ÚselfÚ	embed_dimr   r   r	   Ú
operationsÚ	__class__s   &&&&&&€Ú<C:\Ai_Coding\apps\studio\ComfyUI\comfy\text_encoders\bert.pyr   ÚBertAttention.__init__   s^   ø€ Ü‰ÑÔàŒ
Ø×&Ñ& yÀ$ÈeÐ&ÓcˆŒ
Ø×$Ñ$ YÀÈEÐ$ÓaˆŒØ×&Ñ& yÀ$ÈeÐ&ÓcˆŽ
ó    c                ó”   € V P                  V4      pV P                  V4      pV P                  V4      pV! WEW`P                  V4      pV# ©N)r   r   r   r   )r   ÚxÚmaskÚoptimized_attentionÚqÚkÚvÚouts   &&&&    r   ÚforwardÚBertAttention.forward   s@   € Ø�J‰J�q‹MˆØ�H‰H�Q‹KˆØ�J‰J�q‹Mˆá! !¨¯:©:°tÓ<ˆØˆ
r   )r   r   r   r   ©NN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r!   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   Ú__classdict__s   @@r   r   r      s   ù‡ € õd÷ô r   r   c                   ó8   a a€ ] tR t^t oV 3R ltR tRtVtV ;t# )Ú
BertOutputc                ó„   <€ \         SV `  4        VP                  WWER 7      V n        VP	                  W#WER7      V n        R# )©r   r	   ©Úepsr   r	   N)r
   r   r   ÚdenseÚ	LayerNorm)r   Ú	input_dimÚ
output_dimÚlayer_norm_epsr   r	   r   r   s   &&&&&&&€r   r   ÚBertOutput.__init__   s=   ø€ Ü‰ÑÔØ×&Ñ& yÀEÐ&ÓYˆŒ
Ø#×-Ñ-¨jÐTYÐ-ÓiˆŽr   c                óV   € V P                  V4      pV P                  W,           4      pV# r   )r4   r5   )r   r   Úys   &&&r   r!   ÚBertOutput.forward   s$   € Ø�J‰J�q‹Mˆà�N‰N˜1�5Ó!ˆØˆr   )r5   r4   r$   r,   s   @@r   r/   r/      s   ù‡ € õj÷ò r   r/   c                   ó8   a a€ ] tR t^$t oV 3R ltR tRtVtV ;t# )ÚBertAttentionBlockc                ól   <€ \         SV `  4        \        WWEV4      V n        \	        WW4WV4      V n        R # r   )r
   r   r   r   r/   Úoutput)r   r   r   r8   r   r	   r   r   s   &&&&&&&€r   r   ÚBertAttentionBlock.__init__%   s.   ø€ Ü‰ÑÔÜ! )°EÀ:ÓNˆŒ	Ü  °~ÈfÓaˆŽr   c                óH   € V P                  WV4      pV P                  WA4      # r   )r   r@   ©r   r   r   r   r;   s   &&&& r   r!   ÚBertAttentionBlock.forward*   s"   € Ø�I‰I�aÐ2Ó3ˆØ�{‰{˜1Ó Ð r   )r@   r   r$   r,   s   @@r   r>   r>   $   s   ù‡ € õb÷
!ò !r   r>   c                   ó8   a a€ ] tR t^.t oV 3R ltR tRtVtV ;t# )ÚBertIntermediatec                óT   <€ \         SV `  4        VP                  WW4R 7      V n        R# )r1   N)r
   r   r   r4   )r   r   Úintermediate_dimr   r	   r   r   s   &&&&&&€r   r   ÚBertIntermediate.__init__/   s%   ø€ Ü‰ÑÔØ×&Ñ& yÈ%Ð&Ó_ˆŽ
r   c                óv   € V P                  V4      p\        P                  P                  P	                  V4      # r   )r4   ÚtorchÚnnÚ
functionalÚgelu)r   r   s   &&r   r!   ÚBertIntermediate.forward3   s*   € Ø�J‰J�q‹MˆÜ�x‰x×"Ñ"×'Ñ'¨Ó*Ð*r   )r4   r$   r,   s   @@r   rF   rF   .   s   ù‡ € õ`÷+ò +r   rF   c                   ó8   a a€ ] tR t^8t oV 3R ltR tRtVtV ;t# )Ú	BertBlockc                ó�   <€ \         SV `  4        \        WWEWg4      V n        \	        WWVV4      V n        \        W!WEWg4      V n        R # r   )r
   r   r>   Ú	attentionrF   Úintermediater/   r@   )	r   r   rH   r   r8   r   r	   r   r   s	   &&&&&&&&€r   r   ÚBertBlock.__init__9   sC   ø€ Ü‰ÑÔÜ+¨I¸nÐU[ÓhˆŒÜ,¨YÈ%ÐYcÓdˆÔÜ Ð!1¸nÐU[ÓhˆŽr   c                ój   € V P                  WV4      pV P                  V4      pV P                  WA4      # r   ©rS   rT   r@   rC   s   &&&& r   r!   ÚBertBlock.forward?   s2   € Ø�N‰N˜1Ð$7Ó8ˆØ×Ñ˜aÓ ˆØ�{‰{˜1Ó Ð r   rW   r$   r,   s   @@r   rQ   rQ   8   s   ù‡ € õi÷!ò !r   rQ   c                   ó<   a a€ ] tR t^Dt oV 3R ltRR ltRtVtV ;t# )ÚBertEncoderc	                óÀ   <€ \         S
V `  4        \        P                  P	                  \        V4       U	u. uF  p	\        W#WEWgV4      NK  	  up	4      V n        R # u up	i r   )r
   r   rK   rL   Ú
ModuleListÚrangerQ   Úlayer)r   Ú
num_layersr   rH   r   r8   r   r	   r   Úir   s   &&&&&&&&& €r   r   ÚBertEncoder.__init__E   sy   ø€ Ü‰ÑÔÜ—X‘X×(Ñ(ô  MRð  S]ô  M^ó  *_ñ  M^ð  HI¬)°IÐQVÐhmð  xBö  +Cñ  M^ñ  *_ó  `ˆŽ
ùò  *_s   ¶Ac                ó  € \        VP                  VR JRR7      pVe$   V^ 8  d   \        V P                  4      V,           pR p\	        V P                  4       F&  w  rgV! WV4      pWc8X  g   K  VP                  4       pK(  	  W3# )NT)r   Úsmall_input)r   r	   Úlenr^   Ú	enumerateÚclone)r   r   r   Úintermediate_outputr   rT   r`   Úls   &&&&    r   r!   ÚBertEncoder.forwardI   s€   € Ü<¸Q¿X¹XÈDÐX\ÐL\ÐjnÔoÐàÒ*Ø" QÔ&Ü&)¨$¯*©*£oÐ8KÕ&KÐ#àˆÜ˜dŸj™jÖ)‰DˆAÙ�!Ð.Ó/ˆAØÖ'Ø Ÿw™w›y’ñ *ð ˆÐr   )r^   r#   r$   r,   s   @@r   rZ   rZ   D   s   ù‡ € õ`÷ô r   rZ   c                   ó<   a a€ ] tR t^Wt oV 3R ltRR ltRtVtV ;t# )ÚBertEmbeddingsc
                óæ   <€ \         S
V `  4        V	P                  WWGVR 7      V n        V	P                  W%WxR7      V n        V	P                  W5WxR7      V n        V	P                  WVWxR7      V n        R# ))Úpadding_idxr   r	   r1   r2   N)r
   r   Ú	EmbeddingÚword_embeddingsÚposition_embeddingsÚtoken_type_embeddingsr5   )r   Ú
vocab_sizeÚmax_position_embeddingsÚtype_vocab_sizeÚpad_token_idr   r8   r   r	   r   r   s   &&&&&&&&&&€r   r   ÚBertEmbeddings.__init__X   sx   ø€ Ü‰ÑÔØ)×3Ñ3°JÐWcÐyÐ3ó   AˆÔØ#-×#7Ñ#7Ð8OÐbgÐ#7Ó#wˆÔ Ø%/×%9Ñ%9¸/Ð\aÐ%9Ó%qˆÔ"à#×-Ñ-¨iÐSXÐ-ÓhˆŽr   c                óÊ  € Ve   TpMV P                  WR7      pV\        P                  P                  V P                  P
                  R VP                  ^,           V4      ,          pVe$   WPP                  W5P                  R7      ,          pMBV\        P                  P                  V P                  P
                  ^ ,          V4      ,          pV P                  V4      pV# )N)Ú	out_dtype)
ro   ÚcomfyÚopsÚcast_to_inputrp   ÚweightÚshaperq   r   r5   )r   Úinput_tokensÚembedsÚtoken_type_idsr   r   s   &&&&& r   r!   ÚBertEmbeddings.forward`   s¶   € ØÒØ‰Aà×$Ñ$ \Ð$ÓCˆAØ	ŒU�Y‰Y×$Ñ$ T×%=Ñ%=×%DÑ%DÀ[ÀaÇgÁgÈaÅjÐ%QÐSTÓUÕUˆØÒ%Ø×+Ñ+¨NÇgÁgÐ+ÓNÕN‰Aà”—‘×(Ñ(¨×)CÑ)C×)JÑ)JÈ1Õ)MÈqÓQÕQˆAØ�N‰N˜1ÓˆØˆr   )r5   rp   rq   ro   )NNNr$   r,   s   @@r   rk   rk   W   s   ù‡ € õi÷ô r   rk   c                   óJ   a a€ ] tR t^nt oV 3R ltRRRRRR. 3R ltRtVtV ;t# )Ú
BertModel_c                óþ   <€ \         SV `  4        VR ,          pVR,          p\        VR,          VR,          VR,          VR,          WVW#V4	      V n        \	        VR,          WQR,          VR,          WbW44      V n        R	# )
Úhidden_sizer8   rr   rs   rt   ru   Únum_hidden_layersÚintermediate_sizeÚnum_attention_headsN)r
   r   rk   Ú
embeddingsrZ   Úencoder)r   Úconfig_dictr   r	   r   r   r8   r   s   &&&&&  €r   r   ÚBertModel_.__init__o   s¶   ø€ Ü‰ÑÔØ Õ.ˆ	Ø$Ð%5Õ6ˆä(¨°\Õ)BÀKÐPiÕDjÐlwð  yJõ  mKð  MXð  Ygõ  Mhð  jsð  EJð  T^ó  _ˆŒÜ" ;Ð/BÕ#CÀYÐ\oÕPpÐr}ð  Tõ  sUð  Weð  ntó  AˆŽr   NTc	                óJ  € V P                  WVR 7      p	Rp
Veõ   RVP                  V	P                  4      P                  VP                  ^ ,          ^RVP                  R,          34      P                  VP                  ^ ,          ^VP                  R,          VP                  R,          4      ,
          p
V
P                  V
P                  \        P                  4      \        P                  ! V	P                  4      P                  ) 4      p
V P                  WšV4      w  r›W›3# ))r   r   Ng      ð?éÿÿÿÿ)r‰   Útor   Úreshaper}   ÚexpandÚmasked_fillrK   ÚboolÚfinfoÚmaxrŠ   )r   r~   Úattention_maskr   Ú
num_tokensrg   Úfinal_layer_norm_intermediater   Úembeds_infor   r   r`   s   &&&&&&&&&   r   r!   ÚBertModel_.forwardw   s5  € Ø�O‰O˜L¸uˆOÓEˆØˆØÒ%Ø˜×*Ñ*¨1¯7©7Ó3×;Ñ;¸^×=QÑ=QÐRSÕ=TÐVWÐY[Ð]k×]qÑ]qÐrtÕ]uÐ<vÓw×~Ñ~ð  @N÷  @Tñ  @Tð  UVõ  @Wð  YZð  \j÷  \pñ  \pð  qsõ  \tð  vD÷  vJñ  vJð  KMõ  vNó  Oõ  OˆDØ×#Ñ# D§G¡G¬E¯J©JÓ$7¼%¿+º+ÀaÇgÁgÓ:N×:RÑ:RÐ9RÓSˆDà�|‰|˜AÐ%8Ó9‰ˆØˆtˆr   )r‰   rŠ   r$   r,   s   @@r   rƒ   rƒ   n   s7   ù‡ € õAð 48ÀÐQUÐkoð  PTð  \`ð  np÷ ô r   rƒ   c                   óD   a a€ ] tR t^‚t oV 3R ltR tR tR tRtVt	V ;t
# )Ú	BertModelc                ób   <€ \         SV `  4        \        WW44      V n        VR ,          V n        R# )r†   N)r
   r   rƒ   Úbertr_   )r   r‹   r   r	   r   r   s   &&&&&€r   r   ÚBertModel.__init__ƒ   s)   ø€ Ü‰ÑÔÜ˜{°6ÓFˆŒ	Ø%Ð&9Õ:ˆŽr   c                óB   € V P                   P                  P                  # r   ©rž   r‰   ro   )r   s   &r   Úget_input_embeddingsÚBertModel.get_input_embeddingsˆ   s   € Ø�y‰y×#Ñ#×3Ñ3Ð3r   c                ó:   € WP                   P                  n        R # r   r¡   )r   r‰   s   &&r   Úset_input_embeddingsÚBertModel.set_input_embeddings‹   s   € Ø/9�	‰	×ÑÖ,r   c                ó&   € V P                   ! V/ VB # r   )rž   )r   ÚargsÚkwargss   &*,r   r!   ÚBertModel.forwardŽ   s   € Ø�yŠy˜$Ð) &Ñ)Ð)r   )rž   r_   )r%   r&   r'   r(   r   r¢   r¥   r!   r)   r*   r+   r,   s   @@r   rœ   rœ   ‚   s   ù‡ € õ;ò
4ò:÷*ò *r   rœ   )rK   Úcomfy.ldm.modules.attentionr   Ú	comfy.opsry   rL   ÚModuler   r/   r>   rF   rQ   rZ   rk   rƒ   rœ   © r   r   Ú<module>r¯      sÂ   ðÛ Ý FÛ ô�E—H‘H—O‘Oô ô$�—‘—‘ô ô!˜Ÿ™Ÿ™ô !ô+�u—x‘x—‘ô +ô
!�—‘—‘ô 
!ô�%—(‘(—/‘/ô ô&�U—X‘X—_‘_ô ô.�—‘—‘ô ô(*�—‘—‘ö *r   