+
    „°ñjr2  ã                   ó¢  € ^ RI t ^ RIt^ RIHt ^ RIt ! R R] P                  P                  4      tRR R] P                  P                  P                  /t ! R R	] P                  P                  4      t ! R
 R] P                  P                  4      t ! R R] P                  P                  4      t ! R R] P                  P                  4      t ! R R] P                  P                  4      t ! R R] P                  P                  4      t ! R R] P                  P                  4      t ! R R] P                  P                  4      tR# )é    N)Úoptimized_attention_for_devicec                   ó<   a a€ ] tR t^t oRV 3R lltR tRtVtV ;t# )ÚT5LayerNormc                ó¤   <€ \         SV `  4        \        P                  P	                  \        P
                  ! WVR 7      4      V n        W n        R# )©ÚdtypeÚdeviceN)ÚsuperÚ__init__ÚtorchÚnnÚ	ParameterÚemptyÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsr   r	   Ú
operationsÚ	__class__s   &&&&&&€Ú:C:\Ai_Coding\apps\studio\ComfyUI\comfy\text_encoders\t5.pyr   ÚT5LayerNorm.__init__   s6   ø€ Ü‰ÑÔÜ—h‘h×(Ñ(¬¯ª°[ÐV\Ô)]Ó^ˆŒØ #Öó    c                ó  € VP                  ^4      P                  RRR7      pV\        P                  ! W P                  ,           4      ,          p\
        P                  P                  V P                  V4      V,          # )é   T)Úkeepdiméÿÿÿÿ)	ÚpowÚmeanr   Úrsqrtr   ÚcomfyÚopsÚcast_to_inputr   )r   ÚxÚvariances   && r   ÚforwardÚT5LayerNorm.forward   sX   € Ø—5‘5˜“8—=‘= ¨T�=Ó2ˆØ”—’˜H×'<Ñ'<Õ<Ó=Õ=ˆÜ�y‰y×&Ñ& t§{¡{°AÓ6¸Õ:Ð:r   )r   r   )g�íµ ÷Æ°>NNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r&   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   Ú__classdict__s   @@r   r   r      s   ù‡ € ÷$÷
;ò ;r   r   Úgelu_pytorch_tanhc                 óX   € \         P                  P                  P                  V R R7      # )Útanh)Úapproximate)r   r   Ú
functionalÚgelu)Úas   &r   Ú<lambda>r9      s   € ¤5§8¡8×#6Ñ#6×#;Ñ#;¸AÈ6Ð#;Ô#Rr   Úreluc                   ó8   a a€ ] tR t^t oV 3R ltR tRtVtV ;t# )ÚT5DenseActDensec                ó¬   <€ \         SV `  4        VP                  WR WER7      V n        VP                  W!R WER7      V n        \
        V,          V n        R# ©F©Úbiasr   r	   N)r
   r   ÚLinearÚwiÚwoÚactivationsÚact©r   Ú	model_dimÚff_dimÚff_activationr   r	   r   r   s   &&&&&&&€r   r   ÚT5DenseActDense.__init__   sL   ø€ Ü‰ÑÔØ×#Ñ# I¸EÈÐ#Ó^ˆŒØ×#Ñ# F¸EÈÐ#Ó^ˆŒä˜}Õ-ˆŽr   c                óh   € V P                  V P                  V4      4      pV P                  V4      pV# ©N©rE   rB   rC   )r   r$   s   &&r   r&   ÚT5DenseActDense.forward   s*   € Ø�H‰H�T—W‘W˜Q“ZÓ ˆà�G‰G�A‹JˆØˆr   rM   r(   r0   s   @@r   r<   r<      s   ù‡ € õ.÷ò r   r<   c                   ó8   a a€ ] tR t^$t oV 3R ltR tRtVtV ;t# )ÚT5DenseGatedActDensec                óÞ   <€ \         SV `  4        VP                  WR WER7      V n        VP                  WR WER7      V n        VP                  W!R WER7      V n        \        V,          V n        R# r>   )r
   r   rA   Úwi_0Úwi_1rC   rD   rE   rF   s   &&&&&&&€r   r   ÚT5DenseGatedActDense.__init__%   se   ø€ Ü‰ÑÔØ×%Ñ% i¸eÈ5Ð%Ó`ˆŒ	Ø×%Ñ% i¸eÈ5Ð%Ó`ˆŒ	Ø×#Ñ# F¸EÈÐ#Ó^ˆŒä˜}Õ-ˆŽr   c                óš   € V P                  V P                  V4      4      pV P                  V4      pW#,          pV P                  V4      pV# rL   ©rE   rR   rS   rC   )r   r$   Úhidden_geluÚhidden_linears   &&  r   r&   ÚT5DenseGatedActDense.forward-   s?   € Ø—h‘h˜tŸy™y¨›|Ó,ˆØŸ	™	 !›ˆØÕ'ˆà�G‰G�A‹JˆØˆr   rV   r(   r0   s   @@r   rP   rP   $   s   ù‡ € õ.÷ò r   rP   c                   ó8   a a€ ] tR t^5t oV 3R ltR tRtVtV ;t# )Ú	T5LayerFFc                ó¢   <€ \         SV `  4        V'       d   \        WW5Wg4      V n        M\	        WW5Wg4      V n        \        WWgR 7      V n        R# ©©r   r	   r   N)r
   r   rP   ÚDenseReluDenser<   r   Ú
layer_norm)	r   rG   rH   rI   Ú	gated_actr   r	   r   r   s	   &&&&&&&&€r   r   ÚT5LayerFF.__init__6   sC   ø€ Ü‰ÑÔßÜ"6°yÈ-Ð`fÓ"sˆDÕä"1°)À]Ð[aÓ"nˆDÔä% iÀVÔcˆŽr   c                óZ   € V P                  V4      pV P                  V4      pW,          pV# rL   )r`   r_   )r   r$   Úforwarded_statess   && r   r&   ÚT5LayerFF.forward@   s0   € ØŸ?™?¨1Ó-ÐØ×.Ñ.Ð/?Ó@Ðà	ÕˆØˆr   )r_   r`   r(   r0   s   @@r   r[   r[   5   s   ù‡ € õd÷ò r   r[   c                   óV   a a€ ] tR t^Gt oV 3R lt]RR l4       tR tRR ltRt	Vt
V ;t# )ÚT5Attentionc                ó�  <€ \         SV `  4        VP                  WR WVR7      V n        VP                  WR WVR7      V n        VP                  WR WVR7      V n        VP                  W!R WVR7      V n        W0n        RV n        V'       d>   ^ V n	        ^€V n
        VP                  V P                  V P                  WeR7      V n        R# R# )Fr?   N©r	   r   )r
   r   rA   ÚqÚkÚvÚoÚ	num_headsÚrelative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distanceÚ	Embedding)	r   rG   Ú	inner_dimrn   ro   r   r	   r   r   s	   &&&&&&&&€r   r   ÚT5Attention.__init__H   sÈ   ø€ Ü‰ÑÔð ×"Ñ" 9¸eÈ5Ð"Ó`ˆŒØ×"Ñ" 9¸eÈ5Ð"Ó`ˆŒØ×"Ñ" 9¸eÈ5Ð"Ó`ˆŒØ×"Ñ" 9¸eÈ5Ð"Ó`ˆŒØ"Œà'+ˆÔ$ß"Ø24ˆDÔ/Ø36ˆDÔ0Ø+5×+?Ñ+?À×@cÑ@cÐei×esÑesð  }CÐ+?ó  ,QˆDÖ(ñ #r   c                óÄ  € ^ pV'       dP   V^,          pW@^ 8„  P                  \        P                  4      V,          ,          p\        P                  ! V 4      p M,\        P                  ! V \        P
                  ! V 4      4      ) p V^,          pW8  pV\        P                  ! V P                  4       V,          4      \        P                  ! W5,          4      ,          W%,
          ,          P                  \        P                  4      ,           p\        P                  ! V\        P                  ! Wr^,
          4      4      pV\        P                  ! W`V4      ,          pV# )aR  
Adapted from Mesh Tensorflow:
https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

Translate relative position to a bucket number for relative attention. The relative position is defined as
memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
This should allow for more graceful generalization to longer sequences than the model has been trained on

Args:
    relative_position: an int32 Tensor
    bidirectional: a boolean - whether the attention is bidirectional
    num_buckets: an integer
    max_distance: an integer

Returns:
    a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
)Útor   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges   &&&&    r   Ú_relative_position_bucketÚ%T5Attention._relative_position_bucketX   s  € ð, ÐßØ˜AÕˆKØ°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÕ TÕTÐÜ %§	¢	Ð*;Ó <Ñä!&§¢Ð+<¼e×>NÒ>NÐO`Ó>aÓ!bÐ bÐð   1Õ$ˆ	Ø$Ñ0ˆð &/Ü�IŠIÐ'×-Ñ-Ó/°)Õ;Ó<Ü�hŠh�|Õ/Ó0õ1àÕ&õ(÷ ‰"ŒU�Z‰Z‹.õ	&Ð"ô
 &+§Y¢YØ&¬¯ªÐ8RÐbcÕTcÓ(dó&
Ð"ð 	œEŸKšK¨ÐE_Ó`Õ`ÐØÐr   c                ó¤  € \         P                  ! V\         P                  VR7      R,          p\         P                  ! V\         P                  VR7      R,          pWe,
          pV P                  VRV P                  V P
                  R7      pV P                  W„R7      p	V	P                  . RO4      P                  ^ 4      p	V	P                  4       # )z%Compute binned relative position biasr   T)r�   r‚   rƒ   ©Ú	out_dtype)ºNNNN)Nr�   )r   r   é   )
r   Úarangerw   rˆ   rp   rq   ro   ÚpermuteÚ	unsqueezeÚ
contiguous)
r   Úquery_lengthÚ
key_lengthr	   r   Úcontext_positionÚmemory_positionr€   Úrelative_position_bucketÚvaluess
   &&&&&     r   Úcompute_biasÚT5Attention.compute_biasˆ   s±   € ä Ÿ<š<¨¼E¿J¹JÈvÔVÐW^Õ_ÐÜŸ,š, z¼¿¹ÈFÔSÐT[Õ\ˆØ+Õ>ÐØ#'×#AÑ#AØØØ×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÐ-ÓXˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØ× Ñ Ó"Ð"r   c                óì  € V P                  V4      pV P                  V4      pV P                  V4      pV P                  eK   V P	                  VP
                  ^,          VP
                  ^,          VP                  VP                  4      pVe   Ve
   W#,           pMTpV! WVVP
                  R,          V P                  ,          R,          ,          WpP                  V4      pV P                  V4      V3# )Ng      à?r   )
rj   rk   rl   ro   r™   Úshaper	   r   rn   rm   )	r   r$   ÚmaskÚ	past_biasÚoptimized_attentionrj   rk   rl   Úouts	   &&&&&    r   r&   ÚT5Attention.forward—   sÀ   € Ø�F‰F�1‹IˆØ�F‰F�1‹IˆØ�F‰F�1‹IˆØ×'Ñ'Ò3Ø×)Ñ)¨!¯'©'°!­*°a·g±g¸aµjÀ!Ç(Á(ÈAÏGÉGÓTˆIàÒ ØÒØÕ'‘à �á! !¨1¯7©7°2­;¸¿¹Õ+GÈCÕ*OÕ%PÐRS×UcÑUcÐeiÓjˆØ�v‰v�c‹{˜IÐ%Ð%r   )rk   rn   rm   rj   ro   rq   rp   rl   )Té    é€   ©NNN)r)   r*   r+   r,   r   Ústaticmethodrˆ   r™   r&   r-   r.   r/   r0   s   @@r   rg   rg   G   s/   ù‡ € õQð  ó- ó ð- ò^#÷&ô &r   rg   c                   ó<   a a€ ] tR t^§t oV 3R ltRR ltRtVtV ;t# )ÚT5LayerSelfAttentionc	           	     ón   <€ \         S	V `  4        \        WWEWgV4      V n        \	        WWxR 7      V n        R# r]   )r
   r   rg   ÚSelfAttentionr   r`   )
r   rG   rs   rH   rn   ro   r   r	   r   r   s
   &&&&&&&&&€r   r   ÚT5LayerSelfAttention.__init__¨   s1   ø€ Ü‰ÑÔÜ(¨¸yÐchÐr|Ó}ˆÔÜ% iÀVÔcˆŽr   c                ób   € V P                  V P                  V4      W#VR 7      w  rSW,          pW3# ))r�   rž   rŸ   ©r©   r`   )r   r$   r�   rž   rŸ   Úoutputs   &&&&& r   r&   ÚT5LayerSelfAttention.forward®   s<   € Ø ×.Ñ.¨t¯©¸qÓ/AÈð  xKÐ.ó  LÑˆà	�ˆØˆ|Ðr   r¬   r¤   r(   r0   s   @@r   r§   r§   §   s   ù‡ € õd÷ô r   r§   c                   ó<   a a€ ] tR t^´t oV 3R ltRR ltRtVtV ;t# )ÚT5Blockc                ó  <€ \         SV `  4        \        P                  P	                  4       V n        V P
                  P                  \        WW6WxWš4      4       V P
                  P                  \        WWEW‰V
4      4       R # rL   )	r
   r   r   r   Ú
ModuleListÚlayerÚappendr§   r[   )r   rG   rs   rH   rI   ra   rn   ro   r   r	   r   r   s   &&&&&&&&&&&€r   r   ÚT5Block.__init__µ   sg   ø€ Ü‰ÑÔÜ—X‘X×(Ñ(Ó*ˆŒ
Ø�
‰
×ÑÔ.¨yÀVÐXoÐx~ó  Lô  	MØ�
‰
×Ñœ) I°}ÐQVÐ`jÓkÖlr   c                ór   € V P                   ^ ,          ! WW44      w  rV P                   R,          ! V4      pW3# )r   r   ©r³   )r   r$   r�   rž   rŸ   s   &&&&&r   r&   ÚT5Block.forward»   s2   € Ø—z‘z !–} Q¨iÓM‰ˆØ�J‰J�rŽN˜1ÓˆØˆ|Ðr   r·   r¤   r(   r0   s   @@r   r°   r°   ´   s   ù‡ € õm÷ô r   r°   c                   óF   a a€ ] tR t^Àt oV 3R ltRRRR. 3R ltRtVtV ;t# )ÚT5Stackc                ó  <€ \         SV `  4        \        P                  P	                  \        V4       Uu. uF&  p\        Y#YEYgV'       * ;'       g    V^ 8H  WšVR7
      NK(  	  up4      V n        \        W)W«R7      V n	        R# u upi )r   )ro   r   r	   r   r^   N)
r
   r   r   r   r²   Úranger°   Úblockr   Úfinal_layer_norm)r   Ú
num_layersrG   rs   rH   rI   ra   rn   Úrelative_attentionr   r	   r   Úir   s   &&&&&&&&&&&& €r   r   ÚT5Stack.__init__Á   s¿   ø€ Ü‰ÑÔä—X‘X×(Ñ(ô UZð  [eô  Ufó  gñ  Ufð  PQŒW�Y¨6À)ð  wIô  sI÷  rVð  rVð  OPð  TUñ  OUð  _dð  @J÷  Kñ  Ufñ  gó
ˆŒ
ô !,¨IÈ6Ô iˆÖùò gs   ¶,BNTc                ód  € R pVeõ   RVP                  VP                  4      P                  VP                  ^ ,          ^RVP                  R,          34      P	                  VP                  ^ ,          ^VP                  R,          VP                  R,          4      ,
          pVP                  VP                  \        P                  4      \        P                  ! VP                  4      P                  ) 4      pR p\        VP                  VR JRR7      p	R p
Ve$   V^ 8  d   \        V P                  4      V,           p\        V P                  4       F(  w  r¼V! WW©4      w  rW³8X  g   K  VP                  4       pK*  	  V P!                  V4      pVe   V'       d   V P!                  V4      pW3# )Ng      ð?T)r�   Úsmall_inputr   )rv   r   Úreshaperœ   ÚexpandÚmasked_fillr   ÚboolÚfinfoÚmaxr   r	   Úlenr½   Ú	enumerateÚcloner¾   )r   r$   Úattention_maskÚintermediate_outputÚfinal_layer_norm_intermediater   Úembeds_infor�   ÚintermediaterŸ   rž   rÁ   Úls   &&&&&&&      r   r&   ÚT5Stack.forwardÊ   s¶  € ØˆØÒ%Ø˜×*Ñ*¨1¯7©7Ó3×;Ñ;¸^×=QÑ=QÐRSÕ=TÐVWÐY[Ð]k×]qÑ]qÐrtÕ]uÐ<vÓw×~Ñ~ð  @N÷  @Tñ  @Tð  UVõ  @Wð  YZð  \j÷  \pñ  \pð  qsõ  \tð  vD÷  vJñ  vJð  KMõ  vNó  Oõ  OˆDØ×#Ñ# D§G¡G¬E¯J©JÓ$7¼%¿+º+ÀaÇgÁgÓ:N×:RÑ:RÐ9RÓSˆDàˆÜ<¸Q¿X¹XÈNÐbfÐLfÐtxÔyÐØˆ	àÒ*Ø" QÔ&Ü&)¨$¯*©*£oÐ8KÕ&KÐ#ä˜dŸj™jÖ)‰DˆAÙ˜Q iÓE‰LˆAØÖ'Ø Ÿw™w›y’ñ *ð ×!Ñ! !Ó$ˆØÒ#×(EØ×0Ñ0°Ó>ˆLØˆÐr   )r½   r¾   r(   r0   s   @@r   rº   rº   À   s)   ù‡ € õjð )-À$ÐfjÐrvð  EG÷ ô r   rº   c                   óH   a a€ ] tR t^át oV 3R ltR tR tRR ltRtVt	V ;t
# )ÚT5c                ó\  <€ \         SV `  4        VR ,          V n        VR,          pVR,          VR,          ,          p\        V P                  WVVR,          VR,          VR,          VR,          VR,          R8g  W#V4      V n        W n        VP                  VR	,          WSVR
7      V n        R# )r¿   Úd_modelÚd_kvrn   Úd_ffÚdense_act_fnÚis_gated_actÚ
model_typeÚumt5Ú
vocab_sizeri   N)r
   r   r¿   rº   Úencoderr   rr   Úshared)r   Úconfig_dictr   r	   r   rG   rs   r   s   &&&&&  €r   r   ÚT5.__init__â   sÝ   ø€ Ü‰ÑÔØ% lÕ3ˆŒØ 	Õ*ˆ	Ø Õ'¨+°kÕ*BÕBˆ	ä˜tŸ™°	ÀkÐRXÕFYÐ[fÐguÕ[vð  yDð  ESõ  yTð  Vað  bmõ  Vnð  p{ð  |Hõ  pIð  MSñ  pSð  UZð  dnó  oˆŒØŒ
Ø ×*Ñ*¨;°|Õ+DÀiÐfkÐ*ÓlˆŽr   c                ó   € V P                   # rL   ©rá   )r   s   &r   Úget_input_embeddingsÚT5.get_input_embeddingsì   s   € Ø�{‰{Ðr   c                ó   € Wn         R # rL   rå   )r   Ú
embeddingss   &&r   Úset_input_embeddingsÚT5.set_input_embeddingsï   s   € Ø Žr   c                óB  € Vf   TpM0V P                  WP                  R\        P                  4      R7      pV P                  \        P                  \        P
                  \        P                  39  d   \        P                  ! V4      pV P                  ! V3RV/VB # )Nr   r‹   rÎ   )	rá   Úgetr   Úfloat32r   Úfloat16Úbfloat16Ú
nan_to_numrà   )r   Ú	input_idsrÎ   ÚembedsÚ
num_tokensÚkwargsr$   s   &&&&&, r   r&   Ú
T5.forwardò   st   € ØÒØ‰Aà—‘˜I·±¸GÄUÇ]Á]Ó1S�ÓTˆAØ�:‰:œeŸm™m¬U¯]©]¼E¿N¹NÐKÔKÜ× Ò  Ó#ˆAØ�|Š|˜AÑG¨nÐGÀÑGÐGr   )r   rà   r¿   rá   )NN)r)   r*   r+   r,   r   ræ   rê   r&   r-   r.   r/   r0   s   @@r   rÖ   rÖ   á   s!   ù‡ € õmòò!÷Hô Hr   rÖ   )r   r}   Úcomfy.ldm.modules.attentionr   Ú	comfy.opsr!   r   ÚModuler   r6   r:   rD   r<   rP   r[   rg   r§   r°   rº   rÖ   © r   r   Ú<module>rû      sï   ðÛ Û Ý FÛ ô	;�%—(‘(—/‘/ô 	;ð ÑRØ
ˆE�H‰H×Ñ×$Ñ$ð€ô
�e—h‘h—o‘oô ô˜5Ÿ8™8Ÿ?™?ô ô"�—‘—‘ô ô$^&�%—(‘(—/‘/ô ^&ô@˜5Ÿ8™8Ÿ?™?ô ô
ˆe�h‰h�o‰oô 
ôˆe�h‰h�o‰oô ôBHˆ�‰�‰ö Hr   