+
    „°ñj(  ã                   óÌ  € ^ RI t ^ RIHt ^ RIHt ^ RIHtHtH	t	H
t
HtHt ^ RIHt  ! R R4      t ! R R]4      t ! R	 R
]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P,                  4      t ! R R]P0                  4      t ! R R]P4                  4      t ! R R]P8                  4      tR# )é    N)Úsd1_clip)Ú	AttentionÚRMSNormÚMLPÚprecompute_freqs_cisÚ
apply_ropeÚ_make_scaled_embedding)ÚSPieceTokenizerc                   ó&   a € ] tR t^t o R tRtV tR# )ÚT5GemmaEncoderConfigc                ó"  € R V n         RV n        RV n        ^V n        ^V n        ^V n        ^@V n        RV n        RV n        RV n	        RV n
        ^@V n        RV n        RV n        R	R
.^,          V n        RV n        RV n        RV n        RV n        R# )i è é   i   g�íµ ÷Æ°>Fg     ˆÃ@g      I@i   Úgelu_pytorch_tanhÚsliding_attentionÚfull_attentionNT)Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚhead_dimÚrms_norm_epsÚrms_norm_addÚ
rope_thetaÚattn_logit_softcappingÚquery_pre_attn_scalarÚsliding_windowÚmlp_activationÚlayer_typesÚqkv_biasÚq_normÚk_norm©Úselfs   &Ú;C:\Ai_Coding\apps\studio\ComfyUI\comfy\text_encoders\sa3.pyÚ__init__ÚT5GemmaEncoderConfig.__init__	   sŸ   € Ø ˆŒØˆÔØ!%ˆÔØ!#ˆÔØ#%ˆÔ Ø#%ˆÔ ØˆŒØ ˆÔØ!ˆÔØ!ˆŒØ&*ˆÔ#Ø%'ˆÔ"Ø"ˆÔØ1ˆÔØ/Ð1AÐBÀQÕFˆÔØˆŒØˆŒØˆŒØ ˆÖó    )r   r   r   r   r#   r    r   r   r   r   r"   r!   r   r   r   r   r   r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r'   Ú__static_attributes__Ú__classdictcell__)Ú__classdict__s   @r&   r   r      s   ø‡ € ÷!ð !r)   r   c                   óD   a a€ ] tR t^t oRtRV 3R lltRR ltRtVtV ;t	# )ÚT5GemmaAttentionau  Reuses LlamaAttention projection setup; overrides forward for softcap attention.

T5Gemma applies tanh(QK^T * scale / cap) * cap between the matmul and softmax.
This nonlinearity is incompatible with fused SDPA kernels, so attention is
computed manually. Everything else (projections, RoPE, GQA expansion) is identical
to LlamaAttention so __init__ is inherited unchanged.
c                ó|   <€ \         SV `  WW4R 7       VP                  R,          V n        VP                  V n        R# )©ÚdeviceÚdtypeÚopsNg      à¿)Úsuperr'   r   Úscaler   Úsoftcap©r%   Úconfigr5   r6   r7   Ú	__class__s   &&&&&€r&   r'   ÚT5GemmaAttention.__init__(   s4   ø€ Ü‰Ñ˜°eÐÔEØ×1Ñ1°TÕ9ˆŒ
Ø×4Ñ4ˆŽr)   c                óÖ  € VP                   w  rVpV P                  V4      P                  WVV P                  V P                  4      P                  ^^4      pV P                  V4      P                  WVV P                  V P                  4      P                  ^^4      p	V P                  V4      P                  WVV P                  V P                  4      P                  ^^4      p
\        W‰V4      w  r‰V	P                  V P                  V P                  ,          ^R7      p	V
P                  V P                  V P                  ,          ^R7      p
\        P                  ! W€P                  ,          V	P                  RR4      4      p\        P                  ! W°P                  ,          4      V P                  ,          pVe	   W²,           p\        P                   P"                  P%                  VP'                  4       RR7      P)                  VP*                  4      p\        P                  ! Wº4      P                  ^^4      P-                  WVV P.                  4      pV P1                  V4      R3# )é   )ÚdimNéþÿÿÿéÿÿÿÿ)ÚshapeÚq_projÚviewÚ	num_headsr   Ú	transposeÚk_projÚnum_kv_headsÚv_projr   Úrepeat_interleaveÚtorchÚmatmulr9   Útanhr:   ÚnnÚ
functionalÚsoftmaxÚfloatÚtor6   ÚreshapeÚ
inner_sizeÚo_proj)r%   Úhidden_statesÚattention_maskÚ	freqs_cisÚkwargsÚBÚSÚ_ÚxqÚxkÚxvÚattnÚouts   &&&&,        r&   ÚforwardÚT5GemmaAttention.forward-   sÈ  € Ø×%Ñ%‰ˆˆaØ�[‰[˜Ó'×,Ñ,¨Q°4·>±>À4Ç=Á=ÓQ×[Ñ[Ð\]Ð_`ÓaˆØ�[‰[˜Ó'×,Ñ,¨Q°4×3DÑ3DÀdÇmÁmÓT×^Ñ^Ð_`ÐbcÓdˆØ�[‰[˜Ó'×,Ñ,¨Q°4×3DÑ3DÀdÇmÁmÓT×^Ñ^Ð_`ÐbcÓdˆÜ˜B IÓ.‰ˆØ×!Ñ! $§.¡.°D×4EÑ4EÕ"EÈ1Ð!ÓMˆØ×!Ñ! $§.¡.°D×4EÑ4EÕ"EÈ1Ð!ÓMˆÜ�|Š|˜B§¡�O¨R¯\©\¸"¸bÓ-AÓBˆÜ�zŠz˜$§¡Õ-Ó.°·±Õ=ˆØÒ%ØÕ(ˆDÜ�x‰x×"Ñ"×*Ñ*¨4¯:©:«<¸RÐ*Ó@×CÑCÀBÇHÁHÓMˆÜ�lŠl˜4Ó$×.Ñ.¨q°!Ó4×<Ñ<¸QÀ4Ç?Á?ÓSˆØ�{‰{˜3Ó Ð%Ð%r)   )r9   r:   ©NNN©NN©
r*   r+   r,   r-   Ú__doc__r'   rd   r.   r/   Ú__classcell__©r=   r0   s   @@r&   r2   r2      s   ù‡ € ñ÷5÷
&ô &r)   r2   c                   ó@   a a€ ] tR t^>t oRV 3R lltRR ltRtVtV ;t# )ÚT5GemmaBlockc                óâ  <€ \         SV `  4        \        WWER 7      V n        \	        WWER 7      V n        \        VP                  VP                  RW4R7      V n	        \        VP                  VP                  RW4R7      V n
        \        VP                  VP                  RW4R7      V n        \        VP                  VP                  RW4R7      V n        VR8H  V n        VP                  V n        R# )r4   T©ÚepsÚaddr5   r6   r   N)r8   r'   r2   Ú	self_attnr   Úmlpr   r   r   Úpre_self_attn_layernormÚpost_self_attn_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormÚ
is_slidingr   )r%   r<   Ú
layer_typer5   r6   r7   r=   s   &&&&&&€r&   r'   ÚT5GemmaBlock.__init__?   sÕ   ø€ Ü‰ÑÔÜ)¨&ÀuÔVˆŒÜ�v°EÔCˆŒä'.¨v×/AÑ/AÀv×GZÑGZÐ`dÐmsô  (BˆÔ$Ü(/°×0BÑ0BÈ×H[ÑH[ÐaeÐntô  )CˆÔ%Ü)0°×1CÑ1CÈ×I\ÑI\ÐbfÐouô  *DˆÔ&Ü*1°&×2DÑ2DÈ&×J]ÑJ]ÐcgÐpvô  +EˆÔ'Ø%Ð)<Ñ<ˆŒØ$×3Ñ3ˆÖr)   c                óˆ  € TpV P                   '       Ed/   VP                  ^,          V P                  8”  Ed   VP                  ^,          p\        P                  ! WQP
                  R7      pVP                  ^ 4      VP                  ^4      ,
          P                  4       p\        P                  ! WUVP                  VP
                  R7      pVP                  WpP                  8„  \        P                  ! VP                  4      P                  ) 4       VP                  ^ 4      P                  ^ 4      pVe	   W(,           MTpTp	V P                  V4      pV P                  WVR7      w  rV P                  V4      pW‘,           pTp	V P!                  V4      pV P#                  V4      pV P%                  V4      pW‘,           pV# )r@   ©r5   )r6   r5   ©rY   rZ   )rx   rD   r   rM   Úaranger5   Ú	unsqueezeÚabsÚzerosr6   Úmasked_fill_ÚfinfoÚmaxrt   rr   ru   rv   rs   rw   )r%   ÚxrY   rZ   Ú	attn_maskr]   ÚposÚdistÚsw_maskÚresidualr^   s   &&&&       r&   rd   ÚT5GemmaBlock.forwardK   s[  € Ø"ˆ	Ø�?�?‰?˜qŸw™w q�z¨D×,?Ñ,?Õ?Ø—‘˜•
ˆAÜ—,’,˜q¯©Ô2ˆCØ—M‘M !Ó$ s§}¡}°QÓ'7Õ7×<Ñ<Ó>ˆDÜ—k’k !¨a¯g©g¸a¿h¹hÔGˆGØ× Ñ  ×(;Ñ(;Ñ!;¼e¿kºkÈ!Ï'É'Ó>R×>VÑ>VÐ=VÔWØ×'Ñ'¨Ó*×4Ñ4°QÓ7ˆGØ6DÒ6P˜Ö1ÐV]ˆIØˆØ×(Ñ(¨Ó+ˆØ�~‰~˜aÀYˆ~ÓO‰ˆØ×)Ñ)¨!Ó,ˆØ�LˆØˆØ×*Ñ*¨1Ó-ˆØ�H‰H�Q‹KˆØ×+Ñ+¨AÓ.ˆØ�LˆØˆr)   )rx   rs   rw   ru   rv   rt   rr   r   rf   rg   )	r*   r+   r,   r-   r'   rd   r.   r/   rj   rk   s   @@r&   rm   rm   >   s   ù‡ € ÷
4÷ô r)   rm   c                   ó@   a a€ ] tR t^bt oRtV 3R ltRR ltRtVtV ;t	# )ÚT5GemmaEncoderzSEncoder stack: embed_tokens, layers, norm.
Keys: embed_tokens.*, layers.X.*, norm.*c                ó°  <€ \         SV `  4        Wn        \        WAP                  VP
                  VP
                  R ,          W#4      V n        \        P                  ! \        VP                  4       Uu. uF"  p\        WP                  V,          W#VR7      NK$  	  up4      V n        \        VP
                  VP                  RW#R7      V n        R# u upi )g      à?r4   Tro   N)r8   r'   r<   r	   r   r   Úembed_tokensrP   Ú
ModuleListÚranger   rm   r    Úlayersr   r   Únorm)r%   r<   r5   r6   r7   Úir=   s   &&&&& €r&   r'   ÚT5GemmaEncoder.__init__f   s³   ø€ Ü‰ÑÔØŒä2Ø×"Ñ" F×$6Ñ$6¸×8JÑ8JÈcÕ8QÐSYóbˆÔä—m’mä˜6×3Ñ3Ô4ó%
á4�ô ˜×!3Ñ!3°AÕ!6¸vÐX[×\Ù4ñ%
ó ˆŒô ˜F×.Ñ.°F×4GÑ4GÈTÐZ`ÔnˆŽ	ùò	%
s   Á5(Cc                ó  € Ve   TM)T P                  Y;'       g    \        P                  R7      pVP                  ^,          p	\        P                  ! W˜P
                  R7      P                  ^ 4      p
\        V P                  P                  W P                  P                  VP
                  R7      pR pVeã   RVP                  VP                  4      P                  VP                  ^ ,          ^RVP                  R,          34      P                  VP                  ^ ,          ^W’P                  R,          4      ,
          pVP                  VP                  \        P                   4      \        P"                  ! VP                  4      P$                  ) 4      pR p\'        V P(                  4       F'  w  rïV! WŒVR7      pWä8X  g   K  VP+                  4       pK)  	  V P-                  V4      pVe   V'       d   V P-                  V4      pW�3# )N)Ú	out_dtyper|   g      ð?r}   rC   )r�   rM   Úfloat32rD   r~   r5   r   r   r<   r   r   rT   r6   rU   ÚexpandÚmasked_fillÚboolrƒ   r„   Ú	enumerater’   Úcloner“   )r%   Ú	input_idsrY   ÚembedsÚintermediate_outputÚfinal_layer_norm_intermediater6   Ú
num_layersr…   Úseq_lenÚposition_idsrZ   ÚmaskÚintermediater”   Úlayers   &&&&&&&&        r&   rd   ÚT5GemmaEncoder.forwardr   s’  € àÒ(‰F¨d×.?Ñ.?À	×UkÐUkÔ^c×^kÑ^kÐ.?Ó.lˆØ—'‘'˜!•*ˆÜ—|’| G·H±HÔ=×GÑGÈÓJˆÜ(¨¯©×)=Ñ)=¸|Ï[É[×McÑMcÐlm×ltÑltÔuˆ	ØˆØÒ%Ø˜×*Ñ*¨1¯7©7Ó3×;Ñ;Ø×%Ñ% aÕ(¨!¨R°×1EÑ1EÀbÕ1IÐJóç‰f�^×)Ñ)¨!Õ,¨a°×:NÑ:NÈrÕ:RÓSõTˆDð ×#Ñ# D§G¡G¬E¯J©JÓ$7¼%¿+º+ÀaÇgÁgÓ:N×:RÑ:RÐ9RÓSˆDØˆÜ! $§+¡+Ö.‰HˆAÙ�a¸	ÔBˆAØÖ'Ø Ÿw™w›y’ñ /ð �I‰I�a‹LˆØÒ#×(EØŸ9™9 \Ó2ˆLØˆÐr)   )r<   r�   r’   r“   )NNNTNNrh   rk   s   @@r&   r�   r�   b   s   ù‡ € ñ0õ
o÷ô r)   r�   c                   ó6   a a€ ] tR t^‰t oRtV 3R ltRtVtV ;t# )ÚT5GemmaBodyz2Provides the 'encoder' sub-module.
Keys: encoder.*c                óF   <€ \         SV `  4        \        WW44      V n        R # ©N)r8   r'   r�   Úencoderr;   s   &&&&&€r&   r'   ÚT5GemmaBody.__init__�   s   ø€ Ü‰ÑÔÜ% f°eÓAˆŽr)   )r­   )	r*   r+   r,   r-   ri   r'   r.   r/   rj   rk   s   @@r&   rª   rª   ‰   s   ù‡ € ñ÷Bõ Br)   rª   c                   óL   a a€ ] tR t^’t oRtV 3R ltR tR tRR ltRt	Vt
V ;t# )ÚT5GemmaModelu‹   Top-level model class passed to SDClipModel as model_class.
Module layout: self.model.encoder.* â†’ matches checkpoint keys model.encoder.*c                óˆ   <€ \         SV `  4        \        4       pVP                  V n        W n        \        WSW$4      V n        R # r¬   )r8   r'   r   r   r¢   r6   rª   Úmodel)r%   Úconfig_dictr6   r5   Ú
operationsr<   r=   s   &&&&& €r&   r'   ÚT5GemmaModel.__init__–   s6   ø€ Ü‰ÑÔÜ%Ó'ˆØ ×2Ñ2ˆŒØŒ
Ü  °ÓCˆŽ
r)   c                óB   € V P                   P                  P                  # r¬   ©r²   r­   r�   r$   s   &r&   Úget_input_embeddingsÚ!T5GemmaModel.get_input_embeddings�   s   € Ø�z‰z×!Ñ!×.Ñ.Ð.r)   c                ó:   € WP                   P                  n        R # r¬   r·   )r%   Ú
embeddingss   &&r&   Úset_input_embeddingsÚ!T5GemmaModel.set_input_embeddings    s   € Ø*4�
‰
×ÑÖ'r)   c           
     ó’   € Ve   V^ 8  d   V P                   V,           pV P                  P                  WVVVWpP                   R7      # )N)rY   rŸ   r    r¡   r6   r¢   )r¢   r²   r­   )	r%   rž   rY   rŸ   Ú
num_tokensr    r¡   r6   r[   s	   &&&&&&&&,r&   rd   ÚT5GemmaModel.forward£   sN   € àÒ*Ð/BÀQÔ/FØ"&§/¡/Ð4GÕ"GÐØ�z‰z×!Ñ!Ø¸VØ 3Ø*GØ§O¡Oð	 "ó 5ð 	5r)   )r6   r²   r¢   )NNNNTN)r*   r+   r,   r-   ri   r'   r¸   r¼   rd   r.   r/   rj   rk   s   @@r&   r°   r°   ’   s%   ù‡ € ñWõDò/ò5÷5ô 5r)   r°   c                   ó@   a a€ ] tR t^®t oRRRR/ 3V 3R lltRtVtV ;t# )ÚT5GemmaSDClipModelÚcpuÚlastNc                óD   <€ \         SV `  WV/ VR ^ /\        RRVR7
       R# )ÚpadT)
r5   r§   Ú	layer_idxÚtextmodel_json_configr6   Úspecial_tokensÚmodel_classÚenable_attention_masksÚzero_out_maskedÚmodel_optionsN)r8   r'   r°   )r%   r5   r§   rÇ   r6   rÍ   r=   s   &&&&&&€r&   r'   ÚT5GemmaSDClipModel.__init__¯   s0   ø€ Ü‰Ñ ¸yØ/1¸Ø).°¨
Ü%1Ø04ÀdØ'4ð 	ö 	6r)   © ©r*   r+   r,   r-   r'   r.   r/   rj   rk   s   @@r&   rÂ   rÂ   ®   s   ù‡ € Ø#¨6¸TÈÐ]_÷ 6÷ 6r)   rÂ   c                   ó@   a a€ ] tR t^¸t oR/ 3V 3R lltR tRtVtV ;t# )ÚT5GemmaSDTokenizerNc                ót   <€ VP                  R R4      p\        SV `	  VRRR\        RRRR^^ VRRRR/R7       R# )	Úspiece_modelNFr   Út5gemmaiÿàõÚadd_bosÚadd_eos)Úpad_with_endÚembedding_sizeÚembedding_keyÚtokenizer_classÚhas_start_tokenÚhas_end_tokenÚpad_to_max_lengthÚ
max_lengthÚ
min_lengthÚ	pad_tokenÚtokenizer_dataÚtokenizer_args)Úgetr8   r'   r
   )r%   Úembedding_directoryrâ   Útokenizer_modelr=   s   &&& €r&   r'   ÚT5GemmaSDTokenizer.__init__¹   sQ   ø€ Ø(×,Ñ,¨^¸TÓBˆÜ‰Ñ˜°uÈSØ'0Ä/Ø).¸eÐW\Ø$,¸ÀaØ(6Ø)2°E¸9ÀeÐ(Lð 	ö 	Nr)   c                ó:   € R V P                   P                  4       /# )rÔ   )Ú	tokenizerÚserialize_modelr$   s   &r&   Ú
state_dictÚT5GemmaSDTokenizer.state_dictÂ   s   € Ø §¡× >Ñ >Ó @ÐAÐAr)   rÏ   )	r*   r+   r,   r-   r'   rë   r.   r/   rj   rk   s   @@r&   rÒ   rÒ   ¸   s   ù‡ € Ø+/À÷ N÷Bò Br)   rÒ   c                   ó:   a a€ ] tR t^Æt oR/ 3V 3R lltRtVtV ;t# )ÚSAT5GemmaTokenizerNc                ó6   <€ \         SV `  VVR \        R7       R# )rÕ   )rå   râ   Ú	clip_nameré   N)r8   r'   rÒ   )r%   rå   râ   r=   s   &&&€r&   r'   ÚSAT5GemmaTokenizer.__init__Ç   s!   ø€ Ü‰ÑÐ-@Ø(6À)ÔWið 	ö 	kr)   rÏ   rÐ   rk   s   @@r&   rî   rî   Æ   s   ù‡ € Ø+/À÷ k÷ kr)   rî   c                   ó<   a a€ ] tR t^Ìt oRR/ 3V 3R lltRtVtV ;t# )ÚSAT5GemmaModelrÃ   Nc                óD   <€ \         SV `  ! RR VRVRVRRR\        /VB  R# )r5   r6   rÍ   ÚnamerÕ   Ú
clip_modelNrÏ   )r8   r'   rÂ   )r%   r5   r6   rÍ   r[   r=   s   &&&&,€r&   r'   ÚSAT5GemmaModel.__init__Í   sD   ø€ Ü‰Òñ 	R ð 	R¨eð 	RÀ=ð 	RØ'ð	RÜ4Fð	RØJPô	Rr)   rÏ   rÐ   rk   s   @@r&   ró   ró   Ì   s   ù‡ € Ø#¨4¸r÷ R÷ Rr)   ró   )rM   Útorch.nnrP   Úcomfyr   Úcomfy.text_encoders.llamar   ÚLlamaAttentionr   r   r   r   r	   Ú$comfy.text_encoders.spiece_tokenizerr
   r   r2   ÚModulerm   r�   rª   r°   ÚSDClipModelrÂ   ÚSDTokenizerrÒ   ÚSD1Tokenizerrî   ÚSD1ClipModelró   rÏ   r)   r&   Ú<module>r     sÅ   ðÛ Ý Ý ÷ J÷  JÝ @÷!ñ !ô.&�~ô &ô>!�2—9‘9ô !ôH$�R—Y‘Yô $ôNB�"—)‘)ô Bô5�2—9‘9ô 5ô86˜×-Ñ-ô 6ôB˜×-Ñ-ô Bôk˜×.Ñ.ô kôR�X×*Ñ*ö Rr)   