+
    „°ñj®	  ã                   ó*   € ^ RI t ^ RIt ! R R4      tR# )é    Nc                   óV   a € ] tR t^t o ]R 4       tR	R ltR tR tR
R lt	R t
RtV tR# )ÚSPieceTokenizerc                ó   € \        V 3/ VB # ©N)r   )ÚpathÚkwargss   &,ÚHC:\Ai_Coding\apps\studio\ComfyUI\comfy\text_encoders\spiece_tokenizer.pyÚfrom_pretrainedÚSPieceTokenizer.from_pretrained   s   € ä˜tÑ. vÑ.Ð.ó    Nc                óæ  € W n         W0n        W@n        ^ RIp\        P
                  ! V4      '       d   VP                  4       P                  4       p\        V\        4      '       d/   VP                  WP                   V P                  R7      V n        R# \        P                  P                  V4      '       g   \        R4      hVP                  WP                   V P                  R7      V n        R# )r   N)Úmodel_protoÚadd_bosÚadd_eoszinvalid tokenizer)Ú
model_filer   r   )r   r   Úspecial_tokensÚsentencepieceÚtorchÚ	is_tensorÚnumpyÚtobytesÚ
isinstanceÚbytesÚSentencePieceProcessorÚ	tokenizerÚosr   ÚisfileÚ
ValueError)ÚselfÚtokenizer_pathr   r   r   r   s   &&&&& r	   Ú__init__ÚSPieceTokenizer.__init__	   sÉ   € ØŒØŒØ,ÔÛÜ�?Š?˜>×*Ò*Ø+×1Ñ1Ó3×;Ñ;Ó=ˆNä�n¤e×,Ò,Ø*×AÑAÈn×frÑfrð  }A÷  }Iñ  }IÐAó  JˆDŽNä—7‘7—>‘> .×1Ò1Ü Ð!4Ó5Ð5Ø*×AÑAÈ^×eqÑeqÐ{÷  |Hñ  |HÐAó  IˆDŽNr   c                ó˜   € / p\        V P                  P                  4       4       F   pW!V P                  P                  V4      &   K"  	  V# r   )Úranger   Úget_piece_sizeÚid_to_piece)r   ÚoutÚis   &  r	   Ú	get_vocabÚSPieceTokenizer.get_vocab   s?   € ØˆÜ�t—~‘~×4Ñ4Ó6Ö7ˆAØ12�—‘×*Ñ*¨1Ó-Ó.ñ 8àˆ
r   c                ó.  a€ V P                   eé   ^ R IoRP                  V3R lV P                   P                  4        4       4      pV'       dª   SP	                  W!4      '       d“   SP                  RV R2V4      p. pV Fp  pV'       g   K  WPP                   9   d%   VP                  V P                   V,          4       KA  V P                  P                  VRRR7      pVP                  V4       Kr  	  RV/# V P                  P                  V4      pRV/# )NÚ|c              3   óF   <"  € T F  pSP                  V4      x € K  	  R # 5ir   )Úescape)Ú.0ÚtokenÚres   & €r	   Ú	<genexpr>Ú+SPieceTokenizer.__call__.<locals>.<genexpr>!   s   øé € Ð-gÑLfÀ5¨b¯i©i¸×.>Ð.>ÓLfùs   ƒ!Ú(Ú)F)r   r   Ú	input_ids)
r   r1   ÚjoinÚkeysÚsearchÚsplitÚappendr   ÚencodeÚextend)	r   ÚstringÚspecial_tokens_patternÚpartsÚresultÚpartÚencodedr'   r1   s	   &&      @r	   Ú__call__ÚSPieceTokenizer.__call__   sï   ø€ Ø×ÑÒ*ÛØ%(§X¡XÔ-gÈD×L_ÑL_×LdÑLdÔLfÓ-gÓ%gÐ"ß%¨"¯)©)Ð4J×*SÒ*SØŸ™ 1Ð%;Ð$<¸AÐ!>ÀÓG�Ø�Û!�DßÙ Ø×2Ñ2Ô2ØŸ™ d×&9Ñ&9¸$Õ&?Ö@à"&§.¡.×"7Ñ"7¸ÀeÐUZÐ"7Ó"[˜ØŸ™ gÖ.ñ "ð $ VÐ,Ð,à�n‰n×#Ñ# FÓ+ˆØ˜SÐ!Ð!r   c                óî   € V'       dO   V P                   '       d=   \        V P                   P                  4       4      pV Uu. uF  qDV9  g   K  VNK  	  ppV P                  P	                  V4      # u upi r   )r   ÚsetÚvaluesr   Údecode)r   Ú	token_idsÚskip_special_tokensÚspecial_token_idsÚtids   &&&  r	   rI   ÚSPieceTokenizer.decode2   sa   € ç 4×#6×#6Ð#6Ü # D×$7Ñ$7×$>Ñ$>Ó$@Ó AÐÙ(1ÓR©	 Ð@QÑ5QŸ˜©	ˆIÐRà�~‰~×$Ñ$ YÓ/Ð/ùò Ss   ÁA2ÁA2c                óp   € \         P                  ! \        V P                  P	                  4       4      4      # r   )r   Ú
ByteTensorÚlistr   Úserialized_model_proto)r   s   &r	   Úserialize_modelÚSPieceTokenizer.serialize_model:   s%   € Ü×Ò¤ T§^¡^×%JÑ%JÓ%LÓ MÓNÐNr   )r   r   r   r   )FTN)F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr
   r!   r)   rD   rI   rS   Ú__static_attributes__Ú__classdictcell__)Ú__classdict__s   @r	   r   r      s8   ø‡ € Øñ/ó ð/ôIòò"ô(0÷Oð Or   r   )r   r   r   © r   r	   Ú<module>r^      s   ðÛ Û 	÷7Oó 7Or   