+
    ƒ°ñj6-  ã                   óž   € R t ^ RIt^ RIt^ RIt^ RIt^ RItRtRtR tR t	R t
R tR tR t ! R	 R
4      tR tR tR tR t ! R R4      tR# )zÒ
Pure-Python byte-level BPE tokenizer.
Supports loading from HuggingFace tokenizer.json (LLaMA-style)
and from Mistral tekken JSON blobs.
No dependency on the `transformers`, `tokenizers`, or `regex` packages.
Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+c                 óB   € \         P                  ! V 4      ^ ,          R8H  # )é    ÚL©ÚunicodedataÚcategory©Úcs   &ÚEC:\Ai_Coding\apps\studio\ComfyUI\comfy\text_encoders\bpe_tokenizer.pyÚ
_is_letterr      ó   € Ü×Ò Ó" 1Õ%¨Ñ,Ð,ó    c                 óB   € \         P                  ! V 4      ^ ,          R8H  # )r   ÚNr   r   s   &r
   Ú
_is_numberr      r   r   c                 óN   € V R 9   ;'       g    \         P                  ! V 4      R8H  # )u    	
Â…â€¨â€©ÚZsr   r   s   &r
   Ú_is_whitespacer      s&   € ØÐ-Ñ-×PÐP´×1EÒ1EÀaÓ1HÈDÑ1PÐPr   c                 ó   € . p^ pV\        V 4      8  Edi   RpW,          R8X  dK   \         F@  pWV\        V4      ,            P                  4       V8X  g   K,  WV\        V4      ,            p M	  Ve&   VP                  V4       V\        V4      ,          pK“  TpW,          R9  d8   \	        W,          4      '       g!   \        W,          4      '       g
   V^,          pV\        V 4      8  dk   \	        W,          4      '       dT   V^,          pV\        V 4      8  d#   \	        W,          4      '       d   V^,          pK2  VP                  WV 4       TpEKS  \        W,          4      '       da   V^,           pV\        V 4      8  d0   WR,
          ^8  d#   \        W,          4      '       d   V^,          pK?  VP                  WV 4       TpEKÊ  TpW,          R8X  d
   V^,          pTpV\        V 4      8  dQ   \        W,          4      '       g:   \	        W,          4      '       g#   \        W,          4      '       g   V^,          pK`  WV8”  dA   V\        V 4      8  d   W,          R9   d   V^,          pK(  VP                  WV 4       TpEKŠ  \        W,          4      '       d¶   V^,           pV\        V 4      8  d#   \        W,          4      '       d   V^,          pK2  \        V P                  RW%4      V P                  RW%4      4      pWr8¼  d   V^,           pM&V\        V 4      8  d   WR,
          ^8”  d
   V^,          pVP                  WV 4       TpEKV  VP                  W,          4       V^,          pEKy  V# )r   NÚ'z
Ú ÚÚ
)	ÚlenÚ_CONTRACTIONSÚcasefoldÚappendr   r   r   ÚmaxÚrfind)ÚtextÚpiecesÚiÚcontractionÚsuffixÚjÚpunct_startÚlast_newlines   &       r
   Ú_split_llamar'      s±  € Ø€FØ	€AØ
Œc�$‹i�-ØˆØ�7�cŒ>ß'�Ø˜!œc &›k�/Ð*×3Ñ3Ó5¸Ö?Ø"&¨¬S°«[­Ð"9�KÙñ (ð Ò"Ø�M‰M˜+Ô&Ø”�[Ó!Õ!ˆAÙàˆØ�7˜&Ô ¬°DµG×)<Ò)<ÄZÐPTÕPW×EXÒEXØ��FˆAØŒs�4‹yŒ=œZ¨­×0Ò0Ø��FˆAØ”c˜$“i”-¤J¨t­w×$7Ò$7Ø�Q•’Ø�M‰M˜$ ˜)Ô$ØˆAÚä�d•g×ÒØ�A•ˆAØ”c˜$“i”- A¥E¨A¤I´*¸T½W×2EÒ2EØ�Q•’Ø�M‰M˜$ ˜)Ô$ØˆAÚàˆØ�7�cŒ>Ø��FˆAØˆØ”#�d“)Œm¤N°4µ7×$;Ò$;ÄJÈtÍw×DWÒDWÔ`jÐkoÕkr×`sÒ`sØ��FŠAØŒ?Ø”c˜$“i”- D¥G¨vÔ$5Ø�Q•’Ø�M‰M˜$ ˜)Ô$ØˆAÚä˜$�'×"Ò"Ø�A•ˆAØ”c˜$“i”-¤N°4µ7×$;Ò$;Ø�Q•’Ü˜tŸz™z¨$°Ó5°t·z±zÀ$ÈÓ7MÓNˆLØÔ Ø  1Õ$‘Ø”S˜“Y” 1¥5¨1¤9Ø�Q•�Ø�M‰M˜$ ˜)Ô$ØˆAÚà�‰�d•gÔØ	ˆQ�‹Ø€Mr   c                 ó@   € V \         8w  d   \        R V  24      h\        # )z%Unsupported tokenizer split pattern: )Ú_LLAMA_PATTERNÚ
ValueErrorr'   )Úpattern_strs   &r
   Ú_make_split_patternr,   `   s#   € Ø”nÔ$ÜÐ@ÀÀÐNÓOÐOÜÐr   c            	      ó>  € \        \        \        R 4      \        R4      ^,           4      4      \        \        \        R4      \        R4      ^,           4      4      ,           \        \        \        R4      \        R4      ^,           4      4      ,           p V R,          p^ p\        R4       F=  pW09  g   K  V P                  V4       VP                  RV,           4       V^,          pK?  	  V Uu. uF  p\	        V4      NK  	  pp\        \        W4      4      # u upi )Ú!Ú~ô   Â¡ô   Â¬ô   Â®ô   Ã¿:NNNé   )ÚlistÚrangeÚordr   ÚchrÚdictÚzip)ÚbsÚcsÚnÚbs       r
   Ú_bytes_to_unicoder?   f   sÔ   € Ü
Œu”S˜“Xœs 3›x¨!�|Ó,Ó
-Ü””s˜4“y¤# d£)¨a¥-Ó0Ó1õ2ä””s˜4“y¤# d£)¨a¥-Ó0Ó1õ2€Bð 
ˆA�€BØ	€AÜ�4Ž[ˆØŽ;Ø�I‰I�aŒLØ�I‰I�d˜Q•hÔØ��FŠAñ	 ñ
 Ó	™"�QŒ#ˆaŽ&™"€BÐ	Ü”�B“ÓÐùò 
s   Ã0Dc                   óP   a € ] tR t^ut o RtR
R ltR tR tR tR t	RR lt
R	tV tR# )ÚBPETokenizerz6Byte-level BPE tokenizer with optional BOS prepending.Nc                óè  € Wn         VP                  4        UU	u/ uF  w  r‰W˜bK	  	  up	pV n        W n        W0n        \        VP                  4       4      V n        WPn        W`n	        Wpn
        \        V4      V n        \        VP                  4       \        R R7      p
V
'       dB   \         P"                  ! RRP%                  R V
 4       4      ,           R,           4      V n        R# RV n        R# u up	pi )T)ÚkeyÚreverseÚ(Ú|c              3   óN   "  € T F  p\         P                  ! V4      x € K  	  R # 5i©N)ÚreÚescape)Ú.0Úss   & r
   Ú	<genexpr>Ú(BPETokenizer.__init__.<locals>.<genexpr>‡   s   é € ÐE±_°œrŸyšy¨Ÿ|˜|³_ùs   ‚#%Ú)N)Ú_vocabÚitemsÚ
_inv_vocabÚ_mergesÚ_special_token_idsÚsetÚvaluesÚ_special_idsÚ_byte_encoderÚ_byte_decoderÚ_bos_idr,   Ú_splitÚsortedÚkeysr   rI   ÚcompileÚjoinÚ_special_split)ÚselfÚvocabÚmerges_by_pairÚspecial_token_idsr+   Úbyte_encoderÚbyte_decoderÚbos_idÚkÚvÚsorted_specialss   &&&&&&&&   r
   Ú__init__ÚBPETokenizer.__init__x   s½   € àŒØ,1¯K©K¬MÔ:©M¡D A˜1š4©MÒ:ˆŒØ%ŒØ"3ÔÜÐ 1× 8Ñ 8Ó :Ó;ˆÔØ)ÔØ)ÔØŒä)¨+Ó6ˆŒÜ Ð!2×!7Ñ!7Ó!9¼sÈDÔQˆßÜ"$§*¢*Ø�c—h‘hÑE±_ÓEÓEÕEÈÕKó#ˆDÖð #'ˆDÖùó ;s   šC.c                óØ  € \        V4      ^8:  d   V#  \        R4      pRp\        \        V4      ^,
          4       F^  pV P                  P	                  W,          W^,           ,          3\        R4      4      pWR8  g   KE  TpW,          W^,           ,          3pK`  	  Vf    V# V^ ,          V^,          ,           p. p^ pV\        V4      8  d…   V\        V4      ^,
          8  dL   W,          V^ ,          8X  d8   W^,           ,          V^,          8X  d   VP                  V4       V^,          pKr  VP                  W,          4       V^,          pK”  Tp\        V4      ^8X  g   EKV   V# )é   ÚinfN)r   Úfloatr6   rS   Úgetr   )ra   ÚcharsÚmin_rankÚ	best_pairr!   ÚrÚmergedÚ	new_charss   &&      r
   Ú_bpe_encode_pieceÚBPETokenizer._bpe_encode_pieceŒ   s2  € Üˆu‹:˜Œ?ØˆLØÜ˜U“|ˆHØˆIÜœ3˜u›:¨�>Ö*�Ø—L‘L×$Ñ$ e¥h°¸!µeµÐ%=¼uÀU»|ÓL�Ø–<Ø �HØ!&¥¨5°Qµ­<Ð 8’Iñ	 +ð
 Ò Øð ˆð ˜q•\ I¨a¥LÕ0ˆFØˆIØˆAØ”c˜%“j”.Ø”s˜5“z A•~Ô%¨%­(°iÀµlÔ*BÀuÐQRÍUÅ|ÐW`ÐabÕWcÔGcØ×$Ñ$ VÔ,Ø˜•F’Aà×$Ñ$ U¥XÔ.Ø˜•F’AØˆEÜ�5‹z˜Q�ØØˆr   c                ó  € . pV P                   '       d   V P                   P                  V4      MV.pV FÃ  pV'       g   K  W@P                  9   d%   VP                  V P                  V,          4       KA  V P	                  V4       Fm  pVP                  R 4       Uu. uF  q`P                  V,          NK  	  ppV P                  V4       F%  pVP                  V P                  V,          4       K'  	  Ko  	  KÅ  	  V# u upi )úutf-8)	r`   ÚsplitrT   r   r[   ÚencoderX   rx   rP   )	ra   r   ÚidsÚpartsÚpartÚpiecer>   Ú
byte_charsÚtoks	   &&       r
   Ú_encode_rawÚBPETokenizer._encode_raw¨   sÕ   € ØˆØ37×3F×3FÐ3F�×#Ñ#×)Ñ)¨$Ô/ÈTÈFˆÛˆDßÙØ×.Ñ.Ô.Ø—
‘
˜4×2Ñ2°4Õ8Ö9à!Ÿ[™[¨Ö.�EØAFÇÁÈgÔAVÓ!WÑAV¸A×"4Ñ"4°Q×"7Ð"7ÑAV�JÐ!WØ#×5Ñ5°jÖA˜ØŸ
™
 4§;¡;¨sÕ#3Ö4ó  Bó /ñ ð ˆ
ùò "Xs   ÂC>c                óp   € V P                  V4      pV P                  e   V P                  .V,           pRV/# )NÚ	input_ids)r„   rZ   )ra   r   r~   s   && r
   Ú__call__ÚBPETokenizer.__call__·   s7   € Ø×Ñ˜tÓ$ˆØ�<‰<Ò#Ø—<‘<�. 3Õ&ˆCØ˜SÐ!Ð!r   c                ó,   € \        V P                  4      # rH   )r9   rP   )ra   s   &r
   Ú	get_vocabÚBPETokenizer.get_vocab½   s   € Ü�D—K‘KÓ Ð r   c                óT  € \        4       pV F†  pV P                  P                  VR 4      pW@P                  9   d-   V'       g#   VP	                  VP                  R4      4       KY  K[  V F%  pVP                  V P                  V,          4       K'  	  Kˆ  	  VP                  RRR7      # )Ú r{   Úreplace©Úerrors)	Ú	bytearrayrR   rq   rW   Úextendr}   r   rY   Údecode)ra   Ú	token_idsÚskip_special_tokensÚbufÚtidrL   r	   s   &&&    r
   r”   ÚBPETokenizer.decodeÀ   sŠ   € Ü‹kˆÛˆCØ—‘×#Ñ# C¨Ó,ˆAØ×'Ñ'Ô'ß*Ø—J‘J˜qŸx™x¨Ó0Ö1ñ +ó �AØ—J‘J˜t×1Ñ1°!Õ4Ö5ó ñ ð �z‰z˜'¨)ˆzÓ4Ð4r   )
rZ   rY   rX   rR   rS   rW   r`   rT   r[   rP   rH   )T)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rk   rx   r„   rˆ   r‹   r”   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r
   rA   rA   u   s+   ø‡ € Ù@ô'ò(ò8ò"ò!÷
5ò 
5r   rA   c                 óP  € V P                  R 4      R8X  dX   V P                  R. 4       F?  pVP                  R 4      R8X  g   K  VP                  R/ 4      pRV9   g   K6  VR,          u # 	  R# V P                  R 4      R8X  d#   V P                  R/ 4      pRV9   d
   VR,          # R# )ÚtypeÚSequenceÚpretokenizersÚSplitÚpatternÚRegexN©rq   )ÚpretokÚsubÚpats   &  r
   Ú_extract_patternr®   Í   s•   € Ø‡z�z�&Ó˜ZÔ'Ø—:‘:˜o¨rÖ2ˆCØ�w‰w�v‹ 'Ö)Ø—g‘g˜i¨Ó,�Ø˜c–>Ø˜w�<Ò'ñ	 3ñ ð	 
�‰�FÓ	˜wÔ	&Ø�j‰j˜ BÓ'ˆØ�cŒ>Ø�w•<ÐÙr   c                 óÒ   € V P                  R 4      R8X  dQ   V P                  R. 4      pV'       d7   RV^ ,          9   d)   V^ ,          R,          R,          pVP                  V4      # R# )r¤   ÚTemplateProcessingÚsingleÚSpecialTokenÚidNrª   )Úpost_processorrd   r±   Úbos_strs   &&  r
   Ú_extract_bos_idr¶   Û   s]   € Ø×Ñ˜&Ó!Ð%9Ô9Ø×#Ñ# H¨bÓ1ˆß�n¨¨q­	Ô1Ø˜Q•i Õ/°Õ5ˆGØ$×(Ñ(¨Ó1Ð1Ùr   c           
     ó>  € \         P                  P                  V R4      p\        VRR7      ;_uu_ 4       p\        P
                  ! V4      pRRR4       \        XR,          R,          4      p/ p\        VR,          P                  R. 4      4       F(  w  rgVP                  R^4      w  r‰W‰3V9  g   K#  WeW‰3&   K*  	  / p
VP                  R	. 4       F'  pVR
,          W«R,          &   VR
,          WKR,          &   K)  	  \        VP                  R/ 4      4      pVf   \        RV 24      h\        VP                  R/ 4      V
4      p\        4       pVP                  4        UUu/ uF	  w  ppVVbK  	  ppp\        WEW¬VVVR7      #   + '       g   i     ELA; iu uppi )z?Load a BPETokenizer from a directory containing tokenizer.json.ztokenizer.jsonr{   )ÚencodingNÚmodelrb   Úmergesr   Úadded_tokensr³   ÚcontentÚpre_tokenizerz%Could not extract regex pattern from r´   ©rg   )ÚosÚpathr_   ÚopenÚjsonÚloadr9   Ú	enumeraterq   r|   r®   r*   r¶   r?   rQ   rA   )rÀ   Útok_fileÚfÚdatarb   rc   r!   Ú	merge_strÚar>   rd   rƒ   r¨   rg   re   rh   ri   rf   s   &                 r
   Úfrom_tokenizer_jsonrÊ   ä   s{  € ä�w‰w�|‰|˜DÐ"2Ó3€HÜ	ˆh ×	)Õ	)¨QÜ�yŠy˜‹|ˆ÷ 
*ô ��g•˜wÕ'Ó(€Eà€NÜ! $ w¥-×"3Ñ"3°H¸bÓ"AÖB‰ˆØ�‰˜s AÓ&‰ˆØˆ6˜Ö'Ø%&˜A˜6Ó"ñ Cð
 ÐØ�x‰x˜¨Ö+ˆØ,/°­IÐ˜i�.Ñ)Ø # D¥	ˆ�)�nÓñ ,ô ˜tŸx™x¨¸Ó<Ó=€GØ‚ÜÐ@ÀÀ
ÐKÓLÐLä˜TŸX™XÐ&6¸Ó;Ð=NÓO€Fä$Ó&€LØ%1×%7Ñ%7Ô%9Ô:Ñ%9™T˜Q �A�q’DÑ%9€LÑ:ä˜Ð/@Ø$ l¸6ôCð C÷3 
*×	)Ð	)üó. ;s   ¶FÅ#FÆF	c           
     óÖ  aa€ \         P                  ! V 4      pVR,          p\        4       oSP                  4        UUu/ uF  w  r4WCbK	  	  pppV3R lpVR,          pVR,          V,
          p/ p	VR,           F:  p
V
R,          pW¸8¼  d   K  W·,           V	\        P
                  ! V
R,          4      &   K<  	  / pVR,           FE  p
RV
9   d*   V
R,          V\        P
                  ! V
R,          4      &   K3  V
R,          WÊR	,          &   KG  	  \        VP                  4       4      p\        V4      oSP                  V	4       / p. pSP                  4        Fà  w  ppVV9  d¤   VWæ! V4      &   \        V4      ^8X  d   K)  . p\        ^\        V4      4       FC  pVR
V VVR
 ppVS9   g   K  VS9   g   K  VV,           S9   g   K/  VP                  VVV34       KE  	  VP                  V3R lR7       VP                  V4       K°  \        V\         4      '       d   VP#                  RRR7      MTpVVV&   Kâ  	  VP                  R R7       / p\%        V4       F(  w  pw  pppV! V4      V! V4      3pVV9  g   K#  VVV&   K*  	  / pV FC  p\        V\         4      '       d   VP#                  RRR7      MTpVV9   g   K7  VV,          VV&   KE  	  \'        VVV\(        SVR
R7      # u uppi )zDBuild a BPETokenizer from a Mistral tekken JSON blob (bytes or str).Úconfigc                 óX   <€ R P                  V3R lV P                  R4       4       4      # )rŽ   c              3   óH   <"  € T F  pS\        V4      ,          x € K  	  R # 5irH   )r7   )rK   r	   re   s   & €r
   rM   Ú1from_tekken_json.<locals>.tbts.<locals>.<genexpr>  s   øé € ÐIÑ5H°�|¤C¨£F×+Ò+Ó5Hùs   ƒ"zlatin-1)r_   r”   )r>   re   s   &€r
   ÚtbtsÚfrom_tekken_json.<locals>.tbts  s!   ø€ Ø�w‰wÔI°Q·X±X¸iÔ5HÓIÓIÐIr   Údefault_num_special_tokensÚdefault_vocab_sizerb   ÚrankÚtoken_bytesÚspecial_tokensÚ	token_strNc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# )r   © )ÚxÚcombineds   &€r
   Ú<lambda>Ú"from_tekken_json.<locals>.<lambda>0  s   ø€  h¨q°­t¥n°h¸qÀ½tµnÑ%Er   )rC   r{   r�   r�   c                 ó   € V ^,          # )é   rÙ   )ri   s   &r
   rÜ   rÝ   6  s   €  Q q¦Tr   r¾   )rÂ   Úloadsr?   rQ   Úbase64Ú	b64decoder5   r]   r9   Úupdater   r6   r   Úsortr“   Ú
isinstanceÚbytesr”   rÄ   rA   r)   )rÇ   Úmistral_vocabrÌ   rh   ri   rf   rÐ   Úspecial_token_offsetÚ	max_vocabÚ	raw_vocabÚwru   Úspecial_tokens_dictÚall_specialÚ	bpe_vocabÚmerge_triplesÚtokenrÔ   Úlocalr!   ÚplÚprÚtok_strrc   Ú_ÚpairÚspecial_str_idsrƒ   re   rÛ   s   &                           @@r
   Úfrom_tekken_jsonrø     sÛ  ù€ ä—J’J˜tÓ$€MØ˜8Õ$€Fä$Ó&€LØ%1×%7Ñ%7Ô%9Ô:Ñ%9™T˜Q�A’DÑ%9€LÑ:õJð "Ð">Õ?ÐØÐ+Õ,Ð/CÕC€Ià€IØ˜7×#Ð#ˆØˆf�IˆØŒ>ÙØ89Õ8Pˆ	”&×"Ò" 1 ]Õ#3Ó4Ó5ñ	 $ð ÐØÐ+×,Ð,ˆØ˜AÔØFGÈÅiÐ¤× 0Ò 0°°=Õ1AÓ BÓCà23°Fµ)Ð +¥Ó/ñ	 -ô Ð*×/Ñ/Ó1Ó2€KÜÐ'Ó(€HØ‡O�O�IÔà€IØ€MØ—~‘~Ö'‰ˆˆtØ˜Ô#Ø%)ˆI�d˜5“kÑ"Ü�5‹z˜QŒÙØˆEÜ˜1œc %›jÖ)�Ø˜r ˜ E¨!¨" I�B�Ø˜–> b¨H¦n¸"¸r½'ÀhÖ9NØ—L‘L " b¨$ Ö0ñ *ð �J‰JÔEˆJÔFØ× Ñ  Ö'äAKÈEÔSX×AYÒAY�e—l‘l 7°9�lÔ=Ð_dˆGØ!%ˆI�gÓñ (ð  ×Ñ™>ÐÔ*à€NÜ# MÖ2‰ˆ‰;ˆB��AÙ�R“™$˜r›(Ð#ˆØ�~Ö%Ø#$ˆN˜4Ó ñ 3ð
 €OÛˆÜ;EÀcÌ5×;QÒ;Q�#—*‘*˜W¨Y�*Ô7ÐWZˆØ�iÖØ'0°Õ'9ˆO˜GÓ$ñ ô
 ˜	 >°?ÄNØ$ l¸4ôAð Aùóu ;s   ¿K%c                   ó4   a € ] tR tRt o Rt]R 4       tRtV tR# )ÚLlamaTokenizerFastiH  zHDrop-in replacement for transformers.LlamaTokenizerFast (read-only use).c                ó   € \        V 4      # rH   )rÊ   )rÀ   Úkwargss   &,r
   Úfrom_pretrainedÚ"LlamaTokenizerFast.from_pretrainedK  s   € ä" 4Ó(Ð(r   rÙ   N)	rš   r›   rœ   r�   rž   Ústaticmethodrý   rŸ   r    r¡   s   @r
   rú   rú   H  s   ø‡ € ÙRàñ)ó ö)r   rú   )z'rez'vez'llz'sz'tz'mz'd)rž   rá   rÂ   r¿   rI   r   r)   r   r   r   r   r'   r,   r?   rA   r®   r¶   rÊ   rø   rú   rÙ   r   r
   Ú<module>r      sx   ðñó Û Û 	Û 	Û ð L€Ø=€ò-ò-òQò>òBò÷U5ñ U5òpòòCò@AA÷H)ó )r   