+
    Xåûjû  ã                   óª   € R t ^ RIHtHtHtHtHtHt ^ RIH	t	 ^RI
Ht ^RIHt ]P                  ! ]4      tRRRRR	R
/tRR/tRt ! R R]4      tR.tR# )zTokenization classes for Qwen2.)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPE)ÚTokenizersBackend)ÚloggingÚ
vocab_filez
vocab.jsonÚmerges_filez
merges.txtÚtokenizer_fileztokenizer.jsonzqwen/qwen-tokenizeri €  zn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+c                   óR   a a€ ] tR t^$t o]tRR.t]tRV3R lV 3R lllt	Rt
VtV ;t# )ÚQwen2TokenizerÚ	input_idsÚattention_maskc          
      óš   <€ V ^8„  d   QhRS[ S[S[ S[3,          ,          R,          RS[ S[S[ ,          ,          R,          RS[ RS[ RS[ /# )é   ÚvocabNÚmergesÚ	unk_tokenÚ	eos_tokenÚ	pad_token)ÚstrÚdictÚintÚlist)ÚformatÚ__classdict__s   "€ÚhC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\transformers\models\qwen2\tokenization_qwen2.pyÚ__annotate__ÚQwen2Tokenizer.__annotate__)   sc   ø€ ÷ 9`ñ 9`á‘T™#™s˜(•^Õ# dÕ*ð9`ñ ‘d™3•i• $Õ&ð9`ñ ð	9`ñ ð9`ñ ñ9`ó    c                óÜ  <€ Ve   TMRV n         Ve   TMR^ /V n        T;'       g    . V n        \        \	        V P                  V P                  R R RRRRR7      4      V n        \        P                  ! 4       V P
                  n        \        P                  ! 4       V P
                  n        \        P                  ! \        P                  ! \        \         4      RRR7      \        P                  ! V P                   RR7      .4      V P
                  n        \$        S
V `L  ! RRVR	VR
VRVRV/VB  T P)                  V P*                   U	u. uF  p	\-        V	RR7      NK  	  up	4       R # u up	i )NFú<|endoftext|>Ú )r   r   Údropoutr   Úcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkÚbyte_fallbackÚisolated)ÚbehaviorÚinvert)Úadd_prefix_spaceÚ	use_regexr   Ú	bos_tokenr   r   r.   T)Úspecial© )r.   Ú_vocabÚ_mergesr   r   Ú
_tokenizerr   Ú	ByteLevelÚdecoderr   ÚNFCÚ
normalizerr   ÚSequenceÚSplitr   ÚPRETOKENIZE_REGEXÚpre_tokenizerÚsuperÚ__init__Ú
add_tokensÚall_special_tokensr   )Úselfr   r   r   r0   r   r   r.   ÚkwargsÚtokenÚ	__class__s   &&&&&&&&, €r   r?   ÚQwen2Tokenizer.__init__)   so  ø€ ð 5EÒ4PÑ 0ÐV[ˆÔð Ò ñ ð   ðð 	Œð —|�| ˆŒÜ#ÜØ—k‘kØ—|‘|ØØØ*,Ø#%ØØ#ô	ó
ˆŒô #+×"4Ò"4Ó"6ˆ�‰ÔÜ%0§_¢_Ó%6ˆ�‰Ô"Ü(6×(?Ò(?ä×$Ò$ÜÔ+Ó,Ø'Ø ôô
 ×(Ò(Ø%)×%:Ñ%:Ø#ôð
ó)
ˆ�‰Ô%ô 	‰Òñ 	
Øð	
àð	
ð  ð	
ð  ð		
ð
 .ð	
ð ò	
ð 	�‰Àd×F]ÒF]Ó^ÑF]¸Uœ E°4×8ÑF]Ñ^Ö_ùÒ^s   Å
E))r4   r5   r3   r.   )NNr$   Nr$   r$   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr?   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rE   r   s   @@r   r   r   $   s.   ù‡ € Ø)ÐØ$Ð&6Ð7ÐØ€E÷9`÷ 9`õ 9`r"   r   N)Ú__doc__Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   Úutilsr
   Ú
get_loggerrG   ÚloggerrK   ÚMAX_MODEL_INPUT_SIZESr<   r   Ú__all__r2   r"   r   Ú<module>r[      su   ðñ &ç Z× ZÝ !å >Ý ð 
×	Ò	˜HÓ	%€ð �,Ø�<ØÐ&ðÐ ð /°Ð6Ð ð JÐ ô>`Ð&ô >`ðB Ð
‚r"   