+
    råûjm›  ã                   óZ  € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIHtHt ^ RIH	t	 ^ RI
H
t
 ^ RIHt ^ RIHtHt ^ RIHt ^ RIHtHtHtHtHtHtHt ^ RIt^ R	IHt ^R
IHt ^RIH t H!t!H"t" ]PF                  ! ]$4      t%] ! 4       '       d&   ^ RI&t&^ RI't&^ RI(t&^ RI)t&^ RI*t&^ RI+t&^ RI(H,t, ^ RI-H.t. MRt&]"! 4       '       d   ^ RI/H0t0 ]1]2]3]3,          ,          t4Rt5Rt6]7]8]3]9]]:! R4      R3t;]Px                  ! R]Pz                  4      t>]Px                  ! R]Pz                  4      t?]Px                  ! R]Pz                  ]P€                  ,          4      tA]Px                  ! R]Pz                  4      tB ! R R]C4      tD ! R R]C4      tE]7R]8R]3R]9R]:! R4      R/tFR R  ltGR! R" ltHR# R$ ltIR% R& ltJR' R( ltKR) R* ltL]]R+ R, l4       4       tM]]R- R. l4       4       tN]R/ R0 l4       tOR1 R2 ltPR3 R4 ltQR5 R6 ltRR7 tS]R8 4       tT]R9 4       tURCtVR: R; ltWR< R= ltXRDR> R? lltYR@ tZ ! RA RB4      t[R# )Eé    N)ÚCallableÚMapping)Úcontextmanager)Údatetime)ÚEnum)Ú	lru_cacheÚpartial)Ú
isfunction)ÚAnyÚLiteralÚUnionÚget_argsÚ
get_originÚget_type_hintsÚno_type_check)Úversion)Úlogging)Úis_jinja_availableÚis_torch_availableÚis_vision_available)Ú	Extension©ÚImmutableSandboxedEnvironment)ÚImageu   ó°€€u   ó°€�.z(^(.*?)[\n\s]*(Args:|Returns:|Raises:|\Z)z-\n\s*Args:\n(.*?)[\n\s]*(Returns:|Raises:|\Z)aX  
(?:^|\n)  # Match the start of the args block, or a newline
([ \t]*)(\w+):\s*  # Capture the indentation and the argument name
(.*?)\s*  # Capture the argument description, which can span multiple lines, and strip trailing spacing
(?=\n(?!\1[ \t])[ \t]*\w+:|\Z)  # Stop at the next "name:" that is not indented deeper, or the end of the block
z*\n\s*Returns:\n\s*(.*?)[\n\s]*(Raises:|\Z)c                   ó   € ] tR t^NtRtRtR# )ÚTypeHintParsingExceptionzJException raised for errors in parsing type hints to generate JSON schemas© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__static_attributes__r   ó    ÚbC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\transformers\utils\chat_template_utils.pyr   r   N   ó   † ÝTr%   r   c                   ó   € ] tR t^RtRtRtR# )ÚDocstringParsingExceptionzJException raised for errors in parsing docstrings to generate JSON schemasr   Nr   r   r%   r&   r)   r)   R   r'   r%   r)   ÚintegerÚnumberÚstringÚbooleanÚnullc                ó`   € V ^8„  d   QhR\         R\        \        \        3,          R,          /# )é   Ú
param_typeÚreturnN)ÚtypeÚdictÚstr)Úformats   "r&   Ú__annotate__r7   [   s'   € ÷ (ñ (¤dð (¬t´C¼°H­~ÀÕ/Dñ (r%   c                 ó<  € \         R R/\        R R/\        R R/\        R R/\	        R4      R R/\
        R R/\        R R/\        R R/\        / /	p\        4       '       d   R R	/V\        &   \        4       '       d   ^ RIpR R
/WP                  &   VP                  V 4      # )r3   r*   r+   r,   r-   Nr.   ÚarrayÚobjectÚimageÚaudio)ÚintÚfloatr5   Úboolr3   ÚlistÚtupler4   r   r   r   r   ÚtorchÚTensorÚget)r1   Útype_mappingrB   s   &  r&   Ú_get_json_schema_typerF   [   sª   € äˆf�iÐ Ü�˜Ð!Üˆf�hÐÜˆv�yÐ!ÜˆT‹
�V˜VÐ$Üˆv�wÐÜ�˜Ð Üˆv�xÐ ÜˆRð
€Lô ×ÒØ% wÐ/ˆ”UÑÜ×ÒÛà&,¨gÐ%6ˆ—\‘\Ñ"Ø×Ñ˜JÓ'Ð'r%   c                ó0   € V ^8„  d   QhR\         R\        /# )r0   Úvaluesr2   )r@   r4   )r6   s   "r&   r7   r7   p   s   € ÷ ^ñ ^¬ð ^´ñ ^r%   c                 ó�   € \        \        P                  R  V  4       4      4      pR\        V4      ^8X  d   V^ ,          RV /# TRV /# )c              3   óN   "  € T F  p\         \        V4      ,          x € K  	  R # 5i©N)ÚJSON_SCALAR_TYPESr3   ©Ú.0Úvalues   & r&   Ú	<genexpr>Ú._enum_values_to_json_schema.<locals>.<genexpr>r   s   é € Ð$XÑQWÈÔ%6´t¸E³{×%CÒ%CÓQWùs   ‚#%r3   Úenum)r@   r4   ÚfromkeysÚlen)rH   Úvalue_typess   & r&   Ú_enum_values_to_json_schemarV   p   sI   € ä”t—}‘}Ñ$XÑQWÓ$XÓXÓY€KØ¤c¨+Ó&6¸!Ô&;�K •NÈfÐV\Ð]Ð]ÀÈfÐV\Ð]Ð]r%   c                ó0   € V ^8„  d   QhR\         R\        /# )r0   Úhintr2   )r5   r4   )r6   s   "r&   r7   r7   v   s"   € ÷ [tñ [tœ3ð [t¤4ñ [tr%   c           	      ó  € \        V 4      p\        V 4      pVfÉ   \        V \        4      '       d³   \	        V \
        4      '       d�   V  Uu. uF  q3P                  NK  	  ppV'       g   \        RV P                   R24      h\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d   \        RV P                   RV R24      h\        V4      # Vf"   \        V 4      pVf   \        RV : R	24      hV# V\        J g+   \        \        R
4      '       dí   V\        P                  J dÙ   V Uu. uF  qf\        R 4      Jg   K  \!        V4      NK   	  pp\#        V4      ^8X  d   V^ ,          pM}\$        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       d@   \'        V U	u0 uF  q™R,          kK  	  up	4      p
R\#        V
4      ^8X  d
   V
^ ,          MT
/pMRV/p\        R 4      V9   d   RVR&   V# V\(        J dj   \#        V4      ^ 8”  dZ   \        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d   \        R4      h\        \+        V4      4      # V\*        J d#   V'       g   RR/# RRR\!        V^ ,          4      /# V\,        J dv   V'       g   RR/# \#        V4      ^8X  d)   \        R\/        V 4      P1                  RR4       R24      hRV9   d   \        R4      hRRRV Uu. uF  p\!        V4      NK  	  up/# V\2        J d,   RR/p\#        V4      ^8X  d   \!        V^,          4      VR&   V# \        RV 4      hu upi u upi u up	i u upi )Nz	The Enum z1 has no members, so no value could ever match it.c              3   óF   "  € T F  p\        V4      \        9  x € K  	  R # 5irK   ©r3   rL   rM   s   & r&   rP   Ú#_parse_type_hint.<locals>.<genexpr>�   s   é € ÐHÁ¸Œt�E‹{Ô"3Ö3Ãùó   ‚!TFzC has member values that cannot be written as JSON scalars.
Values: z[
Enum member values must be int, float, str, bool or None to be converted to a JSON schema.zCouldn't convert the type hint ac   to a JSON schema, likely because it is a custom class or object. Supported type hints are basic types like int, float, str and bool, Enum classes, typing.Literal, and list, tuple, dict and Union types containing those. If you need a more complex schema than that, you can write the JSON schema for your tool by hand and pass that instead of the function.Ú	UnionTypec              3   ó†   "  € T F7  qP                  4       R 08H  ;'       d    \        VR ,          \        4      x € K9  	  R# 5i)r3   N)ÚkeysÚ
isinstancer5   )rN   Úsubtypes   & r&   rP   r\   �   s4   é € ÐiÑ`hÐU\—‘“ F 8Ñ+×PÐP´
¸7À6½?ÌCÓ0PÔPÓ`hùs
   ‚A¡ Ar3   ÚanyOfÚnullablec              3   óF   "  € T F  p\        V4      \        9  x € K  	  R # 5irK   r[   )rN   Úargs   & r&   rP   r\   «   s   é € ÐB¹T°cŒt�C‹yÔ 1Ö1»Tùr]   z?Only the valid python literals can be listed in typing.Literal.r9   ÚitemszThe type hint ztyping.Ú a1   is a Tuple with a single element, which we do not automatically convert to JSON schema as it is rarely necessary. If this input can contain more than one element, we recommend using a list[] type instead, or if it really is a single element, remove the tuple[] wrapper and just pass the element directly..znConversion of '...' is not supported in Tuple type hints. Use list[] types for variable-length inputs instead.ÚprefixItemsr:   ÚadditionalPropertieszGCouldn't parse this type hint, likely due to a custom class or object: )r   r   ra   r3   Ú
issubclassr   rO   r   r   ÚanyrV   rF   r   ÚhasattrÚtypesr^   Ú_parse_type_hintrT   ÚallÚsortedr   r@   rA   r5   Úreplacer4   )rX   ÚoriginÚargsÚmemberrH   Újson_schema_typeÚtÚsubtypesÚreturn_dictrb   Úsubtype_namesÚouts   &           r&   ro   ro   v   sU  € Ü˜Ó€FÜ�D‹>€Dà‚~œ* T¬4×0Ò0´ZÀÄd×5KÒ5Ká-1Ó2©T 6—,”,©TˆÐ2ßÜ*Ø˜DŸM™M˜?Ð*[Ð\óð ÷ ‹3ÑHÁÓH�3�3Š3ÑHÁÓH×HÒHÜ*Ø˜DŸM™M˜?ð +Ø!˜(ð #mðmóð ô
 +¨6Ó2Ð2à	Šô 1°Ó6ÐØÒ#Ü*Ø1°$±ð :#ð #óð ð  Ðà	”5‹œW¤U¨K×8Ò8¸VÄuÇÁÓ=Vá15ÓM±¨AÄ$ÀtÃ*Ð9LÔ'Ô$ QÖ'±ˆÐMÜˆx‹=˜AÔà" 1�+‰Kß‹SÑiÑ`hÓi�S�SŠSÑiÑ`hÓi×iÒiô #Á8Ó#LÁ8¸¨F§O OÁ8Ñ#LÓMˆMØ!´s¸=Ó7IÈQÔ7N =°Ö#3ÐTaÐb‰Kð # HÐ-ˆKÜ�‹:˜ÔØ&*ˆK˜
Ñ#ØÐà	”7Ó	œs 4›y¨1œ}ß‹3ÑB¹TÓB�3�3Š3ÑB¹TÓB×BÒBÜ*Ð+lÓmÐmÜ*¬4°«:Ó6Ð6à	”4‹ßØ˜GÐ$Ð$ð ˜G WÔ.>¸tÀA½wÓ.GÐHÐHà	”5‹ßØ˜GÐ$Ð$Üˆt‹9˜Œ>Ü*Ø ¤ T£×!2Ñ!2°9¸bÓ!AÐ Bð C-ð -óð ð �$Œ;Ü*ð#óð ð
 ˜ ÉdÓ0SÉdÈÔ1AÀ!Ö1DÉdÑ0SÐTÐTà	”4‹ð �xÐ ˆÜˆt‹9˜Œ>Ü*:¸4À½7Ó*CˆCÐ&Ñ'Øˆ
ä
"Ð#lÐnrÓ
sÐsùòk 3ùò: Nùò $MùòL 1Ts   ÁM3Å M8ÅM8ÇM=ÌN
c                ó0   € V ^8„  d   QhR\         R\        /# ©r0   Úfuncr2   ©r   r4   )r6   s   "r&   r7   r7   Ô   s   € ÷ )ñ )¬Xð )¼$ñ )r%   c                 ó\  € \        V 4      p\        P                  ! V 4      p\        V R R4      p\	        \        VP                  4      R4      pVR
9   d>   VP                  V,          P                  \        P                  P                  8X  d   TpMRp. pVP                  P                  4        F�  w  rxWu8X  d   K  VP                  \        P                  P                  8X  d   \        RVP                   RV 24      hVP                  \        P                  P                  8X  g   K|  VP                  V4       K�  	  / p	VP                  4        F1  w  rzWu8X  d   K  VR8X  d    \        V
4      W—&   K$  \        V
4      W—&   K3  	  RRRV	/pV'       d   WkR	&   V#   \         d     KW  i ; i)r   Ú	operationNz	Argument z$ is missing a type hint in function r2   r3   r:   Ú
propertiesÚrequired>   ÚclsÚself)r   ÚinspectÚ	signatureÚgetattrÚnextÚiterÚ
parametersÚ
annotationÚ	ParameterÚemptyrg   r   ÚnameÚdefaultÚappendro   )r~   Ú
type_hintsr‡   Ú	func_nameÚfirst_param_nameÚimplicit_arg_namerƒ   Ú
param_nameÚparamr‚   r1   Úschemas   &           r&   Ú"_convert_type_hints_to_json_schemar™   Ô   sˆ  € Ü Ó%€JÜ×!Ò! $Ó'€IÜ˜˜j¨+Ó6€Iô œD ×!5Ñ!5Ó6¸Ó=Ðà˜OÔ+Ø× Ñ Ð!1Õ2×=Ñ=Ä×ARÑAR×AXÑAXÔXà,Ñà ÐØ€HØ&×1Ñ1×7Ñ7Ö9Ñˆ
ØÔ*ÙØ×Ñœw×0Ñ0×6Ñ6Ô6Ü*¨Y°u·z±z°lÐBfÐgpÐfqÐ+rÓsÐsØ�=‰=œG×-Ñ-×3Ñ3Ö3Ø�O‰O˜JÖ'ñ :ð €JØ",×"2Ñ"2Ö"4Ñˆ
ØÔ*ÙØ˜Ô!ðÜ)9¸*Ó)E�
Ó&ô &6°jÓ%AˆJÓ"ñ #5ð �h ¨jÐ9€FßØ%ˆzÑà€Møô ,ô Úðús   Å(FÆF+Æ*F+c                ó†   € V ^8„  d   QhR\         R\        \         R,          \        R,          \         R,          3,          /# )r0   Ú	docstringr2   N)r5   rA   r4   )r6   s   "r&   r7   r7      s5   € ÷ +ñ +¬Sð +´U¼3À½:ÄtÈdÅ{ÔTWÐZ^ÕT^Ð;^Õ5_ñ +r%   c           
     óÂ  € \         P                  V 4      p\        P                  V 4      p\        P                  V 4      pV'       d    VP	                  ^4      P                  4       MRpV'       d   VP	                  ^4      MRpV'       d    VP	                  ^4      P                  4       MRpVe¡   RP                  VP                  R4       Uu. uF  qwP                  4       '       g   K  VNK  	  up4      p\        P                  V4      pV U	u/ uF7  q™^,          \        P                  ! RRV	^,          P                  4       4      bK9  	  p
p	M/ p
WJV3# u upi u up	i )zô
Parses a Google-style docstring to extract the function description,
argument descriptions, and return description.

Args:
    docstring (str): The docstring to parse.

Returns:
    The function description, arguments, and return description.
NÚ
z	\s*\n+\s*Ú )Údescription_reÚsearchÚargs_reÚ
returns_reÚgroupÚstripÚjoinÚsplitÚargs_split_reÚfindallÚreÚsub)r›   Údescription_matchÚ
args_matchÚreturns_matchÚdescriptionÚdocstring_argsÚreturnsÚlineÚmatchesÚmatchÚ	args_dicts   &          r&   Úparse_google_format_docstringrµ      s"  € ô '×-Ñ-¨iÓ8ÐÜ—‘ 	Ó*€JÜ×%Ñ% iÓ0€M÷ 9JÐ#×)Ñ)¨!Ó,×2Ñ2Ô4Èt€Kß,6�Z×%Ñ% aÔ(¸D€Nß0=ˆm×!Ñ! !Ó$×*Ñ*Ô,À4€Gð Ò!ØŸ™°^×5IÑ5IÈ$Ô5OÓ#`Ñ5O¨T×S]ÑS]×S_§D DÑ5OÑ#`ÓaˆÜ×'Ñ'¨Ó7ˆÙX_Ó`ÑX_Èu˜1•XœrŸvšv l°C¸¸q½¿¹Ó9IÓJÒJÑX_ˆ	Ð`ˆ	àˆ	à 7Ð*Ð*ùò $aùâ`s   ÃEÃ+EÄ=Ec                ó0   € V ^8„  d   QhR\         R\        /# r}   r   )r6   s   "r&   r7   r7   !  s   € ÷ G4ñ G4œ(ð G4¤tñ G4r%   c                ó@  € \         P                  ! V 4      p\        V RR4      pV'       g   \        RV R24      hVP	                  4       p\        V4      w  r4p\        V 4      pVR,          P                  RR4      ;pe	   Ve   WWR&   VR,          P                  4        Fá  w  r‰W„9  d   \        RV R	V R
24      hWH,          p
\        P                  ! RV
\        P                  R7      pV'       dŠ   \        P                  ! VP                  ^4      4       Uu. uF+  p\        V\         4      '       d   VP	                  4       MTNK-  	  upV	R&   VP"                  RVP%                  4        P	                  4       p
W©R&   Kã  	  RVRVRV/pVe   W}R&   RRRV/# u upi )aÇ  
This function generates a JSON schema for a given function, based on its docstring and type hints. This is
mostly used for passing lists of tools to a chat template. The JSON schema contains the name and description of
the function, as well as the names, types and descriptions for each of its arguments. `get_json_schema()` requires
that the function has a docstring, and that each argument has a description in the docstring, in the standard
Google docstring format shown below. It also requires that all user-facing arguments have valid Python type hints.
When passing methods, implicit receiver arguments (`self` or `cls`) are ignored.

Although it is not required, a `Returns` block can also be added, which will be included in the schema. This is
optional because most chat templates ignore the return value of the function.

Args:
    func: The function to generate a JSON schema for.

Returns:
    A dictionary containing the JSON schema for the function.

Examples:
```python
>>> def multiply(x: float, y: float):
>>>    '''
>>>    A function that multiplies two numbers
>>>
>>>    Args:
>>>        x: The first number to multiply
>>>        y: The second number to multiply
>>>    '''
>>>    return x * y
>>>
>>> print(get_json_schema(multiply))
{
    "name": "multiply",
    "description": "A function that multiplies two numbers",
    "parameters": {
        "type": "object",
        "properties": {
            "x": {"type": "number", "description": "The first number to multiply"},
            "y": {"type": "number", "description": "The second number to multiply"}
        },
        "required": ["x", "y"]
    }
}
```

The general use for these schemas is that they are used to generate tool descriptions for chat templates that
support them, like so:

```python
>>> from transformers import AutoTokenizer
>>> from transformers.utils import get_json_schema
>>>
>>> def multiply(x: float, y: float):
>>>    '''
>>>    A function that multiplies two numbers
>>>
>>>    Args:
>>>        x: The first number to multiply
>>>        y: The second number to multiply
>>>    return x * y
>>>    '''
>>>
>>> multiply_schema = get_json_schema(multiply)
>>> tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01")
>>> messages = [{"role": "user", "content": "What is 179 x 4571?"}]
>>> formatted_chat = tokenizer.apply_chat_template(
>>>     messages,
>>>     tools=[multiply_schema],
>>>     chat_template="tool_use",
>>>     return_dict=True,
>>>     return_tensors="pt",
>>>     add_generation_prompt=True
>>> )
>>> # The formatted chat can now be passed to model.generate()
```

Each argument description can also have an optional `(choices: ...)` block at the end, such as
`(choices: ["tea", "coffee"])`, which will be parsed into an `enum` field in the schema. Note that this will
only be parsed correctly if it is at the end of the line:

```python
>>> def drink_beverage(beverage: str):
>>>    '''
>>>    A function that drinks a beverage
>>>
>>>    Args:
>>>        beverage: The beverage to drink (choices: ["tea", "coffee"])
>>>    '''
>>>    pass
>>>
>>> print(get_json_schema(drink_beverage))
```
{
    'name': 'drink_beverage',
    'description': 'A function that drinks a beverage',
    'parameters': {
        'type': 'object',
        'properties': {
            'beverage': {
                'type': 'string',
                'enum': ['tea', 'coffee'],
                'description': 'The beverage to drink'
                }
            },
        'required': ['beverage']
    }
}
r   r�   z Cannot generate JSON schema for z because it has no docstring!r‚   r2   Nr®   z< because the docstring has no description for the argument 'Ú'z\(choices:\s*(.*?)\)\s*$)ÚflagsrR   r�   r‹   r3   Úfunction)r†   Úgetdocrˆ   r)   r¤   rµ   r™   Úpoprg   r©   r    Ú
IGNORECASEÚjsonÚloadsr£   ra   r5   r,   Ústart)r~   Údocr“   Úmain_docÚparam_descriptionsÚ
return_docÚjson_schemary   rf   r˜   ÚdescÚenum_choicesÚcÚoutputs   &             r&   Úget_json_schemarÊ   !  sž  € ôX �.Š.˜Ó
€CÜ˜˜j¨+Ó6€IçÜ'Ð*JÈ9È+ÐUrÐ(sÓtÐtØ
�)‰)‹+€CÜ/LÈSÓ/QÑ,€H *ä4°TÓ:€KØ" <Õ0×4Ñ4°X¸tÓDÐDˆÒQØÒ!Ø)3˜Ñ&Ø" <Õ0×6Ñ6Ö8‰ˆØÔ(Ü+Ø2°9°+Ð=yÐz}Ðy~Ð~ð  Aóð ð "Õ&ˆÜ—y’yÐ!<¸dÌ"Ï-É-ÔXˆßÜNRÏjÊjÐYe×YkÑYkÐlmÓYnÔNoÓpÑNoÈ¬:°a¼×+=Ò+=˜aŸg™gœiÀ1ÒDÑNoÑpˆF�6‰NØ×&Ñ&Ð'=¨×);Ñ);Ó)=Ð>×DÑDÓFˆDØ $ˆ}Óñ 9ð �i °¸,ÈÐT€FØÒØ&ˆxÑØ�J 
¨FÐ3Ð3ùò qs   Ä1Fc                óT   € V ^8„  d   QhR\         R,          R\        \         ,          /# )r0   Úchat_templateNr2   )r5   Ú	frozenset)r6   s   "r&   r7   r7   ­  s&   € ÷ Añ A¬3°­:ð A¼)ÄC½.ñ Ar%   c                óº   € V f   \        4       # \        V 4      pVP                  P                  V 4      p\        \        P
                  P                  V4      4      # )aO  Return the set of undeclared variables referenced by a chat template.

Uses ``jinja2.meta.find_undeclared_variables`` so that callers can
automatically distinguish template-level kwargs from processor kwargs
without maintaining a manual allowlist. Needed only to support BC as we
allowed all `kwargs` to be merged into one in the past
)rÍ   Ú_compile_jinja_templateÚenvironmentÚparseÚjinja2ÚmetaÚfind_undeclared_variables)rÌ   ÚcompiledÚasts   &  r&   Ú_get_template_variablesr×   «  sL   € ð ÒÜ‹{ÐÜ& }Ó5€HØ
×
Ñ
×
$Ñ
$ ]Ó
3€CÜ”V—[‘[×:Ñ:¸3Ó?Ó@Ð@r%   c                ó0   € V ^8„  d   QhR\         R\         /# )r0   rÌ   r2   )r5   )r6   s   "r&   r7   r7   ¾  s   € ÷ ñ ¬cð ´cñ r%   c                óÜ  € \        V 4      pVP                  P                  V 4      pVP                  \        P
                  P                  4       Uu0 uF  q3P                  kK  	  ppYBP                  \        P
                  P                  4       Uu0 uF1  p\        VP                  \        4      '       g   K%  VP                  kK3  	  up,          pRP                  V4      # u upi u upi )aO  
Return the literal strings in a chat template (its raw template text, plus every string constant), joined with null
characters so that a substring search can't match across two of them.

Added tokens found in this text are treated as control tokens by `sanitize_control_tokens`. This misses tokens that
the template builds from pieces, like Phi-4-mini's `'<|' + message['role'] + '|>'`. Special tokens are always
control tokens, so only non-special tokens can be missed, and tokenizers whose templates build them should list
their control tokens in `tokenizer.chat_control_tokens` instead.
Ú )rÏ   rÐ   rÑ   Úfind_allrÒ   ÚnodesÚTemplateDataÚdataÚConstra   rO   r5   r¥   )rÌ   rÕ   rÖ   ÚnodeÚliteralss   &    r&   Ú_get_template_literal_textrâ   ¼  sª   € ô ' }Ó5€HØ
×
Ñ
×
$Ñ
$ ]Ó
3€CØ&)§l¡l´6·<±<×3LÑ3LÔ&MÓNÑ&M˜d—	”	Ñ&M€HÐNØ¯©´V·\±\×5GÑ5GÔ(HÓhÑ(H ÌJÐW[×WaÑWaÔcf×Lg”�—”Ñ(HÑhÕh€HØ�9‰9�XÓÐùò OùÚhs   ÁC$Â"C)Â:C)c                ó”   € V ^8„  d   QhR\         \        R3,          R\         \        P                  \        P                  3,          /# )r0   Úcontrol_tokens.r2   )rA   r5   r©   ÚPattern)r6   s   "r&   r7   r7   Ð  s7   € ÷ 
)ñ 
)´´c¸3°hµð 
)ÄEÌ"Ï*É*ÔVX×V`ÑV`ÐJ`ÕDañ 
)r%   c                ó,  € \        V \        RR7      p\        P                  ! RP	                  \        \        P                  V4      4      ;'       g    R4      p\        \        rC\        P                  ! V RVP                   RV RV V R2	4      pW%3# )	a   
Return a regex that matches any of `control_tokens`, and a regex that matches a control token wrapped in the markers
added by `mark_control_tokens`, or a lone marker. Longer tokens are matched first, so a token takes priority over
the tokens it contains.
T)ÚkeyÚreverseÚ|z(?!)Ú(Ú)z|[Ú])
rq   rT   r©   Úcompiler¥   ÚmapÚescapeÚ_CONTROL_TOKEN_STARTÚ_CONTROL_TOKEN_ENDÚpattern)rä   Úlongest_firstÚcontrol_tokens_rerÀ   ÚendÚ
markers_res   &     r&   Ú_get_control_token_patternsr÷   Ï  s„   € ô ˜>¬s¸DÔA€MÜŸ
š
 3§8¡8¬C´·	±	¸=Ó,IÓ#J×#TÐ#TÈfÓUÐÜ%Ô'9ˆ3Ü—’˜u˜g QÐ'8×'@Ñ'@Ð&AÀÀ3À%ÀrÈ%ÈÐQTÐPUÐUVÐWÓX€JØÐ(Ð(r%   c                óP   € V ^8„  d   QhR\         R\        P                  R\         /# )r0   Úobjrô   r2   )r   r©   rå   )r6   s   "r&   r7   r7   Ý  s%   € ÷ ñ œSð ´R·Z±Zð ÄCñ r%   c           	     óÐ  € \        V \        4      '       dJ   V P                  \        R4      P                  \        R4      p VP                  \         R\         2V 4      # \        V \        4      '       d/   V P                  4        UUu/ uF  w  r#V\        W14      bK  	  upp# \        V \        \        34      '       d   V  Uu. uF  p\        W14      NK  	  up# V # u uppi u upi )a¯  
Wrap the control tokens in user-supplied strings with private-use markers, so that `encode_marked_chat` can tell
them apart from the control tokens the chat template writes itself.

Args:
    obj: A string, or a list or dict (like messages, tools or documents) whose values are marked recursively. Dict
        keys aren't marked.
    control_tokens_re: A regex that matches any control token, from `_get_control_token_patterns`.
rh   z\g<0>)ra   r5   rr   rð   rñ   rª   r4   rg   Úmark_control_tokensr@   rA   )rù   rô   rç   rO   s   &&  r&   rû   rû   Ý  sÈ   € ô �#”s×Òà�k‰kÔ.°Ó3×;Ñ;Ô<NÐPRÓSˆØ ×$Ñ$Ô(<Ð'=¸VÔDVÐCWÐ%XÐZ]Ó^Ð^Ü�#”t×ÒØUX×U^ÑU^ÔU`ÔaÑU`ÁzÀs�Ô(¨ÓBÒBÑU`ÒaÐaÜ�#œœe�}×%Ò%ÙKNÓOÉ3À%Ô# EÖ=É3ÑOÐOØ€Jùó bùâOs   Â	CÃC#c                óF   € V ^8„  d   QhR\         R\        \        ,          /# ©r0   Úchatr2   )r5   r@   r=   )r6   s   "r&   r7   r7   ò  s   € ÷ ñ ¬ð ´´Sµ	ñ r%   c                ó¬  € \        \        V P                  4      4      w  r#VP                  V4      pV P	                  RRR7      pV P	                  V^ ,          RR7      p\        VR,          ^R7       F]  w  rxV'       g   K  V^,          ^8H  p	\        WW•4      p
V	'       d'   V P                  V4      V
9   d   \        RV: R24      hWj,          pK_  	  V# )a  
Encode a chat rendered from inputs marked by `mark_control_tokens`. Marked control tokens are encoded as plain text,
without their markers, and the rest of the chat, including the template's own control tokens, is encoded normally.

String methods in the template, like `split()`, can separate the markers around a token. Lone markers like these
are dropped, but still break the text, so that the text on either side can't join into a control token.

Args:
    tokenizer: The tokenizer that rendered the chat.
    chat: The rendered chat.
r�   F©Úadd_special_tokensºé   NN)rÀ   zCan't sanitize z+: the tokenizer produces it from plain text)	r÷   rA   Úadded_tokens_encoderr¦   ÚencodeÚ	enumerateÚ_encode_chat_pieceÚconvert_tokens_to_idsÚ
ValueError)Ú	tokenizerrþ   Ú_rö   ÚpiecesÚnewline_idsÚidsÚiÚpieceÚas_textÚ	piece_idss   &&         r&   Úencode_marked_chatr  ò  sÈ   € ô 0´°i×6TÑ6TÓ0UÓV�M€AØ×Ñ˜dÓ#€FØ×"Ñ" 4¸EÐ"ÓB€KØ
×
Ñ
˜6 !�9¸Ð
Ó
?€CÜ˜f R�j°×2‰ˆßÙØ�a•%˜1‘*ˆÜ& y¸ÓNˆ	ß�y×6Ñ6°uÓ=ÀÔJÜ˜¨u©iÐ7bÐcÓdÐdØÕŠñ 3ð €Jr%   c          	      ót   € V ^8„  d   QhR\         R\        R\        \        ,          R\        \        ,          /# )r0   r  r  r  r2   )r5   r?   r@   r=   )r6   s   "r&   r7   r7     s3   € ÷ ñ ¬ð ´tð Ì$ÌsÍ)ð ÔX\Ô]`ÕXañ r%   c                óÊ   € V'       d   V P                   pM\        V P                  RR7      pV! RV,           4      pVR\        V4       V8X  d   V\        V4      R # V! V4      # )zTEncode a piece from the middle of a chat, ignoring added tokens if `as_text` is set.Fr   r�   N)Ú_encode_without_added_tokensr	   r  rT   )r
  r  r  r  r  r  s   &&&&  r&   r  r    sd   € çØ×7Ñ7‰ä˜×)Ñ)¸eÔDˆñ
 �t˜e•|Ó$€IØÐ#”3�{Ó#Ð$¨Ô3Øœ˜[Ó)Ð+Ð,Ð,Ù�%‹=Ðr%   c                 ó  € . p. pV P                   P                  Wg4      ;_uu_ 4        V P                  ! RR VRVRVRV/VB  F  pVP                  V4       K  	  RP	                  V4      p	RRR4       W—3#   + '       g   i     X	T3# ; i)ÚmessagesÚtoolsÚ	documentsÚadd_generation_promptrh   Nr   )rÐ   Úactivate_trackerÚgenerater‘   r¥   )
Úcompiled_templater  r  r  r  Útemplate_kwargsÚrendered_blocksÚgeneration_indicesÚblockÚrendered_chats
   &&&&&,    r&   Ú_render_with_assistant_indicesr$    s²   € ð €OØÐØ	×	&Ñ	&×	7Ñ	7¸×	\Õ	\Ø&×/Ò/ñ 
Øð
àð
ð  ð
ð #8ð	
ð
 ô
ˆEð ×"Ñ" 5Ö)ñ
ð Ÿ™ Ó0ˆ÷ 
]ð Ð,Ð,÷ 
]Ö	\ð Ð,Ð,Ð,ús   ¨AA7Á7B
	c                 ó   € \        V 4      # rK   )Ú_cached_compile_jinja_template)rÌ   s   &r&   rÏ   rÏ   1  s   € ä)¨-Ó8Ð8r%   c                 óŽ  a€ \        4       '       g   \        R 4      h ! R R\        4      p\        P                  ! \
        P                  4      \        P                  ! R4      8  d   \        R\
        P                   R24      hR pRR lpR p ! R	 R
4      p ! V3R lR\
        P                  P                  4      o\        RRV\
        P                  P                  .R7      pW6P                  R&   W&P                  R&   WFP                  R&   V! 4       VP                  R&   SVP                  R&   VP                  V 4      # )zbapply_chat_template requires jinja2 to be installed. Please install it using `pip install jinja2`.c                   ó´   a a€ ] tR tRt oR0tV3R lV 3R lltV3R lR lt]P                  V3R lR l4       t	V3R	 lR
 lt
]V3R lR l4       tRtVtV ;t# )Ú8_cached_compile_jinja_template.<locals>.AssistantTrackeri=  Ú
generationc                ó    <€ V ^8„  d   QhRS[ /# )r0   rÐ   r   )r6   Ú__classdict__s   "€r&   r7   ÚE_cached_compile_jinja_template.<locals>.AssistantTracker.__annotate__A  s   ø€ ÷ 	,ñ 	,Ñ(Eñ 	,r%   c                óz   <€ \         SV `  V4       VP                  V P                  R 7       RV n        RV n        R# ))r  N)ÚsuperÚ__init__Úextendr  Ú_rendered_blocksÚ_generation_indices)r…   rÐ   Ú	__class__s   &&€r&   r0  ÚA_cached_compile_jinja_template.<locals>.AssistantTracker.__init__A  s7   ø€ ä‰GÑ˜[Ô)Ø×Ñ°×0EÑ0EÐÔFØ$(ˆDÔ!Ø'+ˆDÖ$r%   c                óv   <€ V ^8„  d   QhRS[ P                  P                  RS[ P                  P                  /# )r0   Úparserr2   )rÒ   r7  ÚParserrÜ   Ú	CallBlock)r6   r,  s   "€r&   r7   r-  H  s3   ø€ ÷ 	tñ 	t¡§¡× 4Ñ 4ð 	t¹¿¹×9OÑ9Oñ 	tr%   c                óè   € \        VP                  4      P                  pVP                  R .RR7      p\        P
                  P                  V P                  R4      . . V4      P                  V4      # )zname:endgenerationT)Údrop_needleÚ_generation_support)	r‰   ÚstreamÚlinenoÚparse_statementsrÒ   rÜ   r9  Úcall_methodÚ
set_lineno)r…   r7  r>  Úbodys   &&  r&   rÑ   Ú>_cached_compile_jinja_template.<locals>.AssistantTracker.parseH  se   € Ü˜&Ÿ-™-Ó(×/Ñ/ˆFØ×*Ñ*Ð,@Ð+AÈtÐ*ÓTˆDÜ—<‘<×)Ñ)¨$×*:Ñ*:Ð;PÓ*QÐSUÐWYÐ[_Ó`×kÑkÐlrÓsÐsr%   c                ó|   <€ V ^8„  d   QhRS[ P                  P                  RS[ P                  P                  RS[/# )r0   ÚcontextÚcallerr2   )rÒ   rÜ   ÚEvalContextÚruntimeÚMacror5   )r6   r,  s   "€r&   r7   r-  N  s:   ø€ ÷ 	ñ 	©v¯|©|×/GÑ/Gð 	ÑQW×Q_ÑQ_×QeÑQeð 	Ñjmñ 	r%   c                óä   € V! 4       pV P                  4       '       dS   \        R P                  V P                  4      4      pV\        V4      ,           pV P                  P                  WE34       V# )rh   )Ú	is_activerT   r¥   r2  r3  r‘   )r…   rE  rF  ÚrvÚstart_indexÚ	end_indexs   &&&   r&   r<  ÚL_cached_compile_jinja_template.<locals>.AssistantTracker._generation_supportM  sY   € á“ˆBØ�~‰~×Òä! "§'¡'¨$×*?Ñ*?Ó"@ÓA�Ø'¬#¨b«'Õ1�	Ø×(Ñ(×/Ñ/°Ð0HÔIØˆIr%   c                ó    <€ V ^8„  d   QhRS[ /# )r0   r2   )r?   )r6   r,  s   "€r&   r7   r-  W  s   ø€ ÷ 	]ñ 	]™tñ 	]r%   c                óJ   € V P                   R J;'       g    V P                  R J# rK   )r2  r3  )r…   s   &r&   rK  ÚB_cached_compile_jinja_template.<locals>.AssistantTracker.is_activeW  s'   € Ø×(Ñ(°Ð4×\Ð\¸×8PÑ8PÐX\Ð8\Ð\r%   c                óF   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          /# )r0   r   r!  )r@   r=   )r6   r,  s   "€r&   r7   r-  [  s&   ø€ ÷ 
	0ñ 
	0±D¹µIð 
	0ÑSWÑX[ÕS\ñ 
	0r%   c              3  ó´   "  €  V P                  4       '       d   \        R 4      hWn        W n        Rx € RV n        RV n        R#   RT n        RT n        i ; i5i)z3AssistantTracker should not be reused before closedN)rK  r	  r2  r3  )r…   r   r!  s   &&&r&   r  ÚI_cached_compile_jinja_template.<locals>.AssistantTracker.activate_trackerZ  sU   é € ð	0Ø—>‘>×#Ò#Ü$Ð%ZÓ[Ð[Ø(7Ô%Ø+=Ô(ãà(,�Ô%Ø+/�Ö(øð )-�Ô%Ø+/�Õ(üs   ‚A„1A µAÁAÁA)r3  r2  )r   r    r!   r"   Útagsr0  rÑ   rÒ   Úpass_eval_contextr<  rK  r   r  r$   Ú__classdictcell__Ú__classcell__)r4  r,  s   @@r&   ÚAssistantTrackerr)  =  sa   ù‡ € àˆ~ˆ÷	,ó 	,÷	tð 	tð
 
×	!Ñ	!÷	ó 
"ð	÷	]ð 	]ð 
÷
	0ó 
÷
	0ð 
	0r%   rZ  z3.1.6zLapply_chat_template requires jinja2>=3.1.6 to be installed. Your version is z2. You can upgrade it with `pip install -U jinja2`.c                 ó@   € \         P                  P                  V 4      hrK   )rÒ   Ú
exceptionsÚTemplateError©Úmessages   &r&   Úraise_exceptionÚ7_cached_compile_jinja_template.<locals>.raise_exceptionn  s   € Ü×Ñ×-Ñ-¨gÓ6Ð6r%   c                 ó4   € \         P                  ! WW#VR 7      # ))Úensure_asciiÚindentÚ
separatorsÚ	sort_keys)r¾   Údumps)Úxrc  rd  re  rf  s   &&&&&r&   ÚtojsonÚ._cached_compile_jinja_template.<locals>.tojsonq  s   € ô �zŠz˜!¸vÐhqÔrÐrr%   c                 óJ   € \         P                  ! 4       P                  V 4      # rK   )r   ÚnowÚstrftime)r6   s   &r&   Ústrftime_nowÚ4_cached_compile_jinja_template.<locals>.strftime_nowv  s   € Ü�|Š|‹~×&Ñ& vÓ.Ð.r%   c                   ó,   a € ] tR tRt o R tR tRtV tR# )Ú4_cached_compile_jinja_template.<locals>.TemplateDictiy  c                ó   € \        V/ VB # rK   ©r4   ©r…   rt   Úkwargss   &*,r&   Ú__call__Ú=_cached_compile_jinja_template.<locals>.TemplateDict.__call__|  s   € Ü˜Ð( Ñ(Ð(r%   c                ó.   € \         P                  ! V/ VB # rK   )r4   rS   rt  s   &*,r&   rS   Ú=_cached_compile_jinja_template.<locals>.TemplateDict.fromkeys  s   € Ü—=’= $Ð1¨&Ñ1Ð1r%   r   N)r   r    r!   r"   rv  rS   r$   rX  ©r,  s   @r&   ÚTemplateDictrq  y  s   ø‡ € ò	)÷	2ð 	2r%   r{  c                   óJ   <a € ] tR tRt o V3R ltV3R lt]R 4       tRtV t	R# )Ú9_cached_compile_jinja_template.<locals>.TemplateNamespacei‚  c                 óÆ   <€ V ^ ,          V R,          r\        V / VB pV F  pSP                  V4       K  	  \        P                  P                  P                  W#4       R# )r   r  N)r4   Ú_check_namerÒ   ÚutilsÚ	Namespacer0  )rt   ru  r…   Úattrsr�   ÚTemplateNamespaces   *,   €r&   r0  ÚB_cached_compile_jinja_template.<locals>.TemplateNamespace.__init__„  sQ   ø€ à˜a� $ r¥(�$Ü˜$Ð) &Ñ)ˆEÛ�Ø!×-Ñ-¨dÖ3ñ ä�L‰L×"Ñ"×+Ñ+¨DÖ8r%   c                ó~   <€ SP                  V4       \        P                  P                  P	                  WV4       R # rK   )r  rÒ   r€  r�  Ú__setitem__)r…   r�   rO   rƒ  s   &&&€r&   r†  ÚE_cached_compile_jinja_template.<locals>.TemplateNamespace.__setitem__Œ  s+   ø€ Ø×)Ñ)¨$Ô/Ü�L‰L×"Ñ"×.Ñ.¨t¸5ÖAr%   c                ó¬   € \        V \        4      '       d>   V P                  R 4      '       d%   \        P                  P                  RV : R24      hR# R# )r  zSChat templates cannot store underscore-prefixed attributes on a namespace, but got z. Please rename the attribute.N)ra   r5   Ú
startswithrÒ   r\  ÚSecurityError)r�   s   &r&   r  ÚE_cached_compile_jinja_template.<locals>.TemplateNamespace._check_name�  sR   € ä˜$¤×$Ò$¨¯©¸×)=Ò)=Ü×'Ñ'×5Ñ5ØiÐjnÑiqð r3ð 3óð ñ *>Ñ$r%   r   N)
r   r    r!   r"   r0  r†  Ústaticmethodr  r$   rX  )r,  rƒ  s   @€r&   rƒ  r}  ‚  s$   ù‡ € õ	9õ	Bð 
ñ	ó 
ö	r%   rƒ  T)Útrim_blocksÚlstrip_blocksÚ
extensionsri  r`  rn  r4   Ú	namespace)FNNF)r   ÚImportErrorr   r   rÑ   rÒ   Ú__version__r€  r�  r   ÚextÚloopcontrolsÚfiltersÚglobalsÚfrom_string)rÌ   rZ  r`  ri  rn  r{  Ú	jinja_envrƒ  s   &      @r&   r&  r&  6  s%  ø€ ä×ÒÜØpó
ð 	
ô(0œ9ô (0ôT ‡}‚}”V×'Ñ'Ó(¬7¯=ª=¸Ó+AÔAäØZÔ[a×[mÑ[mÐZnð o?ð ?ó
ð 	
ò
7ôsò
/÷2ñ 2÷œFŸL™L×2Ñ2ô ô, .Ø¨Ð:JÌFÏJÉJ×LcÑLcÐ9dô€Ið #)×Ñ�hÑØ+:×ÑÐ'Ñ(Ø(4×Ñ�nÑ%Ù ,£€I×Ñ�fÑØ%6€I×Ñ�kÑ"Ø× Ñ  Ó/Ð/r%   c                ó0   € V ^8„  d   QhR\         R\         /# rý   )ÚChatType)r6   s   "r&   r7   r7   §  s   € ÷ 	ñ 	¤(ð 	¬xñ 	r%   c                 ó¼   a€ . pV  FR  o\        S\        4      '       d)   \        V3R  l\         4       R4      pVe   RV/SCRV/CoVP	                  S4       KT  	  V# )c              3   ó`   <"  € T F#  pSP                  V4      f   K  SV,          x € K%  	  R # 5irK   )rD   )rN   rç   r_  s   & €r&   rP   Ú*_alias_reasoning_fields.<locals>.<genexpr>¬  s'   øé € ÐhÑ6G¨sÈ7Ï;É;ÐWZÓK[œl˜g cŸlšlÓ6Gùs   ƒ.�.NÚthinkingÚreasoning_content)ra   r4   r‰   Ú_REASONING_FIELDSr‘   )rþ   Úaliased_chatÚ	reasoningr_  s   &  @r&   Ú_alias_reasoning_fieldsr£  §  sg   ø€ à€LÛˆÜ�gœt×$Ò$ÜÔhÕ6GÓhÐjnÓoˆIØÒ$Ø% yÐ\°GÐ\Ð=PÐR[Ñ\�Ø×Ñ˜GÖ$ñ ð Ðr%   c                ó0   € V ^8„  d   QhR\         R\         /# )r0   rO   r2   )r   )r6   s   "r&   r7   r7   ³  s   € ÷ !+ñ !+¤ð !+¬ñ !+r%   c           
      ó  € V e%   \        V 4      \        \        \        \        39   d   V # \        V \        4      '       d   \        P                  V 4      # \        V \        4      '       d   \        P                  V 4      # \        V \        4      '       d   \        P                  V 4      # \        V \        4      '       d8   V P                  4        UUu/ uF  w  r\        V4      \        V4      bK  	  upp# \        V \        4      '       d   V  Uu. uF  p\        V4      NK  	  up# \        V \        4      '       d-   \        ;QJ d    . R V  4       F  NK  	  5# ! R V  4       4      # \        V \        P                  4      '       d   \        V P!                  4       4      # \"        P$                  ! V 4      '       dc   \        V \         4      '       gM   \"        P&                  ! V 4       Uu/ uF+  q3P(                  \        \+        WP(                  4      4      bK-  	  up# \-        \+        V RR 4      4      '       d   \        V P/                  4       4      # R\        V 4      P0                   R2# u uppi u upi u upi )Nc              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irK   )Ú_plain_template_data)rN   Úitems   & r&   rP   Ú'_plain_template_data.<locals>.<genexpr>È  s   é € ÐB¹E°DÔ)¨$×/Ð/»Eùs   ‚Ú
model_dumpÚ<Ú>)r3   r5   r=   r>   r?   ra   Ú__str__Ú__int__Ú	__float__r   rg   r§  r@   rA   ÚnpÚgenericr¨  ÚdataclassesÚis_dataclassÚfieldsr�   rˆ   Úcallablerª  r   )rO   rç   r¨  Úfields   &   r&   r§  r§  ³  sÙ  € ð ‚}œ˜U›¬¬S´%¼Ð'>Ô>àˆÜ	�Eœ3×	Ò	ä�{‰{˜5Ó!Ð!Ü	�Eœ3×	Ò	Ü�{‰{˜5Ó!Ð!Ü	�Eœ5×	!Ò	!Ü�‰˜uÓ%Ð%Ü	�Eœ7×	#Ò	#ØW\×WbÑWbÔWdÔeÑWdÉ)È#Ô$ SÓ)Ô+?ÀÓ+EÒEÑWdÒeÐeÜ	�Eœ4×	 Ò	 Ù7<Ó=±u¨tÔ$ TÖ*±uÑ=Ð=Ü	�Eœ5×	!Ò	!ßŒuÑB¹EÓB�uÐBˆuÑB¹EÓBÓBÐBÜ	�Eœ2Ÿ:™:×	&Ò	&ä# E§J¡J£LÓ1Ð1Ü	×	!Ò	! %×	(Ò	(´¸EÄ4×1HÒ1HäZe×ZlÒZlÐmrÔZsÓtÑZsÐQV—
‘
Ô0´¸Ç
Á
Ó1KÓLÒLÑZsÑtÐtÜ	”'˜% ¨tÓ4×	5Ò	5ä# E×$4Ñ$4Ó$6Ó7Ð7ð ”4˜“;×'Ñ'Ð(¨Ð*Ð*ùó! fùâ=ùò us   Ã I1ÄI7Ç/1I<c                óú   € V ^8„  d   QhR\         \        ,          R\         \        \        ,          ,          R,          R\        R,          R\        R,          R\
        R\
        \        ,          R\
        R	\        /# )
r0   Úconversationsr  Nr  rÌ   Úreturn_assistant_tokens_maskÚcontinue_final_messager  r2   )r@   rš  r4   r   r5   r?   )r6   s   "r&   r7   r7   ×  s€   € ÷ ,ñ ,Üœ•>ð,ä””x•Õ  4Õ'ð,ô ˜$�ð,ô ˜•:ð	,ô
 #'ð,ô !¤3�Jð,ô  ð,ô 	ñ,r%   c                 ó2  a€ V'       d3   \         P                  ! R S4      '       g   \        P                  R4       \	        S4      pVeˆ   . p	V F~  p
\        V
\        4      '       d   V	P                  V
4       K,  \        V
4      '       g   \        P                  ! V
4      '       d   V	P                  \        V
4      4       Ku  \        R4      h	  MRp	Ve+   V F$  p\        V\        4      '       d   K  \        R4      h	  \        V	4      p	\        V4      p\        V4      p. p. pRpV  EFá  p\        VR4      '       d   VP                   p\        V4      pV F)  p\        V\        4      '       d   K  \        RV: R24      h	  \#        V4      pV'       Edr   V'       g   \        R	4      h\        V\$        4      '       d   TMR
pVR,          P'                  V4      ;pf   \        RV R24      hV.pV\(        9   d5   \(         Uu. uF$  pVR,          P'                  V4      V8X  g   K"  VNK&  	  pp\*        ;QJ d    V3R lV 4       F  '       g   K   RM	  RM! V3R lV 4       4      '       g   \        RV R24      h\        V\,        \.        34      '       dC   \1        V4       F(  pRV9   g   K  VR,          pVR,          V,           VR&    M7	  \        R4      hV F$  pVR,          V,          V,           VR,          V&   K&  	  V'       d+   \3        RRVRVRV	RVRV/VB w  ppVP                  V4       MVP4                  ! RRVRV	RVRV/VB pV'       dœ   XP7                  4       V9  g   VP7                  4       V9  d    \        RVP7                  4        RV 24      hVP9                  VP7                  4       4      pVVV\;        V4      ,            V8X  d   VRV pMVRV P=                  4       pVP                  V4       EKä  	  WÍ3# u upi )z\{\%-?\s*generation\s*-?\%\}zareturn_assistant_tokens_mask==True but chat template does not contain `{% generation %}` keyword.Nz†Tools should either be a JSON schema, or a callable function with type hints and a docstring suitable for auto-conversion to a schema.zADocuments should be a list of dicts with 'title' and 'text' keys!zCONTINUE_FINAL_MESSAGE_TAG r  zGChat messages should be dicts, dataclasses or pydantic models, but got Ú.z<continue_final_message is set but the conversation is empty!Úcontentz<continue_final_message is set but the final message has no "z" to continue!c              3   ó,   <"  € T F	  qS9   x € K  	  R # 5irK   r   )rN   r¶  rÌ   s   & €r&   rP   Ú(render_jinja_template.<locals>.<genexpr>  s   øé € ÐK¹?°% Ö-»?ùs   ƒTFz"continue_final_message is set to "z8" but this is not an accepted field in the chat_templateÚtextz^continue_final_message is set but we could not find any text to continue in the final message!r  r  r  r  a6  continue_final_message is set but the final message does not appear in the chat after applying the chat template! This can happen if the chat template deletes portions of the final message. Please verify the chat template and final message in your chat to ensure they are compatible.Final message to continue: z
Rendered chat:
éÿÿÿÿr   )r©   r    ÚloggerÚwarning_oncerÏ   ra   r4   r‘   r
   r†   ÚismethodrÊ   r	  Ú	TypeErrorr§  rm   r  r£  r5   rD   r   rl   r@   rA   Úreversedr$  Úrenderr¤   ÚrindexrT   Úrstrip)r¸  r  r  rÌ   r¹  rº  r  ru  r  Útool_schemasÚtoolÚdocumentÚrenderedÚall_generation_indicesÚcontinue_final_message_tagrþ   r_  Úfinal_messageÚcontinue_fieldsr¶  Úcontent_blockr#  r!  Útag_locs   &&&f&&&,                r&   Úrender_jinja_templaterÔ  ×  sL  ø€ ÷ $¬B¯IªIÐ6UÐWd×,eÒ,eÜ×ÑØoô	
ô
 0°Ó>Ðð ÒØˆÛˆDÜ˜$¤×%Ò%Ø×#Ñ# DÖ)Ü˜D×!Ò!¤W×%5Ò%5°d×%;Ò%;Ø×#Ñ#¤O°DÓ$9Ö:ä ðPóð ò ð ˆàÒÛ!ˆHÜ˜h¬×-Ô-ÜÐ cÓdÐdñ "ô
 (¨Ó5€LÜ$ YÓ/€IÜ! &Ó)€Fà€HØÐØ!>ÐÜˆÜ�4˜×$Ò$à—=‘=ˆDä# DÓ)ˆÛˆGÜ˜g¤t×,Ô,ÜÐ"iÐjqÑitÐtuÐ vÓwÐwñ ô ' tÓ,ˆß!Ð!ßÜ Ð!_Ó`Ð`Ü?IÐJ`Ôbe×?fÒ?fÑ%;ÐluÐ"à!% b¥§¡Ð.DÓ!EÐE�ÒNÜ ØRÐSiÐRjÐjxÐyóð ð  6Ð6ˆOØ%Ô):Ô:å6GÓ"pÑ6G¨UÈ4ÐPRÍ8Ï<É<ÐX]ÓK^ÐboÑKo§5 5Ñ6G�Ð"pß“3ÔK¹?ÓK—3—3’3ÔK¹?ÓK×KÒKÜ Ø8Ð9OÐ8Pð  QIð  Jóð ô ˜M¬D´%¨=×9Ò9Ü%-¨mÖ%<�MØ Ö.à(5°fÕ(=˜Ø0=¸fÕ0EÐHbÕ0b˜ fÑ-Ùñ &=ô %Øxóð ó -�EØ&*¨2¥h¨u¥oÐ8RÕ&R�D˜•H˜U“Oñ -ç'Ü0Nñ 1Ø"3ð1àð1ð #ð1ð $ð	1ð
 '<ð1ð ñ1Ñ-ˆMÐ-ð #×)Ñ)Ð*<Õ=à-×4Ò4ñ Øðà"ðð $ðð '<ð	ð
 ñˆM÷ "Ø×#Ñ#Ó%¨]Ô:Ø*×0Ñ0Ó2¸-ÔGä ð2ð 3@×2EÑ2EÓ2GÐ1HÐHZÐ[hÐZið	kóð ð $×*Ñ*Ð+E×+KÑ+KÓ+MÓNˆGØ˜W w´Ð5OÓ1PÕ'PÐQÐUoÔoà -¨h¨wÐ 7‘ð !.¨h¨wÐ 7× >Ñ >Ó @�Ø�‰˜×&ñ] ð` Ð+Ð+ùòu #qs   È	PÈ-Pc                óV   € \        V \        4      '       g   R# RV 9   d   RV 9   g   R# R# )z^
Check that input is a valid message in a chat, namely a dict with "role" and "content" keys.
FÚroler½  T)ra   r4   r^  s   &r&   Úis_valid_messager×  Y  s)   € ô �gœt×$Ò$ÙØ�gÔ )¨wÔ"6ÙÙr%   c                   ó6   a € ] tR tRt o RtV 3R lR ltRtV tR# )ÚChatid  a  This class is intended to just be used internally for pipelines and not exposed to users. We convert chats
to this format because the rest of the pipeline code tends to assume that lists of messages are
actually a batch of samples rather than messages in the same conversation.c                ó    <€ V ^8„  d   QhRS[ /# )r0   r  rs  )r6   r,  s   "€r&   r7   ÚChat.__annotate__i  s   ø€ ÷ !ñ !¡ñ !r%   c                ó\   € V F  p\        V4      '       d   K  \        R 4      h	  Wn        R# )zQWhen passing chat dicts as input, each dict must have a 'role' and 'content' key.N)r×  r	  r  )r…   r  r_  s   && r&   r0  ÚChat.__init__i  s*   € ÛˆGÜ# G×,Ô,Ü Ð!tÓuÐuñ  ð !Žr%   )r  N)r   r    r!   r"   r#   r0  r$   rX  rz  s   @r&   rÙ  rÙ  d  s   ø‡ € ñR÷!ö !r%   rÙ  )rŸ  rž  r¢  )NNNFFF)\r²  r†   r¾   r©   rn   Úcollections.abcr   r   Ú
contextlibr   r   rR   r   Ú	functoolsr   r	   r
   Útypingr   r   r   r   r   r   r   Únumpyr°  Ú	packagingr   rh   r   Úimport_utilsr   r   r   Ú
get_loggerr   rÂ  rÒ   Újinja2.exceptionsÚ
jinja2.extÚjinja2.metaÚjinja2.nodesÚjinja2.runtimer   Újinja2.sandboxr   Ú	PIL.Imager   r@   r4   r5   rš  rð   rñ   r=   r>   r?   r3   ÚBASIC_TYPESrí   ÚDOTALLrŸ   r¡   ÚVERBOSEr§   r¢   Ú	Exceptionr   r)   rL   rF   rV   ro   r™   rµ   rÊ   r×   râ   r÷   rû   r  r  r$  rÏ   r&  r   r£  r§  rÔ  r×  rÙ  r   r%   r&   Ú<module>rñ     sB  ðó Û Û Û 	Û ß -Ý %Ý Ý ß (Ý ß [× [Ñ [ã Ý å ß UÑ Uð 
×	Ò	˜HÓ	%€á×ÒÛÛÛÛÛÛÝ$Þ<à€Fá×ÒÝà��S˜#�X•Õ€ð $Ð Ø!Ð ð �E˜3  c©4°«:°sÐ;€à—’ÐGÈÏÉÓS€à
�*Š*ÐEÀrÇyÁyÓ
Q€à—
’
ðð ‡I�I�—
‘
Õó€ð �ZŠZÐEÀrÇyÁyÓQ€
ôU˜yô UôU 	ô Uð ˜) U¨H°c¸8ÀTÈ9ÑVZÐ[_ÓV`ÐbhÐiÐ õ(õ*^õ[tõ|)õX+õBG4ðT ØôAó ó ðAð Øôó ó ðð" ô
)ó ð
)õõ*õ:ò -ð$ ñ9ó ð9ð ñi0ó ði0ðZ CÐ õ	õ!+÷H,òD÷	!ó 	!r%   