+
    „°ñjG.  ã                   óN  € ^ RI t ^ RIt^ RIHt ^ RIHu Ht ^ RIHt ^ RI	H
t
 ^ RIt	^RIHt ^RIHtHtHtHtHtHtHt R]! RR^. R!OR	7      R
]! RR^. R"OR	7      R]! RR^. R"OR	7      /t]! ^^^^^RR7      tR]R
]R]/t ! R R]P4                  4      t ! R R]4      t ! R R]]]]P                  P4                  4      tR t ! R R]
P>                  4      t  ! R R]
PB                  4      t" ! R R]
PF                  4      t$ ! R R]
PJ                  4      t&R#R lt'R$R  lt(R# )%é    N)ÚQwen2Tokenizer)Úsd1_clip)ÚQwen35VisionModel)Ú	BaseLlamaÚ	BaseQwen3ÚBaseGenerateÚLlama2_ÚQwen3VL_4BConfigÚQwen3VL_8BConfigÚQwen3VL_32BConfigÚ
qwen3vl_4bi   é   )Úhidden_sizeÚintermediate_sizeÚdepthÚdeepstack_visual_indexesÚ
qwen3vl_8bi€  iÐ  Úqwen3vl_32bi 	  )Ú	num_headsÚ
patch_sizeÚtemporal_patch_sizeÚin_channelsÚspatial_merge_sizeÚnum_position_embeddingsc                   ó<   a a€ ] tR t^t oRV 3R lltR tRtVtV ;t# )ÚQwen3VLDeepstackMergerc                ó2  <€ \         SV `  4        W^,          ,          V n        VP                  V P                  RWER7      V n        VP                  V P                  V P                  WER7      V n        VP                  V P                  W4VR7      V n        R# )é   g�íµ ÷Æ°>)ÚepsÚdeviceÚdtype)r    r!   N)ÚsuperÚ__init__Ú	merge_dimÚ	LayerNormÚnormÚLinearÚ
linear_fc1Ú
linear_fc2)Úselfr   r   Úout_hidden_sizer    r!   ÚopsÚ	__class__s   &&&&&&&€Ú?C:\Ai_Coding\apps\studio\ComfyUI\comfy\text_encoders\qwen3vl.pyr#   ÚQwen3VLDeepstackMerger.__init__   sp   ø€ Ü‰ÑÔØ$¸aÕ(?Õ@ˆŒØ—M‘M $§.¡.°dÀ6�MÓWˆŒ	ØŸ*™* T§^¡^°T·^±^ÈF˜*Ó`ˆŒØŸ*™* T§^¡^°_Ð[`˜*ÓaˆŽó    c                óÀ   € V P                  VP                  RV P                  4      4      pV P                  \        P
                  ! V P                  V4      4      4      # )é   éÿÿÿÿ)r&   Úviewr$   r)   ÚFÚgelur(   )r*   Úxs   &&r.   ÚforwardÚQwen3VLDeepstackMerger.forward"   s@   € Ø�I‰I�a—f‘f˜R §¡Ó0Ó1ˆØ�‰œqŸvšv d§o¡o°aÓ&8Ó9Ó:Ð:r0   )r(   r)   r$   r&   ©NNN)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r#   r8   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r-   Ú__classdict__s   @@r.   r   r      s   ù‡ € ÷b÷;ò ;r0   r   c                   ó6   a a€ ] tR t^'t oRV 3R lltRtVtV ;t# )ÚQwen3VLVisionModelc                ó  <€ \         SV `  WW4R 7       VR,          V n        \        P                  ! V P                   Uu. uF.  p\        V P                  V P                  VR,          W#VR 7      NK0  	  up4      V n        R# u upi )©r    r!   r,   r   r+   N)	r"   r#   r   ÚnnÚ
ModuleListr   r   r   Údeepstack_merger_list)r*   Úconfigr    r!   r,   Ú_r-   s   &&&&& €r.   r#   ÚQwen3VLVisionModel.__init__)   s‚   ø€ Ü‰Ñ˜°eÐÔEØ(.Ð/IÕ(JˆÔ%Ü%'§]¢]à×2Ò2ó4
á2�ô # 4×#3Ñ#3°T×5LÑ5LÈfÐUfÕNgÐpvð  JM÷  NÙ2ñ4
ó &ˆÖ"ùò 4
s   ¿4B)rJ   r   r:   ©r;   r<   r=   r>   r#   r?   r@   rA   rB   s   @@r.   rE   rE   '   s   ù‡ € ÷÷ r0   rE   c                   óZ   a a€ ] tR t^2t oRtV 3R ltR tR tRRRRRR. 3R ltRt	Vt
V ;t# )	ÚQwen3VLr   c                ó.  <€ \         SV `  4        \        V P                  ,          ! R/ VB pVP                  V n        \        WSW$R 7      V n        / \        C\        V P                  ,          CRVP                  /Cp\        WcW$R 7      V n        W n        R# )rG   r+   N© )r"   r#   ÚQWEN3VL_CONFIGSÚ
model_typeÚnum_hidden_layersÚ
num_layersr	   ÚmodelÚQWEN3VL_VISION_COMMONÚQWEN3VL_VISIONr   rE   Úvisualr!   )r*   Úconfig_dictr!   r    Ú
operationsrK   Úvision_configr-   s   &&&&&  €r.   r#   ÚQwen3VL.__init__5   sz   ø€ Ü‰ÑÔÜ  §¡Ö1Ñ@°KÑ@ˆØ ×2Ñ2ˆŒÜ˜V¸%ÔPˆŒ
Ø{Ô0Ð{´NÀ4Ç?Á?Õ4SÐ{ÐUfÐhn×hzÑhzÑ{ˆÜ(¨ÈUÔcˆŒØŽ
r0   c                ó  € VR ,          R8X  dv   \         P                  P                  P                  VR,          ^. RO. ROR7      w  r4V P	                  VP                  V\        P                  R7      V4      w  rVVRVRV/3# R# )	ÚtypeÚimageÚdata)r   Ú
image_meanÚ	image_std)r!   ÚgridÚ	deepstack)ç      à?rg   rg   )NN)ÚcomfyÚtext_encodersÚqwen_vlÚprocess_qwen2vl_imagesrZ   ÚtoÚtorchÚfloat32)r*   Úembedr    ra   re   Úmergedrf   s   &&&    r.   Úpreprocess_embedÚQwen3VL.preprocess_embed>   s‡   € Ø��=˜GÔ#ä×-Ñ-×5Ñ5×LÑLÈUÐSYÍ]Ðgiò  wFò  RaÐLó  b‰KˆEØ $§¡¨E¯H©H°VÄ5Ç=Á=¨HÓ,QÐSWÓ XÑˆFØ˜F D¨+°yÐAÐAÐAØÐr0   c           	     óÌ  € \        V Uu. uF  q3P                  R 4      R8X  g   K  VNK  	  upR R7      p\        V4      ^ 8X  d   R# VP                  pVP                  ^,          p\
        P                  P                  P                  W&V4      p\        P                  ! ^V3\        P                  VR7      pRp	V F‘  pVR,          p
VR,          V
,           pRV^ W«13&   VR	,          R
,          pV	f   V Uu. uF  qÝNK  	  p	pKJ  \        \        V4      4       Uu. uF)  p\        P                  ! Wž,          WÎ,          .^ R7      NK+  	  p	pK“  	  WxV	3# u upi u upi u upi )r`   ra   c                 ó   € V R ,          # )ÚindexrR   )Úes   &r.   Ú<lambda>Ú,Qwen3VL.build_image_inputs.<locals>.<lambda>H   s   € Ð\]Ð^eÖ\fr0   )ÚkeyN)r!   r    ru   ÚsizeTÚextrarf   )Údimr:   )ÚsortedÚgetÚlenr    Úshaperh   ri   rj   Úqwen2vl_mrope_position_idsrm   ÚzerosÚboolÚrangeÚcat)r*   ÚembedsÚembeds_inforv   Úimagesr    ÚseqÚposition_idsÚvisual_pos_masksrf   ÚstartÚendÚdsÚdÚis   &&&            r.   Úbuild_image_inputsÚQwen3VL.build_image_inputsF   s@  € ä¡KÓL¡K˜q·5±5¸³=ÀGÑ3KŸ˜¡KÑLÑRfÔgˆÜˆv‹;˜!ÔØ#Ð#à—‘ˆØ�l‰l˜1�oˆÜ×*Ñ*×2Ñ2×MÑMÈkÐ`fÓgˆô !Ÿ;š;¨¨3 x´u·z±zÈ&ÔQÐØˆ	ÛˆAØ�g•JˆEØ�F•)˜eÕ#ˆCØ-1Ð˜Q  	˜\Ñ*Ø�7•˜KÕ(ˆBØÒ Ù(*›O© 1šQ©�	˜O‘	äNSÔTWÐXZÓT[ÌnÓ]ÉnÈœUŸYšY¨	­°bµeÐ'<À!×DÉn�	Ð]‘	ñ ð ¨yÐ8Ð8ùò) Mùò" ,ùâ]s   ŠE¦EÃ:
EÄ/E!NTc	                óô   € V	P                  R R4      p
V	P                  RR4      pV	P                  RR4      pVe   V
f   V P                  W84      w  r«pV P                  ! V3RVRVRVRVRVR	VR V
R
VRVRV/
V	B # )rŠ   Nr‹   Údeepstack_embedsÚattention_maskr†   Ú
num_tokensÚintermediate_outputÚfinal_layer_norm_intermediater!   r‡   )Úpopr‘   rW   )r*   Ú	input_idsr•   r†   r–   r—   r˜   r!   r‡   ÚkwargsrŠ   r‹   r”   s   &&&&&&&&&,   r.   r8   ÚQwen3VL.forward^   sÙ   € Ø—z‘z .°$Ó7ˆØ!Ÿ:™:Ð&8¸$Ó?ÐØ!Ÿ:™:Ð&8¸$Ó?ÐØÒ ,Ò"6Ø?C×?VÑ?VÐW]Ó?kÑ<ˆLÐ,<Ø�zŠzØñ
à)ð
ð ð
ð "ð	
ð
 !4ð
ð +Hð
ð ð
ð &ð
ð $ð
ð .ð
ð .ð
ð ñ
ð 	
r0   )r!   rW   rV   rZ   )r;   r<   r=   r>   rT   r#   rq   r‘   r8   r?   r@   rA   rB   s   @@r.   rP   rP   2   sD   ù‡ € Ø€Jõòò9ð0 15¸TÈdÐhlð  MQð  Y]ð  km÷ 
ô 
r0   rP   c                 ó0   €  ! R  R\         4      pWn        V# )c                   ó   € ] tR t^utRtR# )Ú%_make_qwen3vl_model.<locals>.Qwen3VL_rR   N)r;   r<   r=   r>   r?   rR   r0   r.   ÚQwen3VL_rŸ   u   s   † Ûr0   r    )rP   rT   )rT   r    s   & r.   Ú_make_qwen3vl_modelr¡   t   s   € ô”7ô à$ÔØ€Or0   c                   óN   a a€ ] tR t^{t oRRR	RR/ R3V 3R lltR
R ltRtVtV ;t# )ÚQwen3VLClipModelÚcpuÚhiddenNTr   c                óN   <€ \         SV `  WV/ VR R/R\        V4      VWVR7       R# )Úpadé[P F)r    ÚlayerÚ	layer_idxÚtextmodel_json_configr!   Úspecial_tokensÚlayer_norm_hidden_stateÚmodel_classÚenable_attention_masksÚreturn_attention_masksÚmodel_optionsN)r"   r#   r¡   )	r*   r    r©   rª   r!   r•   r±   rT   r-   s	   &&&&&&&&€r.   r#   ÚQwen3VLClipModel.__init__|   s7   ø€ Ü‰Ñ ¸yÐ`bØ$°e¸V°_Ð^cÜ%8¸Ó%DÐ]kØ0>ð 	ö 	]r0   c                ó   € \        V\        4      '       d#   \        \        VP	                  4       4      4      pV UUu. uF  qÌ Uu. uF  qÝ^ ,          NK  	  upNK  	  pppV P                  WàP                  4      w  p ppV P                  P                  VV4      w  pppV P                  P                  WòW4WVWxV	V
VVVR7      # u upi u uppi )r   )Úpresence_penaltyrŠ   r‹   r”   )
Ú
isinstanceÚdictÚnextÚiterÚvaluesÚprocess_tokensÚexecution_deviceÚtransformerr‘   Úgenerate)r*   ÚtokensÚ	do_sampleÚ
max_lengthÚtemperatureÚtop_kÚtop_pÚmin_pÚrepetition_penaltyÚseedr´   ÚmtpÚbÚtÚtokens_onlyr†   rL   r‡   rŠ   r‹   rf   s   &&&&&&&&&&&&         r.   r½   ÚQwen3VLClipModel.generate‚   sÑ   € Ü�fœd×#Ò#Üœ$˜vŸ}™}›Ó/Ó0ˆFÙ28Ô9±&¨Q aÓ(¡a ˜!Ÿ˜¡aÔ(±&ˆÑ9Ø$(×$7Ñ$7¸×EZÑEZÓ$[Ñ!ˆ��1�kØ48×4DÑ4D×4WÑ4WÐX^Ð`kÓ4lÑ1ˆÐ&¨	Ø×Ñ×(Ñ(¨¸JÐUZÐchð  CØ:JÐYeØ:JÐ]fð )ó hð 	hùò )ùÓ9s   ¾C
ÁCÁC
ÃC
rR   r3   )g        T)	r;   r<   r=   r>   r#   r½   r?   r@   rA   rB   s   @@r.   r£   r£   {   s-   ù‡ € Ø#¨8¸rÈÐ^bÐrtð  BN÷ ]÷hô hr0   r£   c                   ó>   a a€ ] tR t^�t oRR/ R3V 3R lltRtVtV ;t# )ÚQwen3VLTEModelr¤   Nr   c                ó:   <a€ V3R  lp\         SV `  WSWSR7       R# )c                  ó"   <€ \        R/ V BR S/B # )rT   rR   )r£   )ÚkwrT   s   ,€r.   rw   Ú)Qwen3VLTEModel.__init__.<locals>.<lambda>�   s   ø€ Ô"2Ñ"O°RÑ"OÀJÓ"Or0   )r    r!   ÚnameÚ
clip_modelr±   N©r"   r#   )r*   r    r!   r±   rT   rÓ   r-   s   &&&&f €r.   r#   ÚQwen3VLTEModel.__init__Ž   s   ù€ ÜOˆ
Ü‰Ñ ¸*ÐQ[ÐÖyr0   rR   rN   rB   s   @@r.   rÍ   rÍ   �   s   ù‡ € Ø#¨4¸rÈl÷ z÷ zr0   rÍ   c                   ó>   a a€ ] tR t^“t oR/ RR3V 3R lltRtVtV ;t# )ÚQwen3VLSDTokenizerNr   r   c                ó  <€ \         P                  P                  \         P                  P                  \         P                  P	                  \
        4      4      R 4      p\        SV `  VRWV\        RRRR^RVR7       R# )Úqwen25_tokenizerFiÿàõr¨   )Úpad_with_endÚembedding_directoryÚembedding_sizeÚembedding_keyÚtokenizer_classÚhas_start_tokenÚhas_end_tokenÚpad_to_max_lengthrÀ   Ú
min_lengthÚ	pad_tokenÚtokenizer_dataN)	ÚosÚpathÚjoinÚdirnameÚrealpathÚ__file__r"   r#   r   )r*   rÛ   rä   rÜ   rÝ   Útokenizer_pathr-   s   &&&&& €r.   r#   ÚQwen3VLSDTokenizer.__init__”   sx   ø€ ÜŸ™Ÿ™¤b§g¡g§o¡o´b·g±g×6FÑ6FÄxÓ6PÓ&QÐSeÓfˆÜ‰Ñ˜°eÐQdð  Taô  sAØ).¸eÐW\ÐiqÐ~ð  LRð  cqð 	ö 	rr0   rR   rN   rB   s   @@r.   r×   r×   “   s   ù‡ € Ø+/ÀÐSWÐgs÷ r÷ rr0   r×   c                   óX   a a€ ] tR t^št oR/ R3V 3R lltRR. RRR3V 3R lltRtVtV ;t# )ÚQwen3VLTokenizerNr   c                óp   <aa€ SR 8X  d   RMRoVV3R lp\         SV `  WSVR7       RV n        RV n        R# )r   i 
  r   c                  ó&   <€ \        V / VBR SRS/B # )rÜ   rÝ   )r×   )ÚarÐ   rÜ   rT   s   *,€€r.   rw   Ú+Qwen3VLTokenizer.__init__.<locals>.<lambda>�   s   ø€ Ô%7¸Ð%z¸bÑ%zÐQ_Ð%zÐoyÓ%zr0   )rÛ   rä   rÒ   Ú	tokenizerz4<|im_start|>user
{}<|im_end|>
<|im_start|>assistant
z_<|im_start|>user
<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>
<|im_start|>assistant
N)r"   r#   Úllama_templateÚllama_template_images)r*   rÛ   rä   rT   ró   rÜ   r-   s   &&&f @€r.   r#   ÚQwen3VLTokenizer.__init__›   sG   ú€ Ø!+¨|Ô!;™ÀˆÝzˆ	Ü‰ÑÐ-@Ðfpð  }FÐô  	GØWˆÔð &JˆÖ"r0   FÚ c                ó2  <€ VP                  R R4      p	V	eD   \        V4      ^ 8X  d4   \        V	P                  ^ ,          4       U
u. uF  q©Wª^,            NK  	  pp
T;'       g    VP	                  R4      pV'       d
   VR8X  d   RpV'       d   TpM¥Ve   TpM]\        V4      ^ 8X  d   V P
                  pM@V P                  p\        V4      ^8”  d%   RpVP                  WÝ\        V4      ,          ^4      pVP                  V4      pV'       d)   RV,           R,           W»P                  R4      R ,           p\        SV `,  ! V3R	VR
R/VB p\        \        V4      4      p^ pWï,           F¤  p\        \        V4      4       F‰  p
\        VV
,          ^ ,          \        \         34      '       g   K/  VV
,          ^ ,          R8X  g   KF  \        V4      V8”  g   KX  RR RVV,          RR /3VV
,          R,          ,           VV
&   V^,          pK‹  	  K¦  	  V# u up
i )ra   Nz<|im_start|>r÷   Ú z+<|vision_start|><|image_pad|><|vision_end|>z<|im_start|>system
z<|im_end|>
z<|im_start|>userÚreturn_word_idsÚdisable_weightsTigP r`   rb   Úoriginal_type:r2   NN)r~   r   r„   r€   Ú
startswithrô   rõ   ÚreplaceÚformatru   r"   Útokenize_with_weightsr·   r¸   rµ   ÚintÚfloat)r*   Útextrú   rô   rˆ   Úprevent_empty_textÚskip_templateÚsystem_promptr›   ra   r�   Ú
llama_textÚtemplateÚvision_blockr¾   Úkey_nameÚembed_countÚrr-   s   &&&&&&&&,         €r.   r   Ú&Qwen3VLTokenizer.tokenize_with_weights¢   sï  ø€ Ø—
‘
˜7 DÓ)ˆØÒ¤ V£°Ô!1Ü.3°E·K±KÀµNÔ.CÓDÑ.C¨˜A !�e“nÑ.CˆFÐDà%×HÐH¨¯©¸Ó)Hˆß $¨"¤*ØˆDçØ‰JàÒ)Ø)‘Ü�V“ Ô!Ø×.Ñ.‘à×5Ñ5�Ü�v“; ”?Ø#P�LØ'×/Ñ/°ÌSÐQWË[Õ>XÐZ[Ó\�HØ!Ÿ™¨Ó.ˆJßØ3°mÕCÀnÕTÐWa×brÑbrð  tFó  cGð  cHð  XIõ  I�
ä‘Ò.¨zÑ{È?Ð{ÐlpÐ{ÐtzÑ{ˆÜœ˜V›Ó%ˆØˆØ×!Ð!ˆAÜœ3˜q›6–]�Ü˜a �d 1�g¬¬U |×4Ô4¸¸1½¸a½ÀFÖ9JÜ˜6“{ [Ö0Ø!'¨°&¸&ÀÕ:MÈÐ`gÐ hÐjÐmnÐopÕmqÐrtÕmuÕu˜˜!™Ø# qÕ(šó	 #ñ "ð ˆùò? Es   ÁH)rô   rõ   )	r;   r<   r=   r>   r#   r   r?   r@   rA   rB   s   @@r.   rî   rî   š   s9   ù‡ € Ø+/ÀÈ|÷ Jð ;@ÐPTÐ]_Ðtyð  JOð  _a÷ "÷ "r0   rî   c                 ó,   a €  ! V 3R  lR\         4      pV# )c                   ó>   <a a€ ] tR t^Èt oR/ 3V V3R lltRtVtV ;t# )Ú$tokenizer.<locals>.Qwen3VLTokenizer_Nc                ó*   <€ \         SV `  WSR 7       R# ))rÛ   rä   rT   NrÔ   )r*   rÛ   rä   r-   rT   s   &&&€€r.   r#   Ú-tokenizer.<locals>.Qwen3VLTokenizer_.__init__É   s   ø€ Ü‰GÑÐ1DÐpzÐÖ{r0   rR   rN   )r-   rC   rT   s   @@€r.   ÚQwen3VLTokenizer_r  È   s   ú‡ € Ø/3ÀB÷ 	|÷ 	|ð 	|r0   r  )rî   )rT   r  s   f r.   ró   ró   Ç   s   ø€ ÷|Ô,ô |ð Ðr0   c                 ó4   a aa€  ! V VV3R  lR\         4      pV# )c                   óD   <a a€ ] tR t^Ït oRR/ 3V VVV3R lltRtVtV ;t# )Úte.<locals>.Qwen3VLTEModel_r¤   Nc                ój   <€ Se   SpSe   VP                  4       pSVR&   \        SV `	  WVSR7       R # )NÚquantization_metadata)r    r!   r±   rT   )Úcopyr"   r#   )r*   r    r!   r±   r-   Údtype_llamaÚllama_quantization_metadatarT   s   &&&&€€€€r.   r#   Ú$te.<locals>.Qwen3VLTEModel_.__init__Ð   sB   ø€ ØÒ&Ø#�Ø*Ò6Ø -× 2Ñ 2Ó 4�Ø9T�Ð5Ñ6Ü‰GÑ FÀ}ÐakÐÖlr0   rR   rN   )r-   rC   r  r  rT   s   @@€€€r.   ÚQwen3VLTEModel_r  Ï   s   ú‡ € Ø"'¨tÀ2÷ 	m÷ 	mò 	mr0   r  )rÍ   )r  r  rT   r  s   fff r.   Úter  Î   s   ú€ ÷mñ mœ.ô mð Ðr0   )é   é   é   )é   é   é   )r   )NNr   ))rå   rm   Útorch.nnrH   Útorch.nn.functionalÚ
functionalr5   Útransformersr   rh   r   Úcomfy.text_encoders.qwen_vlÚqwen35r   Úllamar   r   r   r	   r
   r   r   r¶   rY   rX   rS   ÚModuler   rE   rP   r¡   ÚSDClipModelr£   ÚSD1ClipModelrÍ   ÚSDTokenizerr×   ÚSD1Tokenizerrî   ró   r  rR   r0   r.   Ú<module>r1     s/  ðÛ 	ã Ý ß Ð Ý 'å Û "Ý %ß u× uÑ uð ‘$ 4¸4ÀrÒdoÔpØ‘$ 4¸4ÀrÒdoÔpØ‘4 D¸DÈÒepÔqð€ñ
  r°bÈaÐ]^Ø01È4ôQÐ ð  Ð!1°<ÐAQÐS`ÐbsÐt€ô;˜RŸY™Yô ;ôÐ*ô ô?
ˆi˜ L°%·(±(·/±/ô ?
òDôh�x×+Ñ+ô hô$z�X×*Ñ*ô zôr˜×-Ñ-ô rô*�x×,Ñ,ô *ôZö	r0   