+
    „°ñj…N  ã                   ó²  € ^ RI t ^ RIHt ^ RIHu Ht ^ RIHtHt ^ RI	t	^ RI
Ht RR R lltRR ltRR ltR tR	 t ! R
 R]P"                  4      tR tR t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      t ! R R]P"                  4      tR# )é    N)ÚOptionalÚTuple)Úoptimized_attention_for_devicec                ó˜   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        R\        R\        R\        R	\        /	# )
é   ÚimagesÚ
min_pixelsÚ
max_pixelsÚ
patch_sizeÚtemporal_patch_sizeÚ
merge_sizeÚ
image_meanÚ	image_stdÚinterpolation)ÚtorchÚTensorÚintÚlistÚstr)Úformats   "Ú?C:\Ai_Coding\apps\studio\ComfyUI\comfy\text_encoders\qwen_vl.pyÚ__annotate__r   	   sq   € ÷ /Gñ /GÜ�L‰Lð/Gäð/Gô ð/Gô ð	/Gô
 ð/Gô ð/Gô ð/Gô ð/Gô ñ/Gó    c	                 ó¦  € Vf   . ROpVf   . ROpV R,          p V P                   w  ršr¼V P                  ^ ^^^4      p V ^ ,          pW5,          p\        W®,          4      V,          p\        W¾,          4      V,          pVV,          V8”  d�   \        P                  ! W«,          V,          4      p\        V\        P                  ! V
V,          V,          4      V,          4      p\        V\        P                  ! VV,          V,          4      V,          4      pM‡VV,          V8  dz   \        P                  ! WV,          ,          4      p\        P                  ! V
V,          V,          4      V,          p\        P                  ! VV,          V,          4      V,          p\        P                  ! VP                  ^ 4      VV3VRR7      P                  ^ 4      pVP                  4       p\        ^4       F+  pVV,          VV,          ,
          VV,          ,          VV&   K-  	  \        VP                  ^ 4      P                  V^^^4      W4V4      # )NF©ÚsizeÚmodeÚalign_corners)g3<Í4'ÐÞ?gwgí¶MÝ?gy{Îå Ú?)g�‡Bô91Ñ?g•wÝt.¹Ð?g�Ý	U¦Ñ?©.:Né   N)ÚshapeÚpermuteÚroundÚmathÚsqrtÚmaxÚfloorÚceilÚFÚinterpolateÚ	unsqueezeÚsqueezeÚcloneÚrangeÚqwen2vl_patchifyÚrepeat)r   r	   r
   r   r   r   r   r   r   Ú
batch_sizeÚheightÚwidthÚchannelsÚimgÚfactorÚh_barÚw_barÚbetaÚimg_resizedÚ
normalizedÚcs   &&&&&&&&&            r   Úprocess_qwen2vl_imagesr=   	   sç  € ð ÒÚ8ˆ
ØÒÚ8ˆ	à�G�_€FØ*0¯,©,Ñ'€J˜à�^‰^˜A˜q ! QÓ'€Fà
��)€CàÕ$€Fä�&•/Ó" VÕ+€EÜ�%•.Ó! FÕ*€Eàˆu…}�zÔ!Ü�yŠy˜&�.¨JÕ6Ó7ˆÜ�FœDŸJšJ v°¥}°vÕ'=Ó>ÀÕGÓHˆÜ�FœDŸJšJ u¨t¥|°fÕ'<Ó=ÀÕFÓG‰Ø	��˜Ô	#Ü�yŠy˜°¥~Õ6Ó7ˆÜ—	’	˜& 4�-¨&Õ0Ó1°FÕ:ˆÜ—	’	˜% $�,¨Õ/Ó0°6Õ9ˆä—-’-Ø�‰�aÓØ�Uˆ^ØØô	÷
 �gˆaƒjð ð ×"Ñ"Ó$€JÜ�1ŽXˆØ$ Q�¨*°Q­-Õ7¸9ÀQ½<ÕGˆ
�1‹ñ ô ˜J×0Ñ0°Ó3×:Ñ:Ð;NÐPQÐSTÐVWÓXÐZdð  |Fó  Gð  Gr   c                 ó$  € \        W,          4      V,          p\        W#,          4      V,          p\        P                  ! W,          4      V,          p	W—,          V,          V8”  d–   \        P                  ! W,          V,          V,          4      p
\	        V\        P
                  ! W,          V,          4      V,          4      p\	        V\        P
                  ! W*,          V,          4      V,          4      pWx3# W—,          V,          V8  d   \        P                  ! WPV,          V,          ,          4      p
\        P                  ! W,          V,          4      V,          p\        P                  ! W*,          V,          4      V,          pWx3# ©N)r#   r$   r(   r%   r&   r'   )Ú
num_framesr2   r3   r6   Útemporal_factorr	   r
   r7   r8   Út_barr9   s   &&&&&&&    r   Úqwen2vl_video_smart_resizerC   ;   s  € ä�&•/Ó" VÕ+€EÜ�%•.Ó! FÕ*€EÜ�IŠI�jÕ2Ó3°oÕE€EØ…}�uÕ˜zÔ)Ü�yŠy˜*Õ-°Õ5¸ÕCÓDˆÜ�FœDŸJšJ v¥}°vÕ'=Ó>ÀÕGÓHˆÜ�FœDŸJšJ u¥|°fÕ'<Ó=ÀÕFÓGˆð
 ˆ<Ðð	 
�˜Õ	 Ô	+Ü�yŠy˜°FÕ':¸UÕ'BÕCÓDˆÜ—	’	˜&�-¨&Õ0Ó1°FÕ:ˆÜ—	’	˜%�,¨Õ/Ó0°6Õ9ˆØˆ<Ðr   c	                 óš  € V R,          P                  ^ ^^^4      p \        P                  ! WV3VRR7      p \        P                  ! W`P
                  V P                  R7      P                  ^^^^4      p	\        P                  ! WpP
                  V P                  R7      P                  ^^^^4      p
\        W	,
          V
,          W4V4      # ).Fr   ©ÚdeviceÚdtyper   )	r"   r)   r*   r   ÚtensorrF   rG   Úviewr/   )Úframesr2   r3   r   r   r   r   r   r   ÚmeanÚstds   &&&&&&&&&  r   Úprocess_qwen2vl_videorM   K   s¤   € à�G�_×$Ñ$ Q¨¨1¨aÓ0€FÜ�]Š]˜6°¨¸mÐ[`Ôa€FÜ�<Š<˜
¯=©=ÀÇÁÔM×RÑRÐSTÐVWÐYZÐ\]Ó^€DÜ
�,Š,�y¯©¸f¿l¹lÔ
K×
PÑ
PÐQRÐTUÐWXÐZ[Ó
\€CÜ˜V�]¨cÕ1°:ÐT^Ó_Ð_r   c                 ó  € V P                   ^ ,          V,          pV P                   ^,          pV P                   ^,          V,          pV P                   ^,          V,          pV P                  VVVWc,          VVWs,          VV4	      pVP                  ^ ^^^^^^^^4	      pVP                  WF,          V,          WR,          V,          V,          4      p	V	\        P                  ! WFV..V P
                  \        P                  R7      3# )r   rE   )r!   Úreshaper"   r   rH   rF   Úlong)
Úpixel_valuesr   r   r   Úgrid_tÚchannelÚgrid_hÚgrid_wÚpatchesÚflatten_patchess
   &&&&      r   r/   r/   T   sú   € à×Ñ Õ"Ð&9Õ9€FØ× Ñ  Õ#€GØ×Ñ Õ" jÕ0€FØ×Ñ Õ" jÕ0€Fà×"Ñ"ØØØØÕØØØÕØØó
€Gð �o‰o˜a  A q¨!¨Q°°1°aÓ8€GØ—o‘oØ�˜&Õ ØÕ%¨
Õ2°ZÕ?ó€Oð
 œEŸLšL¨6¸6Ð*BÐ)CÈL×L_ÑL_Ôgl×gqÑgqÔrÐrÐrr   c                 ó8  € R p^ pV  EFŽ  pVP                  R4      R9   g   K  VP                  RR 4      p\        V\        4      '       d
   VR,          MTpVP                  R4      pVf:   \        P                  ! ^V3VR7      p\        P
                  ! ^ W‚R7      VRR V13&   VP                  R4      V,           p	\        VP                  4       4      ^,          p
W¨,           p\        P
                  ! W´,           W±V	,
          ,           V,           VR7      VRV	R 13&   W„,           V^ W‰13&   \        V^ ,          ^,          4      ^,          p\        P
                  ! W„,           WŒ,           V,           VR7      P                  ^4      P                  ^\        P                  ! W˜,
          V,          4      4      P                  ^ 4      R W˜,
           V^W‰13&   \        V^ ,          ^,          4      ^,          p\        P
                  ! W„,           WŒ,           V,           VR7      P                  ^ 4      P                  \        P                  ! W˜,
          V,          4      ^4      P                  ^ 4      R W˜,
           V^W‰13&   WJW˜,
          ,
          ,          pEK‘  	  V# )	NÚtypeÚextraÚgridÚindex©rF   ºNNNr   )ÚimageÚvideo)ÚgetÚ
isinstanceÚdictr   ÚzerosÚaranger   r&   r+   r0   r$   r(   Úflatten)Úembeds_infoÚseq_lenrF   Úposition_idsÚoffsetÚerZ   r[   ÚstartÚendÚlen_maxÚ
start_nextÚmax_ds   &&&          r   Úqwen2vl_mrope_position_idsrq   p   s¸  € ð €LØ€FÜˆØ�5‰5�‹=Ð.Ö.Ø—E‘E˜' 4Ó(ˆEÜ$.¨u´d×$;Ò$;�5˜–=ÀˆDØ—E‘E˜'“NˆEØÒ#Ü$Ÿ{š{¨A¨w¨<ÀÔG�Ü*/¯,ª,°q¸%Ô*O�˜Q   ˜YÑ'Ø—%‘%˜“- %Õ'ˆCÜ˜$Ÿ(™(›*“o¨Õ*ˆGØ �ˆJÜ$)§L¢L°Õ1DÀjÐ^aÕTaÕFbÐekÕFkÐtzÔ${ˆL˜˜C™D˜Ñ!Ø).­ˆL˜˜E˜I˜Ñ&Ü˜˜Q� �
“O qÕ(ˆEÜ).¯ª°eµnÀeÅmÐV\ÕF\ÐekÔ)l×)vÑ)vÐwxÓ)y÷  *Añ  *Að  BCô  EI÷  ENò  ENð  PSõ  P[ð  _dõ  Odó  Eeó  *f÷  *nñ  *nð  opó  *qð  r~ð  svõ  s~ð  *ˆL˜˜E˜I˜Ñ&Ü˜˜Q� �
“O qÕ(ˆEÜ).¯ª°eµnÀeÅmÐV\ÕF\ÐekÔ)l×)vÑ)vÐwxÓ)y÷  *Añ  *Aô  BF÷  BKò  BKð  MPõ  MXð  \aõ  Laó  Bbð  deó  *f÷  *nñ  *nð  opó  *qð  r~ð  svõ  s~ð  *ˆL˜˜E˜I˜Ñ&Ø ¥Õ-Õ-‹Fñ# ð$ Ðr   c                   óT   a a€ ] tR t^‹t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚVisionPatchEmbedc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )r   r   r   Úin_channelsÚ	embed_dim©r   )r   Ú__classdict__s   "€r   r   ÚVisionPatchEmbed.__annotate__Œ   s3   ø€ ÷ 
ñ 
áð
ñ !ð
ñ ð	
ñ
 ñ
r   c           
     ó–   <€ \         S	V `  4        Wn        W n        W0n        W@n        W!V.pVP                  VVVVR VVR7      V n        R# )F)Úkernel_sizeÚstrideÚbiasrF   rG   N)ÚsuperÚ__init__r   r   ru   rv   ÚConv3dÚproj)
Úselfr   r   ru   rv   rF   rG   Úopsr{   Ú	__class__s
   &&&&&&&& €r   r   ÚVisionPatchEmbed.__init__Œ   sZ   ø€ ô 	‰ÑÔØ$ŒØ#6Ô Ø&ÔØ"Œà*¸
ÐCˆØ—J‘JØØØ#ØØØØð ó 
ˆŽ	r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úhidden_statesÚreturn©r   r   )r   rx   s   "€r   r   ry   §   s#   ø€ ÷ 6ñ 6¡U§\¡\ð 6±e·l±lñ 6r   c                óÖ   € VP                  RV P                  V P                  V P                  V P                  4      pV P	                  V4      pVP                  RV P
                  4      # ©é   éÿÿÿÿ)rI   ru   r   r   r�   rv   )r‚   r‡   s   &&r   ÚforwardÚVisionPatchEmbed.forward§   sX   € Ø%×*Ñ*Ø�× Ñ  $×":Ñ":¸D¿O¹OÈTÏ_É_ó
ˆð Ÿ	™	 -Ó0ˆØ×!Ñ! " d§n¡nÓ5Ð5r   )rv   ru   r   r�   r   )é   r   r    é   NNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   rŽ   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r„   rx   s   @@r   rs   rs   ‹   s   ù‡ € ÷
õ 
÷66÷ 6ð 6r   rs   c                 óÌ   € V R RV P                   R,          ^,          13,          pV R V P                   R,          ^,          R13,          p\        P                  ! V) V3RR7      # ).N©Údimr�   )r!   r   Úcat)ÚxÚx1Úx2s   &  r   Úrotate_halfr¢   ¯   sZ   € Ø	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�9Š9�r�c˜2�Y BÔ'Ð'r   c                 óþ   € VP                  R4      P                  4       VP                  R4      P                  4       r2W,          \        V 4      V,          ,           pW,          \        V4      V,          ,           pWE3# )r   éþÿÿÿ)r+   Úfloatr¢   )ÚqÚkÚcosÚsinÚq_embedÚk_embeds   &&&&  r   Úapply_rotary_pos_emb_visionr¬   µ   s`   € Ø�}‰}˜RÓ ×&Ñ&Ó(¨#¯-©-¸Ó*;×*AÑ*AÓ*CˆØ�wœ; q›>¨CÕ/Õ0€GØ�wœ; q›>¨CÕ/Õ0€GØÐÐr   c                   óT   a a€ ] tR t^¼t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚVisionRotaryEmbeddingc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r�   Útheta)r   r¥   )r   rx   s   "€r   r   Ú"VisionRotaryEmbedding.__annotate__½   s   ø€ ÷ ñ ™Cð ©ñ r   c                ó<   <€ \         SV `  4        Wn        W n        R # r?   )r~   r   r�   r°   )r‚   r�   r°   r„   s   &&&€r   r   ÚVisionRotaryEmbedding.__init__½   s   ø€ Ü‰ÑÔØŒØŽ
r   c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# )r   Úseqlenrˆ   )r   r   r   )r   rx   s   "€r   r   r±   Â   s   ø€ ÷ ñ ™cð ©e¯l©lñ r   c           
     óB  € R V P                   \        P                  ! ^ V P                  ^\        P                  VR7      V P                  ,          ,          ,          p\        P                  ! WP
                  VP                  R7      p\        P                  ! WC4      pV# )g      ð?)rG   rF   rE   )r°   r   re   r�   r¥   rF   rG   Úouter)r‚   rµ   rF   Úinv_freqÚseqÚfreqss   &&&   r   rŽ   ÚVisionRotaryEmbedding.forwardÂ   sf   € Ø˜$Ÿ*™*¬¯ª°a¸¿¹À1ÌEÏKÉKÐ`fÔ)gÐjn×jrÑjrÕ)rÕsÕtˆÜ�lŠl˜6¯/©/ÀÇÁÔPˆÜ—’˜CÓ*ˆØˆr   )r�   r°   )g     ˆÃ@r’   rš   s   @@r   r®   r®   ¼   s   ù‡ € ÷õ ÷
÷ ð r   r®   c                   óT   a a€ ] tR t^Ét oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚPatchMergerc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   r�   Úcontext_dimÚspatial_merge_sizerw   )r   rx   s   "€r   r   ÚPatchMerger.__annotate__Ê   s"   ø€ ÷ 
ñ 
™Cð 
©cð 
Ásñ 
r   c                ób  <€ \         SV `  4        W#^,          ,          V n        VP                  VRWER7      V n        \
        P                  ! VP                  V P                  V P                  WER7      \
        P                  ! 4       VP                  V P                  WVR7      4      V n	        R# )r   ç�íµ ÷Æ°>©ÚepsrF   rG   rE   N)
r~   r   Úhidden_sizeÚRMSNormÚln_qÚnnÚ
SequentialÚLinearÚGELUÚmlp)r‚   r�   r¿   rÀ   rF   rG   rƒ   r„   s   &&&&&&&€r   r   ÚPatchMerger.__init__Ê   sƒ   ø€ Ü‰ÑÔØ&ÀÕ*AÕBˆÔØ—K‘K °¸f�KÓRˆŒ	Ü—=’=Ø�J‰J�t×'Ñ'¨×)9Ñ)9À&ˆJÓVÜ�GŠG‹IØ�J‰J�t×'Ñ'¨À5ˆJÓIó
ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rŸ   rˆ   r‰   )r   rx   s   "€r   r   rÁ   Ô   s#   ø€ ÷ ñ ™Ÿ™ð ©%¯,©,ñ r   c                ó~   € V P                  V4      P                  RV P                  4      pV P                  V4      pV# r‹   )rÈ   rO   rÆ   rÍ   )r‚   rŸ   s   &&r   rŽ   ÚPatchMerger.forwardÔ   s4   € Ø�I‰I�a‹L× Ñ   T×%5Ñ%5Ó6ˆØ�H‰H�Q‹KˆØˆr   )rÆ   rÈ   rÍ   )r   NNNr’   rš   s   @@r   r½   r½   É   s   ù‡ € ÷
õ 
÷÷ ð r   r½   c                   óX   a a€ ] tR t^Út oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚVisionAttentionc                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   rÆ   Ú	num_headsrw   )r   rx   s   "€r   r   ÚVisionAttention.__annotate__Û   s   ø€ ÷ `ñ `¡Cð `±Cñ `r   c                óø   <€ \         SV `  4        Wn        W n        W,          V n        V P                  R,          V n        VP                  W^,          RW4R7      V n        VP                  WRW4R7      V n        R# )g      à?T©r}   rF   rG   Ng      à¿)	r~   r   rÆ   rÕ   Úhead_dimÚscalingrË   Úqkvr�   )r‚   rÆ   rÕ   rF   rG   rƒ   r„   s   &&&&&&€r   r   ÚVisionAttention.__init__Û   sc   ø€ Ü‰ÑÔØ&ÔØ"ŒØ#Õ0ˆŒØ—}‘}¨Õ,ˆŒà—:‘:˜k¸­?ÀÈf�:ÓbˆŒØ—J‘J˜{¸dÈ6�JÓ_ˆŽ	r   c                ó¢   <€ V ^8„  d   QhRS[ P                  RS[S[S[ P                  S[ P                  3,          ,          RS[ P                  /# ©r   r‡   Úposition_embeddingsrˆ   ©r   r   r   r   )r   rx   s   "€r   r   rÖ   å   sH   ø€ ÷ 'ñ 'á—|‘|ð'ñ &¡e©E¯L©L¹%¿,¹,Ð,FÕ&GÕHð'ñ 
�‰ñ'r   c                ó  € VP                  4       ^8X  d#   VP                  w  rV^pVP                  ^ 4      pMVP                  w  rupV P                  V4      pVP	                  Wu^V P
                  V P                  4      pVP	                  V^V P
                  R4      P                  ^^ ^^4      P                  ^ 4      w  ršpVe   Vw  rÍ\        WšWÍ4      w  ršV	P                  ^ ^4      P                  ^ 4      p	V
P                  ^ ^4      P                  ^ 4      p
VP                  ^ ^4      P                  ^ 4      pVR,          VRR ,
          pWšV3 Uu. uF)  p\        P                  ! WþP                  4       ^R7      NK+  	  pp\        V!   UUUu. uF  w  pppV! VVVV P
                  RR7      NK   	  pppp\        P                  ! V^R7      pVP	                  VR4      pV P!                  V4      pV# u upi u upppi )r   NºrŒ   NNrœ   T)Úskip_reshaper�   )r�   r!   r+   rÛ   rO   rÕ   rÙ   r"   Úunbindr¬   Ú	transposer   ÚsplitÚtolistÚziprž   r�   )r‚   r‡   rß   Ú
cu_seqlensÚoptimized_attentionÚ
seq_lengthÚ_r1   rÛ   Úquery_statesÚ
key_statesÚvalue_statesr¨   r©   ÚlengthsrH   Úsplitsr¦   r§   ÚvÚattn_outputsÚattn_outputs   &&&&&                 r   rŽ   ÚVisionAttention.forwardå   sì  € ð ×ÑÓ !Ô#Ø)×/Ñ/‰MˆJØˆJØ)×3Ñ3°AÓ6‰Mà(5×(;Ñ(;Ñ%ˆJ Aà�h‰h�}Ó%ˆØ�k‰k˜*°!°T·^±^ÀTÇ]Á]ÓSˆØ14·±¸ZÈÈDÏNÉNÐ\^Ó1_×1gÑ1gÐhiÐklÐnoÐqrÓ1s×1zÑ1zÐ{|Ó1}Ñ.ˆ ,àÒ*Ø*‰HˆCÜ'BÀ<Ð]`Ó'fÑ$ˆLà#×-Ñ-¨a°Ó3×=Ñ=¸aÓ@ˆØ×)Ñ)¨!¨QÓ/×9Ñ9¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3×=Ñ=¸aÓ@ˆà˜R•. :¨c¨r ?Õ2ˆàHTÐbnÑGoó
ÙGo¸VŒE�KŠK˜§¡Ó 0°a×8ÑGoð 	ð 
ô  š<õ
á'‘��1�añ    1 a¨¯©Àd×KÙ'ð 	ò 
ô —i’i °!Ô4ˆØ!×)Ñ)¨*°bÓ9ˆØ—i‘i Ó,ˆàÐùò
ùô
s   Å/G7Æ$G<)rÙ   rÆ   rÕ   r�   rÛ   rÚ   ©NNNr’   rš   s   @@r   rÓ   rÓ   Ú   s    ù‡ € ÷`õ `÷'÷ 'ò 'r   rÓ   c                   óH   a a€ ] tR tRt oRV3R lV 3R llltR tRtVtV ;t# )Ú	VisionMLPi  c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   rÆ   Úintermediate_sizerw   )r   rx   s   "€r   r   ÚVisionMLP.__annotate__  s   ø€ ÷  ñ  ¡Cð  ¹Cñ  r   c                óî   <€ \         SV `  4        VP                  WR W4R7      V n        VP                  WR W4R7      V n        VP                  W!R W4R7      V n        \        P                  ! 4       V n        R# )TrØ   N)	r~   r   rË   Ú	gate_projÚup_projÚ	down_projrÉ   ÚSiLUÚact_fn)r‚   rÆ   rú   rF   rG   rƒ   r„   s   &&&&&&€r   r   ÚVisionMLP.__init__  sa   ø€ Ü‰ÑÔØŸ™ KÈÐV\˜ÓjˆŒØ—z‘z +ÀtÐTZ�zÓhˆŒØŸ™Ð$5ÈÐV\˜ÓjˆŒÜ—g’g“iˆŽr   c                óŒ   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      # r?   )rÿ   r  rý   rþ   )r‚   Úhidden_states   &&r   rŽ   ÚVisionMLP.forward  s2   € Ø�~‰~˜dŸk™k¨$¯.©.¸Ó*FÓGÈ$Ï,É,ÐWcÓJdÕdÓeÐer   )r  rÿ   rý   rþ   rö   r’   rš   s   @@r   rø   rø     s   ù‡ € ÷ õ  ÷fò fr   rø   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚVisionBlocki  c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   rÆ   rú   rÕ   rw   )r   rx   s   "€r   r   ÚVisionBlock.__annotate__  s'   ø€ ÷ bñ b¡Cð b¹Cð bÉCñ br   c                óÔ   <€ \         SV `  4        VP                  VR WER7      V n        VP                  VR WER7      V n        \        WWEVR7      V n        \        WWEVR7      V n        R# )rÃ   rÄ   )rF   rG   rƒ   N)	r~   r   rÇ   Únorm1Únorm2rÓ   Úattnrø   rÍ   )r‚   rÆ   rú   rÕ   rF   rG   rƒ   r„   s   &&&&&&&€r   r   ÚVisionBlock.__init__  s\   ø€ Ü‰ÑÔØ—[‘[ °$¸v�[ÓSˆŒ
Ø—[‘[ °$¸v�[ÓSˆŒ
Ü# KÀ6Ð\_Ô`ˆŒ	Ü˜[ÀFÐ]`ÔaˆŽr   c                ó¢   <€ V ^8„  d   QhRS[ P                  RS[S[S[ P                  S[ P                  3,          ,          RS[ P                  /# rÞ   rà   )r   rx   s   "€r   r   r	  #  sH   ø€ ÷ ñ á—|‘|ðñ &¡e©E¯L©L¹%¿,¹,Ð,FÕ&GÕHðñ 
�‰ñr   c                ó¸   € TpV P                  V4      pV P                  WW44      pWQ,           pTpV P                  V4      pV P                  V4      pWQ,           pV# r?   )r  r  r  rÍ   )r‚   r‡   rß   ré   rê   Úresiduals   &&&&& r   rŽ   ÚVisionBlock.forward#  s^   € ð !ˆØŸ
™
 =Ó1ˆØŸ	™	 -ÀjÓfˆØ Õ0ˆà ˆØŸ
™
 =Ó1ˆØŸ™ Ó/ˆØ Õ0ˆàÐr   )r  rÍ   r  r  rö   r’   rš   s   @@r   r  r    s    ù‡ € ÷bõ b÷÷ ò r   r  c                   ód   a a€ ] tR tRt oR	V3R lV 3R llltR tR tR
V3R lR lltRtVt	V ;t
# )ÚQwen2VLVisionTransformeri7  c                óP   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[ RS[ RS[ R	S[ /	# )
r   rÆ   Úoutput_hidden_sizerú   rÕ   Ú
num_layersr   r   rÀ   Úwindow_sizerw   )r   rx   s   "€r   r   Ú%Qwen2VLVisionTransformer.__annotate__8  se   ø€ ÷ /
ñ /
áð/
ñ  ð/
ñ ð	/
ñ
 ð/
ñ ð/
ñ ð/
ñ !ð/
ñ  ð/
ñ ñ/
r   c                ó†  <€ \         SV `  4        Wn        W`n        W€n        W�n        . ROV n        \        VV^VV
VVR7      V n        W,          p\        V^,          4      V n
        \        P                  ! \        V4       Uu. uF  p\        WWJW¼4      NK  	  up4      V n        \!        VVVV
VVR7      V n        R# u upi )é   )r   r   ru   rv   rF   rG   rƒ   )r�   r¿   rÀ   rF   rG   rƒ   N)r  é   é   é   )r~   r   rÆ   r   rÀ   r  Úfullatt_block_indexesrs   Úpatch_embedr®   Úrotary_pos_embrÉ   Ú
ModuleListr.   r  Úblocksr½   Úmerger)r‚   rÆ   r  rú   rÕ   r  r   r   rÀ   r  rF   rG   rƒ   rÙ   rì   r„   s   &&&&&&&&&&&&&  €r   r   Ú!Qwen2VLVisionTransformer.__init__8  sÊ   ø€ ô 	‰ÑÔØ&ÔØ$ŒØ"4ÔØ&ÔÚ%4ˆÔ"ä+Ø!Ø 3ØØ!ØØØô
ˆÔð Õ+ˆÜ3°HÀµMÓBˆÔä—m’mä˜:Ô&ó%
á&�ô ˜¸	È5ÖVÙ&ñ%
ó ˆŒô
 "Ø"Ø#Ø1ØØØô
ˆŽùò%
s   ÂB>c                ó^  € . p^ .p^ pV P                   V P                  ,          V P                  ,          pV EF×  w  rgpWpP                  ,          p	W€P                  ,          p
\        P                  ! Wi,          V
,          4      P                  WiV
4      pWYV,          ,
          pWZV,          ,
          pWœ,           V,          pW­,           V,          p\        P                  ! V^ V^ V3RR4      pVP                  VVVVV4      pVP                  ^ ^^^^4      P                  VWï,          VV4      pVR8g  P                  ^^.4      P                  R4      pVP                  R4      pVVR8g  ,          pVP                  VV,           4       VP                  ^ 4      V P                  ,          V P                  ,          VR,          ,           pVP                  VP                  4       4       WFV	,          V
,          P                  4       ,          pEKÚ  	  \        P                  ! V^ R7      pW#3# )r   Úconstantrœ   iœÿÿÿr�   )r  rÀ   r   r   re   rO   r)   Úpadr"   ÚsumÚappendÚcumsumÚextendrç   Úitemrž   )r‚   Úgrid_thwÚwindow_indexÚcu_window_seqlensÚwindow_index_idÚvit_merger_window_sizerR   rT   rU   Ú
llm_grid_hÚ
llm_grid_wr\   Úpad_hÚpad_wÚnum_windows_hÚnum_windows_wÚindex_paddedÚseqlensÚ	index_newÚcu_seqlens_tmps   &&                  r   Úget_window_indexÚ)Qwen2VLVisionTransformer.get_window_indexi  s  € ØˆØ˜CÐØˆØ!%×!1Ñ!1°T×5LÑ5LÕ!LÐPT×P_ÑP_Õ!_Ðä&.Ñ"ˆF˜FØ×#:Ñ#:Õ:ˆJØ×#:Ñ#:Õ:ˆJä—L’L Õ!4°zÕ!AÓB×JÑJÈ6Ð_iÓjˆEà*Ð:PÕ-PÕPˆEØ*Ð:PÕ-PÕPˆEØ'Õ/Ð4JÕJˆMØ'Õ/Ð4JÕJˆMäŸ5š5 ¨¨E°1°eÐ(<¸jÈ$ÓOˆLØ'×/Ñ/ØØØ&ØØ&óˆLð (×/Ñ/°°1°a¸¸AÓ>×FÑFØØÕ-Ø&Ø&ó	ˆLð $ tÑ+×0Ñ0°!°Q°Ó8×@Ñ@ÀÓDˆGØ'×/Ñ/°Ó3ˆLØ$ \°TÑ%9Õ:ˆIØ×Ñ 	¨OÕ ;Ô<à$Ÿ^™^¨AÓ.°×1HÑ1HÕHÈ4×KbÑKbÕbÐevÐwyÕezÕzˆNØ×$Ñ$ ^×%:Ñ%:Ó%<Ô=Ø¨Õ 3°jÕ @×FÑFÓHÕH‹OñC '/ôF —y’y °1Ô5ˆØÐ.Ð.r   c                óÀ  € . pV EF  w  rEp\         P                  ! WRR 7      P                  ^4      P                  RV4      pVP	                  WPP
                  ,          V P
                  W`P
                  ,          V P
                  4      pVP                  ^ ^^^4      P                  4       p\         P                  ! WbR 7      P                  ^ 4      P                  VR4      pVP	                  WPP
                  ,          V P
                  W`P
                  ,          V P
                  4      pVP                  ^ ^^^4      P                  4       pVP                  \         P                  ! Wx.RR7      P                  V^4      4       EK‚  	  \         P                  ! V^ R7      pVR,          P                  4       p	V P                  W’4      p
W£,          P                  ^4      # )r]   rœ   r�   )r^   râ   )r   re   r+   ÚexpandrO   rÀ   r"   rf   r*  Ústackr0   rž   r&   r!  )r‚   r.  rF   Úpos_idsÚtÚhÚwÚhpos_idsÚwpos_idsÚmax_grid_sizeÚrotary_pos_emb_fulls   &&&        r   Úget_position_embeddingsÚ0Qwen2VLVisionTransformer.get_position_embeddings•  s‡  € Øˆä‰GˆA�!Ü—|’| AÔ5×?Ñ?ÀÓB×IÑIÈ"ÈaÓPˆHØ×'Ñ'Ø×,Ñ,Õ,Ø×'Ñ'Ø×,Ñ,Õ,Ø×'Ñ'ó	ˆHð  ×'Ñ'¨¨1¨a°Ó3×;Ñ;Ó=ˆHä—|’| AÔ5×?Ñ?ÀÓB×IÑIÈ!ÈRÓPˆHØ×'Ñ'Ø×,Ñ,Õ,Ø×'Ñ'Ø×,Ñ,Õ,Ø×'Ñ'ó	ˆHð  ×'Ñ'¨¨1¨a°Ó3×;Ñ;Ó=ˆHà�N‰Nœ5Ÿ;š;¨Ð';ÀÔD×KÑKÈAÈqÓQ×Rñ'  ô* —)’)˜G¨Ô+ˆØ  �×+Ñ+Ó-ˆØ"×1Ñ1°-ÓHÐØ"Õ+×3Ñ3°AÓ6Ð6r   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# )r   rQ   Úimage_grid_thwrˆ   )r   r   r   )r   rx   s   "€r   r   r  ²  s:   ø€ ÷ -ñ -á—l‘lð-ñ !¡§¡Õ.ð-ñ 
�‰ñ	-r   c                ó|  € \        VP                  R RR7      pV P                  V4      pV P                  V4      w  rV\        P
                  ! WdP                  R7      p\        P                  ! V4      pV P                  W$P                  4      pVP                  4       w  r‰V P                  V P                  ,          p
VP                  WŠ,          V
R	4      pWERR3,          pVP                  VR	4      pVP                  WŠ,          V
R	4      pWuRR3,          pVP                  VR	4      p\        P                  ! Ww3R	R7      pVP                  4       VP                  4       3p\        P                  ! VR
,          VR,          ,          VR,          4      P                  ^ \        P                   R7      p\"        P$                  ! VR^ R7      p\'        V P(                  4       F$  w  rÍWÀP*                  9   d   TpMTpV! WGWãR7      pK&  	  V P-                  V4      p\        P.                  ! V4      pWOR3,          pV# )FT)ÚmaskÚsmall_inputr]   r^   rœ   )r�   rG   )Úvalue)rê   r�   )r^   rŒ   )r^   r   )r^   r   )rŒ   r   )r   rF   r   r=  r   rH   Úunique_consecutiverJ  r   rÀ   rO   rž   r¨   r©   Úrepeat_interleaver+  Úint32r)   r(  Ú	enumerater#  r  r$  Úargsort)r‚   rQ   rM  rê   r‡   r/  r0  rß   rh   rì   Úspatial_merge_unitré   ÚiÚblockÚcu_seqlens_nowÚreverse_indicess   &&&             r   rŽ   Ú Qwen2VLVisionTransformer.forward²  s  € ô
 =¸\×=PÑ=PÐW\ÐjnÔoÐà×(Ñ(¨Ó6ˆà*.×*?Ñ*?ÀÓ*OÑ'ˆÜ!ŸLšLÐ):×CWÑCWÔXÐÜ!×4Ò4Ð5FÓGÐà"×:Ñ:¸>×K_ÑK_Ó`Ðà"×'Ñ'Ó)‰
ˆØ!×4Ñ4°t×7NÑ7NÕNÐà%×-Ñ-¨gÕ.KÐM_ÐacÓdˆØ%°A°qÐ&8Õ9ˆØ%×-Ñ-¨g°rÓ:ˆà1×9Ñ9¸'Õ:WÐYkÐmoÓpÐØ1ÀÀ1Ð2DÕEÐØ1×9Ñ9¸'À2ÓFÐÜ#ŸišiÐ)<Ð(RÐXZÔ[ÐØ2×6Ñ6Ó8Ð:M×:QÑ:QÓ:SÐTÐä×,Ò,¨^¸DÕ-AÀNÐSWÕDXÕ-XÐZhÐimÕZnÓo×vÑvØÜ—+‘+ð wó 
ˆ
ô —U’U˜: v°QÔ7ˆ
ä! $§+¡+Ö.‰HˆAØ×.Ñ.Ô.Ø!+‘à!2�Ù! -ÀnÔ~ŠMñ /ð Ÿ™ MÓ2ˆäŸ-š-¨Ó5ˆØ%°qÐ&8Õ9ˆØÐr   )	r#  r  rÆ   r$  r   r   r!  rÀ   r  )r‘   r‘   i\  é   é    r�   r   r   ép   NNNr?   )r“   r”   r•   r–   r   r=  rJ  rŽ   r—   r˜   r™   rš   s   @@r   r  r  7  s*   ù‡ € ÷/
õ /
òb*/òX7÷:-÷ -ò -r   r  )i@  i  Ä r�   r   r   NNÚbilinear)r^  r   i @  i   )r�   r   r   NNr`  )r   Útorch.nnrÉ   Útorch.nn.functionalÚ
functionalr)   Útypingr   r   r$   Úcomfy.ldm.modules.attentionr   r=   rC   rM   r/   rq   ÚModulers   r¢   r¬   r®   r½   rÓ   rø   r  r  © r   r   Ú<module>rh     s·   ðÛ Ý ß Ð ß "Û Ý F÷/Gôdô `òsò8ô6!6�r—y‘yô !6òH(òô
˜BŸI™Iô 
ô�"—)‘)ô ô"2�b—i‘iô 2ôj	f�—	‘	ô 	fô�"—)‘)ô ô8h˜rŸy™yö hr   