+
    ƒ°ñjå~ ã                   ó^  € R t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIH	t	H
t
 ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RItR tR tR tR t ]P0                  P3                  4       '       dö   ^ RIHtHt R]P:                  ! ]4      P<                  9   d»   ]P>                  ]P@                  ]PB                  ]PD                  .t#^ t$]PJ                  PM                  4       '       dg   ]P0                  PO                  ]PJ                  PQ                  4       4      t)])PT                  PW                  R	4      ^ ,          R
8X  d   ])PX                  t$R tM]PZ                  ! R4       Rt0 ]PJ                  Pc                  4       '       d}   ]Pd                  Pf                  Pi                  4       t5]5R8¼  dR   ]5R8  dK   ]PJ                  Pl                  RI8¼  d0   ]PJ                  Pl                  RJ8:  d   Rt0]Pn                  ! R4       ]PJ                  Pp                  t8RKR lt9R t:R t;RLR lt<RMR lt=RNR lt>R t? ! R R4      t@ ! R R4      tA ! R R]P„                  P†                  ]A4      tD ! R  R!4      tE ! R" R#]E4      tFR$ tG ! R% R&]F4      tHRtI ^ R'IJHKtKHLtL RtI]I'       d    ! R( R)]F4      tJ^R*INHOtO ^R+IOHPtPHQtQHRtRHStS R, tTR- tUR.R/ R0]TR1]U/tV]P®                  R2 4       tXROR3 ltYR4 tZR5 t[R6 t\ ! R7 R84      t]RPR9 lt^RQR:R/R; llt_R<^R=^/t` ! R> R?]PÂ                  PÄ                  4      tcRMR@ ltdRLRA lteRRRB ltf ! RC RD]A4      tg/ ]PÐ                  R. 3RE ltiRSRF ltj]P®                  RG 4       tkRTRH ltlR#   ].]/3 d    ]PZ                  ! R4        EL0i ; i    ELš; i  ]M d     ELi ; i)Ua¥  
This file is part of ComfyUI.
Copyright (C) 2024 Stability AI

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
N)ÚargsÚPerformanceFeaturec                  óŽ   € \         P                  P                  4       '       d   R # \        P                  P                  4        R # ©N)ÚtorchÚcompilerÚis_compilingÚcomfyÚmodel_managementÚ)throw_exception_if_processing_interrupted© ó    Ú-C:\Ai_Coding\apps\studio\ComfyUI\comfy\ops.pyÚrun_every_opr   $   s*   € Ü‡~�~×"Ñ"×$Ò$Ùä	×Ñ×DÑDÖFr   c                 óŒ   € W8w  d   \        R V RV 24      hW8X  d   ^# W,          ^ 8w  d   \        RV  RV 24      hW,          # )z'Key/value head count mismatch for GQA: z != z:Query heads must be divisible by key/value heads for GQA: z vs )Ú
ValueError)Úquery_headsÚ	key_headsÚvalue_headss   &&&r   Úgqa_repeat_factorr   *   s^   € ØÔÜÐBÀ9À+ÈTÐR]ÐQ^Ð_Ó`Ð`ØÔÙØÕ !Ô#ÜÐUÐVaÐUbÐbfÐgpÐfqÐrÓsÐsØÕ#Ð#r   c                 óº   € \        W P                  V,          VP                  V,          4      pV^8”  d%   V P                  WCR7      p VP                  WCR7      pW3# )é   ©Údim)r   ÚshapeÚrepeat_interleave)ÚkÚvr   Úhead_dimÚn_reps   &&&& r   Úrepeat_kv_for_gqar    3   sT   € Ü˜k¯7©7°8Õ+<¸a¿g¹gÀhÕ>OÓP€EØˆq„yØ×Ñ ÐÓ4ˆØ×Ñ ÐÓ4ˆØˆ4€Kr   c                 ó6  € \        V4      ^ 8”  d
   V^ ,          MVP                  R4      pVP                  RR4      '       d*   Ve&   \        WV P                  R,          R4      w  rRVR&   \        P
                  P                  P                  ! WV.VO5/ VB # )é    Ú	attn_maskÚ
enable_gqaFéýÿÿÿ)ÚlenÚgetr    r   r   ÚnnÚ
functionalÚscaled_dot_product_attention)Úqr   r   r   Úkwargsr#   s   &&&*, r   r*   r*   :   s   € Ü˜t›9 qœ=��Q–¨f¯j©j¸Ó.E€IØ‡z�z�, ×&Ò&¨9Ò+@Ü   q§w¡w¨r¥{°BÓ7‰ˆØ$ˆˆ|ÑÜ�8‰8×Ñ×;Ò;¸AÀ!ÐUÀdÒUÈfÑUÐUr   )Ú
SDPBackendÚsdpa_kernelÚset_priorityÚ:Úgfx1151c                 ó2  € V P                  4       R8  d0   \        P                  P                  P                  ! WV.VO5/ VB # \        V4      ^ 8”  d
   V^ ,          MVP                  R4      pVP                  RR4      '       dN   VeJ   \        P                  P                  4       '       g&   \        WV P                  R,          R4      w  rRVR&   \        '       dQ   \        R W3 4       4      VP                  4       ,          \        8”  d    VP                  4       VP                  4       r!\!        \"        RR7      ;_uu_ 4        VP                  RR4      '       Edj   VEee   V P                  R,          VP                  R,          8w  Ed;   \        V4      ^8”  d
   V^,          MVP                  RR	4      p\        V4      ^8”  d
   V^,          MVP                  R
R4      p\        P$                  P&                  P)                  WW%WgR4      p\        P$                  P&                  P+                  V4      ;'       g[    \        P$                  P&                  P-                  V4      ;'       g*    \        P$                  P&                  P/                  V4      p	V	'       g&   \        WV P                  R,          R4      w  rRVR&   \        P                  P                  P                  ! WV.VO5/ VB uuRRR4       #   + '       g   i     R# ; i)i   r#   r$   FNc              3   ó¦   "  € T FG  qP                  4       '       d   K  VP                  R,          VP                  R,          ,          x € KI  	  R# 5i)é   Néþÿÿÿéÿÿÿÿ)Úis_contiguousr   )Ú.0Úts   & r   Ú	<genexpr>Ú/scaled_dot_product_attention.<locals>.<genexpr>Z   s7   é € Ð5yÑY_ÐTU×gvÑgv×gxÔ6O°a·g±g¸bµkÀAÇGÁGÈBÅK×6OÒ6OÓY_ùs
   ‚Až3AT)r/   Ú	dropout_pç        Ú	is_causali   r%   )Únelementr   r(   r)   r*   r&   r'   r	   r
   Ú	is_nvidiar    r   ÚSDPA_CONTIGUOUS_KV_L2_SIZEÚsumÚelement_sizeÚ
contiguousr.   ÚSDPA_BACKEND_PRIORITYÚbackendsÚcudaÚ
SDPAParamsÚcan_use_flash_attentionÚcan_use_cudnn_attentionÚcan_use_efficient_attention)
r+   r   r   r   r,   r#   r<   r>   ÚparamsÚsupports_native_gqas
   &&&*,     r   r*   r*   S   sf  € Ø—:‘:“< *Ô,Ü Ÿ8™8×.Ñ.×KÒKÈAÐRSÐeÐVZÒeÐ^dÑeÐeÜ'*¨4£y°1¤}˜D žG¸&¿*¹*À[Ó:Q�	Ø—:‘:˜l¨E×2Ò2°yÒ7LÔUZ×UkÑUk×UuÑUu×UwÒUwÜ,¨Q°1·7±7¸2µ;ÀÓC‘D�AØ+0�F˜<Ñ(ß-Ó-´#Ñ5yÐZ[ÑY_Ó5yÓ2yÐ|}÷  }Kñ  }Kó  }Mõ  3Mô  Pjô  3jØŸ<™<›>¨1¯<©<«>�qÜ Ô!6ÀT×JÖJØ—z‘z ,°×6Ó6¸9Ó;PÐUV×U\ÑU\Ð]_ÕU`Ðde×dkÑdkÐlnÕdoÕUoÜ/2°4«y¸1¬} D¨¦GÀ&Ç*Á*È[ÐZ]ÓB^˜	Ü/2°4«y¸1¬} D¨¦GÀ&Ç*Á*È[ÐZ_ÓB`˜	Ü!&§¡×!4Ñ!4×!?Ñ!?ÀÀaÐT]ÐjnÓ!o˜ä!ŸN™N×/Ñ/×GÑGÈÓO÷ Wð WÜ$Ÿ~™~×2Ñ2×JÑJÈ6ÓR÷Wð Wä$Ÿ~™~×2Ñ2×NÑNÈvÓVð ,÷
  3Ü#4°Q¸1¿7¹7À2½;ÈÓ#K™D˜AØ38˜F <Ñ0Ü Ÿ8™8×.Ñ.×KÒKÈAÐRSÐeÐVZÒeÐ^dÑe÷ K×J×JÓJús   Å	D+LÉ51LÊ'ALÌL	z3Torch version too old to set sdpa backend priority.z$Could not set sdpa backend priority.Fizc ile Tz(working around nvidia conv3d memory bug.c                 ón   € \         P                  P                  WP                  VP                  W#R 7      # ))Únon_blockingÚcopy)r	   r
   Úcast_toÚdtypeÚdevice)ÚweightÚinputrO   rP   s   &&&&r   Úcast_to_inputrV   |   s)   € Ü×!Ñ!×)Ñ)¨&·+±+¸u¿|¹|ÐZfÐ)ÓrÐrr   c                 óh  € V F«  p\        WR 4      pVf   K  \        VRR4      '       g   K*  \        VP                  4       R4      '       d   KL  \        W\        P
                  P                  \        P                  ! VP                  VP                  R7      VP                  R7      4       K­  	  R # )NÚis_metaFÚ_comfy_tensor_file_slice©rR   ©Úrequires_grad)ÚgetattrÚhasattrÚuntyped_storageÚsetattrr   r(   Ú	ParameterÚzerosr   rR   r\   )ÚsÚ
param_keysÚ	param_keyÚparams   &&  r   Úmaterialize_meta_paramrg   €   s˜   € Ûˆ	Ü˜ dÓ+ˆØÔ¤¨°	¸5×!AÔ!AÌ'ÐRW×RgÑRgÓRið  lF÷  KGô  KGÜ�A¤%§(¡(×"4Ñ"4´U·[²[ÀÇÁÐTY×T_ÑT_Ô5`Ðpu÷  qDñ  qDÐ"4ó  #Eö  Fó  r   c                 ó¤   € \        V R R4      e   R# V P                  w  rpW!P                  ,
          V,           VP                  4       R,          8„  # )Ú	_prefetchNFi   )r]   Ú_vÚ	base_addrÚget_watermark)rc   ÚvbarÚallocÚsizes   &   r   Úvbar_above_watermarkrp   ‡   sG   € äˆq�+˜tÓ$Ò0ÙØŸ™Ñ€D�Ø—>‘>Õ! DÕ(¨4×+=Ñ+=Ó+?À8Õ+LÑLÐLr   c                 ó 
  a aaaa a!a"a#a$a%€ R o$R o^ o V'       d5   \         ;QJ d    R S  4       F  '       d   K   RM	  RM! R S  4       4      pVV VV$3R lpVV VV$3R lp/ o#. p	S  EFˆ  p
V
P                  pVe   VP                  fV   \        P                  P                  V
P                  4      o%\        P                  P                  S%V
P                  4      pEMV
S#9  dù   VP                   F[  p\        P                  P                  VP                  4      o%S%\        P                  P                  S%VP                  4      3S#V&   K]  	  \         ;QJ d)    V#3R lVP                   4       F  '       d   K   RM	  RM! V#3R lVP                   4       4      '       g:   V	P                  V4       VP                   F  pS#V,          ^ ,          R3S#V&   K  	  S#P                  V
4      w  o%pV'       d   S%e	   V'       g   RpRS%RV/pV'       d
   Wên        EK·  \        V
R	R
.4       S%e+   \        P                  P                  V
P                  S4      MR p\         P"                  P%                  V
P&                  V
P(                  .4      pR pRpV
P&                  V
P(                  .pV
P*                  R,          o"S"'       d   RMRp\         P,                  P/                  V
VR7      pVf6   S"'       g.   \         P,                  P/                  V
RR7      pVf   S%e   RpTpVe   V.p\1        V
P&                  V
P(                  .V4       F/  w  ppVf   K  VP2                  VP2                  8w  g   K)  RpTpR p M	  \         P"                  P5                  V4      pT! Y¯f   TM^ R4       Vf	   V! V4      pRV3R llo!RV!V"V$V%3R llp\7        V
\8        4      '       d   V
P:                  '       d   V! V
VVVVVR7       R FË  p\=        V
VR,           R 4      pVf   K  V! V
S R4       VP?                  4       pV! V4      pVPA                  VR RRR7       S"'       d   RMRp\         P,                  P/                  VVR7      pVf9   S"'       g1   \         P,                  P/                  VRR7      pVe   RpTpMS%e   RpV! VVVVVVR7       KÍ  	  WþR&   VVR&   VVR&   VVR&   Wên        EK‹  	  V	 F
  pR Vn        K  	  V'       d   S$X3# S$# )Nc              3   ód   "  € T F&  pR F  p\        WR,           . 4      '       * x € K  	  K(  	  R# 5i)rT   Ú	_functionN©rT   Úbias)r]   )r8   rc   re   s   &  r   r:   Ú)cast_modules_with_vbar.<locals>.<genexpr>•   s;   é € ð  DÑQ^ÈAó  qCÐcl¤¨°{Õ+BÀBÓ G×GÑGñ  qCÑGÓQ^ùs   ‚.0FTc                 óÊ  <€ Sf    \         P                  P                  S4      oSe   V'       d   \        S4      ^8w  d   R # Sf   ^ MSP	                  4       pW18  dG   V \         P                  P
                  ^ ,          J d"   \         P                  P                  S4      oR oV\         P                  P
                  ^,          8”  d   W3\         P                  n        R # R # r   )r	   r
   Úget_offload_streamr&   ro   ÚLARGEST_AIMDO_CASTED_WEIGHT)ÚmoduleÚrequired_sizeÚcheck_largestÚcurrent_sizeÚcast_bufferÚcomfy_modulesrS   Úoffload_streams   &&& €€€€r   Úensure_offload_streamÚ5cast_modules_with_vbar.<locals>.ensure_offload_stream—   s¸   ø€ ð Ò!Ü"×3Ñ3×FÑFÀvÓNˆNØÒ!¯¼#¸mÓ:LÐPQÔ:QÙà'Ò/‘q°[×5EÑ5EÓ5GˆØÔ'¨F´e×6LÑ6L×6hÑ6hÐijÕ6kÓ,kÜ"×3Ñ3×FÑFÀvÓNˆNØˆKØœ5×1Ñ1×MÑMÈaÕPÔPØBHÐAXŒE×"Ñ"Ö>ñ Qr   c                 ó$  <€ V ^ 8X  d   R# Sf)   \         P                  ! V 3\         P                  SR7      # \        P                  P                  SS4      o\        P                   P                  SP                  V S4      S4      pSV ,          oV# )r"   N©rR   rS   )	r   ÚemptyÚuint8r	   r
   Úget_aimdo_cast_bufferÚcomfy_aimdoÚaimdo_to_tensorr'   )Úbuffer_sizeÚbufferr~   Úcast_buffer_offsetrS   r€   s   & €€€€r   Úget_cast_bufferÚ/cast_modules_with_vbar.<locals>.get_cast_buffer§   s}   ø€ ð
 ˜!ÔÙàÒ!Ü—;’; ˜~´U·[±[ÈÔPÐPä×,Ñ,×BÑBÀ>ÐSYÓZˆÜ×"Ñ"×2Ñ2°;·?±?À;ÐPbÓ3cÐekÓlˆØ˜kÕ)ÐØˆr   c              3   óD   <"  € T F  pSV,          ^,          x € K  	  R# 5i©r   Nr   )r8   ÚmÚgroup_faultss   & €r   r:   rv   È   s   øé € ÐE±}°!˜<¨�?¨1×-Ò-³}ùs   ƒ Ú	signatureÚresidentrT   ru   Ú	fast_diskúweights-fastÚweights©Úsubsetúweights-loadedc                 óö   <€ V et   \        V RR4      '       d=   Ve   V P                  WRRR7       V.p TpR pMVe   V P                  W2RRR7       R # R # \        P                  P	                  WSW#R7       R # R # )NÚis_lowvram_patchFT©rP   Úcommit)rO   ÚstreamÚr2)r]   Úpreparer	   r
   Úcast_to_gathered)Úxfer_sourceÚ	xfer_destrŸ   Ú
xfer_dest2rO   s   &&&&€r   Úcast_maybe_lowvram_patchÚ8cast_modules_with_vbar.<locals>.cast_maybe_lowvram_patchú   s�   ø€ ØÒ&Ü˜;Ð(:¸E×BÒBØ Ò,Ø#×+Ñ+¨IÀDÐQVÐ+ÔWØ(1 m˜Ø$.˜	Ø%)™
Ø#Ò/Ø#×+Ñ+¨JÀTÐRWÐ+ÔXÙáÜ×&Ñ&×7Ñ7¸Ð]iÐrxÐ7ö  Iñ 'r   c                 ó  <€ Ve   S! V.VS4       R # S	e   S'       d   \         P                  '       dB   \        P                  P	                  WVR7       \        P                  P                  WR7      pS! W!SVR7       R # )N©r™   ro   r˜   )r¥   )r   Úhigh_ramr	   Úpinned_memoryÚ
pin_memoryÚget_pin)
r‘   ÚpinÚsourceÚdestr™   ro   r¦   r•   r€   r“   s
   &&&&&&€€€€r   Ú
handle_pinÚ*cast_modules_with_vbar.<locals>.handle_pin	  sg   ø€ ØŠÙ(¨#¨°°nÔEÙØÒ ¯	´T·]·]°]Ü×#Ñ#×.Ñ.¨qÀdÐ.ÔKÜ×)Ñ)×1Ñ1°!Ð1ÓC�Ù$ V°.ÈT×Rr   r©   Ú_lowvram_functionr�   zpatches-fastÚpatcheszpatches-loadedr¤   Ú	cast_destÚcast_geometryÚ
needs_castr   )r—   Nrt   )!ÚallÚ_combined_linearÚ	_v_weightrˆ   Ú
model_vbarÚ
vbar_faultrj   Úvbar_signature_compareÚ_v_signatureÚlinearsÚappendÚpopri   rg   r   r‰   r	   Úmemory_managementÚtensors_to_geometriesrT   ru   Ú
_pin_stater«   r­   ÚziprR   Úvram_aligned_sizeÚ
isinstanceÚWritableWeightÚread_before_writer]   Úmemory_requiredr¡   )&r   rR   rS   Ú
bias_dtyperO   Úreturn_faultedÚfully_faultedr�   r�   Údropped_groupsrc   Úgroupr”   r‘   Úprefetchr¤   r¶   rµ   r·   r£   r™   r®   Ú
loaded_pinÚdataÚgeometryÚ	dest_sizer±   re   Úlowvram_sourceÚlowvram_sizeÚlowvram_destr~   rŒ   r¦   r•   r’   r€   r“   s&   f&f&f&                         @@@@@@@r   Úcast_modules_with_vbarrØ   �   sž  ÿù€ Ø€NØ€KØÐßß›ñ  DÑQ^ó  DŸŸšñ  DÑQ^ó  Dó  Dˆ÷Yð Y÷ ð ð  €LØ€NÜˆØ×"Ñ"ˆØŠ=˜EŸO™OÒ3Ü#×.Ñ.×9Ñ9¸!¿$¹$Ó?ˆIÜ"×-Ñ-×DÑDÀYÐPQ×P^ÑP^Ó_ŠHð ˜Ô$ØŸœ�AÜ +× 6Ñ 6× AÑ AÀ!Ç$Á$Ó G�IØ'0´+×2HÑ2H×2_Ñ2_Ð`iÐkl×kyÑkyÓ2zÐ&{�L “Oñ '÷ “sÔE°u·}²}ÓE—s—s’sÔE°u·}²}ÓE×EÒEð #×)Ñ)¨%Ô0Ø"Ÿ]œ]˜Ø+7¸­?¸1Õ+=¸uÐ*E˜ Q›ñ +à".×"2Ñ"2°1Ó"5ÑˆI�xß˜yÒ0¿Ø!ˆMà˜Ø˜ð
ˆ÷
 Ø"ŒKÚä˜q 8¨VÐ"4Ô5ØGPÒG\”K×%Ñ%×5Ñ5°a·d±d¸FÔCÐbfˆ	Ü×/Ñ/×EÑEÈÏÉÐRS×RXÑRXÐFZÓ[ˆØˆ	Øˆ
àŸ™ !§&¡&Ð*ˆØ—L‘L Õ-ˆ	ß#,‘°)ˆÜ×!Ñ!×)Ñ)¨!°FÐ)Ó;ˆØŠ;ŸyÜ×,Ñ,×4Ñ4°QÐ?OÐ4ÓPˆJØÒ%¨Ò)>Ø)�Ø �ØŠ?Ø˜'ˆKä! A§H¡H¨a¯f©fÐ"6¸ÖF‰NˆD�(ØŠ|ÙØ�z‰z˜XŸ^™^Ö+Ø!�
Ø%�	Ø �	Ùñ Gô ×+Ñ+×=Ñ=¸kÓJˆ	Ù˜aÒ.?¡ÀQÈÔMØÒÙ'¨	Ó2ˆI÷	I÷	Sò 	Sô ˜!œ^×,Ò,°×0C×0CÐ0CÙ�q˜#˜{¨I¸fÈ9ÕUã+ˆIÜ$ Q¨	Ð4GÕ(GÈÓNˆNØÔ)Ù% aÐ);¸UÔCØ-×=Ñ=Ó?�Ù.¨|Ó<�Ø×&Ñ& |°TÀÈdÐ&ÔSç+4™¸)�Ü×)Ñ)×1Ñ1°.ÈÐ1ÓP�Ø’;§yÜ!&×!4Ñ!4×!<Ñ!<¸^ÐTdÐ!<Ó!e�JØ!Ò-Ø!1˜Ø(™Ø"Ò.Ø!1˜Ù˜>¨3°ÀÐU[Ðbn×oñ# ,ð( !*�ÑØ )ˆ�ÑØ$1ˆ�Ñ!Ø!+ˆ�ÑØ�ñi ól  ˆØˆŽñ  çØ˜}Ð,Ð,ØÐr   c                 óð  aaa€ \        V R R4      pVR,          '       d   V P                  pV P                  p	EMdVR,          p
VR,          '       dè   VR,          e
   VR,          MK\        P                  ! \
        P                  P                  VR,          4      3\        P                  VR7      p\        \
        P                  P                  V P                  V P                  .V
4      \
        P                  P                  VR,          V4      4       F  w  rÍVf   K  VP                  V4       K  	  Tp
\
        P                  P                  VR,          V
4      pV^ ,          pV^,          p	VR,          e   W€n        W�n        R	V n        VR,          V n        VVV3R
 lpVR,          RJpV! V RW�VR,          V4      pV	e   V! V RW“VR,          V4      p	VR,          e   RVR&   S'       d   W‰3# R# )ri   Nr”   r¤   r·   rµ   r¶   r„   r“   Fc                 óØ  <€ \        WR ,           R4      p\        WR,           . 4      pVf   R# TpR p	S'       d   VP                  V8w  g   \        V4      ^ 8”  d	   V	! W#4      pV'       gî   Veê   T	! TSf   TMS4      pV! V4      pS'       d   \        V4      ^ 8X  g	   V'       d€   \        P                  P                  V P                  4      p
\        V\        4      '       d   VP                  VRV
R7      pM+\        P                  P                  W(P                  V
R7      pS'       d   \        V4      ^ 8X  d   XpV'       d   VP                  X4       V F  pV! V4      pK  	  V# )r³   Nrs   c                 óv   € V P                  VR 7      p \        V \        4      '       d   V P                  4       p V # )rZ   )ÚtorÇ   ÚQuantizedTensorÚ
dequantize)ÚtensorrR   s   &&r   Ú
to_dequantÚDresolve_cast_module_with_vbar.<locals>.post_cast.<locals>.to_dequantY  s2   € Ø—Y‘Y U�YÓ+ˆFÜ˜&¤/×2Ò2Ø×*Ñ*Ó,�ØˆMr   Úrecalculate)ÚscaleÚstochastic_rounding)Úseed)r]   rR   r&   r	   ÚutilsÚstring_to_seedÚseed_keyrÇ   rÝ   Úrequantize_from_floatÚfloaträ   Úcopy_)rc   re   ÚxrR   r”   Úupdate_weightÚ
lowvram_fnÚfnsÚorigrà   rå   ÚyÚfÚcompute_dtypeÚreturn_weightsÚwant_requants   &&&&&&       €€€r   Ú	post_castÚ0resolve_cast_module_with_vbar.<locals>.post_castP  s!  ø€ Ü˜QÐ,?Õ ?ÀÓFˆ
Ü�a [Õ0°"Ó5ˆàŠ9Ùàˆò	÷ ˜tŸz™z¨UÔ2´s¸3³xÀ!´|Ù˜1Ó$ˆAß˜JÒ2Ù˜1 }Ò'<™eÀ-ÓPˆAÙ˜1“ˆAß¤ S£¨Q¤·-Ü—{‘{×1Ñ1°!·*±*Ó=�Ü˜d¤O×4Ò4Ø×2Ñ2°1¸MÐ_cÐ2Ód‘AäŸ™×7Ñ7¸¿:¹:ÈDÐ7ÓQ�Aß¤ C£¨A¤Ø�ßØ—
‘
˜1”ÛˆAÙ�!“ŠAñ àˆr   rT   ru   T)r]   rº   Ú_v_biasr   r…   r	   rÂ   rÆ   r†   rÅ   Úinterpret_gathered_likerT   ru   rë   Ú_v_relayoutr¾   )rc   rR   rS   rË   ró   rõ   rô   rÐ   rT   ru   r¤   rµ   Úpre_caströ   rL   rí   s   &&&&fff         r   Úresolve_cast_module_with_vbarrü   6  sû  ú€ ä�q˜+ tÓ,€Hà�
×ÔØ—‘ˆØ�y‰yŠà˜[Õ)ˆ	Ø�L×!Ô!Ø19¸+Õ1FÒ1R˜ Ö-ÔX]×XcÒXcÔej×e|Ñe|÷  fOñ  fOð  PXð  Yhõ  Pió  fjð  elô  ty÷  tñ  tð  HNô  YOˆIÜ'*¬5×+BÑ+B×+ZÑ+ZÐ\]×\dÑ\dÐfg×flÑflÐ[nÐpyÓ+zÜ+0×+BÑ+B×+ZÑ+ZÐ[cÐdsÕ[tÐvó  ,Aö(BÑ#�àÔ(Ø—O‘O HÖ-ñ(Bð "ˆIä×(Ñ(×@Ñ@ÀÈ/ÕAZÐ\eÓfˆØ˜•ˆØ�a�yˆØ�KÕ Ò,Ø ŒKØŒIØ!ˆAŒMØ! +Õ.ˆŒ÷ ðD ˜[Õ)°Ð5€MÙ�q˜( F°8¸JÕ3GÈÓW€FØÒÙ˜˜F D°h¸zÕ6JÈMÓZˆà�ÕÒ(Ø#ˆ�Ñç+ˆFˆ>Ð5°Ð5r   c           
      ó
  € VeT   Vf:   \        V\        4      '       d   VP                  P                  pMVP                  pVf   TpVf   VP
                  pR p\        P                  P                  V4      p	\        V R4      '       d¯   \        P                  P                  V4      '       dŠ   \        V RR.4       V P                  P                  VRR7      p
\        V
\        4      '       d   V
P                  4       p
V P                  e   V P                  P                  VRR7      MR pV! W«R3V4      # \        V R4      '       dÚ   V P                  P
                  V8w  d¿   \        V R4      pR pR pV'       g.   \!        V .W#WI4      p\        P                  P#                  W=4       \%        WW4Wg4      w  r«V'       gV   \'        V R4      R,          e   TpR F,  p\'        WR	,           R 4      pVf   K  VP)                  4        K.  	  \+        V R4       V! W«WÞR 33V4      # V'       dc   W0P                  P
                  8w  g)   V P                  e;   W0P                  P
                  8w  d!   \        P                  P-                  V4      pMR pR pR p
Veù   \.        P0                  '       gã   \        P2                  P5                  V P                  V P                  .4      p\        P                  P7                  WÓVV 4      pVfA   \        P                  P-                  V4      p\        P                  P7                  WÓVV 4      p\        P2                  P9                  V P                  V P                  .V4      pV^ ,          p
V^,          p\;        V P<                  4      ^ 8„  p\;        V P>                  4      ^ 8„  p\        P                  PA                  V P                  R W9VWÚR
7      p
V P                  e/   \        P                  PA                  V P                  R W9VWÛR
7      p\        P                  P#                  W=4       TpT
pV P                  e.   VP                  VR7      pV P>                   F  pV! V4      pK  	  V'       g   V
P                  V8w  dT   V
P                  VR7      p
\        V
\        4      '       d   V
P                  4       p
V P<                   F  pV! V
4      p
K  	  V! W«VVV33V4      # )Nc                 ó*   € V w  r#pV'       d   W#V3# W#3# r   r   )ÚresultÚoffloadablerT   ru   r€   s   &&   r   Úformat_returnÚ'cast_bias_weight.<locals>.format_returnŒ  s    € Ø'-Ñ$ˆ�nß1<�˜nÐ-ÐPÀ6À.ÐPr   rj   rT   ru   T)rR   rP   ri   r“   r³   )rO   rP   rŸ   ÚrrZ   )NNNrt   )!rÇ   rÝ   rL   Ú
orig_dtyperR   rS   r	   r
   Údevice_supports_non_blockingr^   Úis_device_cpurg   rT   rÜ   rÞ   ru   rØ   Úsync_streamrü   r]   Úclear_preparedÚdelattrrx   r   Úcuda_mallocrÂ   rÆ   r�   rù   r&   Úweight_functionÚbias_functionrQ   )rc   rU   rR   rS   rË   r   ró   rõ   r  rO   rT   ru   Ú
prefetchedr€   Úoffload_devicere   rî   Úcast_buffer_sizer~   rL   Úweight_has_functionÚbias_has_functionÚbias_aÚweight_arò   s   &&&&&&&&                 r   Úcast_bias_weightr  }  s;  € ð ÒØŠ=Ü˜%¤×1Ò1ØŸ™×/Ñ/‘àŸ™�ØÒØˆJØŠ>Ø—\‘\ˆFòQô ×)Ñ)×FÑFÀvÓN€Läˆq�$×ÒœE×2Ñ2×@Ñ@À×HÒHô 	˜q 8¨VÐ"4Ô5Ø—‘—‘ 5¨t�Ó4ˆÜ�fœo×.Ò.Ø×&Ñ&Ó(ˆFØ9:¿¹Ò9Kˆq�v‰v�y‰y˜z°ˆyÔ5ÐQUˆÙ˜fÐ,>Ð?ÀÓMÐMä	��D×	Ò	˜aŸh™hŸo™o°Ô7Ü˜Q Ó,ˆ
ØˆØˆßÜ3°Q°C¸È
ÓaˆNÜ×"Ñ"×.Ñ.¨vÔFä4°Q¸vÐS`Óo‰ˆçÜ�q˜+Ó& {Õ3Ò?Ø!'�Û/�	Ü$ QÐ4GÕ(GÈÓN�
ØÔ)Ø×-Ñ-Ö/ñ 0ô �A�{Ô#Ù˜f¨^ÈTÐ,RÐSÐU`ÓaÐa÷ ˜§(¡(§/¡/Ô1ØŸ™Ò+°¿&¹&¿-¹-Ô0GÜ×/Ñ/×BÑBÀ6ÓJ‰àˆà€DØ€FàÒ!¬$×*:×*:Ð*:Ü ×2Ñ2×DÑDÀqÇxÁxÐQR×QWÑQWÐEYÓZÐÜ×,Ñ,×<Ñ<¸^ÐUeÐghÓiˆàÒÜ"×3Ñ3×FÑFÀvÓNˆNÜ×0Ñ0×@Ñ@ÀÐYiÐklÓmˆKÜ×(Ñ(×@Ñ@À1Ç8Á8ÈQÏVÉVÐAUÐWbÓcˆØ˜•ˆØ�a�yˆä˜a×/Ñ/Ó0°1Ñ4ÐÜ˜AŸO™OÓ,¨qÑ0Ðä×#Ñ#×+Ñ+¨A¯H©H°d¸FÐdwð  AOÐ+ó  Z€Fà‡v�vÒÜ×%Ñ%×-Ñ-¨a¯f©f°d¸FÐduð  MÐ-ó  Vˆä	×Ñ×&Ñ& vÔ>à€FØ€Hà‡v�vÒØ�w‰w˜ZˆwÓ(ˆØ—”ˆAÙ�T“7ŠDñ !÷ ˜fŸl™l¨eÔ3Ø—‘ �Ó'ˆÜ�fœo×.Ò.Ø×&Ñ&Ó(ˆFØ×"Ô"ˆAÙ�v“YŠFñ #ñ ˜&¨¸À6Ð(JÐKÈ[ÓYÐYr   c                 ót  € Vf   R # Vw  rEpR pVeL   \        V\        P                  4      '       g,   \        P                  P                  V P                  4       TpVf   R # Vf$   Ve   VP                  pMVf   R # VP                  pVP                  \        P                  P                  V4      4       R # r   )rÇ   r   ÚTensorrˆ   r»   Ú
vbar_unpinrj   rS   Úwait_streamr	   r
   Úcurrent_stream)rc   rT   ru   r€   Úosr  r  rS   s   &&&&    r   Úuncast_bias_weightr  å  s—   € ØÒÙØ)Ñ€B�&Ø€FàÒ¤J¨x¼¿¹×$FÒ$FÜ×Ñ×)Ñ)¨!¯$©$Ô/ØˆØ	‚zÙØ‚~ØÒØ—_‘_‰FàŠ~ÙØ—]‘]ˆFØ‡N�N”5×)Ñ)×8Ñ8¸Ó@ÖAr   c                   ó>   a € ] tR tRt o R tR tV 3R lR ltRtV tR# )ÚCastBiasWeightContextiù  c                ó–   € \        V4      '       d
   V^ ,          MRV n        V P                  f
   RV n        R# \        V/ VB V n        R# )r"   N)NN)r&   Úslfr  Ústate)Úselfr   r,   s   &*,r   Ú__init__ÚCastBiasWeightContext.__init__ü  s9   € Ü! $Ÿiši�4˜–7¨TˆŒØ%)§X¡XÒ%5�\ˆŽ
Ô;KÈTÐ;\ÐU[Ñ;\ˆŽ
r   c                ó~   € V P                   p\        V4      ^8  g   V^,          f   R;V n         V n        VR,          # )é   N:Nr4   N)r   r&   r  )r!  rÿ   s   & r   Ú	__enter__ÚCastBiasWeightContext.__enter__   s7   € Ø—‘ˆÜˆv‹;˜Œ?˜f Q�iÒ/à$(Ð(ˆDŒJ˜œØ�b�zÐr   c                ó   <€ V ^8„  d   QhRR/# )r4   ÚreturnNr   )ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú"CastBiasWeightContext.__annotate__  s   ø€ ÷ (ñ ( $ñ (r   c                óŠ   € V P                   f   R # V P                   V P                  r2R ;V n        V n         \        V.VO5!   R # r   )r  r   r  )r!  Ú_argsr  r   s   &*  r   Ú__exit__ÚCastBiasWeightContext.__exit__  s9   € Ø�8‰8ÒÙØ—X‘X˜tŸz™zˆUØ $Ð$ˆŒ
�T”XÜ˜3Ð' Ö'r   )r  r   N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r"  r&  r0  Ú__static_attributes__Ú__classdictcell__©r+  s   @r   r  r  ù  s   ø‡ € ò]ò÷(ö (r   r  c                   ó*   € ] tR tRtRtRt. t. tRtRt	R# )ÚCastWeightBiasOpi  FNr   )
r2  r3  r4  r5  Úcomfy_cast_weightsÚcomfy_force_forwardr  r  r¹   r6  r   r   r   r:  r:    s   † ØÐØÐØ€OØ€MØÔr   r:  c                   ól   a a€ ] tR tRt oRtRtRV 3R llt]P                  R	R l4       t	R t
RtVtV ;t# )
rÈ   i  zGOne runtime tensor, either realized or cast from writable file backing.Tc                óÆ   <€ \         SV `  4        \        P                  P	                  \        P
                  ! WVR 7      RR7      V n        RV n        / V n        RV n	        R# )r„   Fr[   N)
Úsuperr"  r   r(   ra   r…   rT   ru   Ú_pinsÚ_pin_subset)r!  r   rR   rS   Ú	__class__s   &&&&€r   r"  ÚWritableWeight.__init__  sL   ø€ Ü‰ÑÔÜ—h‘h×(Ñ(¬¯ª°UÐPVÔ)WÐglÐ(ÓmˆŒØˆŒ	ØˆŒ
ØˆÖr   c              #  óæ  "  € V P                   '       g   V P                  R 3x € R # \        V R4      '       g   Q R4       h\        W P                  P                  VRR7      ;_uu_ 4       p Vx € V'       d'   V P
                  f   V P                  V^ ,          4       R R R 4       R #   T'       d)   T P
                  f   T P                  T^ ,          4       i i i ; i  + '       g   i     R # ; i5i)Nri   z!Writable weights require prefetchT)rR   rS   r   )r;  rT   r^   r  rR   r¾   Ú	writeback)r!  rS   ÚwriteÚvaluess   &&& r   ÚaccessÚWritableWeight.access!  s¿   é € à×&×&Ð&Ø—+‘+˜tÐ#Ò#ÙÜ�t˜[×)Ò)ÐNÐ+NÓNÐ)Ü" 4¯{©{×/@Ñ/@ÈÐ]a×bÕbÐflð.Ø’ç˜T×.Ñ.Ò6Ø—N‘N 6¨!¥9Ô-÷ cÑbø÷ ˜T×.Ñ.Ò6Ø—N‘N 6¨!¥9Õ-ð 7�5ú÷	 c×bÐbüs6   ‚A(C1Á*CÁ,B(Á0.CÂ
C1Â(2CÃCÃC.	Ã(	C1c                óÌ  € V P                   pVe    \        P                  P                  W4      MR pVfm   V P                  R,          '       d   RMV P
                  e   RMRp\        P                  P                  WR7       \        P                  P                  W4      pVe•   \        P                  P                  V P                  .V4      ^ ,          pVP                  VRR7       \        P                  P                  VP                  4      P                  4        RV P                   V,          R&   R # \        P                  P#                  V P                  V4       R # )	Nr•   r–   rš   r—   r˜   T)rO   Údirty)rA  r	   r«   r­   rÄ   r¾   r¬   rÂ   rù   rT   rë   r   rG   r  rS   Úsynchronizer@  Úwrite_tensor_file_slice)r!  Úvaluer™   r®   r°   s   &&   r   rE  ÚWritableWeight.writeback.  s  € Ø×!Ñ!ˆØ;AÒ;MŒe×!Ñ!×)Ñ)¨$Ô7ÐSWˆØŠ;Ø'+§¡°{×'CÔ'C‘^Ð]a×]nÑ]nÒ]zÑIYð  AJˆFÜ×Ñ×*Ñ*¨4Ð*Ô?Ü×%Ñ%×-Ñ-¨dÓ;ˆCØŠ?Ü×*Ñ*×BÑBÀDÇKÁKÀ=ÐRUÓVÐWXÕYˆDØ�J‰J�u¨4ˆJÔ0ä�J‰J×%Ñ% e§l¡lÓ3×?Ñ?ÔAØ*.ˆD�J‰J�vÕ˜wÓ'ä×#Ñ#×;Ñ;¸D¿K¹KÈÖOr   )rA  r@  ru   rT   )Úmeta©T)r2  r3  r4  r5  Ú__doc__rÉ   r"  Ú
contextlibÚcontextmanagerrH  rE  r6  r7  Ú__classcell__©rB  r+  s   @@r   rÈ   rÈ     s9   ù‡ € ÙQàÐ÷ ð ×Ñó
.ó ð
.÷Pò Pr   rÈ   c                   ó  a € ] tR tRt o ]R 4       t]RR l4       t ! R R]P                  P                  ]
4      t	 ! R R]P                  P                  ]
4      t ! R	 R
]P                  P                  ]
4      t ! R R]P                  P                  ]
4      t ! R R]P                  P                  ]
4      t ! R R]P                  P                  ]
4      t ! R R]P                  P                   ]
4      t ! R R]P                  P"                  ]
4      t ! R R]P                  P$                  ]
4      t ! R R]P                  P&                  ]
4      t ! R R]P                  P(                  ]
4      t]R 4       tRtV tR# )Údisable_weight_initi>  c                ó  € \        W4      p\        VR R4      '       d   RMVP                  p\        W\        P                  P                  \        P                  ! VP                  W2P                  R7      RR7      4       R# )rX   FN©rS   rR   r[   )	r]   rS   r`   r   r(   ra   rb   r   rR   )rz   Únamerf   rS   s   &&  r   Ú_zero_init_parameterÚ(disable_weight_init._zero_init_parameter?  sd   € ä˜Ó%ˆÜ  ¨	°5×9Ò9‘¸u¿|¹|ˆÜ�œeŸh™h×0Ñ0´·²¸U¿[¹[ÐQW×_jÑ_jÔ1kð  |AÐ0ó  Bö  	Cr   Nc                ó‚  € VP                  R R4      p\        V4      p	VP                  4        F°  w  r«W©R pVR8X  dA   V'       g   VP                  4       p\        P
                  P                  VRR7      V n        KP  VeL   VR8X  dE   VeA   V'       g   VP                  4       p\        P
                  P                  VRR7      V n        KŸ  VP                  V
4       K²  	  V P                  fS   \        P
                  P                  \        P                  ! V4      RR7      V n        VP                  VR,           4       Vez   V P                  fj   \        V RR4      '       dU   \        P
                  P                  \        P                  ! V4      RR7      V n        VP                  VR,           4       R# R# R# R# )Úassign_to_params_buffersFNrT   r[   ru   Úcomfy_need_lazy_init_bias)r'   r&   ÚitemsÚcloner   r(   ra   rT   ru   rÀ   rb   r]   )rz   Ú
state_dictÚprefixÚlocal_metadataÚmissing_keysÚunexpected_keysÚweight_shapeÚ
bias_shaper_  Ú
prefix_lenr   r   Úkeys   &&&&&&&&     r   Ú_lazy_load_from_state_dictÚ.disable_weight_init._lazy_load_from_state_dictE  s]  € ð $2×#5Ñ#5Ð6PÐRWÓ#XÐ Ü˜“[ˆ
Ø×$Ñ$Ö&‰DˆAØ�K�.ˆCØ�hŒß/ØŸ™›	�AÜ %§¡× 2Ñ 2°1ÀEÐ 2Ó J�–ØÒ'¨C°6¬M¸aºmß/ØŸ™›	�AÜ#Ÿh™h×0Ñ0°À%Ð0ÓH�–à×&Ñ& qÖ)ñ 'ð �=‰=Ò Ü!ŸH™H×.Ñ.¬u¯{ª{¸<Ó/HÐX]Ð.Ó^ˆFŒMØ×Ñ ¨Õ 1Ô2àÒ! f§k¡kÒ&9¼gÀfÐNiÐkp×>qÒ>qÜŸ(™(×,Ñ,¬U¯[ª[¸Ó-DÐTYÐ,ÓZˆFŒKØ×Ñ ¨¥Ö0ñ ?rÑ&9Ñ!r   c                   óZ   a a€ ] tR tRt oRV 3R lltV 3R ltR tR tV 3R ltRt	Vt
V ;t# )	Údisable_weight_init.Lineari`  c                óz  <€ \         P                  P                  '       d2   \        V 4      P                  \
        P                  P                  Jd   \        SV `!  WW4V4       R # \        P                  P                  P                  V 4       Wn        W n        R V n        R V n        W0n        WPn        WPn        R # r   )r	   rÂ   Úaimdo_enabledÚtypeÚ_load_from_state_dictrX  ÚLinearr?  r"  r   r(   ÚModuleÚin_featuresÚout_featuresrT   ru   r`  Úweight_comfy_model_dtypeÚbias_comfy_model_dtype)r!  rv  rw  ru   rS   rR   rB  s   &&&&&&€r   r"  Ú#disable_weight_init.Linear.__init__b  sŒ   ø€ ä×+Ñ+×9×9Ð9Ü˜“:×3Ñ3Ô;N×;UÑ;U×;kÑ;kÓkÜ‘Ñ  ¸DÈ%ÔPÙô �H‰H�O‰O×$Ñ$ TÔ*Ø*ÔØ ,ÔØˆDŒKØˆDŒIØ+/Ô*Ø,1Ô)Ø*/Ö'r   c                óJ  <€ \         P                  P                  '       d2   \        V 4      P                  \
        P                  P                  Jd   \        SV `  WW4WVV4      # \
        P                  V VVVVVV P                  V P                  3V P                  3R 7       R# ))rh  ri  N)r	   rÂ   rq  rr  rs  rX  rt  r?  rl  rv  rw  ©	r!  rc  rd  re  Ústrictrf  rg  Ú
error_msgsrB  s	   &&&&&&&&€r   rs  Ú0disable_weight_init.Linear._load_from_state_dictw  s™   ø€ ô ×+Ñ+×9×9Ð9Ü˜“:×3Ñ3Ô;N×;UÑ;U×;kÑ;kÓkÜ‘wÑ4°ZÈØ5AÐT^ó`ð `ä×:Ñ:ØØØØØØØ"×.Ñ.°×0AÑ0AÐBØ ×-Ñ-Ð/ð ;ö 	r   c                ó   € R # r   r   ©r!  s   &r   Úreset_parametersÚ+disable_weight_init.Linear.reset_parametersŠ  ó   € Ùr   c                óÂ   € \        WR R7      ;_uu_ 4       w  r#\        P                  P                  P	                  WV4      uuRRR4       #   + '       g   i     R# ; i©T©r   N)r  r   r(   r)   Úlinear©r!  rU   rT   ru   s   &&  r   Úforward_comfy_cast_weightsÚ5disable_weight_init.Linear.forward_comfy_cast_weights�  s=   € Ü& tÀ×EÕEÉÈ&Ü—x‘x×*Ñ*×1Ñ1°%ÀÓF÷ F×E×EÓEús   –,AÁA	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # ©r"   ©r   r;  r&   r  r  rŠ  r?  Úforward©r!  r   r,   rB  s   &*,€r   r�  Ú"disable_weight_init.Linear.forward‘  óc   ø€ ÜŒNØ×&×&Ð&¬#¨d×.BÑ.BÓ*CÀaÔ*GÌ3Èt×OaÑOaÓKbÐefÔKfØ×6Ò6¸ÐGÀÑGÐGä‘w’¨Ð7°Ñ7Ð7r   )ru   ry  r`  rv  rw  rT   rx  ©TNN)r2  r3  r4  r5  r"  rs  r‚  rŠ  r�  r6  r7  rU  rV  s   @@r   rt  ro  `  s$   ù‡ € ÷	0õ*	ò&	ò	G÷	8õ 	8r   rt  c                   ó>   a a€ ] tR tRt oR tR tV 3R ltRtVtV ;t	# )Údisable_weight_init.Conv1di˜  c                ó   € R # r   r   r�  s   &r   r‚  Ú+disable_weight_init.Conv1d.reset_parameters™  r„  r   c                ó’   € \        WR R7      ;_uu_ 4       w  r#V P                  WV4      uuRRR4       #   + '       g   i     R# ; ir†  ©r  Ú_conv_forwardr‰  s   &&  r   rŠ  Ú5disable_weight_init.Conv1d.forward_comfy_cast_weightsœ  ó3   € Ü& tÀ×EÕEÉÈ&Ø×)Ñ)¨%¸Ó>÷ F×E×EÓEúó	   –5µA	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú"disable_weight_init.Conv1d.forward   r’  r   r   ©
r2  r3  r4  r5  r‚  rŠ  r�  r6  r7  rU  rV  s   @@r   ÚConv1dr•  ˜  ó   ù‡ € ò	ò	?÷	8õ 	8r   r¡  c                   ó>   a a€ ] tR tRt oR tR tV 3R ltRtVtV ;t	# )Údisable_weight_init.Conv2di§  c                ó   € R # r   r   r�  s   &r   r‚  Ú+disable_weight_init.Conv2d.reset_parameters¨  r„  r   c                ó’   € \        WR R7      ;_uu_ 4       w  r#V P                  WV4      uuRRR4       #   + '       g   i     R# ; ir†  r™  r‰  s   &&  r   rŠ  Ú5disable_weight_init.Conv2d.forward_comfy_cast_weights«  rœ  r�  c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú"disable_weight_init.Conv2d.forward¯  r’  r   r   r   rV  s   @@r   ÚConv2dr¤  §  r¢  r   r«  c                   óR   a a€ ] tR tRt oR tRV 3R lltRR ltV 3R ltRtVt	V ;t
# )Údisable_weight_init.Conv3di¶  c                ó   € R # r   r   r�  s   &r   r‚  Ú+disable_weight_init.Conv3d.reset_parameters·  r„  r   c                óú  <€ VR 8X  d#   VRRVP                   ^,          ) R1RR3,          p\        '       d²   VP                  \        P                  \        P
                  39   dƒ   \        P                  ! WV P                  V P                  V P                  V P                  RRRR7	      pVe7   WsP                  RRVP                  ^,
          ,          ,           4      ,          pV# \        SV `8  ! WV.VO5/ VB # )Úcausal_zero:NNNNFT)Ú	benchmarkÚdeterministicÚ
allow_tf32)r   r6   )r   )r   Ú!NVIDIA_MEMORY_CONV_BUG_WORKAROUNDrR   r   Úfloat16Úbfloat16Úcudnn_convolutionÚpaddingÚstrideÚdilationÚgroupsÚreshapeÚndimr?  rš  )	r!  rU   rT   ru   Úautopadr   r,   ÚoutrB  s	   &&&&&*, €r   rš  Ú(disable_weight_init.Conv3d._conv_forwardº  sÞ   ø€ Ø˜-Ô'Ø  1 u§{¡{°1¥~ oÑ&6¸¸1Ð <Õ=�ß0Ó0°V·\±\ÄeÇmÁmÔUZ×UcÑUcÐEdÔ5dÜ×-Ò-¨e¸T¿\¹\È4Ï;É;ÐX\×XeÑXeÐgk×grÑgrð  Dð  TYð  fjô  k�ØÒ#ØŸ<™<¨°$¸#¿(¹(ÀQ½,Õ2GÕ(GÓHÕH�CØ�
ä‘wÒ,¨U¸DÐRÀ4ÒRÈ6ÑRÐRr   c           	     ó”   € \        WR R7      ;_uu_ 4       w  r4V P                  WWBR7      uuRRR4       #   + '       g   i     R# ; i)Tr‡  )r¿  Nr™  )r!  rU   r¿  rT   ru   s   &&&  r   rŠ  Ú5disable_weight_init.Conv3d.forward_comfy_cast_weightsÅ  s6   € Ü& tÀ×EÕEÉÈ&Ø×)Ñ)¨%¸Ð)ÓO÷ F×E×EÓEús	   –6¶A	c                óö   <€ \        4        V P                  '       g<   \        V P                  4      ^ 8”  g"   \        V P                  4      ^ 8”  g   RV9   d   V P
                  ! V/ VB # \        SV `  ! V/ VB # )r"   r¿  rŽ  r�  s   &*,€r   r�  Ú"disable_weight_init.Conv3d.forwardÉ  sl   ø€ ÜŒNØ×&×&Ð&¬#¨d×.BÑ.BÓ*CÀaÔ*GÌ3Èt×OaÑOaÓKbÐefÔKfÐjsÐw}Ôj}Ø×6Ò6¸ÐGÀÑGÐGä‘w’¨Ð7°Ñ7Ð7r   r   r   )r2  r3  r4  r5  r‚  rš  rŠ  r�  r6  r7  rU  rV  s   @@r   ÚConv3dr­  ¶  s    ù‡ € ò	÷		Sô	P÷	8õ 	8r   rÆ  c                   ó>   a a€ ] tR tRt oR tR tV 3R ltRtVtV ;t	# )Údisable_weight_init.GroupNormiÐ  c                ó   € R # r   r   r�  s   &r   r‚  Ú.disable_weight_init.GroupNorm.reset_parametersÑ  r„  r   c           	     óì   € \        WR R7      ;_uu_ 4       w  r#\        P                  P                  P	                  WP
                  W#V P                  4      uuRRR4       #   + '       g   i     R# ; ir†  )r  r   r(   r)   Ú
group_normÚ
num_groupsÚepsr‰  s   &&  r   rŠ  Ú8disable_weight_init.GroupNorm.forward_comfy_cast_weightsÔ  sJ   € Ü& tÀ×EÕEÉÈ&Ü—x‘x×*Ñ*×5Ñ5°e¿_¹_ÈfÐ\`×\dÑ\dÓe÷ F×E×EÓEús   –AA"Á"A3	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú%disable_weight_init.GroupNorm.forwardØ  r’  r   r   r   rV  s   @@r   Ú	GroupNormrÈ  Ð  s   ù‡ € ò	ò	f÷	8õ 	8r   rÒ  c                   ó>   a a€ ] tR tRt oR tR tV 3R ltRtVtV ;t	# )Údisable_weight_init.BatchNorm2diß  c                ó   € R # r   r   r�  s   &r   r‚  Ú0disable_weight_init.BatchNorm2d.reset_parametersà  r„  r   c                ó  € \        WR R7      ;_uu_ 4       w  r#V P                  e2   V P                  P                  VP                  VP                  R7      MRpV P
                  e2   V P
                  P                  VP                  VP                  R7      MRp\        P                  P                  P                  WWRW0P                  V P                  V P                  4      uuRRR4       #   + '       g   i     R# ; i)Tr‡  NrZ  )r  Úrunning_meanrÜ   rS   rR   Úrunning_varr   r(   r)   Ú
batch_normÚtrainingÚmomentumrÎ  )r!  rU   rT   ru   rØ  rÙ  s   &&    r   rŠ  Ú:disable_weight_init.BatchNorm2d.forward_comfy_cast_weightsã  sÙ   € Ü& tÀ×EÕEÉÈ&Ø`d×`qÑ`qÒ`}˜t×0Ñ0×3Ñ3¸5¿<¹<ÈvÏ|É|Ð3Ô\ð  DH�Ø^b×^nÑ^nÒ^z˜d×.Ñ.×1Ñ1¸¿¹ÈVÏ\É\Ð1ÔZð  AE�Ü—x‘x×*Ñ*×5Ñ5°eÈ;Ð`d×fsÑfsÐuy÷  vCñ  vCð  EI÷  EMñ  EMó  N÷ F×E×EÓEús   –CC0Ã0D	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú'disable_weight_init.BatchNorm2d.forwardé  r’  r   r   r   rV  s   @@r   ÚBatchNorm2drÔ  ß  s   ù‡ € ò	ò	N÷	8õ 	8r   rà  c                   ó>   a a€ ] tR tRt oR tR tV 3R ltRtVtV ;t	# )Údisable_weight_init.LayerNormið  c                ó   € R # r   r   r�  s   &r   r‚  Ú.disable_weight_init.LayerNorm.reset_parametersñ  r„  r   c           	     ó  € \        V P                  e   T MR VRR7      ;_uu_ 4       w  r#\        P                  P                  P                  WP                  W#V P                  4      uuR R R 4       #   + '       g   i     R # ; i©NTr‡  )r  rT   r   r(   r)   Ú
layer_normÚnormalized_shaperÎ  r‰  s   &&  r   rŠ  Ú8disable_weight_init.LayerNorm.forward_comfy_cast_weightsô  s^   € Ü&¨t¯{©{Ò/F¡tÈDÐRWÐei×jÕjÑn|ÐouÜ—x‘x×*Ñ*×5Ñ5°e×=RÑ=RÐTZÐbf×bjÑbjÓk÷ k×j×jÓjús   §AA3Á3B	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú%disable_weight_init.LayerNorm.forwardø  r’  r   r   r   rV  s   @@r   Ú	LayerNormrâ  ð  s   ù‡ € ò	ò	l÷	8õ 	8r   rì  c                   óD   a a€ ] tR tRt oR tR tR tV 3R ltRtVt	V ;t
# )Údisable_weight_init.RMSNormiÿ  c                ó   € R V n         R # r   ©ru   r�  s   &r   r‚  Ú,disable_weight_init.RMSNorm.reset_parameters   ó   € ØˆDŒIÙr   c                ó>   € \        V P                  e   T MR VRR7      # ræ  )r  rT   ©r!  rU   s   &&r   Úcast_weightÚ'disable_weight_init.RMSNorm.cast_weight  s   € Ü(°·±Ò1H©ÈdÐTYÐgkÔlÐlr   c                óò   € V P                  V4      ;_uu_ 4       w  r#\        P                  P                  P	                  WP
                  W P                  4      uuR R R 4       #   + '       g   i     R # ; ir   )rõ  r   r(   r)   Úrms_normrè  rÎ  r‰  s   &&  r   rŠ  Ú6disable_weight_init.RMSNorm.forward_comfy_cast_weights  sM   € Ø×!Ñ! %×(Ô(©N¨VÜ—x‘x×*Ñ*×3Ñ3°E×;PÑ;PÐRX×ZbÑZbÓc÷ )×(×(Ó(ús   šA A%Á%A6	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú#disable_weight_init.RMSNorm.forward  r’  r   rð  )r2  r3  r4  r5  r‚  rõ  rŠ  r�  r6  r7  rU  rV  s   @@r   ÚRMSNormrî  ÿ  s    ù‡ € ò	ò	mò	d÷	8õ 	8r   rü  c                   óB   a a€ ] tR tRt oR tRR ltV 3R ltRtVtV ;t	# )Ú#disable_weight_init.ConvTranspose2di  c                ó   € R # r   r   r�  s   &r   r‚  Ú4disable_weight_init.ConvTranspose2d.reset_parameters  r„  r   c                ó–  € ^pV P                  WV P                  V P                  V P                  W0P                  4      p\        WRR7      ;_uu_ 4       w  rV\        P                  P                  P                  WW`P                  V P                  W@P                  V P                  4      uuRRR4       #   + '       g   i     R# ; i)r4   Tr‡  N)Ú_output_paddingrº  r¹  Úkernel_sizer»  r  r   r(   r)   Úconv_transpose2dr¼  ©r!  rU   Úoutput_sizeÚnum_spatial_dimsÚoutput_paddingrT   ru   s   &&&    r   rŠ  Ú>disable_weight_init.ConvTranspose2d.forward_comfy_cast_weights  óŽ   € Ø ÐØ!×1Ñ1Ø D§K¡K°·±¸t×?OÑ?OØ §-¡-ó1ˆNô ' tÀ×EÕEÉÈ&Ü—x‘x×*Ñ*×;Ñ;Ø d¯K©K¸¿¹Ø%§{¡{°D·M±MóC÷ F×E×EÓEúó   ÁAB7Â7C	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú+disable_weight_init.ConvTranspose2d.forward!  r’  r   r   r   r   rV  s   @@r   ÚConvTranspose2drþ    ó   ù‡ € ò	ô		C÷	8õ 	8r   r  c                   óB   a a€ ] tR tRt oR tRR ltV 3R ltRtVtV ;t	# )Ú#disable_weight_init.ConvTranspose1di(  c                ó   € R # r   r   r�  s   &r   r‚  Ú4disable_weight_init.ConvTranspose1d.reset_parameters)  r„  r   c                ó–  € ^pV P                  WV P                  V P                  V P                  W0P                  4      p\        WRR7      ;_uu_ 4       w  rV\        P                  P                  P                  WW`P                  V P                  W@P                  V P                  4      uuRRR4       #   + '       g   i     R# ; i)r   Tr‡  N)r  rº  r¹  r  r»  r  r   r(   r)   Úconv_transpose1dr¼  r  s   &&&    r   rŠ  Ú>disable_weight_init.ConvTranspose1d.forward_comfy_cast_weights,  r
  r  c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Ú+disable_weight_init.ConvTranspose1d.forward7  r’  r   r   r   r   rV  s   @@r   ÚConvTranspose1dr  (  r  r   r  c                   óh   a a€ ] tR tRt oR	V 3R lltV 3R ltR tR
R ltR
R ltV 3R lt	Rt
VtV ;t# )Údisable_weight_init.Embeddingi>  c                óH  <€ \         P                  P                  '       d2   \        V 4      P                  \
        P                  P                  Jd   \        SV `!  WW4WVWxWšV4       R # \        P                  P                  P                  V 4       Wn        W n        W0n        W@n        WPn        W`n        Wpn        Ve   TM\        P&                  ! 4       p\        P                  P)                  \        P*                  ! W3RVR7      RR7      V n        R V n        W°n        R # )NrP  rZ  Fr[   )r	   rÂ   rq  rr  rs  rX  Ú	Embeddingr?  r"  r   r(   ru  Únum_embeddingsÚembedding_dimÚpadding_idxÚmax_normÚ	norm_typeÚscale_grad_by_freqÚsparseÚget_default_dtypera   r…   rT   ru   rx  )r!  r  r  r   r!  r"  r#  r$  Ú_weightÚ_freezerS   rR   Úembedding_dtyperB  s   &&&&&&&&&&&& €r   r"  Ú&disable_weight_init.Embedding.__init__?  sé   ø€ ô ×+Ñ+×9×9Ð9Ü˜“:×3Ñ3Ô;N×;XÑ;X×;nÑ;nÓnÜ‘Ñ  ÀØ!*ÀØ!(°%ô9ñ ä�H‰H�O‰O×$Ñ$ TÔ*Ø"0ÔØ!.ÔØ*ÔØ$ŒMØ&ŒNØ&8Ô#Ø ŒKà',Ò'8™e¼e×>UÒ>UÓ>WˆOÜŸ(™(×,Ñ,Ü—’˜^Ð;ÀFÐRaÔbØ#ð -ó ˆDŒKð ˆDŒIØ,1Ö)r   c           
     ó2  <€ \         P                  P                  '       d2   \        V 4      P                  \
        P                  P                  Jd   \        SV `  WW4WVV4      # \
        P                  V VVVVVV P                  V P                  3R 7       R# ))rh  N)r	   rÂ   rq  rr  rs  rX  r  r?  rl  r  r  r|  s	   &&&&&&&&€r   rs  Ú3disable_weight_init.Embedding._load_from_state_dict[  s�   ø€ ô ×+Ñ+×9×9Ð9Ü˜“:×3Ñ3Ô;N×;XÑ;X×;nÑ;nÓnÜ‘wÑ4°ZÈØ5AÐT^ó`ð `ä×:Ñ:ØØØØØØØ"×1Ñ1°4×3EÑ3EÐFð ;ö r   c                ó   € R V n         R # r   rð  r�  s   &r   r‚  Ú.disable_weight_init.Embedding.reset_parametersl  rò  r   c                ó  € TpV P                   P                  \        P                  8X  g*   V P                   P                  \        P                  8X  d   R p\        WP                  VRR7      ;_uu_ 4       w  rE\        P                  P                  P                  WV P                  V P                  V P                  V P                  V P                  4      P                  VR7      uuR R R 4       #   + '       g   i     R # ; i)NT©rS   rR   r   rZ   )rT   rR   r   r¶  r·  r  rS   r(   r)   Ú	embeddingr   r!  r"  r#  r$  rÜ   )r!  rU   Ú	out_dtypeÚoutput_dtyperT   ru   s   &&&   r   rŠ  Ú8disable_weight_init.Embedding.forward_comfy_cast_weightsp  sÛ   € Ø$ˆLØ�{‰{× Ñ ¤E§M¡MÔ1°T·[±[×5FÑ5FÌ%Ï.É.Ô5XØ �	Ü& t·L±LÈ	Ð_c×dÕdÑhvÐioÜ—x‘x×*Ñ*×4Ñ4°UÀD×DTÑDTÐVZ×VcÑVcÐei×esÑesÐuy÷  vMñ  vMð  OS÷  OZñ  OZó  [÷  ^ñ  ^ð  eqð  ^ó  r÷ e×d×dÓdús   Á7A2C4Ã4D	c                ó$  € V P                   VP                  R4      P                  4       ,          P                  VP                  4      pVP
                  ! . VP                  OVP                  R,          N5!  pVf   V# VP                  VR7      # )r   rZ   r6   )rT   r½  ÚcpurÜ   rS   Úviewr   )r!  rU   r1  rì   s   &&& r   Ú	host_rowsÚ'disable_weight_init.Embedding.host_rowsw  sn   € à—‘˜EŸM™M¨"Ó-×1Ñ1Ó3Õ4×7Ñ7¸¿¹ÓEˆAØ—’Ð1˜Ÿ™Ð1 Q§W¡W¨R¥[Ó1ˆAØ!Ò)�1ÐD¨q¯t©t¸)¨tÓ/DÐDr   c                ó  <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # RV9   d   VP                  R4       \        SV `   ! V/ VB # )r"   r1  )	r   r;  r&   r  r  rŠ  rÁ   r?  r�  r�  s   &*,€r   r�  Ú%disable_weight_init.Embedding.forward}  sw   ø€ ÜŒNØ×&×&Ð&¬#¨d×.BÑ.BÓ*CÀaÔ*GÌ3Èt×OaÑOaÓKbÐefÔKfØ×6Ò6¸ÐGÀÑGÐGà &Ô(Ø—J‘J˜{Ô+Ü‘w’¨Ð7°Ñ7Ð7r   )
ru   r  r!  r"  r  r   r#  r$  rT   rx  )	NNg       @FFNFNNr   )r2  r3  r4  r5  r"  rs  r‚  rŠ  r7  r�  r6  r7  rU  rV  s   @@r   r  r  >  s*   ù‡ € ÷	2õ8	ò"	ô	rô	E÷	8õ 	8r   r  c                ó‚   € V^8X  d   V P                   ! V/ VB # V^8X  d   V P                  ! V/ VB # \        RV 24      h)r4   zunsupported dimensions: )r«  rÆ  r   )rc   Údimsr   r,   s   &&*,r   Úconv_ndÚdisable_weight_init.conv_nd†  sK   € à�1Œ9Ø—8’8˜TÐ, VÑ,Ð,Ø�QŒYØ—8’8˜TÐ, VÑ,Ð,äÐ7¸°vÐ>Ó?Ð?r   r   r   )r2  r3  r4  r5  Ústaticmethodr\  rl  r   r(   rt  r:  r¡  r«  rÆ  rÒ  rà  rì  rü  r  r  r  Úclassmethodr=  r6  r7  r8  s   @r   rX  rX  >  sR  ø‡ € ØñCó ðCð
 ó1ó ð1ô468�—‘—‘Ð"2ô 68ôp8�—‘—‘Ð"2ô 8ô8�—‘—‘Ð"2ô 8ô8�—‘—‘Ð"2ô 8ô48�E—H‘H×&Ñ&Ð(8ô 8ô8�e—h‘h×*Ñ*Ð,<ô 8ô"8�E—H‘H×&Ñ&Ð(8ô 8ô8�%—(‘(×"Ñ"Ð$4ô 8ô&8˜%Ÿ(™(×2Ñ2Ð4Dô 8ô,8˜%Ÿ(™(×2Ñ2Ð4Dô 8ô,F8�E—H‘H×&Ñ&Ð(8ô F8ðP ñ@ó ö@r   rX  c                   óä  € ] tR tRt ! R R]P
                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R	]P                  4      t ! R
 R]P                  4      t	 ! R R]P                  4      t
 ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tRtR# )Úmanual_casti�  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.Lineari‘  Tr   N©r2  r3  r4  r5  r;  r6  r   r   r   rt  rD  ‘  ó   † Ø!Ôr   rt  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.Conv1di”  Tr   NrE  r   r   r   r¡  rH  ”  rF  r   r¡  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.Conv2di—  Tr   NrE  r   r   r   r«  rJ  —  rF  r   r«  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.Conv3diš  Tr   NrE  r   r   r   rÆ  rL  š  rF  r   rÆ  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.BatchNorm2di�  Tr   NrE  r   r   r   rà  rN  �  rF  r   rà  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.GroupNormi   Tr   NrE  r   r   r   rÒ  rP     rF  r   rÒ  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.LayerNormi£  Tr   NrE  r   r   r   rì  rR  £  rF  r   rì  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.ConvTranspose2di¦  Tr   NrE  r   r   r   r  rT  ¦  rF  r   r  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.ConvTranspose1di©  Tr   NrE  r   r   r   r  rV  ©  rF  r   r  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.RMSNormi¬  Tr   NrE  r   r   r   rü  rX  ¬  rF  r   rü  c                   ó   € ] tR tRtRtRtR# )Úmanual_cast.Embeddingi¯  Tr   NrE  r   r   r   r  rZ  ¯  rF  r   r  r   N)r2  r3  r4  r5  rX  rt  r¡  r«  rÆ  rà  rÒ  rì  r  r  rü  r  r6  r   r   r   rB  rB  �  sØ   † ô"Ð$×+Ñ+ô "ô"Ð$×+Ñ+ô "ô"Ð$×+Ñ+ô "ô"Ð$×+Ñ+ô "ô"Ð)×5Ñ5ô "ô"Ð'×1Ñ1ô "ô"Ð'×1Ñ1ô "ô"Ð-×=Ñ=ô "ô"Ð-×=Ñ=ô "ô"Ð%×-Ñ-ô "ô"Ð'×1Ñ1÷ "ð "r   rB  c           
     óò  € \         P                  P                  V P                  .4      ^ ,          P                  pV\
        P                  39  d   R# VP                  pVP                  pVP                  ^8H  pV'       d   VP                  R	V^,          4      pVP                  ^8w  d   R# \         P                  P                  VP                  4      p\
        P                  ! R
VP                  \
        P                  R7      p\
        P                  ! R
VP                  \
        P                  R7      p\
        P                  ! VRRVR7      pVP!                  V4      P#                  4       p	\$        P&                  ! Wƒ\)        V	P                  4      R7      p
\+        V	RV
4      p\-        WW#RVRR7      ;_uu_ 4       w  rÍ\)        VP                  4      p\$        P&                  ! WsVR7      p\+        VRV4      p\
        P.                  P0                  P3                  VVV4      pRRR4       V'       d*   XP                  V^ ,          V^,          X^ ,          34      pX#   + '       g   i     LC; i)zd
Legacy FP8 linear function for backward compatibility.
Uses QuantizedTensor subclass for dispatch.
NrZ  iÀ  )ÚminÚmaxrÀ  )rã   r  Ú
orig_shapeÚTensorCoreFP8LayoutT)rR   rË   r   ró   rõ   r6   r   i@þÿÿ)r	   rÂ   rÃ   rT   rR   r   Úfloat8_e4m3fnr   r¾  r½  r
   Úlora_compute_dtyperS   ÚonesÚfloat32ÚclamprÜ   rD   r_  ÚParamsÚtuplerÝ   r  r(   r)   rˆ  )r!  rU   rR   Úinput_dtypeÚinput_shapeÚ	tensor_3dra  Úscale_weightÚscale_inputÚ	input_fp8Úlayout_params_inputÚquantized_inputÚwru   Úw_shapeÚlayout_params_weightÚquantized_weightÚos   &&                r   Ú
fp8_linearrt  ³  sþ  € ô ×#Ñ#×9Ñ9¸4¿;¹;¸-ÓHÈÕK×QÑQ€EØ”U×(Ñ(Ð)Ô)Ùà—+‘+€KØ—+‘+€KØ—
‘
˜a‘€IçØ—‘˜b +¨a¥.Ó1ˆà‡z�z�Q„ÙÜ×-Ñ-×@Ñ@ÀÇÁÓNÐÜ—:’:˜b¨¯©¼U¿]¹]ÔK€Lä—*’*˜R¨¯©¼E¿M¹MÔJ€KÜ�KŠK˜ 4¨S°eÔ<€EØ—‘˜“×*Ñ*Ó,€IÜ-×4Ò4¸;ÔkpÐqz÷  rAñ  rAó  lBô  CÐÜ% iÐ1FÐH[Ó\€Oä	˜t°%Ð]að  rDð  SW÷  
Xõ  
Xñ  \eð  ]^ô ˜Ÿ™“.ˆÜ2×9Ò9ÀÐqxÔyÐÜ*¨1Ð.CÐEYÓZÐÜ�H‰H×Ñ×&Ñ& Ð8HÈ$ÓOˆ÷ 
X÷ Ø�I‰I�{ 1•~ {°1¥~°w¸qµzÐBÓCˆà€H÷ 
X÷  
Xús   ÇA(I&É&I6	c                   ó@   € ] tR tRt ! R R]P
                  4      tRtR# )Úfp8_opsiÜ  c                   ó,   a € ] tR tRt o R tR tRtV tR# )Úfp8_ops.LineariÝ  c                ó"   € R V n         R V n        R # r   )rj  rk  r�  s   &r   r‚  Úfp8_ops.Linear.reset_parametersÞ  s   € Ø $ˆDÔØ#ˆDÔÙr   c                óÊ  € \        V P                  4      ^ 8X  d.   \        V P                  4      ^ 8X  d    \        W4      pVe   V#  \        WRR7      ;_uu_ 4       w  rE\        P                  P                  P                  WV4      uuRRR4       #   \         d0   p\
        P                  ! RP                  T4      4        Rp?L�Rp?ii ; i  + '       g   i     R# ; i)r"   NzException during fp8 op: {}Tr‡  )r&   r  r  rt  Ú	ExceptionÚloggingÚinfor*  r  r   r(   r)   rˆ  )r!  rU   rÀ  ÚerT   ru   s   &&    r   rŠ  Ú)fp8_ops.Linear.forward_comfy_cast_weightsã  s¸   € Ü�4×'Ñ'Ó(¨AÔ-´#°d×6HÑ6HÓ2IÈQÔ2NðJÜ$ TÓ1�CØ’Ø"˜
ð 'ô
 ' tÀ×EÕEÉÈ&Ü—x‘x×*Ñ*×1Ñ1°%ÀÓF÷ FÒEøô !ô JÜ—L’LÐ!>×!EÑ!EÀaÓ!H×IÑIûðJú÷ F×EÐEús#   ¶B Á,CÂCÂ%C	Ã	CÃC"	)rk  rj  N)r2  r3  r4  r5  r‚  rŠ  r6  r7  r8  s   @r   rt  rx  Ý  s   ø‡ € ò	÷

	Gð 
	Gr   rt  r   N)r2  r3  r4  r5  rB  rt  r6  r   r   r   rv  rv  Ü  s   † ôG�×#Ñ#÷ Gð Gr   rv  )ÚCublasLinearÚcublas_half_matmulc                   óB   € ] tR tRt ! R R]]P                  4      tRtR# )Ú
cublas_opsi÷  c                   ó>   a a€ ] tR tRt oR tR tV 3R ltRtVtV ;t	# )Úcublas_ops.Lineariø  c                ó   € R # r   r   r�  s   &r   r‚  Ú"cublas_ops.Linear.reset_parametersù  ó   € Ùr   c           	     ó°   € \        WR R7      ;_uu_ 4       w  r#\        WW0P                  V P                  4      uuRRR4       #   + '       g   i     R# ; ir†  )r  r‚  Ú_epilogue_strÚhas_biasr‰  s   &&  r   rŠ  Ú,cublas_ops.Linear.forward_comfy_cast_weightsü  s<   € Ü*¨4ÀD×IÕIÉ^ÈfÜ-¨e¸T×CUÑCUÐW[×WdÑWdÓe÷ J×I×IÓIús   –#AÁA	c                óè   <€ \        4        V P                  '       g5   \        V P                  4      ^ 8”  g   \        V P                  4      ^ 8”  d   V P
                  ! V/ VB # \        SV `  ! V/ VB # r�  rŽ  r�  s   &*,€r   r�  Úcublas_ops.Linear.forward   sd   ø€ Ü”Ø×*×*Ð*¬c°$×2FÑ2FÓ.GÈ!Ô.KÌsÐSW×SeÑSeÓOfÐijÔOjØ×:Ò:¸DÐKÀFÑKÐKä ™7š?¨DÐ;°FÑ;Ð;r   r   r   rV  s   @@r   rt  r†  ø  s   ù‡ € òòf÷<õ <r   rt  r   N)r2  r3  r4  r5  r�  rB  rt  r6  r   r   r   r„  r„  ÷  s   † ô	<�\ ;×#5Ñ#5÷ 	<ð 	<r   r„  )Ú	quant_ops)rÝ   ÚQUANT_ALGOSr_  Úget_layout_classc                 óœ   € V P                  ^RR7      w  r\        P                  P                  P	                  V4      P                  V4      # )r4   r   r6   )Úchunkr   r(   r)   ÚsiluÚmul_)rì   ÚgateÚups   &  r   Ú_swiglu_eagerr™    s<   € Ø�w‰w�q˜bˆwÓ!�H€DÜ�8‰8×Ñ×#Ñ# DÓ)×.Ñ.¨rÓ2Ð2r   c                 óü   € V P                  ^RR7      w  rR\        P                  ! VR,          4      ,          \        P                  ! V4      ,          R\        P                  ! VR,          4      ,          ,          # )r4   r   g      @g      9@r6   )r”  r   ÚtanhÚsigmoid)rì   Úhiddenr—  s   &  r   Ú_situ_eagerrž    sV   € à—7‘7˜1 "�7Ó%�L€FØ”%—*’*˜T C�ZÓ(Õ(¬5¯=ª=¸Ó+>Õ>À4Ì%Ï*Ê*ÐU[Ð^bÕUbÓJcÕCcÕdÐdr   Ú	gelu_tanhc                 óX   € \         P                  P                  P                  V R R7      # )r›  )Úapproximate)r   r(   r)   Úgelu©rì   s   &r   Ú<lambda>r¤    s   € œ5Ÿ8™8×.Ñ.×3Ñ3°AÀ6Ð3ÔJr   ÚswigluÚsituc              #   óÒ  "  € VR 8X  dÃ   \        V\        P                  P                  4      '       d™   VP	                  V 4      ;_uu_ 4       w  rEVP
                  pVf^   \        P                  ! V P                  \        P                  8X  d   \        P                  M\        P                  4      P
                  pWF3x € RRR4       R# W#3x € R#   + '       g   i     R# ; i5i)rø  N)
rÇ   r   r(   ru  rõ  rÎ  ÚfinforR   Úfloat64rc  )rì   Ú	input_actÚ
act_weightÚact_epsrT   Ú_rÎ  s   &&&&   r   Ú_input_act_weightr®  $  sš   é € à�JÔ¤:¨j¼%¿(¹(¿/¹/×#JÒ#JØ×#Ñ# A×&Ô&©+¨6Ø—.‘.ˆCØŠ{Ü—k’k°1·7±7¼e¿m¹mÔ3K¤%§-¢-ÔQV×Q^ÑQ^Ó_×cÑc�Ø�+Ò÷	 'Ñ&ð Ð!Ô!÷ '×&Ð&üs   ‚AC'ÁA5CÃC'ÃC$	Ã	C'c                 óê   € Vf   V # VR8X  dB   \        WW#4      ;_uu_ 4       w  rE\        P                  P                  WV4      uuR R R 4       # \        V,          ! V 4      #   + '       g   i     L#; i)Nrø  )r®  r	   Úrmsnormrø  ÚINPUT_ACT_EAGER)rì   rª  r«  r¬  rT   rÎ  s   &&&&  r   Ú_eager_input_actr²  0  s]   € ØÒØˆØ�JÔÜ˜q¨Z×AÔAÁ]ÀfÜ—=‘=×)Ñ)¨!°SÓ9÷ BÒAä˜9Ö% aÓ(Ð(÷ B×Aús   ¢"A"Á"A2	c                 ó<   € Vf   V # \         P                  ! WV4      # r   )r   Úaddcmul)rÀ  ÚresidualÚresidual_scales   &&&r   Ú_linear_residualr·  9  s   € ØÒ"ˆ3ÐT¬¯ª°hÀ^Ó(TÐTr   c                óÒ   € \        \        P                  P                  P                  RR4      ;'       d2    V P
                  \        P                  8H  ;'       d    V P                  # )z¬kitchen's fp16-accumulate GEMM replaces a plain linear when the user opted into
fp16 accumulation and the activation is fp16 on CUDA; weights come through cast_bias_weight.Úallow_fp16_accumulationF)r]   r   rF   rG   ÚmatmulrR   r¶  Úis_cudar£  s   &r   Ú_fp16_linear_wantedr¼  =  sQ   € ô ”E—N‘N×'Ñ'×.Ñ.Ð0IÈ5ÓQ÷ 7ð 7Ø—‘œ5Ÿ=™=Ñ(÷7ð 7Ø-.¯Y©Yð8r   c                 ó  € \         P                  P                  '       g   \        V\        4      '       g   \
        # \        WWE4      ;_uu_ 4       w  rEVP                  WW4VWgR 7      uuRRR4       #   + '       g   i     R# ; i))rª  Úinput_act_weightÚinput_act_epsrµ  r¶  N)r	   r
   Úin_trainingrÇ   rÝ   ÚNotImplementedr®  Úlinear_input_act)rì   rT   ru   rª  r«  r¬  rµ  r¶  s   &&&&&&&&r   Ú_try_quantized_linear_input_actrÃ  D  se   € ä×Ñ×)×)Ð)´¸FÄO×1TÒ1TÜÐÜ	˜1¨×	=Ô	=ÑAVÀ*Ø×&Ñ&Ø˜yÐU\Øð 'ó 
÷ 
>×	=×	=Ó	=ús   ÁA3Á3B	c                   óT   a € ] tR tRt o RtRt]R 4       tR tR t	R t
RR	 ltR
tV tR# )ÚCombinedLineariO  aM  A residency-scoped combination of consecutive VBAR linears, not a parameter-owning module.

All members must belong to one prefetch unit, so they fault, resolve and miss together.
Once combined, the members' weights are gone and the model is committed to the
combined weight until a miss reloads them. The model owns the split path.
Fc                ó<   € V P                   ^ ,          P                  # r�  )r¿   Ú_full_precision_mmr�  s   &r   rÇ  Ú!CombinedLinear._full_precision_mmY  s   € ð �|‰|˜A�×1Ñ1Ð1r   c                ó`   € Wn         R V n        R V n        V P                    F	  pWn        K  	  R # r   )r¿   rj   rº   r¹   )r!  r¿   rˆ  s   && r   r"  ÚCombinedLinear.__init___  s)   € ØŒØˆŒØˆŒØ—l”lˆFØ&*Ö#ó #r   c                óì  € . pV P                    F�  p\        V\        4      '       dS   \        VP                  \        4      '       d3   VP
                  '       g!   VP                  '       g   VP                  e    R# VP                  R9  d    R# VP                  VP                  4       KŸ  	  V P                   ^ ,          P                  ^ ,          \        R V P                    4       4      \        R V P                    4       4      3p\        P                  P                  V\         P"                  P%                  WA4      4      pVf   R# W@n
        WPn        V P                    F
  pRVn        K  	  R# )zMConcatenate the members' resident weights in place; they are gone afterwards.Nc              3   óF   "  € T F  qP                   ^,          x € K  	  R# 5ir�   ©rj   ©r8   rc   s   & r   r:   Ú)CombinedLinear.prepare.<locals>.<genexpr>s  s   é € Ð'F¹°A¯©¨Q¯ª»ùó   ‚!c              3   óF   "  € T F  qP                   ^,          x € K  	  R# 5i)r4   NrÍ  rÎ  s   & r   r:   rÏ  s  s   é € ÐKjÑ]iÐXYÏDÉDÐQRÏGÊGÓ]iùrÐ  )Úint8_tensorwiseÚasym_w4a8_int8Ú	w6a8_int8)r¿   rÇ   ÚMixedPrecisionOprº   rÝ   rÇ  r<  ru   Úquant_formatrÀ   rj   r\  rB   r	   rÂ   Úcat_gatheredrˆ   r   r‰   )r!  rS   r—   rc   r   rT   s   &&    r   r¡   ÚCombinedLinear.preparef  s
  € àˆØ—”ˆAä˜qÔ"2×3Ò3¼:ÀaÇkÁkÔSb×;cÒ;cØ×+×+Ð+¨q×/D×/DÐ/DÈÏÉÒHZÚà�~‰~Ð%WÔWÚØ�N‰N˜1Ÿ;™;Ö'ñ ð �\‰\˜!�_×Ñ Õ"¤CÑ'F¸¿ºÓ'FÓ$FÌÑKjÐ]a×]iÒ]iÓKjÓHjÐkˆÜ×(Ñ(×5Ñ5°g¼{×?PÑ?P×?`Ñ?`ÐabÓ?kÓlˆØŠ>ÙØŒØŒØ—”ˆAØˆAŽKó r   c                ó&   € V P                  VR 4      # r   )rÂ  )r!  rì   s   &&r   r�  ÚCombinedLinear.forward|  s   € Ø×$Ñ$ Q¨Ó-Ð-r   Nc           
     óz  € \        4        V P                  f   \        # V P                  '       dt   \	        WW44      p\
        P                  P                  P                  WP                  P                  VP                  R7      P                  4       4      p\        WuV4      # \        WP                  R W#VWV4      # )NrZ   )r   rº   rÁ  rÇ  r²  r   r(   r)   rˆ  rÜ   rR   rÞ   r·  rÃ  )r!  rì   rª  r«  r¬  rµ  r¶  rÀ  s   &&&&&&& r   rÂ  ÚCombinedLinear.linear_input_act  s”   € äŒØ�>‰>Ò!Ü!Ð!Ø×"×"Ð"Ü  ¨zÓCˆAÜ—(‘(×%Ñ%×,Ñ,¨Q·±×0AÑ0AÈÏÉÐ0AÓ0P×0[Ñ0[Ó0]Ó^ˆCÜ# C°>ÓBÐBÜ.¨q·.±.À$È	Ð_fØ/7óIð 	Ir   )rj   rº   r¿   ©Nr=   NN)r2  r3  r4  r5  rR  Úcomfy_gemv_optimizedÚpropertyrÇ  r"  r¡   r�  rÂ  r6  r7  r8  s   @r   rÅ  rÅ  O  s?   ø‡ € ñð !Ðàñ2ó ð2ò
+òò,.÷
Iò 
Ir   rÅ  c                 óò  € \        V \        4      '       d   V P                  WW4WV4      # \        4        V P                  p\        V\
        4      p\        P                  P                  '       g@   \        V R R4      '       g-   \        V RR4      '       g   V'       g0   \        V4      '       g   V ! \        WW44      4      p	\        W•V4      # \        YRV'       d   VP                  MRVR7      ;_uu_ 4       w  rzV'       g   \        WW¢W4WV4      uuRRR4       # \!        WW¢W4WV4      p	V	\"        Jd   V	uuRRR4       #  RRR4       V ! \        WW44      4      p	\        W•V4      #   + '       g   i     L.; i)r<  FrÇ  TN©r   ró   rõ   )rÇ   rÅ  rÂ  r   rT   rÝ   r	   r
   rÀ  r]   r¼  r²  r·  r  rR   Úlinear_input_act_rÃ  rÁ  )rˆ  rì   rª  r«  r¬  rµ  r¶  rT   Ú	quantizedrÀ  ru   s   &&&&&&&    r   rÂ  rÂ  Œ  sR  € ä�&œ.×)Ò)Ø×&Ñ& q°ZÈ(ÓcÐcÜ„NØ�]‰]€FÜ˜6¤?Ó3€IÜ×Ñ×*×*Ð*¬g°fÐ>SÐUZ×.[Ò.[Ô_fÐgmð  pDð  FK÷  `Lò  `LßÔ!4°Q×!7Ò!7ÙÔ% a°JÓHÓIˆÜ ¨~Ó>Ð>ä	˜v°dß8A¨Q¯WªWÀtØ,5÷
7õ 
7á:H¸6ßÜ$ Q°ÀØ%-ó?÷	
7ò 
7ô .¨a¸È*Ø.6óHˆà”nÓ$Ø÷
7ò 
7ð %÷
7ñ Ô! !°
ÓDÓ
E€CÜ˜C¨>Ó:Ð:÷
7÷ 
7ús   Ã8E&ÄE&Å&E6	Úfp16_accumulationc          
     óü  € Vf/   Vf+   \         P                  P                  P                  WV4      # \	        WW#WEWg4      p	V	\
        Jd   V	# \        WWE4      p \        P                  P                  '       gR   \        V\        4      '       g<   V'       d4   \        V 4      '       d#   \        P                  P                  WW&VR7      # \!        \         P                  P                  P                  WV4      Wg4      # )zí``linear(act(x))`` with optional residual addcmul, fused where the weight supports it.

Linear weights are already prepared. An RMSNorm module in act_weight owns
its norm-weight cast context; a tensor uses the supplied act_eps directly.
)rµ  r¶  )r   r(   r)   rˆ  rÃ  rÁ  r²  r	   r
   rÀ  rÇ   rÝ   r¼  r�  ÚckÚfp16_linearr·  )
rì   rT   ru   rª  r«  r¬  rµ  r¶  rä  rÀ  s
   &&&&&&&&$ r   râ  râ  ¨  sÇ   € ð Ò˜XÒ-Ü�x‰x×"Ñ"×)Ñ)¨!°TÓ:Ð:ä
)¨!°TÀjØ*2óD€Cà
”.Ó Øˆ
Ü˜ zÓ;€Aä×"Ñ"×.×.Ð.Ü˜v¤×7Ò7ß!Ô&9¸!×&<Ò&<Ü�|‰|×'Ñ'¨°4Ð[iÐ'ÓjÐjäœEŸH™H×/Ñ/×6Ñ6°qÀ$ÓGÈÓbÐbr   rÓ  rÔ  c                   óŠ   a € ] tR tRt o Rt]R 4       t]]P                  P                  P                  R 4       4       tRtV tR# )ÚQuantLinearFunciÅ  aå  Custom autograd function for quantized linear: quantized forward, optionally FP8 backward.

When training_fp8_bwd is enabled:
  - Forward: quantize input per layout (FP8/NVFP4), use quantized matmul
  - Backward: all matmuls use FP8 tensor cores via torch.mm dispatch
  - Cached input is FP8 (half the memory of bf16)

When training_fp8_bwd is disabled:
  - Forward: quantize input per layout, use quantized matmul
  - Backward: dequantize weight to compute_dtype, use standard matmul
c                óv  € VP                   pVP                  4       P                  ^ R4      pVe   \        P                  ! W„VR7      p	MTp	VP
                  '       d   VP                  4       MTp
Ve#   VP
                  '       d   VP                  4       MTp\        P                  P                  P                  WšV4      p\        V4      ^8”  d   VP                  ^ VRR 4      pWpn        VRJV n        W`n        VP
                  V n        \         P"                  P$                  V n        V P&                  '       dd   \)        V	\        4      '       d   VP+                  R4      '       d   W�n        M\        P                  ! VR4      V n        V P/                  V4       V# RV n        V P/                  W4       V# )r"   N©rã   ÚTensorCoreFP8ÚTensorCoreFP8E4M3Layoutr5   r6   )r   ÚdetachÚflattenrÝ   Ú
from_floatr\   r   r(   r)   rˆ  r&   Ú	unflattenrh  rŒ  ró   Úweight_requires_gradr	   r
   Útraining_fp8_bwdÚfp8_bwdrÇ   Ú
startswithÚq_inputÚsave_for_backward)ÚctxÚinput_floatrT   ru   Úlayout_typeÚinput_scaleró   rh  Úinprö  ro  ÚbÚoutputs   &&&&&&&      r   r�  ÚQuantLinearFunc.forwardÒ  sm  € à!×'Ñ'ˆØ× Ñ Ó"×*Ñ*¨1¨bÓ1ˆð Ò"Ü%×0Ò0°ÈÔU‰GàˆGà%×3×3Ð3ˆF�M‰MŒO¸ˆØ!Ò-°$×2D×2DÐ2DˆD�K‰KŒMÈ$ˆä—‘×$Ñ$×+Ñ+¨G¸Ó:ˆô ˆ{Ó˜aÔØ×%Ñ% a¨°S°bÐ)9Ó:ˆFð &ŒØ 4Ð'ˆŒØ)ÔØ#)×#7Ñ#7ˆÔ Ü×,Ñ,×=Ñ=ˆŒà�;�;ˆ;ä˜'¤?×3Ò3¸×8NÑ8NÈ×8_Ò8_Ø%•ô .×8Ò8¸Ð>WÓX�”Ø×!Ñ! &Ô)ð
 ˆð ˆCŒKØ×!Ñ! +Ô6àˆr   c                ó¼  € V P                   pVP                  ^ R4      P                  V4      pV P                  '       dÞ   V P                  w  p\
        P                  ! VR4      p\        V\
        4      '       d%   VP                  P                  R4      '       d   TpMq\        V\
        4      '       d6   \
        P                  ! VP                  4       P                  V4      R4      pM&\
        P                  ! VP                  V4      R4      pV P                  pMŒV P                  w  r„Tp\        V\
        4      '       d!   VP                  4       P                  V4      pMVP                  V4      pV P                  '       d"   VP                  ^ R4      P                  V4      MRp\        P                  ! WV4      p	\        V P                   4      ^8”  d    V	P#                  ^ V P                   RR 4      p	Rp
V P                  '       d&   \        P                  ! VP%                  4       V4      p
RpV P&                  '       d   VP)                  ^ R7      pWšVRRR3# )r"   ÚTensorCoreFP8E5M2Layoutrì  rí  Nr   r5   r6   )ró   rï  rÜ   rô  Úsaved_tensorsrÝ   rð  rÇ   Ú_layout_clsrõ  rÞ   rö  rò  r   Úmmr&   rh  rñ  r9   rŒ  rB   )rø  Úgrad_outputró   Úgrad_2drT   Úgrad_mmÚ	weight_mmÚinput_mmrù  Ú
grad_inputÚgrad_weightÚ	grad_biass   &&          r   ÚbackwardÚQuantLinearFunc.backwardû  sß  € ð ×)Ñ)ˆØ×%Ñ% a¨Ó,×/Ñ/°Ó>ˆð �;�;ˆ;Ø×'Ñ'‰GˆFä%×0Ò0°Ð:SÓTˆGÜ˜&¤/×2Ò2°v×7IÑ7I×7TÑ7TÐUd×7eÒ7eØ"‘	Ü˜F¤O×4Ò4Ü+×6Ò6°v×7HÑ7HÓ7J×7MÑ7MÈmÓ7\Ð^wÓx‘	ä+×6Ò6°v·y±yÀÓ7OÐQjÓk�	Ø—{‘{‰Hà"%×"3Ñ"3ÑˆKàˆGÜ˜&¤/×2Ò2Ø"×-Ñ-Ó/×2Ñ2°=ÓA‘	à"ŸI™I mÓ4�	ØGJ×G_×G_ÐG_�{×*Ñ*¨1¨bÓ1×4Ñ4°]ÔCÐeiˆHô —X’X˜gÓ1ˆ
Üˆs�‰Ó !Ô#Ø#×-Ñ-¨a°·±ÀÀ"Ð1EÓFˆJàˆØ×#×#Ð#ÜŸ(š( 7§9¡9£;°Ó9ˆKàˆ	Ø�<�<ˆ<ØŸ™¨˜Ó*ˆIà¨	°4¸¸tÐCÐCr   r   N)r2  r3  r4  r5  rR  r?  r�  r   ÚautogradÚfunctionÚonce_differentiabler  r6  r7  r8  s   @r   ré  ré  Å  sM   ø‡ € ñ
ð ñ&ó ð&ðP Ø
‡^�^×Ñ×0Ñ0ñ'Dó 1ó ö'Dr   ré  c           	     ó(  € V'       d)   V P                  4        F  pVP                  V4       K  	  V P                  P                  4        F…  w  rEVf   K  V! V4      p\        P
                  ! 4       '       g'   VP                  4       '       d   VP                  4       pV P                  V\        P                  P                  VRR7      4       K‡  	  V P                  P                  4        F   w  rGVf   K  V! V4      V P                  V&   K"  	  V # )zWRe-wrap Parameters after fn so .to()/.cuda() propagate through QuantizedTensor weights.Fr[   )ÚchildrenÚ_applyÚ_parametersra  r   Úis_inference_mode_enabledÚis_inferencerb  Úregister_parameterr(   ra   Ú_buffers)rz   ÚfnÚrecurseÚchildrk  rf   ÚpÚbufs   &&&     r   Ú_quantized_applyr  (  sÑ   € çØ—_‘_Ö&ˆEØ�L‰L˜Öñ 'à×(Ñ(×.Ñ.Ö0‰
ˆØŠ=ÙÙˆu‹IˆÜ×/Ò/×1Ò1°q·~±~×7GÒ7GØ—‘“	ˆAØ×!Ñ! #¤u§x¡x×'9Ñ'9¸!È5Ð'9Ó'QÖRñ 1ð —O‘O×)Ñ)Ö+‰ˆØŒ?Ù#% c£7ˆF�O‰O˜CÓ ñ ,ð €Mr   c
                ó²  aaaa€ V P                   R,          oV P                   R,          p
V P                  pSP                  R4      pSP                  S R2R4      pVf#   \        P
                  ! RV 24       RV n        R# S R2.oR7VVVV3R llpSP                  S R2R4      pVe3   \        P                  ! VP                  4       P                  4       4      pVf:   \        P                  P                  VP                  SV
R	7      R
R7      V n        EMŒVP                  RR4      V n        VP                  RR
4      V n        V P$                  '       g   V P"                  V n        V P                   V9   d   RV n        V P                   f   \'        RV 24      h\(        V P                   ,          pVR,          V n        \-        V P*                  4      pV P                   R89   d   RV! R4      /pEMªV P                   R8X  d0   V! R\        P.                  4      pVf   \'        RV 24      hRV/pEMjV P                   R8X  d>   V! R4      pV! R\        P0                  4      pVe   Vf   \'        RV 24      hRVRV/pEMV P                   R8X  d¦   V! R4      pVf   \'        RV 24      hRV/pVP                  R/ 4      p\3        V\4        4      '       g   / pVP                  RVP                  RR
4      4      '       d4   RVR&   \7        VP                  RVP                  RR4      4      4      VR&   EMfV P                   R8X  dš   V! R4      pVf   \'        R V 24      hVP                  R/ 4      p\3        V\4        4      '       g   / pRVR\7        VP                  RVP                  RR4      4      4      R!^@R"VP                  R"VP                  R"R#4      4      /pEM¼V P                   \8        9   Ed�   \8        V P                   ,          pVP:                  ^,          ^,          V P<                  ^,          V,          8w  dG   \'        V P                    R$V R%VP:                  ^,           R&V P<                  ^,           R'V R(2
4      hV! R)4      pVf   \'        R*V P                    R+V 24      hVP>                  \        P@                  8X  d    VPC                  \        P0                  4      pVP                  R/ 4      p\3        V\4        4      '       g   / pRVR,V! R-4      R.V! R/4      R0\7        VP                  R0VP                  R0^4      4      4      R\7        VP                  RVP                  RR4      4      4      /pM\'        R1V P                    24      hVPD                  ! R9/ VBR2V
R3V P<                  /B p\        P                  P                  \G        VP                  SVR4,          R	7      V P*                  V4      R
R7      V n        V	'       dŠ   VR5,           F|  pVR:9   d   K  S V 2pSP                  VR4      pVf   K*  V PI                  V\        P                  P                  VP                  SR67      R
R7      4       SPK                  V4       K~  	  V! SSWEWgV4       S F  pVV9   g   K  VPM                  V4       K  	  R# );aP  Shared _load_from_state_dict body for quantized-weight modules.

Pops weight (+ scales, +/- extras), populates module.weight as a Parameter
or Parameter-wrapped QuantizedTensor, then calls super_load and strips
consumed keys from missing_keys. Reads compute_dtype from factory_kwargs
and disabled formats from module._disabled_formats.
rS   rR   Ú.rT   NzMissing weight for layer c                 ó²   <€ S V  2pSP                  VR 4      pVe:   VP                  SR7      pVe   VP                  VR7      pSP                  V4       V# )N©rS   rZ   )rÁ   rÜ   r6  rÀ   )r[  rR   rk  r   rS   Úmanually_loaded_keysrd  rc  s   &&  €€€€r   Ú	pop_scaleÚ)_load_quantized_module.<locals>.pop_scaleO  s_   ø€ Ø�˜˜ÐˆØ�N‰N˜3 Ó%ˆØŠ=Ø—‘˜F�Ó#ˆAØÒ Ø—F‘F �FÓ'�Ø ×'Ñ'¨Ô,Øˆr   Úcomfy_quantrZ  Fr[   r*  Úfull_precision_matrix_multTz&Unknown quantization format for layer Úcomfy_tensor_layoutrã   Úweight_scaleÚmxfp8z%Missing MXFP8 block scales for layer Únvfp4Úweight_scale_2zMissing NVFP4 scales for layer Úblock_scalerÒ  z$Missing INT8 weight scale for layer rL   ÚconvrotÚconvrot_groupsizeé   Úconvrot_w4a4z,Missing ConvRot W4A4 weight scale for layer Úquant_group_sizeÚlinear_dtypeÚint4z layer z: packed weight width z does not match K=z at z bitsÚweight_s_relzMissing z& group scale (weight_s_rel) for layer Ú	s_channelÚweight_s_channelÚcodebookÚweight_codebookÚ
group_sizez!Unsupported quantization format: r  r^  Ú	storage_tÚ
parametersr#  r   )r`  Úfloat8_e5m2r   >   r*  r-  )'Úfactory_kwargsÚ_disabled_formatsÚrstriprÁ   r}  ÚwarningrT   ÚjsonÚloadsÚnumpyÚtobytesr   r(   ra   rÜ   r'   rÖ  Ú_full_precision_mm_configrÇ  r   r‘  rú  r’  Úfloat8_e8m0fnur`  rÇ   ÚdictÚintÚ_GROUPED_INT8_FORMATSr   Ú_orig_shaperR   r†   r6  re  rÝ   r  rÀ   Úremove)rz   Ú
super_loadrc  rd  re  r}  rf  rg  r~  Úload_extra_paramsró   Údisabled_formatsÚ
layer_namerT   r%  Ú
layer_confÚqconfigÚ
layout_clsÚscalesÚbsÚtsrã   Úparams_confÚbitsrL   Ú
param_namere   rj   rk  rS   r$  s   &&ff&&&&&&                   @@r   Ú_load_quantized_moduler[  :  sy  û€ ð ×"Ñ" 8Õ,€FØ×)Ñ)¨'Õ2€MØ×/Ñ/ÐØ—‘˜sÓ#€Jà�^‰^˜v˜h fÐ-¨tÓ4€FØ‚~Ü�ŠÐ3°J°<Ð@ÔAØˆŒÙØ%˜h fÐ-Ð.Ð÷ò ð —‘ 6 (¨+Ð 6¸Ó=€JØÒÜ—Z’Z 
× 0Ñ 0Ó 2× :Ñ :Ó <Ó=ˆ
àÒÜŸ™×*Ñ*¨6¯9©9¸FÈ-¨9Ó+XÐhmÐ*ÓnˆŽà(Ÿn™n¨X°tÓ<ˆÔØ+5¯>©>Ð:VÐX]Ó+^ˆÔ(Ø×(×(Ð(Ø(.×(HÑ(HˆFÔ%Ø×ÑÐ"2Ô2Ø(,ˆFÔ%Ø×ÑÒ&ÜÐEÀjÀ\ÐRÓSÐSä˜f×1Ñ1Õ2ˆØ$Ð%:Õ;ˆÔÜ% f×&8Ñ&8Ó9ˆ
ð ×ÑÐ"BÔBØ™y¨Ó8Ð9ŠFØ× Ñ  GÔ+Ù˜>¬5×+?Ñ+?Ó@ˆBØŠzÜ Ð#HÈÈÐ!UÓVÐVØ˜r�]ŠFØ× Ñ  GÔ+ÙÐ+Ó,ˆBÙ˜>¬5×+>Ñ+>Ó?ˆBØŠz˜RšZÜ Ð#BÀ:À,Ð!OÓPÐPØ˜r =°"Ð5ŠFØ× Ñ Ð$5Ô5Ù˜nÓ-ˆEØŠ}Ü Ð#GÈ
À|Ð!TÓUÐUØ˜uÐ%ˆFØ$Ÿ.™.¨°2Ó6ˆKÜ˜k¬4×0Ò0Ø �Ø�~‰~˜i¨¯©¸ÀEÓ)J×KÒKØ$(��yÑ!Ü.1Ø—N‘NÐ#6¸¿¹ÐH[Ð]`Ó8aÓbó/�Ð*Ñ+ùð × Ñ  NÔ2Ù˜nÓ-ˆEØŠ}Ü Ð#OÐPZÈ|Ð!\Ó]Ð]Ø$Ÿ.™.¨°2Ó6ˆKÜ˜k¬4×0Ò0Ø �à˜Ø#¤SØ—N‘NÐ#6¸¿¹ÐH[Ð]`Ó8aÓbó&ð # BØ 
§¡¨~¸{¿¹È~Ð_eÓ?fÓ gðŠFð × Ñ Ô$9Õ9ô )¨×)<Ñ)<Õ=ˆDØ�|‰|˜A� Õ" f×&8Ñ&8¸Õ&;¸dÕ&BÔBÜ Ø×*Ñ*Ð+¨7°:°,Ð>TÐU[×UaÑUaÐbcÕUdÐTeð f(Ø(.×(:Ñ(:¸1Õ(=Ð'>¸dÀ4À&ÈðOóð ñ ˜nÓ-ˆEØŠ}Ü  8¨F×,?Ñ,?Ð+@Ð@fÐgqÐfrÐ!sÓtÐtØ�{‰{œeŸk™kÔ)ØŸ
™
¤5×#6Ñ#6Ó7�Ø$Ÿ.™.¨°2Ó6ˆKÜ˜k¬4×0Ò0Ø �à˜Ø™YÐ'9Ó:Ø™IÐ&7Ó8Øœc *§.¡.°¸{¿¹È|Ð]_Ó?`Ó"aÓbØ#¤SØ—N‘NÐ#6¸¿¹ÐH[Ð]`Ó8aÓbó&ð‰Fô Ð@À×ATÑATÐ@UÐVÓWÐWà×"Ò"Ñe VÑe¸ÐeÐRX×RdÑRdÒeˆÜŸ™×*Ñ*Ü˜FŸI™I¨V¸7À;Õ;O˜IÓPÐRX×RdÑRdÐflÓmØð +ó 
ˆŒ÷
 Ø% l×3Ð3�
ØÐ!CÔCÙØ%˜h z lÐ3�	Ø—^‘^ I¨tÓ4�Ø’:ÙØ×)Ñ)¨*´e·h±h×6HÑ6HÈÏÉÐV\ÈÓI]ÐmrÐ6HÓ6sÔtØ$×+Ñ+¨IÖ6ñ 4ñ ˆz˜6 >¸<ÐZdÔeÛ#ˆØ�,ÖØ×Ñ Ö$ó $r   c                ód  € \        V R4      '       g   \        P                  ! RV 24       V# \        V RR4      pVe   WQV R2&   V P                  f   V# \        V P                  \        4      '       Ed©   VP                  V P                  P                  V R24      4       RV P                  /p\        V RR4      '       d   RVR	&   \        V P                  R
R4      pV P                  R8X  d*   \        VRR4      '       d   RVR&   \        VRR4      VR&   MoV P                  R8X  d*   \        VRR4      VR&   \        VRR4      pVR8w  d   W†R&   M5V P                  \        9   d!   \        VR^4      VR&   \        VRR4      VR&   V'       d   VP                  V4       \        P                  ! \        \        P                  ! V4      P!                  R4      4      \        P"                  R7      W R2&   V F  p	\        W	R4      p
V
f   K  W¡V V	 2&   K  	  V# V P                  W R2&   V# )z”Shared state_dict body. extra_quant_conf merges into the comfy_quant JSON;
extra_quant_params names attributes written as additional top-level keys.rT   z(Warning: state dict on uninitialized op ru   Nr*  rG  FTr(  Ú_paramsrÒ  r/  r0  r1  r2  r4  r5  r;  úutf-8rZ   r'  )r^   r}  rB  r]   rT   rÇ   rÝ   Úupdaterc  rÖ  rK  r   rß   ÚlistrC  ÚdumpsÚencoder†   )rz   Úsdrd  Úextra_quant_confÚextra_quant_paramsru   Ú
quant_confrL   r4  r[  rN  s   &&&&&      r   Ú_quantized_weight_state_dictrg  Ì  s  € ô �6˜8×$Ò$Ü�ŠÐBÀ6À(ÐKÔLØˆ	Ü�6˜6 4Ó(€DØÒØ"ˆfˆX�Tˆ?ÑØ‡}�}ÒØˆ	Ü�&—-‘-¤×1Ó1Ø
�	‰	�&—-‘-×*Ñ*¨f¨X°VÐ+<Ó=Ô>Ø × 3Ñ 3Ð4ˆ
Ü�6Ð6¸×>Ò>Ø7;ˆJÐ3Ñ4Ü˜Ÿ™¨	°4Ó8ˆØ×ÑÐ"3Ô3¼ÀÈ	ÐSX×8YÒ8YØ$(ˆJ�yÑ!Ü.5°fÐ>QÐSVÓ.WˆJÐ*Ò+Ø× Ñ  NÔ2Ü.5°fÐ>QÐSVÓ.WˆJÐ*Ñ+Ü" 6¨>¸6ÓBˆLØ˜vÔ%Ø-9˜>Ñ*øØ× Ñ Ô$9Ô9Ü'.¨v°|ÀRÓ'HˆJ�|Ñ$Ü.5°fÐ>QÐSVÓ.WˆJÐ*Ñ+ßØ×ÑÐ.Ô/Ü%*§\¢\´$´t·z²zÀ*Ó7M×7TÑ7TÐU\Ó7]Ó2^Ôfk×fqÑfqÔ%rˆˆX�[Ð!Ñ"Û&ˆDÜ˜F¨$Ó/ˆEØÔ Ø(-�f�X˜d˜VÐ$Ó%ñ 'ð €Ið !'§¡ˆˆX�VÐÑØ€Ir   c                   ó.   a € ] tR tRt o RtRtR tRtV tR# )rÕ  iô  NFc                óÊ   € V P                   \        9   ;'       dJ    V P                  '       * ;'       d1    V P                   V P                  9  ;'       d    V P                   V9  # r   )rÖ  r‘  rG  r@  )r!  rP  s   &&r   Úcan_use_quantized_matmulÚ)MixedPrecisionOp.can_use_quantized_matmulû  sd   € Ø×!Ñ!¤[Ñ0÷ >ð >Ø×6Ñ6Ô6÷>ð >à×%Ñ%¨T×-CÑ-CÑC÷>ð >ð ×%Ñ%Ð-=Ñ=ð	?r   r   )	r2  r3  r4  r5  rÖ  rÞ  rj  r6  r7  r8  s   @r   rÕ  rÕ  ô  s   ø‡ € Ø€Lð !Ð÷?ð ?r   rÕ  c                 ó<   a aaaa€  ! VVVVV 3R  lR\         4      oS# )c                   óð   <€ ] tR tRt Zt Zt Zt Zt ! V V3R lR]P                  P                  ]4      t ! V V3R lR]P                  P                  ]4      t ! V 3R lR]P                  4      tRtR	# )
Ú.mixed_precision_ops.<locals>.MixedPrecisionOpsi  c                   ó¤   <a a€ ] tR tRt o ZtRV3R lVV 3R llltR tV 3R ltRRRR	/R
 ltR t	RR lt
R tRR ltRR ltRR ltRtVtV ;t# )Ú5mixed_precision_ops.<locals>.MixedPrecisionOps.Lineari	  Nc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )r4   rv  rw  ru   ©rJ  Úbool)r*  r+  s   "€r   r,  ÚBmixed_precision_ops.<locals>.MixedPrecisionOps.Linear.__annotate__  s"   ø€ ÷ 7ñ 7©Cð 7¹sð 7É$ñ 7r   c                óp  <€ \         SV `  4        R VRSP                  /V n        Wn        W n        W!3V n        V'       dE   \        P                  P                  \        P                  ! V3/ V P                  B 4      V n        MV P                  RR4       RV n        SP                  V n        RV n        R# ©rS   rR   ru   NF)r?  r"  Ú_compute_dtyper?  rv  rw  rL  r   r(   ra   r…   ru   r  Útensor_classrÇ  rG  )r!  rv  rw  ru   rS   rR   ÚMixedPrecisionOpsrB  s   &&&&&&€€r   r"  Ú>mixed_precision_ops.<locals>.MixedPrecisionOps.Linear.__init__  s˜   ø€ Ü‘Ñ Ô"à'/°¸ÐBS×BbÑBbÐ&c�Ô#à#.Ô Ø$0Ô!Ø$0Ð#>�Ô ßÜ %§¡× 2Ñ 2´5·;²;¸|Ñ3cÈt×ObÑObÑ3cÓ d�D•Ià×+Ñ+¨F°DÔ9à$(�Ô!Ø*;×*NÑ*N�Ô'Ø16�Ö.r   c                ó   € R # r   r   r�  s   &r   r‚  ÚFmixed_precision_ops.<locals>.MixedPrecisionOps.Linear.reset_parameters  r‰  r   c                ó8   <€ \        V \        SV `  .VO5R R/  R# )rO  TN)r[  r?  rs  )r!  r   rB  s   &*€r   rs  ÚKmixed_precision_ops.<locals>.MixedPrecisionOps.Linear._load_from_state_dict   s   ø€ Ü& t¬U©WÑ-JÐjÈTÒjÐeiÔjr   Údestinationrd  Ú c               ó.   € Ve   TM/ p\        WVRR7      # )N)re  )rû  Úpre_quant_scale©rg  ©r!  r  rd  r   r,   rc  s   &$$*, r   rc  Ú@mixed_precision_ops.<locals>.MixedPrecisionOps.Linear.state_dict#  s   € Ø$/Ò$;‘[À�Ü3°D¸fÐY{Ô|Ð|r   c                óV   € \         P                  P                  P                  WV4      # r   )r   r(   r)   rˆ  r‰  s   &&&&r   Ú_forwardÚ>mixed_precision_ops.<locals>.MixedPrecisionOps.Linear._forward'  s   € Ü—x‘x×*Ñ*×1Ñ1°%ÀÓFÐFr   c                ó:  € V'       gp   \        V VR VVR7      ;_uu_ 4       w  rVV P                  '       d'   \        V\        4      '       d   VP	                  4       pV P                  WV4      uuRRR4       # \        V RV P                  P                  VP                  VP                  R VR R7      ;_uu_ 4       w  rVVP                  VP                  R7      pV P                  WV4      uuRRR4       #   + '       g   i     LŽ; i  + '       g   i     R# ; i)Trá  N)rU   rR   rS   rË   r   ró   rõ   rZ   )
r  rÇ  rÇ   rÝ   rÞ   r‡  rT   rR   rS   rÜ   )r!  rU   ró   rõ   Úweight_only_quantrT   ru   s   &&&&&  r   rŠ  ÚPmixed_precision_ops.<locals>.MixedPrecisionOps.Linear.forward_comfy_cast_weights*  sï   € ÷ )Ü.ØØØ$(Ø&3Ø%1÷õ ñ (˜&Ø×2×2Ð2´zÀ&Ì/×7ZÒ7ZØ%+×%6Ñ%6Ó%8˜FØ#Ÿ}™}¨U¸DÓA÷ò ô +ØØØŸ+™+×+Ñ+Ø Ÿ<™<Ø$Ÿ{™{Ø $Ø"/Ø!%÷	õ 	ñ $�fØ#ŸY™Y¨U¯[©[˜YÓ9�FØŸ=™=¨¸Ó=÷	ò 	÷÷ ú÷	÷ 	ð 	ús   ¡AC6Â;0D	Ã6D	Ä	D	c           
     óŒ  € \        4        \        V R R4      pVe<   V\        P                  P	                  WAP
                  VP                  4      ,          pVP                  pRpVP                  p\        V RR4      RJ;'       d‰    \        V\        4      '       * ;'       dl    V P                  '       * ;'       dS    \        V RR4      '       * ;'       d9    \        V P                  4      ^ 8H  ;'       d    \        V P                  4      ^ 8H  p\        P                  ! \        V RR4      / 4      P                  RR4      p	VP                   '       d’   V'       dŠ   V	'       d‚   \#        V VRVRR7      ;_uu_ 4       w  r«\        V R	R4      pVe+   \        P                  P	                  WÁP
                  R4      p\$        P'                  WW°P(                  WÇ4      uuRRR4       # V'       d²   V	'       dª   VP*                  ^8¼  d   VP-                  RVR,          4      MTpVP*                  ^8X  dm   VP*                  ^8¬  p\        V R	R4      pVe+   \        P                  P	                  WÁP
                  R4      p\        P.                  ! WÐP(                  VR
7      pT;'       d*    V	'       * ;'       d    \        V P0                  \        4      pV P3                  VV\        V\        4      VR7      pV'       d5   VP-                  . VRR OV P0                  P                  ^ ,          N54      pV#   + '       g   i     EL^; i)r‚  NFrú  Úcomfy_force_cast_weightsrÖ  Úquantize_inputTrá  rû  rë  )rõ   rŠ  r6   )r   r]   r	   r
   Úcast_to_devicerS   rR   r   rÇ   rÝ   rÇ  r&   r  r  r‘  r'   r\   r  ré  Úapplyrú  r¾  r½  rð  rT   rŠ  )r!  rU   r   r,   r‚  rh  Úreshaped_ndró   Ú_use_quantizedrŽ  rT   ru   rã   Úinput_reshapedrŠ  rþ  s   &&*,            r   r�  Ú=mixed_precision_ops.<locals>.MixedPrecisionOps.Linear.forwardJ  sð  € Ü”ô #*¨$Ð0AÀ4Ó"H�Ø"Ò.Ø!¤E×$:Ñ$:×$IÑ$IÈ/×[gÑ[gÐin×itÑitÓ$uÕu�Eà#Ÿk™k�Ø#�à %§¡�ô ˜D -°Ó6¸dÐB÷ Tð TÜ" 5¬/Ó:Ô:÷Tð TØCG×CZÑCZÔ?Z÷Tð Tä Ð&@À%ÓHÔH÷Tð Tô ˜×,Ñ,Ó-°Ñ2÷Tð Tô 8;¸4×;MÑ;MÓ7NÐRSÑ7Sð	 ô "-§¢´¸¸~ÈtÓ1TÐVXÓ!Y×!]Ñ!]Ð^nÐptÓ!u�ð ×'×'Ð'¯N¿~Ü.ØØØ$(Ø&3Ø%)÷õ ñ (˜&Ü '¨¨m¸TÓ B˜Ø Ò,Ü$)×$:Ñ$:×$IÑ$IÈ%×Q]ÑQ]Ð_cÓ$d˜Eä.×4Ñ4Ø!¨4×1AÑ1AÀ5ó ÷ò ÷  "§nð LQÏ:É:ÐYZÌ? U§]¡]°2°{À2µÔ%GÐ`e�Nð &×*Ñ*¨aÔ/Ø&+§j¡j°A¡o˜ä '¨¨m¸TÓ B˜Ø Ò,Ü$)×$:Ñ$:×$IÑ$IÈ%×Q]ÑQ]Ð_cÓ$d˜EÜ /× :Ò :¸>×K[ÑK[ÐchÔ i˜à$2×$vÐ$v¸>Ô7I×$vÐ$vÌjÐY]×YdÑYdÔfuÓNvÐ!Ø×8Ñ8ØØ!Ü!+¨E´?Ó!CØ&7ð	 9ó �÷ Ø#Ÿ^™^Ð,U¨k¸#¸2Ð.>Ð,UÀÇÁ×@QÑ@QÐRSÕ@TÑ,UÓV�Fà�÷U÷ ð ús   Å<AL2Ì2M	c                óR   € \        V\        4      '       d   VP                  4       # V# r   )rÇ   rÝ   rÞ   )r!  rT   Úinplacer,   s   &&&,r   Úconvert_weightÚDmixed_precision_ops.<locals>.MixedPrecisionOps.Linear.convert_weight�  s#   € Ü˜f¤o×6Ò6Ø!×,Ñ,Ó.Ð.à!�Mr   c                ód  € \        V R R4      eD   V P                  P                  VRVRR7      P                  V P                  P                  4      pM%VP                  V P                  P                  4      pV'       d   V# VRJ g   Q h\
        P                  P                  VRR7      V n        R# )rú  Nrâ   T)rã   rä   Úinplace_opsFr[   )r]   rT   ré   rÜ   rR   r   r(   ra   )r!  rT   Úinplace_updaterå   Úreturn_weightr,   s   &&&&&,r   Ú
set_weightÚ@mixed_precision_ops.<locals>.MixedPrecisionOps.Linear.set_weight“  sµ   € Ü˜4 °Ó5ÒAØ!Ÿ[™[×>Ñ>¸vÈ]Ðptð  CGÐ>ó  H÷  Kñ  Kð  LP÷  LWñ  LW÷  L]ñ  L]ó  ^‘Fà#ŸY™Y t§{¡{×'8Ñ'8Ó9�Fß Ø!�Mà%¨Ó.Ð.Ð.Ü#Ÿh™h×0Ñ0°ÀuÐ0ÓM�–r   c                ó   € \        WV4      # r   ©r  ©r!  r  r  s   &&&r   r  Ú<mixed_precision_ops.<locals>.MixedPrecisionOps.Linear._applyž  ó   € Ü'¨°'Ó:Ð:r   )	rÇ  rG  rL  ru   r?  rv  rw  rx  rT   r“  )NFF©F)FNFrQ  )r2  r3  r4  r5  r@  r"  r‚  rs  rc  r‡  rŠ  r�  r—  r�  r  r6  r7  rU  ©rB  r+  ry  Údisableds   @@€€r   rt  rp  	  s^   ú‡ € Ù (Ð÷7ö 7ò"õkð}°Dð }Àô }òGô>ò@AôF"ô	N÷;ô ;r   rt  c                   óü   <a a€ ] tR tRt oRt ZtRV3R lVV 3R llltR tRR ltV 3R lt	V3R	 lR
 lt
R tR tV3R lR lt]P                  R 4       tV3R lR ltR tV3R lR ltRRRR/R ltRtVtV ;t# )Ú9mixed_precision_ops.<locals>.MixedPrecisionOps.MoEExpertsi¡  uã  Container for E quantized expert weights, indexed via expert_weight(i).

The bank lives on self.weight as a single 3D tensor â€” either a
compute_dtype Parameter or a Parameter wrapping a QuantizedTensor
with leading expert dim.

State-dict layout matches mixed_precision_ops.Linear with a leading
expert dim:
    {prefix}.weight          quant data (storage_t), leading dim = E
    {prefix}.weight_scale    block / per-tensor scale
    {prefix}.weight_scale_2  [E] or scalar           NVFP4 only
    {prefix}.bias            [E, out_features]       optional, compute_dtype
    {prefix}.comfy_quant     json -> {{"format": "...", "num_experts": E}}

Without comfy_quant the weight loads as a plain compute_dtype 3D Parameter [E, out, in].
Nc                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[/# )r4   Únum_expertsrv  rw  ru   rr  )r*  r+  s   "€r   r,  ÚFmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts.__annotate__µ  s+   ø€ ÷ +ñ +©Cð +¹cð +ÑQTð +Ñ\`ñ +r   c                ó¨  <€ \         SV `  4        Wn        W n        W0n        WV3V n        R VRSP                  /V n        V'       dE   \        P                  P                  \        P                  ! W3/ V P                  B 4      V n        MV P                  RR4       RV n        RV n        RV n        SP"                  V n        RV n        RV n        R# rv  )r?  r"  rª  rv  rw  rL  rw  r?  r   r(   ra   r…   ru   r  rT   rÖ  rú  rÇ  rG  Ú_resident_bank)	r!  rª  rv  rw  ru   rS   rR   ry  rB  s	   &&&&&&&€€r   r"  ÚBmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts.__init__µ  sº   ø€ Ü‘Ñ Ô"Ø#.Ô Ø#.Ô Ø$0Ô!Ø$/¸{Ð#K�Ô Ø'/°¸ÐBS×BbÑBbÐ&c�Ô#ßÜ %§¡× 2Ñ 2´5·;²;¸{Ñ3pÐ\`×\oÑ\oÑ3pÓ q�D•Ià×+Ñ+¨F°DÔ9ð #�”Ø$(�Ô!Ø#'�Ô Ø*;×*NÑ*N�Ô'Ø16�Ô.Ø&*�Ö#r   c                ó   € R # r   r   r�  s   &r   r‚  ÚJmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts.reset_parametersÉ  r‰  r   c                ó   € \        WV4      # r   r   r¡  s   &&&r   r  Ú@mixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts._applyÌ  r£  r   c                ó  <€ VP                  V R 2R4      pVeB   \        P                  ! VP                  4       P	                  4       4      P                  R4      MRpVP                  V R2R4      pV\
        9   g   VR8X  dí   Veé   VP                  ^8X  dØ   R F£  pVP                  V V 2R4      pVf   K  VP                  ^8”  g   K2  VP                  ^ ,          V P                  8X  g   KV  VP                  ! V P                  VP                  ^,          ,          .VP                  R,          O5!  W V 2&   K¥  	  V P                  V P                  ,          V P                  3V n        \        V \        S	V `<  W.VO5RR/  R# )	r'  Nr*  r*  rÒ  :r4   NNrO  F)rT   r*  r6  r8  )r'   rC  rD  rE  rF  rK  r¾  r   rª  r½  rw  rv  rL  r[  r?  rs  )
r!  rc  rd  r   rR  rÖ  rã   r[  r9   rB  s
   &&&*     €r   rs  ÚOmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts._load_from_state_dictÏ  si  ø€ Ø'Ÿ^™^¨v¨h°kÐ,BÀDÓI�
ØYcÒYoœtŸzšz¨*×*:Ñ*:Ó*<×*DÑ*DÓ*FÓG×KÑKÈHÔUÐuy�Ø"Ÿ™¨&¨°Ð'>ÀÓE�ØÔ#8Ô8¸\ÐM^Ô=^ÐchÒctÐy~÷  zDñ  zDð  HIô  zIã ^˜Ø&ŸN™N¨f¨X°d°VÐ+<¸dÓC˜Øœ=¨Q¯V©V°a®Z¸A¿G¹GÀA½JÈ$×JZÑJZÖ<ZØ<=¿IºIÀd×FVÑFVÐYZ×Y`ÑY`ÐabÕYcÕFcÐ<rÐfg×fmÑfmÐnpÕfqÓ<r˜J¨°$°Ð'8Ó9ñ !_ð )-×(8Ñ(8¸4×;LÑ;LÕ(LÈd×N^ÑN^Ð'_�DÔ$Ü& t¬U©WÑ-JÈJÐÐaeÒÐy~Ôr   c                ó    <€ V ^8„  d   QhRS[ /# )r4   Úi)rJ  )r*  r+  s   "€r   r,  r«  Ü  s   ø€ ÷ &ñ &¡sñ &r   c                ó    € \        V P                  \        4      '       d   V P                  V P                  V4      # V P                  V,          # )z>Expert i's weight (Tensor or per-expert QuantizedTensor view).)rÇ   rT   rÝ   Ú_expert_qt_from)r!  r¶  s   &&r   Úexpert_weightÚGmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts.expert_weightÜ  s8   € ä˜dŸk™k¬?×;Ò;Ø×/Ñ/°·±¸QÓ?Ð?Ø—{‘{ 1•~Ð%r   c           	     óÎ   € \        V P                  \        4      '       d:   \        V R V P                  P                  VP
                  VP                  RR7      # \        WRR7      # )NT)rU   rR   rS   rË   r   r‡  )rÇ   rT   rÝ   r  rR   rS   rô  s   &&r   Ú
_cast_bankÚDmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts._cast_bankâ  sd   € ä˜dŸk™k¬?×;Ò;Ü0°¸TÈÏÉ×IZÑIZÐch×coÑcoð  }B÷  }Hñ  }Hð  VZô  [ð  [Ü,¨TÀdÔKÐKr   c           	     ó¢  € \        VP                  VP                  \        P                  ! VP
                  VR 7      4      p\        P                  ! V P                  V P                  V P                  3W!P                  R7      p\        ^RV P                  V P                  ,          VP                  4       ,          ,          4      p\        ^ V P                  V4       Fl  p\        We,           V P                  4      pV P!                  W6V4      P#                  4       P%                  Wv,
          V P                  V P                  4      WFV% Kn  	  V# ))r  r„   i   )rÝ   Ú_qdatar  ÚdataclassesÚreplacer]  r   r…   rª  rw  rv  rS   r]  rC   Úranger\  Ú
_bank_rowsrÞ   r6  )r!  rT   rR   ÚflatrÀ  ÚstepÚfirstÚlasts   &&&     r   Ú_dequantize_bankÚJmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts._dequantize_bankè  s  € ä& v§}¡}°f×6HÑ6HÌ+×J]ÒJ]Ð^d×^lÑ^lÐy~ÔJó  A�Ü—k’k 4×#3Ñ#3°T×5FÑ5FÈ×HXÑHXÐ"YÐaf×o|Ño|Ô}�Ü˜1˜y¨d×.?Ñ.?À$×BRÑBRÕ.RÐUX×UeÑUeÓUgÕ.gÕhÓi�Ü" 1 d×&6Ñ&6¸Ö=�EÜ˜u�|¨T×-=Ñ-=Ó>�DØ&*§o¡o°dÀ4Ó&H×&SÑ&SÓ&U×&ZÑ&ZÐ[_Õ[gÐim×izÑizð  }A÷  }Mñ  }Mó  'N�C˜d’Oñ >ð �
r   c                ó2   <€ V ^8„  d   QhRS[ RS[RS[RS[ /# )r4   rT   rÆ  rÇ  r)  ©rÝ   rJ  )r*  r+  s   "€r   r,  r«  ò  s0   ø€ ÷ {ñ {©ð {Áð {ÉCð {ÑTcñ {r   c           
     ó  € VP                   p\        W P                  ,          W0P                  ,          4      p\        P                  ! V4       Uu/ uFÉ  p\
        P                  ! \        WFP                  4      4      '       g   K4  \        WFP                  4      P                  ^8¼  g   KZ  \        WFP                  4      P                  ^ ,          VP                  P                  ^ ,          8X  g   K¢  VP                  \        WFP                  4      V,          bKË  	  pp\        VP                  V,          VP                  \        P                  ! V3RW2,
          V P                  ,          V P                  3/VB 4      # u upi )zJExperts [first, last) of a flat [E * out, in] bank as one QuantizedTensor.r^  )r]  Úslicerw  rÀ  Úfieldsr   Ú	is_tensorr]   r[  r¾  r   r¿  rÝ   r  rÁ  rv  )r!  rT   rÆ  rÇ  rL   Úrowsrò   Úper_rows   &&&&    r   rÃ  ÚDmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts._bank_rowsò  sè  € àŸ™�Ü˜U×%6Ñ%6Õ6¸×?PÑ?PÕ8PÓQ�ÜJU×J\ÒJ\Ð]cÔJdó  wÑJdÀQÔhm×hwÒhwÔxð  AG÷  IOñ  IOó  yP÷  iQÔ@ô  V]ð  ^d÷  flñ  fló  Vm÷  Vrñ  Vrð  vwñ  VwÔ@ô  |Cð  DJ÷  LRñ  LRó  |S÷  |Yñ  |Yð  Z[õ  |\ð  `f÷  `mñ  `m÷  `sñ  `sð  tuõ  `vñ  |vÔ@˜1Ÿ6™6¤7¨6·6±6Ó#:¸4Õ#@Ò@ÑJd�ð  wÜ& v§}¡}°TÕ':¸F×<NÑ<NÔP[×PcÒPcÐdjñ  QzÐy}õ  zFð  JN÷  J[ñ  J[õ  y[ð  ]a÷  ]mñ  ]mð  xnð  Qzð  ryñ  Qzó  {ð  {ùò ws   Á1FÂ!FÂ-AFÃ5+Fc              3  ó|  "  € V P                  V4      ;_uu_ 4       w  r#V P                  '       dM   \        V\        4      '       d7   VP                  P
                  ^8X  d   V P                  W!P                  4      pW#3V n         V x € RV n         RRR4       R#   RT n        i ; i  + '       g   i     R# ; i5i)u{   Cast the whole bank once; expert_linear inside reuses the cast.
Not re-entrant â€” do not nest calls on the same instance.
N)	r¼  rÇ  rÇ   rÝ   r¿  r¾  rÈ  rR   r­  r‰  s   &&  r   Úbank_residentÚGmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts.bank_residentù  s–   é € ð
 —_‘_ U×+Ô+©~°Ø×.×.Ð.´:¸fÄo×3VÒ3VÐ[a×[hÑ[h×[mÑ[mÐqrÔ[rØ!%×!6Ñ!6°v¿{¹{Ó!K˜Ø+1¨.�DÔ'ð3Ø"š
à.2˜Õ+÷ ,Ñ+øð /3˜Õ+ú÷ ,×+Ð+üs5   ‚B<œA)B(ÂBÂ
B(Â
B<Â	B%Â%B(Â(B9	Â3	B<c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# )r4   rU   r¶  r)  )r   r  rJ  )r*  r+  s   "€r   r,  r«    s/   ø€ ÷ Lñ L©5¯<©<ð L¹Cð LÁEÇLÁLñ Lr   c                óè   € \        V RR4      pVe   Vw  rEV P                  WWR4      # V P                  V4      ;_uu_ 4       w  rEV P                  WWR4      uuRRR4       #   + '       g   i     R# ; i)z6Linear against expert i's weight (with optional bias).r­  N)r]   Ú_expert_linear_implr¼  )r!  rU   r¶  r”   rT   ru   s   &&&   r   Úexpert_linearÚGmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts.expert_linear  sc   € ä" 4Ð)9¸4Ó@�ØÒ'Ø#+‘L�FØ×3Ñ3°EÀ4ÓKÐKØ—_‘_ U×+Ô+©~°Ø×3Ñ3°EÀ4ÓK÷ ,×+×+Ó+ús   ÁA Á A1	c                ó  € \        V\        4      '       d   V P                  W$4      pM\        W$,          VR R7      pVe   \        W4,          VR R7      MRp\        V\        4      '       dì   V P                  '       * ;'       d6    VP
                  P                  4       ;'       d    VP                  4       ^8H  pV'       dz   \        V P                  ,          P                  RR4      '       d!   \        P                  ! WP                  4      MTp\        P                  P                  P!                  W…V4      # \        VP#                  4       VR R7      p\        P                  P                  P!                  WV4      # )F)rP   NrŽ  T)rÇ   rÝ   r¸  rV   rÇ  rT  Úsupports_fast_matmulr   r‘  rÖ  r'   rð  rú  r   r(   r)   rˆ  rÞ   )	r!  rU   rT   ru   r¶  Úqwrý  Úuse_fastÚqins	   &&&&&    r   rØ  ÚMmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts._expert_linear_impl  s7  € Ü˜f¤o×6Ò6Ø×-Ñ-¨fÓ8‘Bä& v¥y°%¸eÔD�BØAEÒAQ”M $¥'¨5°uÕ=ÐW[�ä˜b¤/×2Ò2à ×3Ñ3Ô3÷ -ð -ØŸM™M×>Ñ>Ó@÷-ð -à!ŸI™I›K¨1Ñ,ð ÷
  ÜU`Ðae×arÑarÕUs×UwÑUwð  yIð  KO÷  VPò  VPœo×8Ò8¸×@PÑ@PÔQð  V[˜Ü$Ÿx™x×2Ñ2×9Ñ9¸#À1ÓEÐEÜ& r§}¡}£¸ÀEÔJ�BÜ—x‘x×*Ñ*×1Ñ1°%¸QÓ?Ð?r   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[ /# )r4   rT   r¶  r)  rË  )r*  r+  s   "€r   r,  r«  #  s'   ø€ ÷ eñ e©oð eÁ#ð eÉ/ñ er   c           	     ó  € VP                   P                  ^8X  d   V P                  WV^,           4      # VP                  pRVP                  P                  4       '       d   VP                  V,          MVP                  RVP                  RV P                  V P                  3/p\        VR4      '       d   VP                  V,          VR&   \        VR4      '       d   VP                  VR&   \        VR4      '       d   VP                  VR&   \        VR4      '       d   VP                  VR&   \        VR4      '       d   VP                  VR&   \        VP                   V,          VP                   \#        V4      ! R	/ VB 4      # )
zDBuild a per-expert QuantizedTensor by indexing into a resident bank.rã   r  r^  r.  r3  r/  r0  r4  r   )r¿  r¾  rÃ  r]  rã   r   r  rw  rv  r^   r.  r3  r/  r0  r4  rÝ   r  rr  )r!  rT   r¶  rL   r,   s   &&&  r   r¸  ÚImixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts._expert_qt_from#  sE  € à—=‘=×%Ñ%¨Ô*ØŸ?™?¨6°a¸!µeÓ<Ð<ØŸ™�à°·±×0@Ñ0@×0BÒ0B˜VŸ\™\¨!ž_ÈÏÉØ  &×"3Ñ"3Ø  4×#4Ñ#4°d×6FÑ6FÐ"Gð�ô
 ˜6 =×1Ò1Ø,2×,>Ñ,>¸qÕ,A�F˜=Ñ)Ü˜6Ð#5×6Ò6Ø17×1HÑ1H�FÐ-Ñ.Ü˜6 9×-Ò-Ø(.¯©�F˜9Ñ%Ü˜6Ð#6×7Ò7Ø28×2JÑ2J�FÐ.Ñ/Ü˜6 >×2Ò2Ø-3×-@Ñ-@�F˜>Ñ*Ü& v§}¡}°QÕ'7¸×9KÑ9KÌTÐRXÌ\ÑMcÐ\bÑMcÓdÐdr   r  rd  r€  c               óF   € Ve   TM/ p\        WVRV P                  /R7      # )Nrª  )rd  )rg  rª  r„  s   &$$*, r   rc  ÚDmixed_precision_ops.<locals>.MixedPrecisionOps.MoEExperts.state_dict9  s*   € Ø$/Ò$;‘[À�Ü3°D¸fÐXeÐgk×gwÑgwÐWxÔyÐyr   )rÇ  rG  rL  r­  ru   r?  rv  rú  rª  rw  rÖ  rT   r“  rQ  )r2  r3  r4  r5  rR  r@  r"  r‚  r  rs  r¹  r¼  rÈ  rÃ  rS  rT  rÔ  rÙ  rØ  r¸  rc  r6  r7  rU  r¥  s   @@€€r   Ú
MoEExpertsr¨  ¡  s¡   ú‡ € ññ" !)Ð÷+ö +ò(ô;õ@÷&ð &òLò÷{ð {ð ×&Ñ&ñ3ó 'ð3÷Lð Lò@÷&eð eð,z°Dð zÀ÷ zô zr   ræ  c                   óh   <a a€ ] tR tRt oVV 3R ltRRRR/R ltRV 3R lltRV 3R	 lltR
tVt	V ;t
# )Ú8mixed_precision_ops.<locals>.MixedPrecisionOps.Embeddingi=  c           
     ó,  <€ V R 2pVP                  V R2R4      p	V	e3   \        P                  ! V	P                  4       P	                  4       4      p	V	e   V	P                  R4      MRp
. pV
R9   Ed’   W�9   Ed‹   W n        \        V
,          pVR,          V n        \        V P                  4      pVP                  V4      pVP                  V4       V R2pVP                  VR4      pVe"   VP                  4       pVP                  V4       / pV
R8X  d<   V	P                  RR4      '       d$   R	VR&   \        V	P                  R
R4      4      VR
&   VP                  ! RRVe   TM%\        P                  ! R\        P                   R7      RSP"                  RV P$                  V P&                  3/VB p\        P(                  P+                  \-        VP/                  VR,          R7      VR,          V4      RR7      V n        M[V	eX   \        P2                  ! \5        \        P6                  ! V	4      P9                  R4      4      \        P:                  R7      W R2&   \<        SV `}  WW4WVV4       V F  pVV9   g   K  VPA                  V4       K  	  R# )rT   r'  Nr*  rÒ  r)  r*  r/  FTr0  r1  rã   rZ   r  r^  r<  r[   r^  )r`  r>  rÒ  r   )!rÁ   rC  rD  rE  rF  r'   rÖ  r‘  rú  r’  rÀ   rê   rJ  re  r   rb  rc  rw  r  r  r(   ra   rÝ   rÜ   rT   rß   r`  ra  rb  r†   r?  rs  rM  )r!  rc  rd  re  r}  rf  rg  r~  Ú
weight_keyrR  rÖ  r$  rS  rT  rT   Ú	scale_keyrã   ÚextrarL   r   ry  rB  s   &&&&&&&&            €€r   rs  ÚNmixed_precision_ops.<locals>.MixedPrecisionOps.Embedding._load_from_state_dict>  sm  ø€ Ø &˜x vÐ.�
Ø'Ÿ^™^¨v¨h°kÐ,BÀDÓI�
ØÒ)Ü!%§¢¨J×,<Ñ,<Ó,>×,FÑ,FÓ,HÓ!I�Jð <FÒ;Q˜zŸ~™~¨hÔ7ÐW[�Ø')Ð$àÐ#VÕVÐ[eÕ[sØ(4Ô%Ü)¨,Õ7�GØ'.Ð/DÕ'E�DÔ$Ü!1°$×2BÑ2BÓ!C�JØ'Ÿ^™^¨JÓ7�FØ(×/Ñ/°
Ô;à#) (¨,Ð 7�IØ&ŸN™N¨9°dÓ;�EØÒ(Ø %§¡£˜Ø,×3Ñ3°IÔ>à�EØ#Ð'8Ô8¸Z¿^¹^ÈIÐW\×=]Ò=]à+/˜˜iÑ(Ü58¸¿¹ÐH[Ð]`Ó9aÓ5b˜Ð1Ñ2Ø'×.Ò.ñ Ø',Ò'8™e¼e¿jºjÈÔSX×S`ÑS`Ô>aðà#4×#CÑ#Cðð %)×$7Ñ$7¸×9KÑ9KÐ#Lðð  ñ	�Fô #(§(¡(×"4Ñ"4Ü'¨¯	©	¸ÀÕ8L¨	Ó(MÈwÐWlÕOmÐouÓvØ&+ð #5ó #-�D•Kð  Ò+ä9>¿ºÜœTŸZšZ¨
Ó3×:Ñ:¸7ÓCÓDÌEÏKÉKô:Y�J ¨Ð5Ñ6ô ‘Ñ-¨jÀ.ÐZfð  zDô  EÛ-�AØ˜LÖ(Ø$×+Ñ+¨AÖ.ó .r   r  Nrd  r€  c               ó*   € Ve   TM/ p\        WV4      # r   rƒ  r„  s   &$$*, r   rc  ÚCmixed_precision_ops.<locals>.MixedPrecisionOps.Embedding.state_dicto  s   € Ø$/Ò$;‘[À�Ü3°D¸fÓEÐEr   c           	     ón  <€ V P                   p\        V\        4      '       g   \        S	V `  WR 7      # V P
                  R8w  d   V P                  WR 7      # VP                  R4      P                  4       pVP                  pVP                  P                  4       ^8¼  d   VP                  V,          MVP                  p\        P                  ! WVP                  VP                  4      R7      pVP                   V,          P                  VP                  4      p\#        V P$                  4      P'                  Wu\(        P*                  ! VP-                  4       VP                  R7      4      pVP.                  ! . VP0                  OVP0                  R,          N5!  pVf   V# VP                  VR7      # )©r1  rÒ  rë  r#  rZ   r6   )rT   rÇ   rÝ   r?  r7  rÖ  rŠ  r½  r5  r]  rã   r   rÀ  rÁ  rÜ   rS   r¿  r’  rú  Údequantize_embeddingr   ÚarangeÚnumelr6  r   )
r!  rU   r1  rT   ÚidxrL   rã   rÐ  rì   rB  s
   &&&      €r   r7  ÚBmixed_precision_ops.<locals>.MixedPrecisionOps.Embedding.host_rowss  s]  ø€ ØŸ™�Ü! &¬/×:Ò:Ü ™7Ñ,¨UÐ,ÓHÐHØ×$Ñ$Ð(9Ô9à×:Ñ:¸5Ð:ÓVÐVØ—m‘m BÓ'×+Ñ+Ó-�ØŸ™�Ø-3¯\©\×-=Ñ-=Ó-?À1Ô-D˜Ÿ™ SÖ)È&Ï,É,�Ü$×,Ò,¨V¿8¹8ÀEÇLÁLÓ;QÔR�Ø—}‘} SÕ)×,Ñ,¨U¯\©\Ó:�Ü$ T×%5Ñ%5Ó6×KÑKÈDÔZ_×ZfÒZfÐgj×gpÑgpÓgrð  |A÷  |Hñ  |Hô  [Ió  J�Ø—F’FÐ5˜EŸK™KÐ5¨¯©°­Ó5�Ø%Ò-�qÐH°1·4±4¸i°4Ó3HÐHr   c                óÊ  <€ V P                   p\        V\        4      '       Edž   \        V P                  4      ^ 8X  Edƒ   \        WP                  VP                  RR7      ;_uu_ 4       w  rE\        V\        4      '       d&   VP                  pVP                  pVP                  pMVP                  pRpV P                  R8X  dH   \        V P                  4      P                  WFV4      pVf   TMVP                  VR7      uuRRR4       # \         P"                  P$                  P'                  WV P(                  V P*                  V P,                  V P.                  V P0                  4      pRRR4       Ve   TMVP                  P2                  p	XP                  V	R7      pXe   W‡P                  V	R7      ,          pV# \4        S
V `m  WR7      #   + '       g   i     Ll; i)r"   Tr/  NrÒ  rZ   rñ  )rT   rÇ   rÝ   r&   r  r  rS   rR   r]  rã   r¿  rÖ  r’  rú  rò  rÜ   r   r(   r)   r0  r   r!  r"  r#  r$  r  r?  rŠ  )r!  rU   r1  rT   ÚqdataÚ_biasrL   rã   rì   Útarget_dtyperB  s   &&&       €r   rŠ  ÚSmixed_precision_ops.<locals>.MixedPrecisionOps.Embedding.forward_comfy_cast_weightsƒ  s…  ø€ ØŸ™�ô ˜f¤o×6Ó6¼3¸t×?SÑ?SÓ;TÐXYÕ;YÜ.¨t¿L¹LÐPV×P\ÑP\Ðjn×oÕoñ  tBÐtyÜ% e¬_×=Ò=Ø%*§]¡]˜FØ$*§L¡L˜EØ$)§L¡L™Eà%+§^¡^˜FØ$(˜Eð  ×,Ñ,Ð0AÔAÜ 0°×1AÑ1AÓ B× WÑ WÐX]ÐglÓ m˜AØ(1Ò(9¡1¸q¿t¹tÈ)¸tÓ?T÷ pÒoô "ŸH™H×/Ñ/×9Ñ9Ø!¨$×*:Ñ*:¸D¿M¹MØ ŸN™N¨D×,CÑ,CÀTÇ[Á[óR˜÷ pð" 1:Ò0E¡9È6Ï>É>×KdÑKd�LØŸ™ <˜Ó0�AØÒ(Ø§¡¨| Ó <Õ<˜Ø�Hô ‘wÑ9¸%Ð9ÓUÐU÷1 p×oús   Á*BGÄA GÇG"	)rú  rÖ  rT   r   )r2  r3  r4  r5  rs  rc  r7  rŠ  r6  r7  rU  )rB  r+  ry  s   @@€r   r  rè  =  s3   ú‡ € ö//ðbF°Dð FÀô F÷I÷ V÷ Vr   r  r   N)r2  r3  r4  r5  Ú_quant_configrw  rÇ  Ú	_disabledr   r(   ru  rÕ  rt  ræ  rB  r  r6  )ry  ró   r¦  Úfull_precision_mmÚquant_configs   €€€€€r   ry  rn    sx   ø† Ù$ˆÙ&ˆÙ.ÐÙˆ	÷V	;ð V	;�U—X‘X—_‘_Ð&6ô V	;÷pZ	zð Z	z˜Ÿ™Ÿ™Ð*:ô Z	z÷xc	V˜×-Ñ-÷ c	Vð c	Vr   ry  )rB  )rÿ  ró   rþ  r¦  ry  s   ffff@r   Úmixed_precision_opsr     s#   ü€ ÷]Vó ]VœKô ]Vð~ Ðr   c                 ó8  € \        4       p\        P                  P                  V 4      '       g   VP	                  R 4       \        P                  P                  V 4      '       g   VP	                  R4       \        P                  P                  V 4      '       g#   VP	                  R4       VP	                  R4       \        P                  P                  V 4      '       g8   VP	                  R4       VP	                  R4       VP                  \        4       V# )r,  r+  r`  r>  rÒ  r2  )
Úsetr	   r
   Úsupports_nvfp4_computeÚaddÚsupports_mxfp8_computeÚsupports_fp8_computeÚsupports_int8_computer_  rK  )rS   r¦  s   & r   Úget_disabled_quant_formatsr  ¤  sÀ   € Ü‹u€HÜ×!Ñ!×8Ñ8¸×@Ò@Ø�‰�WÔÜ×!Ñ!×8Ñ8¸×@Ò@Ø�‰�WÔÜ×!Ñ!×6Ñ6°v×>Ò>Ø�‰�_Ô%Ø�‰�]Ô#Ü×!Ñ!×7Ñ7¸×?Ò?Ø�‰Ð&Ô'Ø�‰�^Ô$Ø�‰Ô-Ô.Ø€Or   c              #   óV  "  € \        V4      p. p V P                  4        FW  p\        V\        4      '       g   K  VP	                  V4      '       g   K4  VP                  WDP                  34       R Vn        KY  	  Rx € V F  w  rEWTn        K  	  R#   T F  w  rEYTn        K  	  i ; i5i)FN)r  ÚmodulesrÇ   rÕ  rj  rÀ   rÇ  )ÚmodelrS   r¦  Úpreviousrz   rþ  s   &&    r   Úuse_quantized_matmulr  ´  s�   é € ä)¨&Ó1€HØ€Hð:Ø—m‘m–oˆFÜ˜&Ô"2×3Ô3¸×8WÑ8WÐX`×8aÔ8aØ—‘ ×)BÑ)BÐ CÔDØ,1�Ö)ñ &ó 	ã)1Ñ%ˆFØ(9Ö%ó *2ø«Ñ%ˆFØ(9Ö%ò *2üs'   ‚B)‘&B ¼B Á+B Â B)ÂB&Â&B)c           
      ó¦  € V'       dÜ   \        VR 4      '       dÊ   VP                  '       d¸   \        P                  ! R4       \	        V4      p\        P                  ! RP                  RP                  \        P                  ! 4       V,
          4      \        V4      ^ 8”  d!   RP                  RP                  V4      4      MR4      4       \        VP                  WR7      # \        P                  P                  V4      pV'       d:   V'       g$   \        P                  \         P"                  9   d   V'       g   \$        # \        P&                  \         P"                  9   dW   \(        '       dK   V \*        P,                  8X  d6   V\*        P,                  8X  g   Vf   \        P                  ! R4       \.        # Ve   W8X  d   \0        # \2        # )rÿ  z Using mixed precision operationszNative ops: {} {}z, z, emulated ops: {}r€  )r¦  zUsing cublas ops)r^   rÿ  r}  r~  r  r*  Újoinr‘  Úkeysr&   r   r	   r
   r  r   ÚFp8MatrixMultiplicationr   Úfastrv  Ú	CublasOpsÚCUBLAS_IS_AVAILABLEr   r¶  r„  rX  rB  )Úweight_dtyperó   Úload_deviceÚdisable_fast_fp8Úfp8_optimizationsÚmodel_configr¦  Úfp8_computes   &&&&&&  r   Úpick_operationsr  Ã  sc  € ßœ ¨n×=Ò=À,×B[×B[ÐB[Ü�ŠÐ7Ô8Ü-¨kÓ:ˆÜ�ŠÐ(×/Ñ/°·	±	¼+×:JÒ:JÓ:LÈxÕ:WÓ0Xô  ORð  S[ó  O\ð  _`ô  O`ÐZn×ZuÑZuÐvz×vÑvð  AIó  wJô  [Kð  fhó  iô  	jÜ" <×#<Ñ#<¸mÔ_Ð_ä×(Ñ(×=Ñ=¸kÓJ€Kçß	Ô0×HÑHÌDÏIÉIÔUßäˆô 	×$Ñ$¬¯	©	Ô1ßÓØœŸ™Ô%Ø	œ%Ÿ-™-Ô	'¨=Ò+@ä�ŠÐ'Ô(ÜÐàÒ Ô =Ü"Ð"äÐr   )r4   é	   )r4   é
   )FTr¤  rQ  )NNNNFNF)Nr=   rÝ  )NNr=   NN)Nr   r   )NFFN)mrR  r   r}  rS  rÀ  ÚinspectÚcomfy.model_managementr	   Úcomfy.cli_argsr   r   Úcomfy.floatrC  Úcomfy.memory_managementÚcomfy.pinned_memoryÚcomfy.rmsnormÚcomfy.utilsÚcomfy_aimdo.model_vbarrˆ   Úcomfy_aimdo.torchr   r   r    r*   rG   Úis_availableÚtorch.nn.attentionr-   r.   r“   r=  ÚFLASH_ATTENTIONÚCUDNN_ATTENTIONÚEFFICIENT_ATTENTIONÚMATHrE   rA   r
   Úis_amdÚget_device_propertiesÚget_torch_deviceÚpropsÚgcnArchNameÚsplitÚL2_cache_sizerB  ÚModuleNotFoundErrorÚ	TypeErrorrµ  r@   rF   ÚcudnnÚversionÚcudnn_versionÚtorch_version_numericr~  rQ   rV   rg   rp   rØ   rü   r  r  r  r:  r(   ru  rÈ   rX  rB  rt  rv  r  r„  r�  r‚  ÚImportErrorr€  r�  rÝ   r‘  r_  r’  r™  rž  r±  rT  r®  r²  r·  r¼  rÃ  rÅ  rÂ  râ  rK  r  ÚFunctionré  r  r[  rg  rÕ  r·  r   r  r  r  r   r   r   Ú<module>r=     sß  ðñó$ Û Û Û Û Û ß 3Û Û Û Û Û Û ã Û òGò$òòVð+<Ø‡z�z×Ñ× Ò ß>Ø˜W×.Ò.¨{Ó;×FÑFÔFà×*Ñ*Ø×*Ñ*Ø×.Ñ.Ø—‘ð	%Ð!ð *+Ð&Ø×%Ñ%×,Ñ,×.Ò.ØŸ
™
×8Ñ8¸×9OÑ9O×9`Ñ9`Ó9bÓc�Ø×$Ñ$×*Ñ*¨3Ó/°Õ2°iÔ?Ø16×1DÑ1DÐ.ófð0 �OŠOÐQÔRð %*Ð !ð	Ø×Ñ×'Ñ'×)Ò)ØŸ™×,Ñ,×4Ñ4Ó6ˆØ˜UÔ" }°uÔ'<À%×BXÑBX×BnÑBnÐrxÔBxð  ~C÷  ~Tñ  ~T÷  ~jñ  ~jð  nuô  ~uà04Ð-Ø�LŠLÐCÔDð ×
 Ñ
 ×
(Ñ
(€ôsòFòMôcôLD6ôNeZòPB÷((ñ (÷*ñ ô'P�U—X‘X—_‘_Ð&6ô 'P÷RO@ñ O@ôd
 "Ð%ô  "òF'ôRGˆkô Gð& Ð ð	ß;ØÐ÷ ô<�[ô <õ& ÷ó ò3ò
eð ÑJØˆmØ
ˆKð€ð ×Ññ"ó ð"ô)òUò8ò
÷:Iñ :Iôz;ñ8cØOSõcð4 *¨1¨k¸1Ð=Ð ô_D�e—n‘n×-Ñ-ô _DôFô$O%ôd%ôP?Ð'ô ?ð &(°u·~±~ÐY^Ðikô `ôDð  ×Ññ:ó ð:öøðo: 	˜YÐ'ô <Ø‡O‚OÐ:×;ð<ûð	Úûðv ô 	Úð	úsK   ÁM7 Á0A8M7 Ã)A)M7 ÅM7 Å(N ÆA<N Ê
N! Í7NÎNÎNÎ!N,Î+N,