+
    „°ñj\×  ã                   ód  € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RI	H
t
 ^ RIHtHtHt ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHtHt ^ RIHtHt ^ RIt^ RIt^ RI t ^ RI!H"t"H#t# ^ RI$H%t% ^ R	I&H't'H(t(H)t) ^ R
IH*t*  ! R R]PV                  PX                  4      t-R5R lt.R6R lt/ ! R R]P`                  Pb                  4      t2 ! R R]P                  Pf                  4      t4R t5R7R R llt6R8R R llt7 ! R R]Pp                  Pr                  4      t:R9R lt;R t<R t=R t>R t?R t@R  tAR! tBR" tCR# tDR$ tER% tFR& tGR' tHR( tI ! R) R*](P”                  4      tK ! R+ R,](P”                  4      tL ! R- R.](P”                  4      tM ! R/ R0](P”                  4      tN ! R1 R2]'4      tOR3 R4 ltPR# ):é    N)Útrange)ÚImageÚ	ImageDrawÚ	ImageFont)Úoverride)ÚCONDRegularÚCONDList)ÚargsÚPerformanceFeature)ÚadaptersÚadapter_maps)ÚBypassInjectionManager)ÚComfyExtensionÚioÚui)ÚProgressBarc                   óH   a a€ ] tR t^t oRtRR/V 3R lltRR ltRtVtV ;t	# )ÚTrainGuiderz:
CFGGuider with modifications for training specific logic
Ú
offloadingFc               ó4   <€ \         SV `  ! V/ VB  Wn        R # ©N)ÚsuperÚ__init__r   )Úselfr   r
   ÚkwargsÚ	__class__s   &$*,€Ú<C:\Ai_Coding\apps\studio\ComfyUI\comfy_extras\nodes_train.pyr   ÚTrainGuider.__init__"   s   ø€ Ü‰Ò˜$Ð) &Ò)Ø$Žó    c
                óö  € \         P                  P                  V P                  VP                  V P
                  V P                  V P                  '       * V P                  R 7      w  V n        V n        V n	        \        P                  P                  4        V P                  P                  p
Ve+   \         P                  P                  WQP                  V
4      pVP                  V
4      pVP                  V
4      pVP                  V
4      p\         P                   P#                  V P                  W P                  P%                  4       R7        V P                  P'                  4        V P)                  VVV
VVVVVVV	R7
      pV P                  P+                  4        \         P                  P-                  V P
                  V P                  4       V =V =	V#   T P                  P+                  4        i ; i))Úforce_full_loadÚforce_offload)ÚdeviceÚdtype)Úlatent_shapes)ÚcomfyÚsampler_helpersÚprepare_samplingÚmodel_patcherÚshapeÚcondsÚmodel_optionsr   Úinner_modelÚloaded_modelsÚtorchÚcudaÚempty_cacheÚload_deviceÚprepare_maskÚtoÚsamplersÚcast_to_load_optionsÚmodel_dtypeÚpre_runÚinner_sampleÚcleanupÚcleanup_models)r   ÚnoiseÚlatent_imageÚsamplerÚsigmasÚdenoise_maskÚcallbackÚdisable_pbarÚseedr%   r#   Úoutputs   &&&&&&&&&&  r   Úouter_sampleÚTrainGuider.outer_sample&   s«  € ô ×!Ñ!×2Ñ2Ø×"Ñ"Ø—‘Ø—
‘
Ø×"Ñ"Ø$(§O¡OÔ 3Ø"Ÿo™oð 3ó ñ 	9ˆÔ˜$œ* dÔ&8ô 	�
‰
×ÑÔ Ø×#Ñ#×/Ñ/ˆàÒ#Ü ×0Ñ0×=Ñ=ØŸk™k¨6óˆLð —‘˜Ó ˆØ#—‘ vÓ.ˆØ—‘˜6Ó"ˆÜ�‰×+Ñ+Ø×Ñ v×5GÑ5G×5SÑ5SÓ5Uð 	,ô 	
ð	)Ø×Ñ×&Ñ&Ô(Ø×&Ñ&ØØØØØØØØØØ+ð 'ó ˆFð ×Ñ×&Ñ&Ô(ä×Ñ×,Ñ,¨T¯Z©Z¸×9KÑ9KÔLØÐØÐØˆøð ×Ñ×&Ñ&Õ(ús   Å5G ÇG8)r+   r-   r.   r   )NNFNN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   rE   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   Ú__classdict__s   @@r   r   r      s    ù‡ € ñð%¨÷ %÷9ô 9r   r   c                 ó¦  € / pV P                  4        Fµ  w  rETp\        V\        4      '       d   \        WQVR 7      pM†\        V\        P
                  4      '       d$   Ve   VP                  ^ 4      V8X  d	   WQ,          pMC\        V\        \        34      '       d(   \        V4      V8X  d   V Uu. uF  quV,          NK  	  ppWcV&   K·  	  V# u upi )©Ú	full_size)
ÚitemsÚ
isinstanceÚdictÚmake_batch_extra_option_dictr/   ÚTensorÚsizeÚlistÚtupleÚlen)ÚdÚindiciesrS   Únew_dictÚkÚvÚnewvÚis   &&&     r   rW   rW   b   s¬   € Ø€HØ—‘–	‰ˆØˆÜ�aœ×ÒÜ/°ÀyÔQ‰DÜ˜œ5Ÿ<™<×(Ò(ØÒ  A§F¡F¨1£I°Ô$:Ø•{�øÜ˜œD¤%˜=×)Ò)¬c°!«f¸	Ô.AÙ"*Ó+¡(˜Q�a—D�D¡(ˆDÐ+Ø�‹ñ ð €Oùò ,s   Â1Cc           
      óò  € \        V R 4      '       d9   \        V R4      '       g'   \        V 4       F  w  r#\        W1 RV 24       K  	  V # \        V R4      '       Ed•   \        V P	                  4       4       EFv  w  rE\        V\        4      '       d   \        WQ RV 24       K.  \        V\        P                  4      '       d   VP                  4       W&   Kb  \        V\        4      '       dT   VP                   Uu. uF5  p\        V\        P                  4      '       d   VP                  4       MTNK7  	  upVn        KË  \        V\        4      '       dP   \        VP                  \        P                  4      '       d#   VP                  P                  4       Vn        EK-  EK0  \        V\        \        34      '       g   EKO  \        V4       F  w  r#\        W1 RV RV 24       K  	  EKy  	  V # u upi )Ú__iter__rT   Ú.)ÚhasattrÚ	enumerateÚprocess_cond_listrZ   rT   rU   rV   r/   rX   Úcloner	   ÚcondÚdetachr   r[   )r]   ÚprefixÚindexÚitemr`   ra   Úts   &&     r   ri   ri   q   sh  € Üˆq�*×Ò¤g¨a°×&9Ò&9Ü$ Qž<‰KˆEÜ˜d h¨a°¨wÐ$7Ö8ñ (àˆÜ	��G×	Ó	Ü˜Ÿ™›—O‰DˆAÜ˜!œT×"Ò"Ü! ! x¨q°° _Ö5Ü˜AœuŸ|™|×,Ò,Ø—w‘w“y�“Ü˜Aœx×(Ò(ØTU×TZÒTZÓ[ÑTZÈq¬
°1´e·l±l×(CÒ(C˜!Ÿ(™(œ*ÈÒJÑTZÑ[�–Ü˜Aœ{×+Ò+Ü˜aŸf™f¤e§l¡l×3Ò3ØŸV™VŸ]™]›_�A—Fò 4ä˜A¤¤e˜}×-Õ-Ü#,¨Q¦<‘K�EÜ% d¨h°a¸°s¸!¸E¸7Ð,CÖDô $0ñ $ð €Hùò \s   Ã=;G4c            	       ó‚   a € ] tR t^‡t o R^^^^ ]P
                  RRR3	R ltR tRR ltR t	R t
R tR	 tRR
 ltRtV tR# )ÚTrainSamplerNFc                ó2  € Wn         W n        W0n        W@n        W`n        WPn        Wpn        W€n        W�n        V
V n	        V'       d   \        P                  P                  4       MR V n        V
e   V P                  V
4       R # R V n        R V n        R V n        R # r   )Úloss_fnÚ	optimizerÚloss_callbackÚ
batch_sizeÚtotal_stepsÚgrad_accrC   Útraining_dtypeÚreal_datasetÚbucket_latentsr/   ÚampÚ
GradScalerÚgrad_scalerÚ_init_bucket_dataÚbucket_offsetsÚbucket_weightsÚ
num_images)r   rt   ru   rv   rw   ry   rx   rC   rz   r{   r|   Úuse_grad_scalers   &&&&&&&&&&&&r   r   ÚTrainSampler.__init__ˆ   s‰   € ð ŒØ"ŒØ*ÔØ$ŒØ&ÔØ ŒØŒ	Ø,ÔØ7CÔð ð 	Ô÷ 6Eœ5Ÿ9™9×/Ñ/Ô1È$ˆÔàÒ%Ø×"Ñ" >Ö2à"&ˆDÔØ"&ˆDÔØ"ˆDŽOr   c                ó~  € ^ .V n         . pV Fi  pVP                  VP                  ^ ,          4       V P                   P                  V P                   R,          VP                  ^ ,          ,           4       Kk  	  V P                   R,          V n        \        P
                  ! V\        P                  R7      V n        R# )z3Initialize bucket offsets and weights for sampling.©r$   Néÿÿÿÿ)r�   Úappendr*   rƒ   r/   ÚtensorÚfloat32r‚   )r   r|   Úbucket_sizesÚlats   &&  r   r€   ÚTrainSampler._init_bucket_data­   sˆ   € à ˜cˆÔØˆÛ!ˆCØ×Ñ §	¡	¨!¥Ô-Ø×Ñ×&Ñ& t×':Ñ':¸2Õ'>ÀÇÁÈ1ÅÕ'MÖNñ "ð ×-Ñ-¨bÕ1ˆŒä#Ÿlšl¨<¼u¿}¹}ÔMˆÖr   c
                óŠ  € VP                   P                  P                  W#VR 4      p
VP                   P                  P                  \        P                  ! V4      \        P                  ! V4      VR 4      pV Uu. uF  qÅV,          NK  	  upVP
                  R&   \        WvVR7      p\        P                  ! V
P                  P                  V P                  R7      ;_uu_ 4        V! V
P                  R4      VP                  R4      3/ VB pV P                  VP                  4       VP                  4       4      pRRR4       V	'       d]   XV P                  ,          pV P                  e,   V P                  P!                  V4      P#                  4        V# VP#                  4        X# u upi   + '       g   i     L{; i)FÚpositiverR   r‡   TN)r-   Úmodel_samplingÚnoise_scalingr/   Ú
zeros_liker+   rW   Úautocastr#   Útyperz   Úrequires_grad_rt   Úfloatry   r   ÚscaleÚbackward)r   Ú
model_wrapÚbatch_sigmasÚbatch_noiseÚbatch_latentrk   r^   Ú
extra_argsÚdataset_sizeÚbwdÚxtÚx0rc   Úbatch_extra_argsÚx0_predÚlossÚbwd_losss   &&&&&&&&&&       r   Úfwd_bwdÚTrainSampler.fwd_bwd¸   sx  € ð ×#Ñ#×2Ñ2×@Ñ@Ø |°Uó
ˆð ×#Ñ#×2Ñ2×@Ñ@Ü×Ò˜\Ó*Ü×Ò˜[Ó)ØØó	
ˆñ :BÓ'B¹°A¨Q¯¨¹Ñ'Bˆ
×Ñ˜Ñ$Ü7Ø¨Lô
Ðô �^Š^˜BŸI™IŸN™N°$×2EÑ2E×FÖFÙ Ø×!Ñ! $Ó'Ø×+Ñ+¨DÓ1ñð #ñˆGð
 —<‘< §¡£°·±³Ó<ˆD÷ G÷ Ø˜dŸm™mÕ+ˆHØ×ÑÒ+Ø× Ñ ×&Ñ& xÓ0×9Ñ9Ô;ð ˆð ×!Ñ!Ô#Øˆùò% (C÷
 G×Fús   Á<F-Ã(AF2Æ2G	c                ó  € \        V4       Uu. uFJ  pVP                  P                  P                  \        P
                  ! R4      P                  4       4      NKL  	  pp\        P                  ! V4      P                  V4      # u upi )z)Generate random sigma values for a batch.©é   )	Úranger-   r‘   Úpercent_to_sigmar/   Úrandro   rŠ   r4   )r   rš   rw   r#   Ú_r›   s   &&&&  r   Ú_generate_batch_sigmasÚ#TrainSampler._generate_batch_sigmasâ   sz   € ô ˜:Ô&ó	
ñ '�ð ×"Ñ"×1Ñ1×BÑBÜ—
’
˜4Ó ×%Ñ%Ó'öñ 'ð	 	ð 
ô �|Š|˜LÓ)×,Ñ,¨VÓ4Ð4ùò
s   ŽABc                ó:  € \         P                  ! V P                  ^4      P                  4       pV P                  V,          pVP
                  ^ ,          p	V P                  V,          p
\        V P                  V	4      p\         P                  ! V	4      RV P                  4       pV Uu. uF  qÚV,           NK  	  ppWŒ,          P                  V4      pVP                  RV/4      P                  VP                  4      pV P                  WVP                  4      pV P                  VVVVVVVV P                   RR7	      pV P"                  '       d    V P#                  VP                  4       4       VP%                  RVP                  4       R RV/4       R# u upi )z)Execute one training step in bucket mode.NÚsamplesT©r    r¥   ú.4fÚbucket)r/   Úmultinomialr‚   ro   r|   r*   r�   Úminrw   ÚrandpermÚtolistr4   Úgenerate_noiser#   r°   r§   rƒ   rv   Úset_postfix)r   rš   rk   rž   Únoisegenr=   ÚpbarÚ
bucket_idxÚbucket_latentÚbucket_sizeÚbucket_offsetÚactual_batch_sizeÚrelative_indicesÚidxÚabsolute_indicesr�   rœ   r›   r¥   s   &&&&&&&            r   Ú_train_step_bucket_modeÚ$TrainSampler._train_step_bucket_modeì   sy  € ô ×&Ò& t×':Ñ':¸AÓ>×CÑCÓEˆ
Ø×+Ñ+¨JÕ7ˆØ#×)Ñ)¨!Õ,ˆØ×+Ñ+¨JÕ7ˆô   §¡°Ó=ÐÜ Ÿ>š>¨+Ó6Ð7IÐ8IÐJ×QÑQÓSÐá;KÓLÑ;K°C¨C×/Ð/Ñ;KÐÐLà$Õ6×9Ñ9¸,ÓGˆØ×-Ñ-¨y¸,Ð.GÓH×KÑKØ×Ñó
ˆð ×2Ñ2°:ÐR^×ReÑReÓfˆà�|‰|ØØØØØØØØ�O‰OØð ó 

ˆð ××ÐØ×Ñ˜tŸy™y›{Ô+Ø×Ñ˜& T§Y¡Y£[°Ð$5¸À*ÐMÖNùò+ Ms   Â*Fc                ó`  € \         P                  ! V4      RV P                   P                  4       p\         P                  ! V U	u. uF  q•V	,          NK  	  up	4      p
VP                  RV
/4      P                  V
P                  4      pV P                  V\        V P                  V4      V
P                  4      pV P                  VVVV
VVVVRR7	      pV P                  '       d    V P                  VP                  4       4       VP                  RVP                  4       R /4       R# u up	i )zGExecute one training step in standard (non-bucket, non-multi-res) mode.Nr³   Tr´   r¥   rµ   )r/   r¹   rw   rº   Ústackr»   r4   r#   r°   r¸   r§   rv   ro   r¼   )r   rš   rk   rž   r½   r=   rŸ   r¾   r^   rc   r�   rœ   r›   r¥   s   &&&&&&&&      r   Ú_train_step_standard_modeÚ&TrainSampler._train_step_standard_mode  s  € ä—>’> ,Ó/Ð0A°$·/±/ÐB×IÑIÓKˆÜ—{’{¹XÓ#F¹X¸°§O O¹XÑ#FÓGˆØ×-Ñ-¨y¸,Ð.GÓH×KÑKØ×Ñó
ˆð ×2Ñ2°:¼sÀ4Ç?Á?ÐT`Ó?aÐco×cvÑcvÓwˆà�|‰|ØØØØØØØØØð ó 

ˆð ××ÐØ×Ñ˜tŸy™y›{Ô+Ø×Ñ˜& T§Y¡Y£[°Ð$5Ð7Ö8ùò' $Gs   ÁD+c                óÊ  € \         P                  ! V4      RV P                   P                  4       p^ p	V Fë  p
V P                  V
,          P                  V4      pVP                  RV/4      P                  VP                  4      pVP                  P                  P                  \         P                  ! R4      P                  4       4      p\         P                  ! V.4      P                  VP                  4      pV P                  VVVVVV
.VVRR7	      pWž,          p	Kí  	  W�P                  ,          \!        V4      ,          p	V P"                  e+   V P"                  P%                  V	4      P'                  4        MV	P'                  4        V P(                  '       d    V P)                  V	P                  4       4       VP+                  RV	P                  4       R /4       R# )zIExecute one training step in multi-resolution mode (real_dataset is set).Nr³   Fr´   r¥   rµ   rª   )r/   r¹   rw   rº   r{   r4   r»   r#   r-   r‘   r­   r®   ro   rŠ   r§   ry   r\   r   r˜   r™   rv   r¼   )r   rš   rk   rž   r½   r=   rŸ   r¾   r^   Ú
total_lossrn   Úsingle_latentrœ   r›   r¥   s   &&&&&&&&       r   Ú_train_step_multires_modeÚ&TrainSampler._train_step_multires_mode'  s™  € ä—>’> ,Ó/Ð0A°$·/±/ÐB×IÑIÓKˆØˆ
ÛˆEØ ×-Ñ-¨eÕ4×7Ñ7¸ÓEˆMØ"×1Ñ1Ø˜MÐ*óç‰b�×%Ñ%Ó&ð ð ×&Ñ&×5Ñ5×FÑFÜ—J’J˜tÓ$×)Ñ)Ó+óð ô
 !Ÿ<š<¨¨Ó7×:Ñ:¸=×;OÑ;OÓPˆLØ—<‘<ØØØØØØ�ØØØð  ó 
ˆDð ÕŠJñ- ð.  §-¡-Õ/´#°h³-Õ?ˆ
Ø×ÑÒ'Ø×Ñ×"Ñ" :Ó.×7Ñ7Õ9à×ÑÔ!Ø××ÐØ×Ñ˜zŸ™Ó0Ô1Ø×Ñ˜& Z§_¡_Ó%6°sÐ$;Ð=Ö>r   c	           
     óÄ  € \        VP                  4      Vn        VP                  R ,          p	VP                  ^ 4      p
\        P                  P                  4        \        V P                  4      p\        V P                  RR\        P                  P                  '       * R7      ;p EFÿ  p\        P                  P                  V P                  VR,          ,           4      pV P                   e   V P#                  WW>Wl4       M7V P$                  f   V P'                  WW>WjV4       MV P)                  WW>WjV4       V^,           V P*                  ,          ^ 8X  Ed;   V P,                  e&   V P,                  P/                  V P0                  4       V P0                  P2                   Fk  pVR,           F[  pVP4                  f   K  VP4                  P6                  P9                  VP6                  P:                  4      VP4                  n        K]  	  Km  	  V P,                  eA   V P,                  P=                  V P0                  4       V P,                  P?                  4        MV P0                  P=                  4        V P0                  PA                  4        VP?                  ^4       EK  	  \        P                  P                  4        \        PB                  ! V4      # )r�   zTraining LoRAg{®Gáz„?)ÚdescÚ	smoothingÚdisableiè  Úparams)"ri   r+   rY   r/   r0   r1   r   rx   r   r&   ÚutilsÚPROGRESS_BAR_ENABLEDÚcomfy_extrasÚnodes_custom_samplerÚNoise_RandomNoiserC   r|   rÇ   r{   rË   rÐ   ry   r   Úunscale_ru   Úparam_groupsÚgradÚdatar4   r$   ÚstepÚupdateÚ	zero_gradr“   )r   rš   r?   rž   rA   r<   r=   r@   rB   rk   rŸ   Úui_pbarr¾   rc   r½   rÝ   Úparams   &&&&&&&&&        r   ÚsampleÚTrainSampler.sampleK  s0  € ô -¨Z×-=Ñ-=Ó>ˆ
ÔØ×Ñ 
Õ+ˆØ—{‘{ 1“~ˆÜ�
‰
×ÑÔ Ü˜d×.Ñ.Ó/ˆäØ× Ñ Ø$ØÜ!ŸK™K×<Ñ<Ô<ô	ð ˆDó ð ô $×8Ñ8×JÑJØ—	‘	˜A �HÕ$óˆHð ×"Ñ"Ò.Ø×,Ñ,¨Z¸zÐUaÕhØ×"Ñ"Ò*Ø×.Ñ.¨zÀÐWcÐswÕxà×.Ñ.¨zÀÐWcÐswÔxà�A•˜Ÿ™Õ&¨!Õ+Ø×#Ñ#Ò/Ø×$Ñ$×-Ñ-¨d¯n©nÔ=Ø$(§N¡N×$?Ô$?�LØ!-¨h×!7Ð!7˜Ø Ÿ:™:Ò-Ù$Ø*/¯*©*¯/©/×*<Ñ*<¸U¿Z¹Z×=MÑ=MÓ*N˜Ÿ
™
žó "8ñ %@ð
 ×#Ñ#Ò/Ø×$Ñ$×)Ñ)¨$¯.©.Ô9Ø×$Ñ$×+Ñ+Õ-à—N‘N×'Ñ'Ô)Ø—‘×(Ñ(Ô*Ø�N‰N˜1×ñAôB 	�
‰
×ÑÔ Ü×Ò Ó-Ð-r   )rw   r|   r�   r‚   ry   r   rv   rt   rƒ   ru   r{   rC   rx   rz   )T)NNF)rG   rH   rI   rJ   r/   Úbfloat16r   r€   r§   r°   rÇ   rË   rÐ   rå   rL   rM   ©rP   s   @r   rr   rr   ‡   sY   ø‡ € ð
 ØØØØØ—~‘~ØØØô##òJ	Nô(òT5ò!OòF9ò0"?÷H3.ò 3.r   rr   c                   óD   a a€ ] tR tRt oV 3R ltR tR tR tRtVt	V ;t
# )ÚBiasDiffi�  c                ó0   <€ \         SV `  4        Wn        R # r   )r   r   Úbias)r   rì   r   s   &&€r   r   ÚBiasDiff.__init__‚  s   ø€ Ü‰ÑÔØŽ	r   c                ó�   € VP                   pVP                  V P                  4      V P                  ,           P                  V4      # r   )r$   r4   rì   )r   ÚbÚ	org_dtypes   && r   Ú__call__ÚBiasDiff.__call__†  s1   € Ø—G‘Gˆ	Ø—‘�T—Y‘Y“ $§)¡)Õ+×/Ñ/°	Ó:Ð:r   c                ót   € V P                   P                  4       V P                   P                  4       ,          # r   )rì   ÚnelementÚelement_size©r   s   &r   Úpassive_memory_usageÚBiasDiff.passive_memory_usageŠ  s'   € Ø�y‰y×!Ñ!Ó# d§i¡i×&<Ñ&<Ó&>Õ>Ð>r   c                óF   € V P                  VR 7       V P                  4       # ))r#   )r4   r÷   )r   r#   s   &&r   Úmove_toÚBiasDiff.move_to�  s   € Ø�‰�vˆÔØ×(Ñ(Ó*Ð*r   )rì   )rG   rH   rI   rJ   r   rñ   r÷   rú   rL   rM   rN   rO   s   @@r   rê   rê   �  s   ù‡ € õò;ò?÷+ò +r   rê   c                 óZ  € R Rr2\         P                  ! RW#3R4      p\        P                  ! V4      p\	        V P                  4       4      \        V P                  4       4      rvV P                  4        Uu. uF  qˆV,
          Wv,
          ,          NK  	  p	p^ V\        V	^ ,          V,          4      ,
          3p
\        V	R,          ^R7       FV  w  r¸\        W±^,
          ,          V,          4      pV\        Wƒ,          4      ,
          pVP                  W¬V3.R^R7       WÍ3p
KX  	  V# u upi )iô  i,  ÚRGBÚwhiteºr«   NN©ÚstartÚblue©ÚfillÚwidth)
r   Únewr   ÚDrawr¸   ÚvaluesÚmaxÚintrh   Úline)Úloss_mapÚstepsr  ÚheightÚimgÚdrawÚmin_lossÚmax_lossÚlÚscaled_lossÚ
prev_pointrc   ÚxÚys   &&            r   Údraw_loss_graphr  ’  sú   € Ø˜ˆ6Ü
�)Š)�E˜E˜?¨GÓ
4€CÜ�>Š>˜#Ó€Dä˜XŸ_™_Ó.Ó/´°X·_±_Ó5FÓ1GˆhØCKÇ?Á?ÔCTÓUÑCT¸a˜•L XÕ%8×9Ð9ÑCT€KÐUà�Vœc +¨a¥.°6Õ"9Ó:Õ:Ð;€JÜ˜+ b�/°×3‰ˆÜ�˜Q•Y• %Õ'Ó(ˆØ”S˜�“_Õ$ˆØ�	‰	�: 1˜vÐ&¨V¸1ˆ	Ô=Ø�VŠ
ñ	 4ð €Jùò Vs   Á6D(c                óL   € V ^8„  d   QhR\         P                  P                  /# )é   Úmodel)r/   ÚnnÚModule)Úformats   "r   Ú__annotate__r  ¤  s   € ÷ ñ Ü�8‰8�?‰?ñr   c                 óæ  € Vf   . pM³\        V R4      '       d¢   \        V \        P                  P                  \        P                  P
                  \        P                  P                  34      '       gE   VP                  V 4       \        P                  ! RV RV P                  P                   R24       V# T;'       g    RpV P                  4        F  w  r4\        WAV RV 24       K  	  V# )NÚforwardzFound module with forward: ú (Ú)Úrootrf   )rg   rU   r/   r  Ú
ModuleListÚ
SequentialÚ
ModuleDictr‰   ÚloggingÚdebugr   rG   Únamed_childrenÚ*find_all_highest_child_module_with_forward)r  ÚresultÚnameÚ	next_nameÚchilds   &&&  r   r+  r+  ¤  sÇ   € ð ‚~Ø‰Ü	�˜	×	"Ò	"¬:Ø”—‘×#Ñ#¤U§X¡X×%8Ñ%8¼%¿(¹(×:MÑ:MÐN÷,ò ,ð 	�‰�eÔÜ�ŠÐ3°D°6¸¸E¿O¹O×<TÑ<TÐ;UÐUVÐWÔXØˆØ�>ˆ>�6€DØ!×0Ñ0Ö2Ñˆ	Ü2°5ÀTÀFÈ!ÈIÈ;ÐBWÖXñ 3à€Mr   c                óz   € V ^8„  d   QhR\         P                  R\        R\        \         P                  ,          /# )r  r  ÚdepthÚreturn)r  r  r
  rZ   )r  s   "r   r  r  µ  s0   € ÷ #ñ #Ü�9‰9ð#Ü ð#ä	Œ"�)‰)…_ñ#r   c           
     óä  € Vf   . pT;'       g    Rp\        V \        P                  \        P                  \        P                  34      p\        V R4      ;'       d    V'       * pV'       dW   V^,          pW18X  dH   VP                  V 4       \        P                  ! RV RV RV P                  P                   R24       V# V P                  4        F  w  rx\        W�W#V RV 24       K  	  V# )ad  
Find modules at a specific depth level for gradient checkpointing.

Args:
    model: The model to search
    depth: Target depth level (1 = top-level blocks, 2 = their children, etc.)
    result: Accumulator for results
    current_depth: Current recursion depth
    name: Current module name for logging

Returns:
    List of modules at the target depth
r$  r!  zFound module at depth z: r"  r#  rf   )rU   r  r%  r&  r'  rg   r‰   r(  r)  r   rG   r*  Úfind_modules_at_depth)	r  r1  r,  Úcurrent_depthr-  Úis_containerÚhas_forwardr.  r/  s	   &&&&&    r   r4  r4  µ  sÙ   € ð  ‚~ØˆØ�>ˆ>�6€Dô ˜e¤b§m¡m´R·]±]ÄBÇMÁMÐ%RÓS€LÜ˜% Ó+×@Ð@°LÔ0@€KçØ˜ÕˆØÔ!Ø�M‰M˜%Ô Ü�MŠMÐ2°5°'¸¸D¸6ÀÀEÇOÁO×D\ÑD\ÐC]Ð]^Ð_Ô`ØˆMð "×0Ñ0Ö2Ñˆ	Ü˜e¨FÀdÀVÈ1ÈYÈKÐCXÖYñ 3ð €Mr   c                   ó\   a € ] tR tRt o Rt]V 3R lR l4       t]V 3R lR l4       tRtV t	R# )	ÚOffloadCheckpointFunctioniÛ  zñ
Gradient checkpointing that works with weight offloading.

Forward: no_grad -> compute -> weights can be freed
Backward: enable_grad -> recompute -> backward -> weights can be freed

For single input, single output modules (Linear, Conv*).
c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r  r  ©r/   rX   )r  rP   s   "€r   r  Ú&OffloadCheckpointFunction.__annotate__æ  s   ø€ ÷ !ñ !™Ÿ™ñ !r   c                ó¸   € V P                  V4       W n        \        P                  ! 4       ;_uu_ 4        V! V4      uuR R R 4       #   + '       g   i     R # ; ir   )Úsave_for_backwardÚ
forward_fnr/   Úno_grad)Úctxr  r?  s   &&&r   r!  Ú!OffloadCheckpointFunction.forwardå  s3   € à×Ñ˜aÔ Ø#ŒÜ�]Š]�_�_Ù˜a“=÷ �_�_‹_ús   µAÁA	c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r  Úgrad_outr;  )r  rP   s   "€r   r  r<  í  s   ø€ ÷ ñ ¡§¡ñ r   c                óL  € V P                   w  pV P                  pR V n        \        P                  ! 4       ;_uu_ 4        VP	                  4       P                  R4      pV! V4      pVP                  V4       VP                  pR R R 4       ???XR 3#   + '       g   i     L; i)NT)Úsaved_tensorsr?  r/   Úenable_gradrl   r–   r™   rÞ   )rA  rD  r  r?  Ú
x_detachedr  Úgrad_xs   &&     r   r™   Ú"OffloadCheckpointFunction.backwardì  s†   € à×Ñ‰ˆØ—^‘^ˆ
ð ˆŒä×Ò× Õ ØŸ™›×2Ñ2°4Ó8ˆJÙ˜:Ó&ˆAØ�J‰J�xÔ Ø—_‘_ˆF÷	 !ð ˆz˜:à�tˆ|Ð÷ !× ús   ¿ABÂB#	© N)
rG   rH   rI   rJ   rK   Ústaticmethodr!  r™   rL   rM   rè   s   @r   r9  r9  Û  s2   ø‡ € ñð ÷!ó ð!ð ÷ó ör   r9  c                 ó6  aa€ \        V R 4      '       g   R# V P                  oV'       dV   \        V \        P                  \        P
                  \        P                  \        P                  34      '       d   V3R lpMV3R loV3R lpSV n        W n        R# )r!  Nc                 ó0   <€ \         P                  V S4      # r   )r9  Úapply)r  Úorg_forwards   &€r   Úcheckpointing_fwdÚ patch.<locals>.checkpointing_fwd  s   ø€ Ü,×2Ñ2°1°kÓBÐBr   c                 ó   <€ S! V / VB # r   rK  )r
   r   rP  s   &&€r   ÚfwdÚpatch.<locals>.fwd  s   ø€ Ù Ð/¨Ñ/Ð/r   c                  ó\   <€ \         P                  P                  P                  SWR R7      # )F)Úuse_reentrant)r/   r×   Ú
checkpoint)r
   r   rT  s   *,€r   rQ  rR    s&   ø€ Ü—;‘;×)Ñ)×4Ñ4°S¸$ÐV[Ð4Ó\Ð\r   )	rg   r!  rU   r  ÚLinearÚConv1dÚConv2dÚConv3drP  )Úmr   rQ  rT  rP  s   && @@r   Úpatchr^     se   ù€ Ü�1�i× Ò ÙØ—)‘)€K÷ ”j ¤R§Y¡Y´·	±	¼2¿9¹9ÄbÇiÁiÐ$P×QÒQö	Cõ	0õ	]ð  €A„MØ!†Ir   c                 óT   € \        V R 4      '       d   V P                  V n        V =R# R# )rP  N)rg   rP  r!  )r]  s   &r   Úunpatchr`    s$   € Üˆq�-× Ò Ø—M‘MˆŒ	ØŠMñ !r   c                óL   € . pV  F  pVP                  VR,          4       K  	  V# )z¢Process latents for bucket mode training.

Args:
    latents: list[{"samples": tensor}] where each tensor is (Bi, C, Hi, Wi)

Returns:
    list of latent tensors
r³   ©r‰   )Úlatentsr|   Úlatent_dicts   &  r   Ú_process_latents_bucket_modere    s,   € ð €NÛˆØ×Ñ˜k¨)Õ4Ö5ñ àÐr   c                ó
  € \        V 4      ^8X  d   V ^ ,          R,          # . pV  FZ  pVR,          pVP                  ^ ,          pV^8w  d$   V F  pVP                  VR,          4       K  	  KI  VP                  V4       K\  	  V# )z´Process latents for standard (non-bucket) mode training.

Args:
    latents: list of latent dicts or single latent dict

Returns:
    Processed latents (tensor or list of tensors)
r³   N)r\   r*   r‰   )rc  Úlatent_listÚlatentÚbsÚ
sub_latents   &    r   Ú_process_latents_standard_moderk  *  s�   € ô ˆ7ƒ|�qÔØ�q�z˜)Õ$Ð$à€KÛˆØ˜	Õ"ˆØ�\‰\˜!�_ˆØ�Œ7Û$�
Ø×"Ñ" :¨dÕ#3Ö4ó %ð ×Ñ˜vÖ&ñ ð Ðr   c                óÂ   € \        V 4      ^8X  d
   V ^ ,          # . pV  F=  p\        V\        4      '       d   VP                  V4       K,  VP	                  V4       K?  	  V# )z�Process conditioning - either single list or list of lists.

Args:
    positive: list of conditioning

Returns:
    Flattened conditioning list
)r\   rU   rZ   Úextendr‰   )r�   Úflat_positiverk   s   &  r   Ú_process_conditioningro  B  s\   € ô ˆ8ƒ}˜ÔØ˜�{Ðð €MÛˆÜ�dœD×!Ò!Ø× Ñ  Ö&à× Ñ  Ö&ñ	 ð
 Ðr   c                ó¤  € V'       dš   V  Uu. uF  q3P                  V4      NK  	  p p\        V 4      p\        R V  4       4      pRp\        P                  ! RV RV R24       \        V 4       F+  w  rx\        P                  ! RV RVP                   24       K-  	  WV3# \        V \        4      '       d    \        4       p	V  Uu. uF  q3P                  V4      NK  	  p pV  F  p
V	P                  V
P                  4       K   	  \        P                  ! RV	 24       \        V	4      ^8”  d   R	pMRp\        P                  ! V ^ R
7      p \        V 4      pMm\        V \        P                  4      '       d(   V P                  V4      p V P                  ^ ,          pRpM&\        P                  ! R\        V 4       24       ^ pRpWV3# u upi u upi )a  Convert latents to dtype and compute image counts.

Args:
    latents: Latents (tensor, list of tensors, or bucket list)
    dtype: Target dtype
    bucket_mode: Whether bucket mode is enabled

Returns:
    tuple: (processed_latents, num_images, multi_res)
c              3   óF   "  € T F  qP                   ^ ,          x € K  	  R# 5i©r   N)r*   )Ú.0rp   s   & r   Ú	<genexpr>Ú-_prepare_latents_and_count.<locals>.<genexpr>g  s   é € Ð5©W¨Ÿ™ Ÿš«Wùs   ‚!FzBucket mode: z
 buckets, z total samplesz	  Bucket z: shape zLatent shapes: T)ÚdimzInvalid latents type: )r4   r\   Úsumr(  r)  rh   r*   rU   rZ   ÚsetÚaddr/   ÚcatrX   Úerrorr•   )rc  r$   Úbucket_moderp   Únum_bucketsrƒ   Ú	multi_resrc   r�   Ú
all_shapesrh  s   &&&        r   Ú_prepare_latents_and_countr€  X  s‰  € ÷ á(/Ó0© 1—4‘4˜–;©ˆÐ0Ü˜'“lˆÜÑ5©WÓ5Ó5ˆ
Øˆ	ä�Š˜ k ]°*¸Z¸LÈÐWÔXÜ Ö(‰FˆAÜ�MŠM˜I a S¨°·±°Ð<Ö=ñ )à IÐ-Ð-ô �'œ4× Ò Ü“Uˆ
Ù(/Ó0© 1—4‘4˜–;©ˆÐ0ÛˆFØ�N‰N˜6Ÿ<™<Ö(ñ ä�Š˜¨
 |Ð4Ô5Üˆz‹?˜QÔØ‰IàˆIÜ—i’i ¨QÔ/ˆGÜ˜“\‰
Ü	�GœUŸ\™\×	*Ò	*Ø—*‘*˜UÓ#ˆØ—]‘] 1Õ%ˆ
Ø‰	ä�ŠÐ.¬t°G«}¨oÐ>Ô?Øˆ
Øˆ	à 	Ð)Ð)ùòA 1ùò 1s   �GÃGc                óø   € V'       d   V # \         P                  ! RV R\        V 4       24       \        V 4      ^8X  d   V^8”  d	   W,          # \        V 4      V8w  d   \        R\        V 4       RV R24      hV # )a7  Validate conditioning count matches image count, expand if needed.

Args:
    positive: Conditioning list
    num_images: Number of images
    bucket_mode: Whether bucket mode is enabled

Returns:
    Validated/expanded conditioning list

Raises:
    ValueError: If conditioning count doesn't match image count
zTotal Images: z, Total Captions: zNumber of positive conditions (z#) does not match number of images (z).)r(  r)  r\   Ú
ValueError)r�   rƒ   r|  s   &&&r   Ú!_validate_and_expand_conditioningrƒ  ˆ  s   € ÷ Øˆä‡M‚M�N : ,Ð.@ÄÀXÃÀÐPÔQÜ
ˆ8ƒ}˜Ô˜j¨1œnØÕ$Ð$Ü	ˆX‹˜*Ô	$ÜØ-¬c°(«m¨_Ð<_Ð`jÐ_kÐkmÐnó
ð 	
ð €Or   c                ó  € V R8X  d   / ^ 3# \         P                  ! RV 4      p\        V P                  R4      ^ ,          P                  R4      R,          4      p/ pV'       d    \        P
                  P                  V4      pW23# )z�Load existing LoRA weights if provided.

Args:
    existing_lora: LoRA filename or "[None]"

Returns:
    tuple: (existing_weights dict, existing_steps int)
ú[None]ÚlorasÚ_steps_r¯   rˆ   )Úfolder_pathsÚget_full_path_or_raiser
  Úsplitr&   r×   Úload_torch_file)Úexisting_loraÚ	lora_pathÚexisting_stepsÚexisting_weightss   &   r   Ú_load_existing_lorar�  £  sx   € ð ˜Ô Ø�1ˆuˆä×3Ò3°G¸]ÓK€Iä˜×,Ñ,¨YÓ7¸Õ:×@Ñ@ÀÓEÀbÕIÓJ€NØÐßÜ Ÿ;™;×6Ñ6°yÓAÐØÐ+Ð+r   c                óŽ  € V R2pV P                   P                  p/ p\        P                  ! RV RV 24       \	        V4      ^8¼  Ed    \        VP                  V R2R4      4      p	VP                  V R2R4      p
Rp\         F  pVP                  WWš4      pVf   K   M	  Vf   \        V,          pVe!   VP                  4       P                  V4      pM-XP                  V P                   VRR7      P                  V4      pVP                  4        F  w  rïWøV R	V 2&   K  	  VP                  4       P                  R
4      V3# \         P"                  P%                  \         P&                  ! V P                   P                  VR
R7      4      p\)        V4      P                  4       P                  R
4      pVW� R2&   VV3# )ap  Create a weight adapter for a module with weight.

Args:
    module: The module to create adapter for
    module_name: Name of the module
    existing_weights: Dict of existing LoRA weights
    algorithm: Algorithm name for new adapters
    lora_dtype: dtype for LoRA weights
    rank: Rank for new LoRA adapters

Returns:
    tuple: (train_adapter, lora_params dict)
ú.weightzCreating weight adapter for z with shape z.alphaç      ð?z.dora_scaleN)ÚrankÚalpharf   T©r$   Úrequires_gradz.diff)Úweightr*   r(  r)  r\   r—   Úgetr   Úloadr   Úto_trainr4   Úcreate_trainÚnamed_parametersÚtrainr–   r/   r  Ú	ParameterÚzerosrê   )ÚmoduleÚmodule_namer�  Ú	algorithmÚ
lora_dtyper”  Úkeyr*   Úlora_paramsr•  Ú
dora_scaleÚexisting_adapterÚadapter_clsÚtrain_adapterr-  Ú	parameterÚdiffÚdiff_modules   &&&&&&            r   Ú_create_weight_adapterr®  ¸  sÇ  € ð  ˆM˜Ð
!€CØ�M‰M×Ñ€EØ€Kä‡M‚MÐ0°°°\À%ÀÐIÔJä
ˆ5ƒz�Q…ÜÐ&×*Ñ*¨c¨U°&¨>¸3Ó?Ó@ˆØ%×)Ñ)¨S¨E°Ð*=¸tÓDˆ
ð  Ðß#ˆKØ*×/Ñ/Ø¨uó Ðð  Ô+Ùñ $ð Ò#Ü& yÕ1ˆKàÒ'Ø,×5Ñ5Ó7×:Ñ:¸:ÓF‰Mð (×4Ñ4Ø—‘ D°ð 5ó ç‰b�‹nð ð  -×=Ñ=Ö?‰OˆDØ3<˜;˜- q¨¨Ð/Ó0ñ  @ð ×"Ñ"Ó$×3Ñ3°DÓ9¸;ÐFÐFô �x‰x×!Ñ!Ü�KŠK˜Ÿ™×+Ñ+°:ÈTÔRó
ˆô ˜t“n×*Ñ*Ó,×;Ñ;¸DÓAˆØ-1ˆ�m 5Ð)Ñ*Ø˜KÐ'Ð'r   c                óú   € \         P                  P                  \         P                  ! V P                  P
                  VRR7      4      p\        V4      P                  4       P                  R4      pV R2V/pWE3# )zÖCreate a bias adapter for a module with bias.

Args:
    module: The module with bias
    module_name: Name of the module
    lora_dtype: dtype for LoRA weights

Returns:
    tuple: (bias_module, lora_params dict)
Tr–  z.diff_b)	r/   r  rŸ  r   rì   r*   rê   rž  r–   )r¡  r¢  r¤  rì   Úbias_moduler¦  s   &&&   r   Ú_create_bias_adapterr±  ô  sk   € ô �8‰8×ÑÜ�Š�F—K‘K×%Ñ%¨ZÀtÔLó€Dô ˜4“.×&Ñ&Ó(×7Ñ7¸Ó=€KØ!�] 'Ð*¨DÐ1€KØÐ#Ð#r   c           	     óü  € / p. pV P                   P                  4        FØ  w  rx\        VR4      '       g   K  VP                  eH   \	        W‡WW44      w  ršVP                  V
4       V R2pV P                  W¹4       VP                  V	4       \        VR4      '       g   K‚  VP                  f   K’  \        W‡V4      w  rÍVP                  V4       V R2pV P                  W¼4       VP                  V4       KÚ  	  WV3# )a0  Setup all LoRA adapters on the model.

Args:
    mp: Model patcher
    existing_weights: Dict of existing LoRA weights
    algorithm: Algorithm name for new adapters
    lora_dtype: dtype for LoRA weights
    rank: Rank for new LoRA adapters

Returns:
    tuple: (lora_sd dict, all_weight_adapters list)
Úweight_functionr’  rì   ú.bias)
r  Únamed_modulesrg   r˜  r®  rá   Úadd_weight_wrapperr‰   rì   r±  )Úmpr�  r£  r¤  r”  Úlora_sdÚall_weight_adaptersÚnr]  ÚadapterrÖ   r¥  Úbias_adapterÚbias_paramss   &&&&&         r   Ú_setup_lora_adaptersr¾    sì   € ð €GØÐà—‘×&Ñ&Ö(‰ˆÜ�1Ð'×(Ô(Ø�x‰xÒ#Ü"8ØÐ*°zó#‘�ð —‘˜vÔ&Ø˜˜7�m�Ø×%Ñ% cÔ3Ø#×*Ñ*¨7Ô3ä�q˜&×!Ô! a§f¡fÔ&8Ü,@ÀÀzÓ,RÑ)�Ø—‘˜{Ô+Ø˜˜5�k�Ø×%Ñ% cÔ8Ø#×*Ñ*¨<Ö8ñ! )ð$ Ð'Ð'r   c           	     ó   € / p. p\        4       pV P                  P                  4        EFN  w  r‰\        V	R4      '       g   K  V	P                  e¤   \        W˜WW44      w  r«VP                  V4       VP                  V
4       V R2p\        V
\        4      '       d,   V P                  WÊ4       \        P                  ! RV 24       M,VP                  WÊRR7       \        P                  ! RV 24       \        V	R4      '       g   Kß  V	P                  f   Kï  \        W˜V4      w  rÞVP                  V4       V R2pV P                  WÍ4       VP                  V4       \        P                  ! R	V 24       EKQ  	  WVV3# )
aG  Setup LoRA adapters in bypass mode.

In bypass mode:
    - Weight adapters (lora/lokr/oft) use bypass injection (forward hook)
    - Bias/norm adapters (BiasDiff) still use weight wrapper (direct modification)

This is useful when the base model weights are quantized and cannot be
directly modified.

Args:
    mp: Model patcher
    existing_weights: Dict of existing LoRA weights
    algorithm: Algorithm name for new adapters
    lora_dtype: dtype for LoRA weights
    rank: Rank for new LoRA adapters

Returns:
    tuple: (lora_sd dict, all_weight_adapters list, bypass_manager)
r³  r’  z:[BypassMode] Added 1D weight adapter (weight wrapper) for r“  )Ústrengthz/[BypassMode] Added weight adapter (bypass) for rì   r´  z5[BypassMode] Added bias adapter (weight wrapper) for )r   r  rµ  rg   r˜  r®  rá   r‰   rU   rê   r¶  r(  r)  Úadd_adapterrì   r±  )r·  r�  r£  r¤  r”  r¸  r¹  Úbypass_managerrº  r]  r»  rÖ   r¥  r¼  r½  s   &&&&&          r   Ú_setup_lora_adapters_bypassrÃ  ,  sZ  € ð( €GØÐÜ+Ó-€Nà—‘×&Ñ&×(‰ˆÜ�1Ð'×(Ô(Ø�x‰xÒ#Ü"8ØÐ*°zó#‘�ð —‘˜vÔ&Ø#×*Ñ*¨7Ô3à˜˜7�m�ô ˜g¤x×0Ò0Ø×)Ñ)¨#Ô7Ü—M’MÐ$^Ð_bÐ^cÐ"dÕeà"×.Ñ.¨sÀcÐ.ÔJÜ—M’MÐ$SÐTWÐSXÐ"YÔZä�q˜&×!Ô! a§f¡fÔ&8ä,@ÀÀzÓ,RÑ)�Ø—‘˜{Ô+Ø˜˜5�k�Ø×%Ñ% cÔ8Ø#×*Ñ*¨<Ô8Ü—’Ð UÐVYÐUZÐ[×\ñ5 )ð8 ¨Ð7Ð7r   c                ó>  € V R8X  d!   \         P                  P                  WR7      # V R8X  d!   \         P                  P                  WR7      # V R8X  d!   \         P                  P	                  WR7      # V R8X  d!   \         P                  P                  WR7      # R# )zÚCreate optimizer based on name.

Args:
    optimizer_name: Name of optimizer ("Adam", "AdamW", "SGD", "RMSprop")
    parameters: Parameters to optimize
    learning_rate: Learning rate

Returns:
    Optimizer instance
ÚAdam)ÚlrÚAdamWÚSGDÚRMSpropN)r/   ÚoptimrÅ  rÇ  rÈ  rÉ  )Úoptimizer_nameÚ
parametersÚlearning_rates   &&&r   Ú_create_optimizerrÎ  c  s†   € ð ˜ÔÜ�{‰{×Ñ 
ÐÓ=Ð=Ø	˜7Ô	"Ü�{‰{× Ñ  Ð Ó>Ð>Ø	˜5Ô	 Ü�{‰{�‰˜zˆÓ<Ð<Ø	˜9Ô	$Ü�{‰{×"Ñ" :Ð"Ó@Ð@ñ 
%r   c                ó.  € V R8X  d   \         P                  P                  4       # V R8X  d   \         P                  P                  4       # V R8X  d   \         P                  P	                  4       # V R8X  d   \         P                  P                  4       # R# )z¡Create loss function based on name.

Args:
    loss_function_name: Name of loss function ("MSE", "L1", "Huber", "SmoothL1")

Returns:
    Loss function instance
ÚMSEÚL1ÚHuberÚSmoothL1N)r/   r  ÚMSELossÚL1LossÚ	HuberLossÚSmoothL1Loss)Úloss_function_names   &r   Ú_create_loss_functionrÙ  x  ss   € ð ˜UÔ"Ü�x‰x×ÑÓ!Ð!Ø	˜tÔ	#Ü�x‰x�‰Ó Ð Ø	˜wÔ	&Ü�x‰x×!Ñ!Ó#Ð#Ø	˜zÔ	)Ü�x‰x×$Ñ$Ó&Ð&ñ 
*r   c                ó¬  € \         P                  ! \        V4      4      p\        P                  P                  V4      pV^ ,          P                  p	V'       de   V^ ,          R,          P                  ! V.^.V	^,
          ,          O5!  p
V P                  VP                  RV
/4      V
VVVP                  R7       R# V'       d^   V^ ,          P                  ! V.^.V	^,
          ,          O5!  pV P                  VP                  RV/4      VVVVP                  R7       R# V P                  VP                  RV/4      VVVVP                  R7       R# )a#  Execute the training loop.

Args:
    guider: The guider object
    train_sampler: The training sampler
    latents: Latent tensors
    num_images: Number of images
    seed: Random seed
    bucket_mode: Whether bucket mode is enabled
    multi_res: Whether multi-resolution mode is enabled
:Nr«   Nr³   )rC   N)r/   rŠ   r¬   rÙ   rÚ   rÛ   ÚndimÚrepeatrå   r»   rC   )ÚguiderÚtrain_samplerrc  rƒ   rC   r|  r~  r?   r<   rÛ  Údummy_latents   &&&&&&&    r   Ú_run_training_looprà  ‹  s*  € ô �\Š\œ% 
Ó+Ó,€FÜ×-Ñ-×?Ñ?ÀÓE€EØ�1�:�?‰?€Dçà˜q•z "•~×,Ò,¨ZÐG¸1¸#¸tÀA½v½,ÓGˆØ�‰Ø× Ñ  )¨\Ð!:Ó;ØØØØ—‘ð 	ö 	
÷ 
à˜!•*×#Ò# JÐ>°!°°d¸1µfµÓ>ˆØ�‰Ø× Ñ  )¨WÐ!5Ó6ØØØØ—‘ð 	ö 	
ð 	�‰Ø× Ñ  )¨WÐ!5Ó6ØØØØ—‘ð 	ö 	
r   c                   ó@   a € ] tR tRt o ]R 4       t]R 4       tRtV tR# )ÚTrainLoraNodei»  c           !     óh  € \         P                  ! R RRRR\         P                  P                  RRR7      \         P                  P                  RRR7      \         P
                  P                  R	R
R7      \         P                  P                  R^^RRR7      \         P                  P                  R^^RRR7      \         P                  P                  R^^RRR7      \         P                  P                  RRRRRRR7      \         P                  P                  R^^^€RR7      \         P                  P                  R. RJORRR 7      \         P                  P                  R!. RKOR"R#R 7      \         P                  P                  R$^ ^ R%R&R7      \         P                  P                  R'. RLOR(R*R 7      \         P                  P                  R+R(R).R(R,R 7      \         P                  P                  R-R.R/R07      \         P                  P                  R1\        \        P                  ! 4       4      \        \        P                  ! 4       4      ^ ,          R2R 7      \         P                  P                  R3RR4R07      \         P                  P                  R5^^^R6R7      \         P                  P                  R7R.R8R07      \         P                  P                  R9\        P                  ! R:4      R;.,           R;R<R 7      \         P                  P                  R=R.R>R07      \         P                  P                  R?R.R@R07      .\         P                  ! RA4      P!                  RBRCRD7      \         P                  ! RE4      P!                  RFRGRD7      \         P                  P!                  RRHRD7      .RI7      # )Mrâ  z
Train LoRAúmodel/trainingTr  zThe model to train the LoRA on.©Útooltiprc  zEThe Latents to use for training, serve as dataset/input of the model.r�   z.The positive conditioning to use for training.rw   i'  z#The batch size to use for training.©Údefaultr¸   r	  ræ  Úgrad_accumulation_stepsi   z>The number of gradient accumulation steps to use for training.r  i † z*The number of steps to train the LoRA for.rÍ  gü©ñÒMb@?gH¯¼šò×z>r“  z&The learning rate to use for training.)rè  r¸   r	  rà   ræ  r”  zThe rank of the LoRA layers.ru   rÇ  z"The optimizer to use for training.)Úoptionsrè  ræ  Úloss_functionrÐ  z&The loss function to use for training.rC   l   ÿÿÿÿ zbThe seed to use for training (used in generator for LoRA weight initialization and noise sampling)rz   Úbf16Úfp32z The dtype to use for training. 'none' preserves the model's native compute dtype instead of overriding it. For fp16 models, GradScaler is automatically enabled.r¤  zThe dtype to use for lora.Úquantized_backwardFztWhen using training_dtype 'none' and training on quantized model, doing backward with quantized matmul when enabled.©rè  ræ  r£  z"The algorithm to use for training.Úgradient_checkpointingz(Use gradient checkpointing for training.Úcheckpoint_depthz'Depth level for gradient checkpointing.r   z@Offload model weights to CPU during training to save GPU memory.rŒ  r†  r…  z9The existing LoRA to append to. Set to None for new LoRA.r|  zeEnable resolution bucket mode. When enabled, expects pre-bucketed latents from ResolutionBucket node.Úbypass_modez¼Enable bypass mode for training. When enabled, adapters are applied via forward hooks instead of weight modification. Useful for quantized models where weights cannot be directly modified.Ú
LORA_MODELÚlorazLoRA weights©Údisplay_nameræ  ÚLOSS_MAPr  zLoss historyzTotal training steps)Únode_idrö  ÚcategoryÚis_experimentalÚis_input_listÚinputsÚoutputs)rÇ  rÅ  rÈ  rÉ  )rÐ  rÑ  rÒ  rÓ  )rì  rí  Únone)r   ÚSchemaÚModelÚInputÚLatentÚConditioningÚIntÚFloatÚComboÚBooleanrZ   r   Úkeysrˆ  Úget_filename_listÚCustomÚOutput©Úclss   &r   Údefine_schemaÚTrainLoraNode.define_schema¼  s®  € ä�yŠyØ#Ø%Ø%Ø Øä—‘—‘˜wÐ0Q�ÓRÜ—	‘	—‘ØØcð  ó ô —‘×%Ñ%ØÐ(Xð &ó ô —‘—‘Ø ØØØØAð ó ô —‘—‘Ø-ØØØØ\ð ó ô —‘—‘ØØØØØHð ó ô —‘—‘Ø#Ø"Ø!ØØ"ØDð ó ô —‘—‘ØØØØØ:ð ó ô —‘—‘ØÚ?Ø#Ø@ð	 ó ô —‘—‘Ø#Ú>Ø!ØDð	 ó ô —‘—‘ØØØØ*ð Að ó ô —‘—‘Ø$Ú4Ø"ð ð	 ó ô —‘—‘Ø Ø# VÐ,Ø"Ø8ð	 ó ô —
‘
× Ñ Ø(Ø!ð Sð !ó ô
 —‘—‘ØÜ ¤×!2Ò!2Ó!4Ó5Ü ¤×!2Ò!2Ó!4Ó5°aÕ8Ø@ð	 ó ô —
‘
× Ñ Ø,Ø ØFð !ó ô
 —‘—‘Ø&ØØØØEð ó ô —
‘
× Ñ Ø Ø!Ø^ð !ó ô
 —‘—‘Ø#Ü(×:Ò:¸7ÓCÀxÀjÕPØ$ØWð	 ó ô —
‘
× Ñ Ø!Ø!ð Dð !ó ô
 —
‘
× Ñ Ø!Ø!ð [ð !ó ðgxôt —	’	˜,Ó'×.Ñ.Ø!'°ð /ó ô —	’	˜*Ó%×,Ñ,Ø!+°^ð -ó ô —‘—‘¨7Ð<R�ÓSðôH
ð H	
r   c                ó6  a.€ V^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV	^ ,          pV
^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV^ ,          pV\         P                  n        V'       d   \        V4      pM\	        V4      p\        V4      p\        P                  ! R4      ;_uu_ 4        VP                  RR7      pRp\        P                  ! V4      pVR8w  d)   \        P                  ! V4      pVP                  V4       M VP                  P                  4       pV\        P                  8X  db   \        P                  pV\        P                  8w  d   Rp\         P"                  \$        P&                  9   d   \(        P*                  ! R4       M\        P                  pVR9  d   TM\        P                  p\-        VVV4      w  ppp\/        VVV4      pVP                  P1                  R4      P3                  4        \5        V4      w  pp Rp!V'       d*   \(        P6                  ! R4       \9        VVWýV4      w  p"p#p!M\;        VVWýV4      w  p"p#\=        VV"P?                  4       V4      p	\A        V4      p$V'       d]   \C        VP                  PD                  VR7      p%\(        PF                  ! R	\I        V%4       R
V 24       V% F  p&\K        V&VR7       K  	  \        PL                  PO                  4        \         P                  PQ                  V.RV'       * R7       \        PL                  PO                  4        R. /o.V.3R lp'V'       d   \S        V$V	V'VVWV,          VVVVR7
      p(M%\S        T$T	T'TTWV,          TTV'       d   TMRVR7
      p(\U        VVR7      p)V)PW                  V4       Rp*V!e^   V!PY                  VP                  4      p*V* F  p+V+P[                  V4       K  	  \(        P6                  ! RV!P]                  4        R24        R\         P                  n/        \a        V)V(VVVVV4       R\         P                  n/        V*e1   V* F  p+V+Pc                  V4       K  	  \(        P6                  ! R4       VP                  Pe                  4        F  p&\g        V&4       K  	  ?(?	V" F,  p,V"V,,          Pi                  V4      Pk                  4       V"V,&   K.  	  V# F  p-V-P1                  R4       ?-K  	  ?#\l        Pn                  ! V"S.VV ,           4      uuRRR4       #   R\         P                  n/        T*e1   T* F  p+T+Pc                  T4       K  	  \(        P6                  ! R4       TP                  Pe                  4        F  p&\g        T&4       K  	  i ; i  + '       g   i     R# ; i)r   FT)Úforce_deepcopyrþ  a  WARNING: FP16 model detected with fp16_accumulation enabled. This combination can be numerically unstable during training and may cause NaN values. Suggested fixes: 1) Set training_dtype to 'bf16', or 2) Disable fp16_accumulation (remove from --fast flags).NzUsing bypass mode for training)r1  z!Gradient checkpointing: patching z modules at depth )r   g@Œµx¯D)Úmemory_requiredr!   r¥   c                 ó8   <€ SR ,          P                  V 4       R# )r¥   Nrb  )r¥   r  s   &€r   rv   Ú,TrainLoraNode.execute.<locals>.loss_callbackÒ  s   ø€ Ø˜Õ ×'Ñ'¨Ö-r   )rv   rw   ry   rx   rC   rz   r|   r„   )rv   rw   ry   rx   rC   rz   r{   r„   z[BypassMode] Injected z bypass hooksz![BypassMode] Ejected bypass hooksr   )8r&   Úmodel_managementÚtraining_fp8_bwdre  rk  ro  r/   Úinference_moderj   Únode_helpersÚstring_to_torch_dtypeÚset_model_compute_dtyper  Ú	get_dtypeÚfloat16rç   r   ÚFp16Accumulationr
   Úfastr(  Úwarningr€  rƒ  r–   rž  r�  r)  rÃ  r¾  rÎ  r  rÙ  r4  Údiffusion_modelÚinfor\   r^  r0   r1   Úload_models_gpurr   r   Ú	set_condsÚcreate_injectionsÚinjectÚget_hook_countÚin_trainingrà  ÚejectÚmodulesr`  r4   rl   r   Ú
NodeOutput)/r  r  rc  r�   rw   r  ré  rÍ  r”  ru   rë  rC   rz   r¤  rî  r£  rð  rñ  r   rŒ  r|  rò  rË  rØ  r·  r„   r$   r7   Úlatents_dtyperƒ   r~  r�  rŽ  rÂ  r¸  r¹  Ú	criterionÚmodules_to_patchr]  rv   rÞ  rÝ  Úbypass_injectionsÚ	injectionrä   r»  r  s/   &&&&&&&&&&&&&&&&&&&&&&                        @r   ÚexecuteÚTrainLoraNode.executeH  s‰  ø€ ð4 �a•ˆØ •]ˆ
Ø�a•ˆØ"9¸!Õ"<ÐØ% aÕ(ˆØ�A�wˆØ" 1�ˆØ*¨1Õ-ÐØ�A�wˆØ'¨Õ*ˆØ •]ˆ
Ø/°Õ2ÐØ˜a•Lˆ	Ø!7¸Õ!:ÐØ •]ˆ
Ø+¨AÕ.ÐØ% aÕ(ˆØ! !•nˆØ! !•nˆà2DŒ×ÑÔ/÷ Ü2°7Ó;‰Gä4°WÓ=ˆGô )¨Ó2ˆä×!Ò! %×(Õ(à—‘¨D�Ó1ˆBØ#ˆOÜ%×;Ò;¸JÓGˆJØ Ô'Ü$×:Ò:¸>ÓJ�Ø×*Ñ*¨5Õ1ð !Ÿh™h×0Ñ0Ó2�Ø¤%§-¡-Ô/Ü!ŸM™M�Eð "¤U§^¡^Ô3Ø*.˜ä)×:Ñ:¼d¿i¹iÔGÜŸšðLôøô "ŸN™N�Eð &+°'Ô%9™E¼u¿~¹~ˆMÜ-GØ˜¨ó.Ñ*ˆG�Z ô
 9¸À:È{Ó[ˆHð �H‰H×#Ñ# EÓ*×0Ñ0Ô2ô 0CÀ=Ó/QÑ,Ð˜nð "ˆNßÜ—’Ð>Ô?Ü?ZØÐ(¨)Àó@Ñ<�Ð,©nô 0DØÐ(¨)Àó0Ñ,�Ð,ô
 *Ø §¡Ó 0°-óˆIô .Ð.@ÓAˆI÷ &Ü#8Ø—H‘H×,Ñ,Ð4Dô$Ð ô —’Ð@ÄÐEUÓAVÐ@WÐWiÐjzÐi{Ð|Ô}Û)�AÜ˜!¨
×3ñ *ô �J‰J×"Ñ"Ô$ô ×"Ñ"×2Ñ2Ø� dÀ
¼Nð 3ô ô �J‰J×"Ñ"Ô$ð  �|ˆHõ.÷ Ü ,ØØØ"/Ø)Ø4Ø %Õ ?ØØ#(Ø#*Ø$3ô!‘ô !-ØØØ"/Ø)Ø4Ø %Õ ?ØØ#(ß,5¡¸4Ø$3ô!�ô ! °
Ô;ˆFØ×Ñ˜XÔ&ð !%ÐØÒ)Ø$2×$DÑ$DÀRÇXÁXÓ$NÐ!Û!2�IØ×$Ñ$ RÖ(ñ "3ä—’Ð 6°~×7TÑ7TÓ7VÐ6WÐWdÐeÔfðØ59”×&Ñ&Ô2Ü"ØØ!ØØØØØôð 6;”×&Ñ&Ô2à$Ò0Û%6˜	Ø!Ÿ™¨Ö+ñ &7ä—M’MÐ"EÔFØŸ™×)Ñ)Ö+�AÜ˜A–Jñ ,à˜yã �Ø!(¨¥×!2Ñ!2°:Ó!>×!EÑ!EÓ!G�˜“ñ !ó /�Ø×&Ñ& uÔ-Úñ /ð $ô —=’= ¨(°E¸NÕ4JÓK÷{ )Ò(øðR 6;”×&Ñ&Ô2à$Ò0Û%6˜	Ø!Ÿ™¨Ö+ñ &7ä—M’MÐ"EÔFØŸ™×)Ñ)Ö+�AÜ˜A–Jò ,ú÷_ )×(Ð(ús.   Ä
J+XÎ6CXÑ9&VÒC#XÖA7XØXØX	rK  N©	rG   rH   rI   rJ   Úclassmethodr  r0  rL   rM   rè   s   @r   râ  râ  »  s2   ø‡ € ØñI
ó ðI
ðV ñULó öULr   râ  c                   óD   a € ] tR tRt o ]R 4       t]RR l4       tRtV tR# )ÚLoraModelLoaderi!  c                óŽ  € \         P                  ! R RRR\         P                  P                  RRR7      \         P                  ! R4      P                  RR	R7      \         P
                  P                  R
RRRRR7      \         P                  P                  RRRR7      .\         P                  P                  RRR7      .R7      # )r5  zLoad LoRA Modelzmodel/loadersTr  z0The diffusion model the LoRA will be applied to.rå  ró  rô  z/The LoRA model to apply to the diffusion model.Ústrength_modelr“  g      Y@zGHow strongly to modify the diffusion model. This value can be negative.rç  ÚbypassFz‰When enabled, applies LoRA in bypass mode without modifying base model weights. Useful for training and when model weights are offloaded.rï  zThe modified diffusion model.rõ  )rø  rö  rù  rú  rü  rý  g      YÀ)r   rÿ  r   r  r
  r  r  r  r  s   &r   r  ÚLoraModelLoader.define_schema"  sÐ   € ä�yŠyØ%Ø*Ø$Ø ä—‘—‘ØÐ%Wð ó ô —	’	˜,Ó'×-Ñ-ØÐ$Uð .ó ô —‘—‘Ø$ØØØØeð ó ô —
‘
× Ñ ØØ!ð hð !ó ðô* —‘—‘Ø!(Ð2Qð  ó ðô3
ð 	
r   c                ó
  € V^ 8X  d   \         P                  ! V4      # V'       d&   \        P                  P	                  VRW#^ 4      w  rVM$\        P                  P                  VRW#^ 4      w  rV\         P                  ! V4      # rr  )r   r*  r&   ÚsdÚload_bypass_lora_for_modelsÚload_lora_for_models)r  r  rô  r7  r8  Ú
model_lorar¯   s   &&&&&  r   r0  ÚLoraModelLoader.executeD  so   € à˜QÔÜ—=’= Ó'Ð'çÜ!ŸH™H×@Ñ@Ø�t˜T°1ó‰MˆJ˜ô "ŸH™H×9Ñ9Ø�t˜T°1ó‰MˆJô �}Š}˜ZÓ(Ð(r   rK  N©Fr2  rè   s   @r   r5  r5  !  s,   ø‡ € Øñ
ó ð
ðB ó)ó ö)r   r5  c                   óD   a € ] tR tRt o ]R 4       t]RR l4       tRtV tR# )ÚSaveLoRAiT  c                ó  € \         P                  ! R R.RRRR\         P                  ! R4      P                  RRR7      \         P                  P                  R	R
RR7      \         P
                  P                  RRRR7      .. R7      # )rB  zexport lorazSave LoRA Weightszmodel/mergingTró  rô  z>The LoRA model to save. Do not use the model with LoRA layers.rå  rm   zloras/ComfyUI_trained_loraz*The prefix to use for the saved LoRA file.rï  r  zYOptional: The number of steps the LoRA has been trained for, used to name the saved file.)Úoptionalræ  )rø  Úsearch_aliasesrö  rù  rú  Úis_output_noderü  rý  )r   rÿ  r
  r  ÚStringr  r  s   &r   r  ÚSaveLoRA.define_schemaU  s“   € ä�yŠyØØ)˜?Ø,Ø$Ø Øä—	’	˜,Ó'×-Ñ-ØØ\ð .ó ô —	‘	—‘ØØ8ØHð  ó ô
 —‘—‘ØØ!Øwð ó ðð  ô/
ð 	
r   Nc                ó:  € \         P                  ! 4       p\         P                  ! W$4      w  rVrxp	Vf   V RVR R2p
MV RV RVR R2p
\        P                  P                  WZ4      p
\        P                  P                  W4       \        P                  ! 4       # )Nr¯   Ú05z_.safetensorsr‡  )rˆ  Úget_output_directoryÚget_save_image_pathÚosÚpathÚjoinÚsafetensorsr/   Ú	save_filer   r*  )r  rô  rm   r  Ú
output_dirÚfull_output_folderÚfilenameÚcounterÚ	subfolderÚfilename_prefixÚoutput_checkpoints   &&&&       r   r0  ÚSaveLoRA.executeq  s–   € ä!×6Ò6Ó8ˆ
ä×,Ò,¨VÓ@ñ 	JÐ g¸/ð Š=Ø#+ *¨A¨g°b¨\¸Ð GÑà#+ *¨A¨e¨W°G¸GÀB¸<À}Ð UÐÜŸG™GŸL™LÐ);ÓOÐÜ×Ñ×#Ñ# DÔ<Ü�}Š}‹Ðr   rK  r   r2  rè   s   @r   rB  rB  T  s+   ø‡ € Øñ
ó ð
ð6 óó ör   rB  c                   óD   a € ] tR tRt o ]R 4       t]RR l4       tRtV tR# )ÚLossGraphNodei€  c                ó6  € \         P                  ! R . RORRRR\         P                  ! R4      P                  RRR7      \         P                  P                  RR	R
R7      .. \         P
                  P                  \         P
                  P                  .R7	      # )r[  zPlot Loss Graphrä  Tr÷  r¥   zLoss map from training node.rå  rW  Ú
loss_graphz&Prefix for the saved loss graph image.rï  )	rø  rE  rö  rù  rú  rF  rü  rý  Úhidden)ztraining chartztraining visualizationz	plot loss)r   rÿ  r
  r  rG  ÚHiddenÚpromptÚextra_pnginfor  s   &r   r  ÚLossGraphNode.define_schema�  s�   € ä�yŠyØ#ÚTØ*Ø%Ø Øä—	’	˜*Ó%×+Ñ+ØÐ$Bð ,ó ô —	‘	—‘Ø%Ø(ØDð  ó ð		ð Ü—I‘I×$Ñ$¤b§i¡i×&=Ñ&=Ð>ô%
ð 	
r   Nc                ó\  € VR ,          pRRrv^(p\         P                  ! RWh,           Wx,           3R4      p	\        P                  ! V	4      p
\	        V4      \        V4      rËV Uu. uF  qÝV,
          WË,
          ,          NK  	  pp\        V4      pW‡\        V^ ,          V,          4      ,
          3p\        VR,          ^R7       FZ  w  ppV\        VV,          V,          4      ,           pV\        W×,          4      ,
          pV
P                  VVV3.R^R7       VV3pK\  	  V
P                  V^ 3W‡3.R	^R7       V
P                  W‡3Wh,           V3.R	^R7       R
p \        P                  ! R^4      pV
P                  ^V^,          3RVR	R7       V
P                  V^,          V^
,           3RVR	R7       V
P                  V^,
          ^ 3VR VR	R7       V
P                  V^,
          V^
,
          3VR VR	R7       \        P                   ! V	4      P#                  \        P$                  4      R,          p\&        P(                  ! V4      R,          p\*        P,                  ! \.        P0                  ! VV R7      R7      # u upi   \         d    \        P                  ! 4       p EL6i ; i)r¥   i   ià  rý   rþ   rÿ   r   r  r  ÚblackNz	arial.ttfÚLoss)Úfontr  ÚStepsz.2fg     ào@r  )r   r   )r   r  r   r  r¸   r	  r\   r
  rh   r  r   ÚtruetypeÚIOErrorÚload_defaultÚtextÚnpÚarrayÚastyper‹   r/   Ú
from_numpyr   r*  r   ÚPreviewImage)r  r¥   rW  r`  ra  Úloss_valuesr  r  Úmarginr  r  r  r  r  r  r  r  rc   r  r  rf  Ú	img_arrayÚ
img_tensors   &&&&&                  r   r0  ÚLossGraphNode.execute˜  sn  € à˜6•lˆØ˜SˆvØˆä�iŠiØ�E•N F¥OÐ4°gó
ˆô �~Š~˜cÓ"ˆä  Ó-¬s°;Ó/?�(ÙGRÓSÁ{À!˜H�¨Õ)<×=Ð=Á{ˆÐSä�KÓ ˆà¤s¨;°q­>¸FÕ+BÓ'CÕCÐDˆ
Ü˜k¨"�o°Q×7‰DˆAˆqØœ˜Q �Y¨Õ.Ó/Õ/ˆAØœ˜Q�Z›Õ(ˆAØ�I‰I�z A q 6Ð*°¸qˆIÔAØ˜Q˜ŠJñ	 8ð 	�	‰	�F˜A�; Ð 0Ð1¸Àqˆ	ÔIØ�	‰	ØÐ ¥°Ð7Ð8¸wÈað 	ô 	
ð ˆð	,Ü×%Ò% k°2Ó6ˆDð
 	�	‰	�1�f •kÐ" F°¸Gˆ	ÔDØ�	‰	�5˜A•:˜v¨�{Ð+¨W¸4Àgˆ	ÔNð 	�	‰	�6˜B•; Ð" x° n¸DÀwˆ	ÔOØ�	‰	Ø�b�[˜& 2�+Ð&¨8°C¨.ÀÈ7ð 	ô 	
ô
 —H’H˜S“M×(Ñ(¬¯©Ó4°uÕ<ˆ	Ü×%Ò% iÓ0°Õ7ˆ
ô �}Š}¤§¢°
ÀÔ DÔEÐEùòK Tøô& ô 	,Ü×)Ò)Ó+‹Dð	,ús   Á$JÅJ Ê J+Ê*J+rK  ©NNr2  rè   s   @r   r[  r[  €  s-   ø‡ € Øñ
ó ð
ð, ó0Fó ö0Fr   r[  c                   ó<   a € ] tR tRt o ]V 3R lR l4       tRtV tR# )ÚTrainingExtensioniÏ  c                óT   <€ V ^8„  d   QhRS[ S[S[P                  ,          ,          /# ©r  r2  )rZ   r•   r   Ú	ComfyNode)r  rP   s   "€r   r  ÚTrainingExtension.__annotate__Ñ  s"   ø€ ÷ 
ñ 
¡T©$©r¯|©|Õ*<Õ%=ñ 
r   c              ƒ  ó6   "  € \         \        \        \        .# 5ir   )râ  r5  rB  r[  rö   s   &r   Úget_node_listÚTrainingExtension.get_node_listÐ  s   é € ô ÜÜÜð	
ð 	
ùs   ‚rK  N)rG   rH   rI   rJ   r   r~  rL   rM   rè   s   @r   rx  rx  Ï  s   ø‡ € Ø÷
ó ö
r   rx  c                ó$   € V ^8„  d   QhR\         /# rz  ©rx  )r  s   "r   r  r  Ú  s   € ÷ ñ Ô 1ñ r   c               ƒ   ó   "  € \        4       # 5ir   r�  rK  r   r   Úcomfy_entrypointrƒ  Ú  s   é € ÜÓÐùs   ‚r   )Ú rv  )r«   Nr   Nr@  )Qr(  rM  Únumpyrl  rP  r/   Útorch.nnr  Útorch.utils.checkpointÚ	tqdm.autor   ÚPILr   r   r   Útyping_extensionsr   Úcomfy.samplersr&   Úcomfy.sampler_helpersÚcomfy.sdÚcomfy.utilsÚcomfy.model_managementÚcomfy.condsr   r	   Úcomfy.cli_argsr
   r   Ú!comfy_extras.nodes_custom_samplerrÙ   rˆ  r  Úcomfy.weight_adapterr   r   Úcomfy.weight_adapter.bypassr   Úcomfy_api.latestr   r   r   r   rÚ   ÚGuider_Basicr   rW   ri   r5   ÚSamplerrr   r  rê   r  r+  r4  ÚautogradÚFunctionr9  r^  r`  re  rk  ro  r€  rƒ  r�  r®  r±  r¾  rÃ  rÎ  rÙ  rà  r{  râ  r5  rB  r[  rx  rƒ  rK  r   r   Ú<module>rš     sz  ðÛ Û 	ã Û Û Ý Û Ý ß +Ñ +Ý &ã Û Û Û Û ß -ß 3Û (Û Û ß 7Ý >ß 3Ñ 3Ý #ôB�,×3Ñ3×@Ñ@ô BôJôô,w.�5—>‘>×)Ñ)ô w.ôt+ˆu�x‰x�‰ô +ò"÷$÷"#ôL" §¡× 7Ñ 7ô "ôJ"ò*òòò0ò,-*ò`ò6,ò*9(òx$ò&"(òJ48ònAò*'ò&-
ô`cL�B—L‘Lô cLôL0)�b—l‘lô 0)ôf)ˆr�|‰|ô )ôXIF�B—L‘Lô IFô^
˜ô 
÷r   