+
    Ñåûjk  ã                   ó@  € R t ^ RIHt ^ RIt^ RIHt ^RIHtHtHtH	t	H
t
HtHtHtHtHtHtHtHtHtHtHtHt RR.t ! R R]4      tRR	] R
]
 R] R] R] R2,           ]n         R R ltR R lt]	! ]R7      RR R ll4       t]P8                  ! RR7      ]n         R# )z'Implementation for the NAdam algorithm.)ÚcastN)ÚTensor)Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ_functional_api_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_params_docÚ_stack_if_compilingÚ
_to_scalarÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚNAdamÚnadamc            	       ó|   a a€ ] tR t^!t oRRRRRRRRR/V3R lV 3R lllltV 3R	 ltR
 t]RR l4       tRt	Vt
V ;t# )r   FÚforeachNÚmaximizeÚ
capturableÚdifferentiablec                ó”   <€ V ^8„  d   QhRS[ RS[S[,          RS[S[S[3,          RS[RS[RS[RS[RS[R	,          R
S[RS[RS[RR	/# )é   ÚparamsÚlrÚbetasÚepsÚweight_decayÚmomentum_decayÚdecoupled_weight_decayr   Nr   r   r   Úreturn)r   Úfloatr   ÚtupleÚbool)ÚformatÚ__classdict__s   "€ÚMC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\torch\optim\nadam.pyÚ__annotate__ÚNAdam.__annotate__"   s”   ø€ ÷ )+ñ )+áð)+ñ ‘F�Nð)+ñ ‘U™E�\Õ"ð	)+ñ
 ð)+ñ ð)+ñ ð)+ñ !%ð)+ñ ˜•ð)+ñ ð)+ñ ð)+ñ ð)+ð 
ñ)+ó    c               ó  <€ \        V\        4      '       d!   VP                  4       ^8w  d   \        R4      hRV8:  g   \        RV 24      hRV8:  g   \        RV 24      hRV^ ,          u;8:  d   R8  g   M \        RV^ ,           24      hRV^,          u;8:  d   R8  g   M \        RV^,           24      hRV8:  g   \        RV 24      hRV8:  g   \        R	V 24      hR
VRVRVRVRVRVRV	RVRV
RV/
p\        SV `  W4       R# )é   zTensor lr must be 1-elementç        zInvalid learning rate: zInvalid epsilon value: ç      ð?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: zInvalid momentum_decay value: r   r    r!   r"   r#   r$   r   r   r   r   N)Ú
isinstancer   ÚnumelÚ
ValueErrorÚsuperÚ__init__)Úselfr   r   r    r!   r"   r#   r$   r   r   r   r   ÚdefaultsÚ	__class__s   &&&&&&&&$$$$ €r+   r7   ÚNAdam.__init__"   s.  ø€ ô �bœ&×!Ò! b§h¡h£j°A¤oÜÐ:Ó;Ð;Ø�bŒyÜÐ6°r°dÐ;Ó<Ð<Ø�cŒzÜÐ6°s°eÐ<Ó=Ð=Ø�e˜A•hÖ$ Ö$ÜÐBÀ5ÈÅ8À*ÐMÓNÐNØ�e˜A•hÖ$ Ö$ÜÐBÀ5ÈÅ8À*ÐMÓNÐNØ�lÔ"ÜÐ;¸L¸>ÐJÓKÐKØ�nÔ$ÜÐ=¸nÐ=MÐNÓOÐOà�"Ø�UØ�3Ø˜LØ˜nØ$Ð&<Ø˜Ø�wØ˜*Ø˜nð
ˆô 	‰Ñ˜Ö*r.   c                ó¾  <€ \         SV `  V4       V P                   EF»  pVP                  R R4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VR,           EFO  pV P                  P                  V. 4      p\        V4      ^ 8w  g   K2  \        P                  ! VR,          4      '       gp   \        VR,          4      pVR,          '       d,   \        P                  ! V\        4       VP                  R	7      M\        P                  ! V\        4       R
7      VR&   \        P                  ! VR,          4      '       d   Ké  VR,          pVR,          '       d,   \        P                  ! V\        4       VP                  R	7      M\        P                  ! V\        4       R
7      VR&   EKR  	  EK¾  	  R# )r   Fr   Nr   r   r$   r   Ústep©ÚdtypeÚdevice©r?   Ú
mu_product)r6   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorr&   Útensorr   r@   )r8   rF   ÚgroupÚpÚp_stateÚstep_valÚmu_prod_valr:   s   &&     €r+   rC   ÚNAdam.__setstate__M   sr  ø€ Ü‰Ñ˜UÔ#Ø×&Õ&ˆEØ×Ñ˜Z¨Ô/Ø×Ñ˜Y¨Ô-Ø×Ñ˜\¨5Ô1Ø×ÑÐ-¨uÔ5Ø×ÑÐ5°uÔ=Ø˜8—_‘_�ØŸ*™*Ÿ.™.¨¨BÓ/�Ü�w“< 1Ö$Ü Ÿ?š?¨7°6­?×;Ò;Ü#(¨°­Ó#9˜ð
  % \×2Ô2ô "ŸLšLØ (Ô0AÓ0CÈAÏHÉHõô "'§¢¨hÔ>OÓ>QÔ!Rð   ™ô !Ÿ?š?¨7°<Õ+@×AÔAØ&-¨lÕ&;˜ð
  % \×2Ô2ô "ŸLšLØ +Ô3DÓ3FÈqÏxÉxõô "'§¢¨kÔARÓATÔ!Uð   Ô-ô %ó 'r.   c                óN  € R pVR,           EF  p	V	P                   f   K  V\        P                  ! V	4      ,          pVP                  V	4       V	P                   P                  '       d   \        R4      hVP                  V	P                   4       V P                  V	,          p
\        V
4      ^ 8X  Ed   VR,          '       d,   \        P                  ! R\        4       V	P                  R7      M\        P                  ! R\        4       R7      V
R&   VR,          '       d,   \        P                  ! R\        4       V	P                  R7      M\        P                  ! R\        4       R7      V
R	&   \        P                  ! V	\        P                  R
7      V
R&   \        P                  ! V	\        P                  R
7      V
R&   VP                  V
R,          4       VP                  V
R,          4       VP                  V
R	,          4       VP                  V
R,          4       EK  	  V# )Fr   z'NAdam does not support sparse gradientsr   r>   r1   rA   r=   r2   rB   )Úmemory_formatÚexp_avgÚ
exp_avg_sq© )ÚgradrI   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorrF   rH   Úzerosr   r@   rK   ÚonesÚ
zeros_likeÚpreserve_format)r8   rL   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmu_productsÚstate_stepsÚhas_complexrM   rF   s   &&&&&&&&   r+   Ú_init_groupÚNAdam._init_groupk   s¨  € ð ˆØ�x—‘ˆAØ�v‰vÔ!Øœu×/Ò/°Ó2Õ2�Ø ×'Ñ'¨Ô*Ø—6‘6×#×#Ð#Ü&Ð'PÓQÐQØ—‘˜QŸV™VÔ$àŸ
™
 1��ä�u“: •?ð ! ×.Ô.ô Ÿš BÔ.?Ó.AÈ!Ï(É(ÕSä"Ÿ\š\¨#Ô5FÓ5HÔIð ˜&‘Mð ! ×.Ô.ô Ÿ
š
 2Ô->Ó-@ÈÏÉÕRä"Ÿ\š\¨#Ô5FÓ5HÔIð ˜,Ñ'ô (-×'7Ò'7Ø¬×)>Ñ)>ô(�E˜)Ñ$ô +0×*:Ò*:Ø¬×)>Ñ)>ô+�E˜,Ñ'ð —‘  iÕ 0Ô1Ø×"Ñ" 5¨Õ#6Ô7Ø×"Ñ" 5¨Õ#6Ô7Ø×"Ñ" 5¨¥=×1ñI !ðJ Ðr.   c                ó*  € V P                  4        RpVe.   \        P                  ! 4       ;_uu_ 4        V! 4       pRRR4       V P                   F¬  p. p. p. p. p. p. p	\	        \
        \        \        3,          VR,          4      w  r«V P                  VVVVVVV	4      p\        VVVVVV	V
VVR,          VR,          VR,          VR,          VR,          VR,          VR	,          VR
,          VR,          VR7       K®  	  V#   + '       g   i     LÎ; i)z�Perform a single optimization step.

Args:
    closure (Callable, optional): A closure that reevaluates the model
        and returns the loss.
Nr    r   r"   r#   r!   r   r$   r   r   r   )Úbeta1Úbeta2r   r"   r#   r!   r   r$   r   r   r   rf   )	Ú'_accelerator_graph_capture_health_checkrI   Úenable_gradrD   r   r'   r&   rg   r   )r8   ÚclosureÚlossrL   r`   ra   rb   rc   rd   re   rj   rk   rf   s   &&           r+   r=   Ú
NAdam.step�   s'  € ð 	×4Ñ4Ô6àˆØÒÜ×"Ò"×$Õ$Ù“y�÷ %ð ×&Ô&ˆEØ-/ÐØ"$ˆEØ%'ˆHØ(*ˆKØ(*ˆKØ(*ˆKÜ¤¤e¬U lÕ 3°U¸7µ^ÓD‰LˆEà×*Ñ*ØØ ØØØØØóˆKô Ø ØØØØØØØØ˜•;Ø" >Õ2Ø$Ð%5Õ6Ø˜%•LØ˜zÕ*Ø',Ð-EÕ'FØ˜iÕ(Ø  Õ.Ø$Ð%5Õ6Ø'÷%ñ' 'ðP ˆ÷W %×$ús   ´DÄD	rV   )gü©ñÒMb`?)gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g:Œ0âŽyE>é    gü©ñÒMbp?F©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r7   rC   rg   r   r=   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r:   r*   s   @@r+   r   r   !   s\   ù‡ € ñ)+ð  $ð)+ð ð)+ð !ð)+ð  %÷)+ö )+õVò<0ðd "ó6ó "÷6ð 6r.   a¼  Implements NAdam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)},
                \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}                   \\
            &\hspace{13mm} \: \lambda \text{ (weight decay)}, \:\psi \text{ (momentum decay)}    \\
            &\hspace{13mm} \: \textit{decoupled\_weight\_decay}, \:\textit{maximize}             \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0 \leftarrow 0 \text{ ( second moment)}                                 \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1}                                       \\
            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{10mm}\textbf{if} \: \textit{decoupled\_weight\_decay}                       \\
            &\hspace{15mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}                    \\
            &\hspace{10mm}\textbf{else}                                                          \\
            &\hspace{15mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
            &\hspace{5mm} \mu_t \leftarrow \beta_1 \big(1 - \frac{1}{2}  0.96^{t \psi} \big)     \\
            &\hspace{5mm} \mu_{t+1} \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{(t+1)\psi}\big)\\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow \mu_{t+1} m_t/(1-\prod_{i=1}^{t+1}\mu_i)\\[-1.ex]
            & \hspace{11mm} + (1-\mu_t) g_t /(1-\prod_{i=1}^{t} \mu_{i})                         \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Incorporating Nesterov Momentum into Adam`_.
    z
    Args:
        aï  
        lr (float, Tensor, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        momentum_decay (float, optional): momentum decay (default: 4e-3)
        decoupled_weight_decay (bool, optional): whether to decouple the weight
            decay as in AdamW to obtain NAdamW. If True, the algorithm does not
            accumulate weight decay in the momentum nor variance. (default: False)
        z	
        zÏ

    .. _Incorporating Nesterov Momentum into Adam:
        https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101

    c          $      ó‚  € V ^8„  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\        R\        R	\        \        ,          R
\        R\        R\        R\        R\        R\        R\        R\        RR/# ©r   r   ra   rb   rc   rd   re   rj   rk   r   r"   r#   r!   r$   r   r   r   rf   r%   N©Úlistr   r&   r(   )r)   s   "r+   r,   r,     sì   € ÷ cñ cÜ”�Lðcä”�<ðcô ”6�lðcô ”f•ð	cô
 ”f•ðcô ”f•ðcô ðcô ðcô 	”�ðcô ðcô ðcô 
ðcô !ðcô ðcô  ð!cô" ð#cô$ ð%cð& 
ñ'cr.   c                ó  € \         P                  P                  4       '       g   \        V4      p\	        V 4       EF¿  w  ppV'       g
   VV,          M	VV,          ) pVV,          pVV,          pVV,          pVV,          p\         P
                  ! V4      '       dY   \         P                  ! V4      p\         P                  ! V4      p\         P                  ! V4      p\         P                  ! V4      p\         P                  P                  4       '       g‹   V'       dƒ   \        4       pVP                  P                  VP                  P                  u;8X  d   VP                  P                  8X  d   M MVP                  P                  V9   g   \        R V R24      hV^,          pV'       d   TpM\        V4      p^VV,          ,
          pV	^ 8w  d;   V'       d    VP                  ^W‰,          ,
          4       MVP                  VV	R7      pVRRRVV
,          ,          ,          ,
          ,          pVRRRV^,           V
,          ,          ,          ,
          ,          pVV,          pVP!                  V^V,
          4       VP                  V4      P#                  VV^V,
          R7       VP%                  V4      P'                  4       pV'       g	   V'       d‡   VP                  V4      pVV,          pVV) RV,
          ,          RV,
          ,          ,          pVV) V,          RV,
          ,          ,          pVP)                  VV4       VP)                  VV4       EK*  \        V4      V,          pVP+                  V4       VP)                  VVV) RV,
          ,          R\        V4      ,
          ,          R7       VP)                  VV\-        \.        V) V,          RV,
          ,          4      R7       EKÂ  	  R# )zVIf capturable=True, params, mu_products and state_steps must be on supported devices: Ú.©Úalphar2   ç      à?ç¸…ëQ¸î?)ÚvalueN)rI   ÚjitÚis_scriptingr   Ú	enumeraterX   Úview_as_realÚcompilerÚis_compilingr
   r@   ÚtypeÚAssertionErrorr   Úmul_ÚaddÚlerp_Úaddcmul_ÚdivÚsqrtÚaddcdiv_Úadd_r   r&   )r   ra   rb   rc   rd   re   rj   rk   r   r"   r#   r!   r$   r   r   r   rf   ÚiÚparamrW   rT   rU   rB   Ústep_tÚcapturable_supported_devicesr=   Úbias_correction2ÚmuÚmu_nextÚdenomÚmu_product_nexts   &&&&&&$$$$$$$$$$$              r+   Ú_single_tensor_nadamrž     s(  € ô( �9‰9×!Ñ!×#Ò#Ü˜‹^ˆä˜f×%‰ˆˆ5ß'ˆu�QŽx¨e°A­h¨YˆØ˜1•+ˆØ  •^ˆ
Ø  •^ˆ
Ø˜Q•ˆä×Ò˜E×"Ò"Ü×&Ò& uÓ-ˆEÜ×%Ò% dÓ+ˆDÜ×(Ò(¨Ó1ˆGÜ×+Ò+¨JÓ7ˆJô �~‰~×*Ñ*×,Ò,·Ü+LÓ+NÐ(à—‘×!Ñ! Z×%6Ñ%6×%;Ñ%;ÖQ¸v¿}¹}×?QÑ?Q×QØ—L‘L×%Ñ%Ð)EÔEä$ð-Ø-IÐ,JÈ!ðMóð ð 	�!�ˆçØ‰Dä˜fÓ%ˆDà˜u d�{�?Ðà˜1Ôß%à—
‘
˜1˜rÕ0Õ0Õ1à—x‘x ¨\�xÓ:�ð �c˜C 4¨D°>Õ,AÕ#BÕCÕCÕDˆØ˜3 ¨°$¸µ(¸nÕ1LÕ(MÕ!NÕNÕOˆð 	�bÕˆ
ð 	�‰�d˜A �IÔ&Ø�‰˜Ó×'Ñ'¨¨d¸!¸e½)Ð'ÔDØ—‘Ð/Ó0×5Ñ5Ó7ˆçŸZØ—I‘I˜c“NˆEð )¨7Õ2ˆOØ˜B˜3 #¨¥(Õ+¨s°ZÕ/?Õ@ÕAˆDØ "  w¥°#¸Õ2GÕ!HÕIˆGØ�N‰N˜4 Ô'Ø�N‰N˜7 E×*ä(¨Ó4°wÕ>ˆOØ�J‰J�sŒOØ�N‰NØØØ�s˜c B�hÕ'¨3´¸JÓ1GÕ+GÕHð ô ð
 �N‰NØØÜœ5 B 3¨¥=°S¸?Õ5JÕ"KÓLð ÷ óQ &r.   c          $      ó‚  € V ^8„  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\        R\        R	\        \        ,          R
\        R\        R\        R\        R\        R\        R\        R\        RR/# r{   r|   )r)   s   "r+   r,   r,   €  sì   € ÷ \ñ \Ü”�Lð\ä”�<ð\ô ”6�lð\ô ”f•ð	\ô
 ”f•ð\ô ”f•ð\ô ð\ô ð\ô 	”�ð\ô ð\ô ð\ô 
ð\ô !ð\ô ð\ô  ð!\ô" ð#\ô$ ð%\ð& 
ñ'\r.   c                óv  a,€ \        V 4      ^ 8X  d   R# V'       d   \        R4      h\        P                  P	                  4       '       g|   V'       dt   \        RR7      o,\        ;QJ d+    V,3R l\        WVRR7       4       F  '       d   K   RM 	  RM! V,3R l\        WVRR7       4       4      '       g   \        RS, R	24      h\        V4      p\        P                  ! WW#WE.4      pVP                  4        EFŸ  w  w  ppppppp\        \        \        ,          V4      p\        \        \        ,          V4      p\        \        \        ,          V4      p\        \        \        ,          V4      p\        \        \        ,          V4      p\        \        \        ,          V4      pV'       d   \        VVVV4       V'       d   \        P                   ! V4      p\        P                  P	                  4       '       gJ   V^ ,          P"                  '       d1   \        P$                  ! V\        P&                  ! R
RR7      R
R7       M\        P$                  ! V^4       V	^ 8w  di   V'       d&   \        P(                  ! V^W‰,          ,
          4       M;V'       d   \        P$                  ! VVV	R7       M\        P*                  ! VVV	R7      p\        P,                  ! VV^V,
          4       \        P(                  ! VV4       \        P.                  ! VVV^V,
          4       \        P0                  ! V4      pV'       EdC   \        P2                  ! VV
4      p \        P4                  ! RV 4      p!\        P(                  ! V!R4       \        P$                  ! V!R
4       \        P(                  ! V!V4       \        P$                  ! V V
4       \        P4                  ! RV 4      p"\        P(                  ! V"R4       \        P$                  ! V"R
4       \        P(                  ! V"V4       ? \        P4                  ! VV4      p#\        P6                  ! V#R
4       \        P8                  ! V#4       \        P:                  ! V#4       M°V U$u. uF#  p$^V\=        V$4      ,          ,
          R,          NK%  	  p#p$V U$u. uF1  p$VR
RR\=        V$4      V
,          ,          ,          ,
          ,          NK3  	  p!p$V U$u. uF8  p$VR
RR\=        V$4      ^,           V
,          ,          ,          ,
          ,          NK:  	  p"p$\        P(                  ! VV!4       \        P>                  ! VV#4       \        P$                  ! VV4       ?#V'       Ed   \        P6                  ! V!R
4       \        P(                  ! V!V4       \        P@                  ! VR
4      p%\        P8                  ! V%4       \        P>                  ! V!V%4       T!p&?%\        P2                  ! VV"4      p%\        P(                  ! V"V4       \        P6                  ! V%R
4       \        P>                  ! V"V%4       T"p'?%\        P2                  ! V&V4      p(\        P.                  ! V(V'V4       \        PB                  ! VV(V4       EK«  \E        \        VV!RR7       U)U*u. uF=  w  p)p*\=        V4      R
V*,
          ,          R
\=        V)4      ,
          ,          R,          NK?  	  up*p)4      p&\E        \        VV"RR7       U)U+u. uF=  w  p)p+\=        V4      V+,          R
\=        V)4      V+,          ,
          ,          R,          NK?  	  up+p)4      p'\        PB                  ! VVVV&4       \        PB                  ! VVVV'4       EK¢  	  R# u up$i u up$i u up$i u up*p)i u up+p)i )rq   Nz#_foreach ops don't support autogradF)Úsupports_xlac              3   ó  <"  € T F{  w  rpVP                   P                  VP                   P                  u;8H  ;'       d    VP                   P                  8H  Mu ;'       d    VP                   P                  S9   x € K}  	  R # 5irr   )r@   r‹   )Ú.0rM   Úmpr=   r˜   s   &   €r+   Ú	<genexpr>Ú&_multi_tensor_nadam.<locals>.<genexpr>Ÿ  sf   øé € ð 
ñ  R‘��tð �H‰H�M‰M˜RŸY™YŸ^™^×?Ô?¨t¯{©{×/?Ñ/?Ô?÷ >ð >Ø—‘—‘Ð!=Ñ=ô>ãQùs   ƒA BÁ$"BT)ÚstrictzWIf capturable=True, params, mu_products, and state_steps must be on supported devices: r   r2   Úcpu)r@   r€   rƒ   r‚   g      à¿éÿÿÿÿ)#rH   rŒ   rI   r‰   rŠ   r
   ÚallÚzipr   r   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   r}   r   r   Ú_foreach_negÚis_cpuÚ_foreach_add_rK   Ú_foreach_mul_Ú_foreach_addÚ_foreach_lerp_Ú_foreach_addcmul_Ú_foreach_sqrtÚ_foreach_mulÚ_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_sqrt_r   Ú_foreach_div_Ú_foreach_subÚ_foreach_addcdiv_r   )-r   ra   rb   rc   rd   re   rj   rk   r   r"   r#   r!   r$   r   r   r   rf   Úgrouped_tensorsÚgrouped_params_Úgrouped_grads_Úgrouped_exp_avgs_Úgrouped_exp_avg_sqs_Úgrouped_mu_products_Úgrouped_state_steps_Ú_Úgrouped_paramsÚgrouped_gradsÚgrouped_exp_avgsÚgrouped_exp_avg_sqsÚgrouped_mu_productsÚgrouped_state_stepsÚexp_avg_sq_sqrtÚexponentÚmusÚmu_nextsÚbias_correction_sqrtr=   rœ   Ústep_size_gradsÚstep_size_expavgÚ	numeratorrB   rš   r›   r˜   s-   &&&&&&$$$$$$$$$$$                           @r+   Ú_multi_tensor_nadamrÔ   €  s«  ø€ ô( ˆ6ƒ{�aÔÙçÜÐBÓCÐCô �>‰>×&Ñ&×(Ò(¯ZÜ'HØô(
Ð$÷ ‹sô 
ô  # 6¸ÈDÕQó
�s�sŠsô 
ô  # 6¸ÈDÕQó
÷ 
ò 
ô
 !ðVà/Ð0°ð3óð ô 
�B‹€Bä×BÒBØ	˜¨{ÐHó€Oð ×"Ñ"×$ñ		ñ 	ØØØØØØØÜœd¤6�l¨OÓ<ˆÜœT¤&�\¨>Ó:ˆÜ¤¤V¥Ð.?Ó@ÐÜ"¤4¬¥<Ð1EÓFÐÜ"¤4¬¥<Ð1EÓFÐÜ"¤4¬¥<Ð1EÓFÐ÷ ÜØ Ð/?ÐATô÷ Ü!×.Ò.¨}Ó=ˆMô �~‰~×*Ñ*×,Ò,Ð1DÀQÕ1G×1N×1NÐ1NÜ×ÒØ#¤U§\¢\°#¸eÔ%DÈCöô ×ÒÐ 3°QÔ7à˜1Ôß%ä×#Ò# N°A¸Õ8IÕ4IÕJ÷ Ü×'Ò'Ø% ~¸\öô %*×$6Ò$6Ø% ~¸\ô%�Mô
 	×ÒÐ-¨}¸aÀ%½iÔHä×ÒÐ/°Ô7Ü×ÒØ °¸qÀ5½yô	
ô  ×-Ò-Ð.AÓBˆ÷
 ˆ:ä×)Ò)Ð*=¸~ÓNˆHÜ×$Ò$ T¨8Ó4ˆCÜ×Ò  TÔ*Ü×Ò  SÔ)Ü×Ò  UÔ+ô ×Ò ¨.Ô9Ü×)Ò)¨$°Ó9ˆHÜ×Ò ¨$Ô/Ü×Ò ¨#Ô.Ü×Ò ¨%Ô0ð ä#(×#5Ò#5°eÐ=PÓ#QÐ ä×ÒÐ 4°cÔ:Ü×ÒÐ 4Ô5Ü× Ò Ð!5Õ6ñ DWó$ÙCV¸4��Uœj¨Ó.Õ.Õ.°3×6Ð6ÑCVð !ð $ñ
 0óá/�Dð ˜˜s d¬z¸$Ó/?À.Õ/PÕ&QÕRÕR×SÐSÙ/ð ð ñ 0óñ 0�Dð Ø˜ ¬*°TÓ*:¸QÕ*>À.Õ)PÕ QÕRÕR÷Tð Tá/ð ð ô 	×ÒÐ/°Ô5ä×Ò˜OÐ-AÔBÜ×Ò˜O¨SÔ1ð !çˆ:ä×Ò  SÔ)Ü×Ò  RÔ(ä×&Ò&Ð':¸CÓ@ˆEÜ×Ò Ô&Ü×Ò  UÔ+à!ˆOàô ×&Ò&Ð':¸HÓEˆEÜ×Ò ¨"Ô-ô ×Ò  sÔ+Ü×Ò ¨%Ô0à'Ðàô ×*Ò*¨?¸MÓJˆIÜ×#Ò# IÐ/?ÐAQÔRô ×#Ò# N°I¸×Oä1ô +.Ð.AÀ3ÈtÕ*Tôá*T™˜
 Bô   “^ s¨R¥xÕ0°C¼*ÀZÓ:PÕ4PÕQÐUW×WÐWÙ*TòóˆOô  3ô 03Ø+¨X¸dõ0ô
ñ0Ñ+˜
 Gô # 2›Ø!õ"à¤¨JÓ!7¸'Õ!AÕAõCð ÷ð ñ0ò
ó Ðô ×#Ò#ØØØØô	ô ×#Ò#ØØ ØØ ÷	óC %ùò`$ùòùòùóbùó
s    Ò)^ Ò;7^%Ó8>^*Ú;A^/ÜA^5)Úsingle_tensor_fnc          &      óœ  € V ^8„  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\         \        ,          R\        R\        R	,          R
\        R\        R\        R\        R\        R\        R\        \        ,          R\        R\        R\        RR	/# )r   r   ra   rb   rc   rd   re   r$   r   Nr   r   rf   r   rj   rk   r   r"   r#   r!   r%   )r}   r   r(   r&   )r)   s   "r+   r,   r,   `  sû   € ÷ @ñ @Ü”�Lð@ä”�<ð@ô ”6�lð@ô ”f•ð	@ô
 ”f•ð@ô ”f•ð@ô !ð@ô �D�[ð@ô ð@ô ð@ô ð@ô ð@ô  ð!@ô" ð#@ô$ 	”�ð%@ô& ð'@ô( ð)@ô* 
ð+@ð, 
ñ-@r.   c                óT  € \         ;QJ d    R  V 4       F  '       d   K   RM	  RM! R  V 4       4      '       g   \        R4      h\         ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   \        R4      hVf   \        W	RR7      w  ppV'       d0   \        P                  P                  4       '       d   \        R4      hV'       d,   \        P                  P                  4       '       g   \        pM\        pV! V VVVVVVVVVVVVVVV	V
R	7       R# )
c              3   óV   "  € T F  p\        V\        P                  4      x € K!  	  R # 5irr   ©r3   rI   r   ©r£   Úts   & r+   r¥   Únadam.<locals>.<genexpr>w  ó   é € Ð@±K¨qŒz˜!œUŸ\™\×*Ð*³Kùó   ‚')FTzPAPI has changed, `state_steps` argument must contain a list of singleton tensorsc              3   óV   "  € T F  p\        V\        P                  4      x € K!  	  R # 5irr   rÙ   rÚ   s   & r+   r¥   rÜ   |  rÝ   rÞ   zPAPI has changed, `mu_products` argument must contain a list of singleton tensorsN)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)rj   rk   r   r"   r#   r   r$   r!   r   r   rf   )rª   r[   r   rI   r…   r†   rÔ   rž   )r   ra   rb   rc   rd   re   r$   r   r   r   rf   r   rj   rk   r   r"   r#   r!   rÅ   Úfuncs   &&&&&&&&&&&&$$$$$$  r+   r   r   _  sÿ   € ÷0 ‹3Ñ@±KÓ@�3�3Š3Ñ@±KÓ@×@Ò@ÜØ^ó
ð 	
÷ ‹3Ñ@±KÓ@�3�3Š3Ñ@±KÓ@×@Ò@ÜØ^ó
ð 	
ð ‚Ü1Ø¨eô
‰
ˆˆ7÷ ”5—9‘9×)Ñ)×+Ò+ÜÐSÓTÐTç”u—y‘y×-Ñ-×/Ò/Ü"‰ä#ˆáØØØØØØØØØØ!Ø%ØØ5ØØØ%Ø÷#r.   )Ú	optimizer)FNFFFF)Ú__doc__Útypingr   rI   r   râ   r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__r   rž   rÔ   r   r)   rV   r.   r+   Ú<module>ræ      sÙ   ðá .å ã Ý ÷÷ ÷ ÷ õ ð* �GÐ
€ôsˆIô sðn&ðN	à	ˆð 	ð 
ˆð 	Ø	ˆð 	Ø	Ðð 	Ø	Ðð ð!õO>ð „õFcõL\ñ~  Ð1EÔFö@ó Gð@ðF $×*Ò*°WÔ=€†r.   