+
    „°ñj4  ã                   ó  € ^ RI t ^ RIHt ^ RIHt ^ RIt^ RIHu Ht	 ^ RI
t^RIHtHtHt ]R 4       t ! R R]P"                  P$                  4      t ! R R	]P"                  P$                  4      t ! R
 R]4      t ! R R]4      tR# )é    N)Úcache)ÚOptional)ÚWeightAdapterBaseÚWeightAdapterTrainBaseÚweight_decomposec                 ó2   € \         P                  ! R4       R# )z0One-time warning about LoHa bypass inefficiency.z‹LoHa bypass mode is inefficient: full weight diff is computed each forward pass. Consider using LoRA or LoKr for training with bypass mode.N)ÚloggingÚwarning© ó    Ú=C:\Ai_Coding\apps\studio\ComfyUI\comfy\weight_adapter\loha.pyÚ_warn_loha_bypass_inefficientr      s   € ô ‡O‚Oð	Eör   c                   óf   a € ] tR t^t o ]]P                  ! ^4      3R l4       t]R 4       tRt	V t
R# )Ú
HadaWeightc                ód   € V P                  W!WCV4       W,          W4,          ,          V,          pV# ©N)Úsave_for_backward)ÚctxÚw1uÚw1dÚw2uÚw2dÚscaleÚdiff_weights   &&&&&& r   ÚforwardÚHadaWeight.forward   s-   € à×Ñ˜c¨°%Ô8Ø�	 c¥iÕ0°EÕ9ˆØÐr   c                ó  € V P                   w  r#rEpW,          pWV,          ,          pWrP                  ,          pVP                  V,          p	WV,          ,          pWtP                  ,          p
VP                  V,          p?W‰W«R 3# r   )Úsaved_tensorsÚT)r   Úgrad_outr   r   r   r   r   ÚtempÚgrad_w1uÚgrad_w1dÚgrad_w2uÚgrad_w2ds   &&          r   ÚbackwardÚHadaWeight.backward   st   € à&)×&7Ñ&7Ñ#ˆ�3˜UØÕ#ˆØ �9Õ%ˆØŸ%™%•<ˆØ—5‘5˜4•<ˆà �9Õ%ˆØŸ%™%•<ˆØ—5‘5˜4•<ˆàØ 8°tÐ;Ð;r   r   N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚstaticmethodÚtorchÚtensorr   r&   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r   r      s6   ø‡ € ØØ/4¯|ª|¸A«ó ó ðð
 ñ<ó ö<r   r   c                   óf   a € ] tR t^+t o ]]P                  ! ^4      3R l4       t]R 4       tRt	V t
R# )ÚHadaWeightTuckerc           	     ó¨   € V P                  WW$WeV4       \        P                  ! R WV4      p\        P                  ! R WFV4      p	W‰,          V,          # )zi j ..., j r, i p -> p r ...)r   r.   Úeinsum)
r   Út1r   r   Út2r   r   r   Úrebuild1Úrebuild2s
   &&&&&&&&  r   r   ÚHadaWeightTucker.forward,   sJ   € à×Ñ˜b s°¸%Ô@ä—<’<Ð >ÀÈÓMˆÜ—<’<Ð >ÀÈÓMˆàÕ" UÕ*Ð*r   c                óì  € V P                   w  r#rErgpW,          p\        P                  ! R WV4      p	\        P                  ! RW—4      p
W¡,          p?
\        P                  ! RW›4      p\        P                  ! RW´P                  4      p??	\        P                  ! RW-4      p\        P                  ! R WÓP                  4      p?\        P                  ! R W#4      p	\        P                  ! RW”4      p
W¡,          p?
\        P                  ! RW›4      p\        P                  ! RW·P                  4      p??	\        P                  ! RW]4      p\        P                  ! R WÖP                  4      p?WüVVVVR3# )zi j ..., j r -> i r ...zi j ..., i r -> r j ...zr j ..., i j ... -> r izi r ..., i j ... -> r jN)r   r.   r7   r   )r   r    r8   r   r   r9   r   r   r   r!   ÚrebuildÚgrad_wr"   Ú	grad_tempr#   Úgrad_t1r$   r%   Úgrad_t2s   &&                 r   r&   ÚHadaWeightTucker.backward5   s3  € à.1×.?Ñ.?Ñ+ˆ�#˜3 UØÕ#ˆä�|Š|Ð5°rÓ?ˆÜ—,’,Ð8¸$ÓDˆàÕ#ˆØä—<’<Ð 9¸4ÓHˆÜ—L’LÐ!:¸FÇEÁEÓJˆ	Ø�Dä—<’<Ð 9¸2ÓIˆÜ—,’,Ð8¸)ÇUÁUÓKˆØä�|Š|Ð5°rÓ?ˆÜ—,’,Ð8¸$ÓDˆàÕ#ˆØä—<’<Ð 9¸4ÓHˆÜ—L’LÐ!:¸FÇEÁEÓJˆ	Ø�Dä—<’<Ð 9¸2ÓIˆÜ—,’,Ð8¸)ÇUÁUÓKˆØØ (¨G°X¸xÈÐMÐMr   r   Nr(   r2   s   @r   r5   r5   +   s8   ø‡ € ØØ7<·|²|ÀA³ó +ó ð+ð ñNó öNr   r5   c                   ó>   a a€ ] tR t^Xt oV 3R ltR tR tRtVtV ;t	# )ÚLohaDiffc                óÐ  <€ \         S
V `  4        Vw  r#rErgr‰\        P                  P	                  V4      V n        \        P                  P	                  V4      V n        \        P                  P	                  V4      V n        \        P                  P	                  V4      V n        R V n	        VeU   VeQ   RV n	        \        P                  P	                  V4      V n
        \        P                  P	                  V4      V n        MRV n
        RV n        VP                  ^ ,          V n        \        P                  P	                  \        P                  ! V4      R R7      V n        R# )FNT)Úrequires_grad)ÚsuperÚ__init__r.   ÚnnÚ	ParameterÚ	hada_w1_aÚ	hada_w1_bÚ	hada_w2_aÚ	hada_w2_bÚ
use_tuckerÚhada_t1Úhada_t2ÚshapeÚrankr/   Úalpha)ÚselfÚweightsÚw1aÚw1brU   Úw2aÚw2br8   r9   Ú_Ú	__class__s   &&        €r   rI   ÚLohaDiff.__init__Y   sþ   ø€ Ü‰ÑÔà/6Ñ,ˆ�%˜c rô Ÿ™×+Ñ+¨CÓ0ˆŒÜŸ™×+Ñ+¨CÓ0ˆŒÜŸ™×+Ñ+¨CÓ0ˆŒÜŸ™×+Ñ+¨CÓ0ˆŒàˆŒØŠ>˜bšnØ"ˆDŒOÜ Ÿ8™8×-Ñ-¨bÓ1ˆDŒLÜ Ÿ8™8×-Ñ-¨bÓ1ˆD�Lð  ˆDŒLØˆDŒLð —I‘I˜a•LˆŒ	Ü—X‘X×'Ñ'¬¯ª°UÓ(;È5Ð'ÓQˆŽ
r   c           	     ó.  € VP                   pV P                  V P                  ,          pV P                  '       dY   \        P                  V P                  V P                  V P                  V P                  V P                  V P                  V4      pMA\        P                  V P                  V P                  V P                  V P                  V4      pVP                  V4      VP                  VP                  4      ,           pVP                  V4      # r   )ÚdtyperU   rT   rP   r5   ÚapplyrQ   rL   rM   rR   rN   rO   r   ÚtoÚreshaperS   )rV   ÚwÚ	org_dtyper   r   Úweights   &&    r   Ú__call__ÚLohaDiff.__call__r   sÄ   € Ø—G‘Gˆ	à—
‘
˜TŸY™YÕ&ˆØ�?�?ˆ?Ü*×0Ñ0Ø—‘Ø—‘Ø—‘Ø—‘Ø—‘Ø—‘Øó‰Kô %×*Ñ*Ø—‘ §¡°·±ÀÇÁÐPUóˆKð
 —‘�kÓ" [×%8Ñ%8¸¿¹Ó%AÕAˆà�y‰y˜Ó#Ð#r   c                óB   € \        R V P                  4        4       4      # )z4Calculates memory usage of the trainable parameters.c              3   ój   "  € T F)  qP                  4       VP                  4       ,          x € K+  	  R # 5ir   )ÚnumelÚelement_size)Ú.0Úparams   & r   Ú	<genexpr>Ú0LohaDiff.passive_memory_usage.<locals>.<genexpr>Œ   s'   é € ÐWÑEV¸E—;‘;“= 5×#5Ñ#5Ó#7×7Ò7ÓEVùs   ‚13)ÚsumÚ
parameters©rV   s   &r   Úpassive_memory_usageÚLohaDiff.passive_memory_usageŠ   s   € äÑWÀTÇ_Á_ÔEVÓWÓWÐWr   )	rU   rQ   rR   rL   rM   rN   rO   rT   rP   )
r)   r*   r+   r,   rI   rg   rt   r0   r1   Ú__classcell__)r]   r3   s   @@r   rE   rE   X   s   ù‡ € õRò2$÷0Xò Xr   rE   c                   ó˜   a € ] tR t^�t o RtR t]RR l4       tR t]RV 3R lR ll4       t	]
P                  R3R ltV 3R	 lR
 ltRtV tR# )ÚLoHaAdapterÚlohac                ó   € Wn         W n        R # r   ©Úloaded_keysrW   )rV   r|   rW   s   &&&r   rI   ÚLoHaAdapter.__init__’   s   € Ø&ÔØŽr   c           
     ód  € VP                   ^ ,          pVP                   R,          P                  4       p\        P                  ! WBVP                  \        P
                  R7      p\        P                  ! W%VP                  \        P
                  R7      p\        P                  P                  P                  VR4       \        P                  P                  P                  VR4       \        P                  ! WBVP                  \        P
                  R7      p\        P                  ! W%VP                  \        P
                  R7      p	\        P                  P                  P                  VR4       \        P                  P                  P                  V	R4       \        WgW8V	RRR34      # )r   :é   NN)Údevicer`   gš™™™™™¹?g        g{®Gáz„?N)rS   rk   r.   Úemptyr€   Úfloat32rJ   ÚinitÚnormal_Ú	constant_rE   )
Úclsrf   rT   rU   Úout_dimÚin_dimÚmat1Úmat2Úmat3Úmat4s
   &&&&      r   Úcreate_trainÚLoHaAdapter.create_train–   s  € à—,‘,˜q•/ˆØ—‘˜bÕ!×'Ñ'Ó)ˆÜ�{Š{˜7°·±ÄeÇmÁmÔTˆÜ�{Š{˜4°·±ÄUÇ]Á]ÔSˆÜ�‰�‰×Ñ˜d CÔ(Ü�‰�‰×Ñ  cÔ*Ü�{Š{˜7°·±ÄeÇmÁmÔTˆÜ�{Š{˜4°·±ÄUÇ]Á]ÔSˆÜ�‰�‰×Ñ˜d CÔ(Ü�‰�‰×Ñ˜d DÔ)Ü˜ U°$¸¸dÀDÐIÓJÐJr   c                ó,   € \        V P                  4      # r   )rE   rW   rs   s   &r   Úto_trainÚLoHaAdapter.to_train¤   s   € Ü˜Ÿ™Ó%Ð%r   Nc                óš   <€ V ^8„  d   QhRS[ RS[S[ S[P                  3,          RS[RS[P                  RS[S[ ,          RS[R,          /# )é   ÚxÚlorarU   Ú
dora_scaler|   Úreturnrx   )ÚstrÚdictr.   ÚTensorÚfloatÚsetr   )Úformatr3   s   "€r   Ú__annotate__ÚLoHaAdapter.__annotate__¨   sb   ø€ ÷ *ñ *áð*ñ ‘3™Ÿ™Ð$Õ%ð*ñ ð	*ñ
 —L‘Lð*ñ ™•Xð*ñ 
�-Õ	 ñ*r   c                ó†  € Vf   \        4       pRP                  V4      pRP                  V4      pRP                  V4      pRP                  V4      p	RP                  V4      p
RP                  V4      pWbP                  4       9   d¹   R pR pW¢P                  4       9   d3   W*,          pW+,          pVP                  V
4       VP                  V4       W&,          W',          VW(,          W),          VVV3pVP                  V4       VP                  V4       VP                  V4       VP                  V	4       V ! W^4      # R # )Nz{}.hada_w1_az{}.hada_w1_bz{}.hada_w2_az{}.hada_w2_bz
{}.hada_t1z
{}.hada_t2)rœ   r�   ÚkeysÚadd)r†   r”   r•   rU   r–   r|   Úhada_w1_a_nameÚhada_w1_b_nameÚhada_w2_a_nameÚhada_w2_b_nameÚhada_t1_nameÚhada_t2_namerQ   rR   rW   s   &&&&&&         r   ÚloadÚLoHaAdapter.load§   s(  € ð ÒÜ›%ˆKà'×.Ñ.¨qÓ1ˆØ'×.Ñ.¨qÓ1ˆØ'×.Ñ.¨qÓ1ˆØ'×.Ñ.¨qÓ1ˆØ#×*Ñ*¨1Ó-ˆØ#×*Ñ*¨1Ó-ˆØŸY™Y›[Ô(ØˆGØˆGØŸy™y›{Ô*ØÕ,�ØÕ,�Ø—‘ Ô-Ø—‘ Ô-ð Õ$ØÕ$ØØÕ$ØÕ$ØØØð	ˆGð �O‰O˜NÔ+Ø�O‰O˜NÔ+Ø�O‰O˜NÔ+Ø�O‰O˜NÔ+Ù�{Ó,Ð,ár   c	           
     ój  € V P                   p	V	^ ,          p
V	^,          pV	^,          e#   V	^,          VP                  ^ ,          ,          pMRpV	^,          pV	^,          pV	^,          pV	^,          Ee8   V	^,          pV	^,          p\        P                  ! R\        P
                  P                  VVP                  V4      \        P
                  P                  W±P                  V4      \        P
                  P                  W¡P                  V4      4      p\        P                  ! R\        P
                  P                  VVP                  V4      \        P
                  P                  WáP                  V4      \        P
                  P                  WÑP                  V4      4      pMÎ\        P                  ! \        P
                  P                  W¡P                  V4      \        P
                  P                  W±P                  V4      4      p\        P                  ! \        P
                  P                  WÑP                  V4      \        P
                  P                  WáP                  V4      4      p VV,          P                  VP                  4      pVe   \        VVVVVVV4      pV# W! W<,          V,          P                  VP                  4      4      ,          p V#   \         d=   p\        P                  ! RP!                  T P"                  TT4      4        Rp?T# Rp?ii ; i)r   Nç      ð?úi j k l, j r, i p -> p r k lzERROR {} {} {})rW   rS   r.   r7   ÚcomfyÚmodel_managementÚcast_to_devicer€   Úmmrc   r   Útyper`   Ú	Exceptionr	   Úerrorr�   Úname)rV   rf   ÚkeyÚstrengthÚstrength_modelÚoffsetÚfunctionÚintermediate_dtypeÚoriginal_weightÚvrX   rY   rU   rZ   r[   r–   r8   r9   Úm1Úm2Ú	lora_diffÚes   &&&&&&&&&             r   Úcalculate_weightÚLoHaAdapter.calculate_weightÔ   s°  € ð �L‰LˆØ��dˆØ��dˆØˆQ�4ÒØ�a•D˜3Ÿ9™9 Q�<Õ'‰EàˆEà��dˆØ��dˆØ�q•Tˆ
ØˆQ�4ÓØ�1•ˆBØ�1•ˆBÜ—’Ø.Ü×&Ñ&×5Ñ5Ø˜Ÿ™Ð'9óô ×&Ñ&×5Ñ5ØŸ™Ð(:óô ×&Ñ&×5Ñ5ØŸ™Ð(:óóˆBô —’Ø.Ü×&Ñ&×5Ñ5Ø˜Ÿ™Ð'9óô ×&Ñ&×5Ñ5ØŸ™Ð(:óô ×&Ñ&×5Ñ5ØŸ™Ð(:óó‰Bô —’Ü×&Ñ&×5Ñ5ØŸ™Ð(:óô ×&Ñ&×5Ñ5ØŸ™Ð(:óó	ˆBô —’Ü×&Ñ&×5Ñ5ØŸ™Ð(:óô ×&Ñ&×5Ñ5ØŸ™Ð(:óó	ˆBð	FØ˜b�×)Ñ)¨&¯,©,Ó7ˆIØÒ%Ü)ØØØØØØ&Øó�ð ˆð ˜( XÕ%5¸Õ$B×#HÑ#HÈÏÉÓ#VÓWÕW‘ð ˆøô ô 	FÜ�MŠMÐ*×1Ñ1°$·)±)¸SÀ!ÓD×EÐEØˆûð	Fús   É;7K+ Ê44K+ Ë+L2Ë61L-Ì-L2c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r“   r”   Úbase_outr—   )r.   rš   )r�   r3   s   "€r   rž   rŸ   *  s3   ø€ ÷ P-ñ P-‘5—<‘<ð P-©5¯<©<ð P-¹E¿L¹Lñ P-r   c                óz  € \        4        RR\        P                  \        P                  \        P                  \        P
                  .pV P                  pV^ ,          pV^,          pV^,          pV^,          pV^,          p	V^,          p
V^,          pVP                  ^ ,          pVe	   W|,          MR\        V RR4      ,          pVP                  VP                  R7      pVP                  VP                  R7      pVP                  VP                  R7      pV	P                  VP                  R7      p	\        V RR4      p\        V R^ 4      p\        V R/ 4      pV
e~   Vez   V
P                  VP                  R7      p
VP                  VP                  R7      p\        P                  ! R	W¦V4      p\        P                  ! R	W¹V4      pVV,          V,          pM WV,          pW‰,          pVV,          V,          pV'       d™   W?^,           ,          p\        V R
RV,          4      p\        V RR4      pVP                  4       ^8X  dS   Ve(   VP                  ! VP                  ^ ,          V.VO5!  pM:VP                  ! . VP                  O^.V,          O5!  pM\        P                  p/ pV! VV3/ VB # )a¦  
Additive bypass component for LoHa: h(x) = diff_weight @ x

WARNING: Inefficient - computes full Hadamard product each forward.

Note:
    Does not access original model weights - bypass mode is designed
    for quantized models where weights may not be accessible.

Args:
    x: Input tensor
    base_out: Output from base forward (unused, for API consistency)

Reference: LyCORIS functional/loha.py bypass_forward_diff
Nr¬   Ú
multiplier)r`   Úis_convFÚconv_dimÚkw_dictr­   Úkernel_sizeÚin_channels)r   )r   ÚFÚlinearÚconv1dÚconv2dÚconv3drW   rS   Úgetattrrb   r`   r.   r7   ÚdimÚview)rV   r”   rÅ   Ú	FUNC_LISTr½   rX   rY   rU   rZ   r[   r8   r9   rT   r   rÈ   rÉ   rÊ   r¾   r¿   r   ÚoprË   rÌ   s   &&&                    r   ÚhÚLoHaAdapter.h*  sg  € ô  	&Ô'ð ˜4¤§¡¬1¯8©8´Q·X±X¼q¿x¹xÐHˆ	à�L‰Lˆà��dˆØ��dˆØ�!•ˆØ��dˆØ��dˆØˆq�TˆØˆq�Tˆð �y‰y˜�|ˆØ!&Ò!2�–¸¼wØ�, ó@
õ 
ˆð
 �f‰f˜1Ÿ7™7ˆfÓ#ˆØ�f‰f˜1Ÿ7™7ˆfÓ#ˆØ�f‰f˜1Ÿ7™7ˆfÓ#ˆØ�f‰f˜1Ÿ7™7ˆfÓ#ˆô ˜$ 	¨5Ó1ˆÜ˜4 ¨QÓ/ˆÜ˜$ 	¨2Ó.ˆð Š>˜bšnØ—‘˜QŸW™W�Ó%ˆBØ—‘˜QŸW™W�Ó%ˆBÜ—’Ð<¸bÀsÓKˆBÜ—’Ð<¸bÀsÓKˆBØ �7 eÕ+‰Kà•ˆBØ•ˆBØ �7 eÕ+ˆKçØ a�<Õ(ˆBÜ! $¨°t¸hµÓGˆKÜ! $¨°tÓ<ˆKð �‰Ó  AÔ%ØÒ*Ø"-×"2Ò"2Ø#×)Ñ)¨!Õ,¨kð#Ø<Gó#‘Kð #.×"2Ò"2ð #Ø$×*Ñ*ð#Ø/0¨c°H­nó#�Køô —‘ˆBØˆGá�!�[Ñ, GÑ,Ð,r   r{   )r   r¬   r   )r)   r*   r+   r,   rµ   rI   Úclassmethodr�   r�   r©   r.   r‚   rÂ   r×   r0   r1   r2   s   @r   rx   rx   �   sc   ø‡ € Ø€Dòð óKó ðKò&ð ÷*ñ *ó ð*ðh !Ÿ=™=ØôT÷lP-ö P-r   rx   )r	   Ú	functoolsr   Útypingr   r.   Útorch.nn.functionalrJ   Ú
functionalrÍ   Úcomfy.model_managementr®   Úbaser   r   r   r   ÚautogradÚFunctionr   r5   rE   rx   r   r   r   Ú<module>râ      s‚   ðÛ Ý Ý ã ß Ð Û ß MÑ Mð ñó ðô<�—‘×(Ñ(ô <ô.*N�u—~‘~×.Ñ.ô *NôZ4XÐ%ô 4Xônk-Ð#ö k-r   