+
    „°ñjI-  ã                   ón   € ^ RI t ^ RIHt ^ RIt^ RIt^RIHtHtH	t	H
t
  ! R R]4      t ! R R]4      tR# )é    N)ÚOptional)ÚWeightAdapterBaseÚWeightAdapterTrainBaseÚweight_decomposeÚfactorizationc                   óh   a a€ ] tR t^t oV 3R ltR tR tV3R lR ltV3R lR ltR t	R	t
VtV ;t# )
ÚOFTDiffc                ó¬  <€ \         SV `  4        Vw  r#rE\        P                  P	                  V4      V n        Ve-   \        P                  P	                  V4      V n        RV n        MRV n        VP                  w  V n	        V n
        p\        V4      V n        \        P                  P	                  \        P                  ! V4      RR7      V n        R # )NTF)Úrequires_grad)ÚsuperÚ__init__ÚtorchÚnnÚ	ParameterÚ
oft_blocksÚrescaleÚrescaledÚshapeÚ	block_numÚ
block_sizeÚfloatÚ
constraintÚtensorÚalpha)ÚselfÚweightsÚblocksr   r   Ú_Ú	__class__s   &&    €Ú<C:\Ai_Coding\apps\studio\ComfyUI\comfy\weight_adapter\oft.pyr   ÚOFTDiff.__init__   s�   ø€ Ü‰ÑÔà$+Ñ!ˆ˜ô  Ÿ(™(×,Ñ,¨VÓ4ˆŒØÒÜ Ÿ8™8×-Ñ-¨gÓ6ˆDŒLØ ˆD�Mà!ˆDŒMØ-3¯\©\Ñ*ˆŒ˜œ¨Ü ›,ˆŒÜ—X‘X×'Ñ'¬¯ª°UÓ(;È5Ð'ÓQˆŽ
ó    c                óT  € VP                   p\        P                  ! V P                  V P                  P
                  R 7      pV P                  V P                  P                  ^^4      ,
          pTpV P                  '       dG   \        P                  ! V4      R,           pW`P                  8”  d   W@P                  ,          V,          pW5,           W5,
          P                  4       P                  4       ,          pVP                  vr‰VP                  VP                   R7      p
V
P                  ^ V P                  V P                  34      p
\        P                  ! RVV
4      P!                  ^ ^4      pV P"                  '       d   V P$                  V,          pVP                  V4      # ))Údeviceç:Œ0âŽyE>)Údtypeúk n m, k n ... -> k m ...)r&   r   Úeyer   r   r$   Ú	transposer   Únormr   Úinverser   ÚtoÚ	unflattenr   ÚeinsumÚflattenr   r   )r   ÚwÚ	org_dtypeÚIÚqÚnormed_qÚq_normÚrr   r   Ú
org_weightÚweights   &&          r    Ú__call__ÚOFTDiff.__call__   s4  € Ø—G‘Gˆ	Ü�IŠI�d—o‘o¨d¯o©o×.DÑ.DÔEˆð �O‰O˜dŸo™o×7Ñ7¸¸1Ó=Õ=ˆØˆØ�?�?ˆ?Ü—Z’Z “] TÕ)ˆFØŸ™Ô'ØŸ™Õ.°Õ7�à�\˜a�l×1Ñ1Ó3×;Ñ;Ó=Õ=ˆð —G‘Gˆ	ˆØ—T‘T §¡�TÓ(ˆ
Ø×)Ñ)¨!¨d¯n©n¸d¿o¹oÐ-NÓOˆ
ä—’Ø'ØØó
÷ ‰'�!�Q‹-ð	 	ð
 �=�=ˆ=Ø—\‘\ FÕ*ˆFØ�y‰y˜Ó#Ð#r"   c                óæ  € V P                   P                  WR7      p\        P                  ! V P                  WR7      pW3P                  ^^4      ,
          pTpV P                  '       dG   \        P                  ! V4      R,           pWpP                  8”  d   WPP                  ,          V,          pWF,           WF,
          P                  4       P                  4       ,          pVP                  V4      # ©z9Compute the orthogonal rotation matrix R from OFT blocks.©r$   r&   r%   )
r   r,   r   r(   r   r)   r   r*   r   r+   )	r   r$   r&   r   r2   r3   r4   r5   r6   s	   &&&      r    Ú_get_orthogonal_matrixÚOFTDiff._get_orthogonal_matrix<   s±   € à—‘×#Ñ#¨6Ð#Ó?ˆÜ�IŠI�d—o‘o¨fÔBˆð ×%Ñ% a¨Ó+Õ+ˆØˆð �?�?ˆ?Ü—Z’Z “] TÕ)ˆFØŸ™Ô'ØŸ™Õ.°Õ7�ð �\˜a�l×1Ñ1Ó3×;Ñ;Ó=Õ=ˆØ�t‰t�E‹{Ðr"   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )é   ÚxÚbase_outÚreturn©r   ÚTensor)ÚformatÚ__classdict__s   "€r    Ú__annotate__ÚOFTDiff.__annotate__O   s.   ø€ ÷ *ñ *‘5—<‘<ð *©5¯<©<ð *¹E¿L¹Lñ *r"   c                ó.   € \         P                  ! V4      # )z‡
OFT has no additive component - returns zeros matching base_out shape.

OFT only transforms the output via g(), it doesn't add to it.
)r   Ú
zeros_like)r   rB   rC   s   &&&r    ÚhÚ	OFTDiff.hO   s   € ô ×Ò Ó)Ð)r"   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rA   ÚyrD   rE   )rG   rH   s   "€r    rI   rJ   W   s#   ø€ ÷ +ñ +‘5—<‘<ð +¡E§L¡Lñ +r"   c                ó6  € V P                  VP                  VP                  4      p\        V RR4      p\        P
                  ! V P                  VP                  VP                  R7      pW#,          ^V,
          V,          ,           p\        V RVP                  4       ^8„  4      pV'       d   VP                  ^R4      pVP                  Ev rgVP                  ! . VOV P                  NV P                  N5!  p\        P                  ! RW(4      p	V	P                  ! . VOVN5!  p
V P                  '       dI   V P                  P                  VP                  VP                  R7      pW«P!                  R4      ,          p
V'       d   V
P                  ^R4      p
V
# )z‡
Output transformation for OFT: applies orthogonal rotation.

OFT transforms output channels using block-diagonal orthogonal matrices.
Ú
multiplierç      ð?r=   Úis_convúk n m, ... k n -> ... k méÿÿÿÿ)r>   r$   r&   Úgetattrr   r(   r   Údimr)   r   Úreshaper   r.   r   r   r,   Úview)r   rQ   r6   rS   r2   rU   Úbatch_shapeÚout_featuresÚ	y_blockedÚout_blockedÚoutr   s   &&          r    ÚgÚ	OFTDiff.gW   s@  € ð ×'Ñ'¨¯©°!·'±'Ó:ˆô ˜T <°Ó5ˆ
Ü�IŠI�d—o‘o¨a¯h©h¸a¿g¹gÔFˆØ�N˜a *�n°Õ1Õ1ˆô ˜$ 	¨1¯5©5«7°Q©;Ó7ˆçà—‘˜A˜rÓ"ˆAð &'§W¡WÑ"ˆð —I’IÐL˜{ÐL¨D¯N©NÐL¸D¿O¹OÓLˆ	ô —l’lÐ#>ÀÓMˆð ×!Ò!Ð= ;Ð=°Ó=ˆð �=�=ˆ=Ø—l‘l—o‘o¨Q¯X©X¸Q¿W¹W�oÓEˆGØŸ™ RÓ(Õ(ˆCçà—-‘-  2Ó&ˆCàˆ
r"   c                óB   € \        R V P                  4        4       4      # )z4Calculates memory usage of the trainable parameters.c              3   ój   "  € T F)  qP                  4       VP                  4       ,          x € K+  	  R # 5i©N)ÚnumelÚelement_size)Ú.0Úparams   & r    Ú	<genexpr>Ú/OFTDiff.passive_memory_usage.<locals>.<genexpr>†   s'   é € ÐWÑEV¸E—;‘;“= 5×#5Ñ#5Ó#7×7Ò7ÓEVùs   ‚13)ÚsumÚ
parameters©r   s   &r    Úpassive_memory_usageÚOFTDiff.passive_memory_usage„   s   € äÑWÀTÇ_Á_ÔEVÓWÓWÐWr"   )r   r   r   r   r   r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r9   r>   rM   ra   ro   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r   rH   s   @@r    r	   r	      s6   ù‡ € õRò $ò:÷&*ð *÷+ð +÷ZXò Xr"   r	   c                   óž   a € ] tR t^‰t o RtR t]RR l4       tR t]RV 3R lR ll4       t	]
P                  R3R ltR	 tV 3R
 lR ltRtV tR# )Ú
OFTAdapterÚoftc                ó   € Wn         W n        R # re   ©Úloaded_keysr   )r   r}   r   s   &&&r    r   ÚOFTAdapter.__init__Œ   s   € Ø&ÔØŽr"   c                óÂ   € VP                   ^ ,          p\        WB4      w  rV\        P                  ! WeWQP                  \        P
                  R7      p\        VRVR34      # )r   r=   N)r   r   r   Úzerosr$   Úfloat32r	   )Úclsr8   Úrankr   Úout_dimr   r   Úblocks   &&&&    r    Úcreate_trainÚOFTAdapter.create_train�   sO   € à—,‘,˜q•/ˆÜ -¨gÓ <Ñˆ
Ü—’Ø :·m±mÌ5Ï=É=ô
ˆô ˜˜t U¨DÐ1Ó2Ð2r"   c                ó,   € \        V P                  4      # re   )r	   r   rn   s   &r    Úto_trainÚOFTAdapter.to_train™   s   € Ü�t—|‘|Ó$Ð$r"   Nc                óš   <€ V ^8„  d   QhRS[ RS[S[ S[P                  3,          RS[RS[P                  RS[S[ ,          RS[R,          /# )rA   rB   Úlorar   Ú
dora_scaler}   rD   ry   )ÚstrÚdictr   rF   r   Úsetr   )rG   rH   s   "€r    rI   ÚOFTAdapter.__annotate__�   sb   ø€ ÷ )ñ )áð)ñ ‘3™Ÿ™Ð$Õ%ð)ñ ð	)ñ
 —L‘Lð)ñ ™•Xð)ñ 
�,Õ	ñ)r"   c                ój  € Vf   \        4       pRP                  V4      pRP                  V4      pR pWbP                  4       9   d.   W&,          pVP                  ^8X  d   VP	                  V4       MR pVf   R # R p	WrP                  4       9   d   W',          p	VP	                  V4       W‰W43p
V ! WZ4      # )Nz{}.oft_blocksz
{}.rescale)r�   rG   ÚkeysÚndimÚadd)r‚   rB   rŒ   r   r�   r}   Úblocks_nameÚrescale_namer   r   r   s   &&&&&&     r    ÚloadÚOFTAdapter.loadœ   s©   € ð ÒÜ›%ˆKØ%×,Ñ,¨QÓ/ˆØ#×*Ñ*¨1Ó-ˆàˆØŸ)™)›+Ô%ØÕ&ˆFØ�{‰{˜aÔØ—‘ Õ,à�ØŠ>ÙàˆØŸ9™9›;Ô&ØÕ(ˆGØ�O‰O˜LÔ)à EÐ6ˆÙ�;Ó(Ð(r"   c	           	     ó   € V P                   p	V	^ ,          p
V	^,          pV	^,          pVf   ^ pV	^,          p\        P                  P                  W¡P                  V4      p
Ve+   \        P                  P                  W±P                  V4      pV
P
                  vrïp \        P                  ! WúP                  V
P                  R7      pWªP                  ^^4      ,
          pTpV^ 8”  d5   \        P                  ! V4      R,           pVV8”  d   VV,          V,          pVV,           VV,
          P                  4       P                  4       ,          pVP                  V4      p\        P                  ! WñP                  VP                  R7      pVP
                  vpp\        P                  ! RVV,          VV,          ,
          VP                  ! Wï.VO5!  4      P                  ! R.VO5!  pVe   \!        VVVVVVV4      pV# W! VV,          P#                  VP                  4      4      ,          p V#   \$         d=   p\&        P(                  ! RP+                  T P,                  TT4      4        Rp?T# Rp?ii ; i)r   Nr=   r%   r'   zERROR {} {} {}rW   )r   ÚcomfyÚmodel_managementÚcast_to_devicer$   r   r   r(   r&   r)   r*   r   r+   r,   r.   r[   r   ÚtypeÚ	ExceptionÚloggingÚerrorrG   Úname)r   r8   ÚkeyÚstrengthÚstrength_modelÚoffsetÚfunctionÚintermediate_dtypeÚoriginal_weightÚvr   r   r   r�   r   r   r   r2   r3   r4   r5   r6   ÚI_wr   Ú	lora_diffÚes   &&&&&&&&&                 r    Úcalculate_weightÚOFTAdapter.calculate_weight¼   s7  € ð �L‰LˆØ�1•ˆØ�A•$ˆØ�!•ˆØŠ=ØˆEØ�q•Tˆ
ä×'Ñ'×6Ñ6Ø—M‘MÐ#5ó
ˆð ÒÜ×,Ñ,×;Ñ;ØŸ™Ð(:óˆGð %+§L¡LÐ!ˆ	 ð"	Fä—	’	˜*¯]©]À&Ç,Á,ÔOˆAà×)Ñ)¨!¨QÓ/Õ/ˆAØˆHØ�qŒyÜŸš A›¨Õ-�Ø˜E”>Ø  5�y¨6Õ1�Hà�X• ! h¥,×!5Ñ!5Ó!7×!?Ñ!?Ó!AÕAˆAØ—‘�V“ˆAä—)’)˜J¯}©}ÀFÇLÁLÔQˆCØŸ™ˆIˆA�ÜŸšØ+Ø�X• ¨C¥Õ/Ø—’˜IÐ:°EÓ:ó÷ ‰dð	ð ð	ð ó	ˆIð
 Ò%Ü)ØØØØØØ&Øó�ð ˆð ˜( H¨yÕ$8×#>Ñ#>¸v¿|¹|Ó#LÓMÕM‘ð ˆøô ô 	FÜ�MŠMÐ*×1Ñ1°$·)±)¸SÀ!ÓD×EÐEØˆûð	Fús   ÂE%H6 È.H6 È6I=É1I8É8I=c                óÈ  € V P                   pV^ ,          P                  WR7      pV^,          pVf   ^ pVP                  w  rgp\        P                  ! WqVR7      p	WDP                  ^^4      ,
          p
T
pV^ 8”  d3   \        P                  ! V
4      R,           pWÅ8”  d   W¥,          V,          pW›,           W›,
          P                  4       P                  4       ,          pWÖV3# r<   )	r   r,   r   r   r(   r)   r*   r   r+   )r   r$   r&   rª   r   r   r   r   r   r2   r3   r4   r5   r6   s   &&&           r    r>   Ú!OFTAdapter._get_orthogonal_matrixþ   sÅ   € à�L‰LˆØ�1•—‘ �Ó4ˆØ�!•ˆØŠ=ØˆEà#)§<¡<Ñ ˆ	˜qÜ�IŠI�j°uÔ=ˆð ×%Ñ% a¨Ó+Õ+ˆØˆð �1Œ9Ü—Z’Z “] TÕ)ˆFØŒ~Ø�9 vÕ-�ð �\˜a�l×1Ñ1Ó3×;Ñ;Ó=Õ=ˆØ˜ZÐ'Ð'r"   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rP   rE   )rG   rH   s   "€r    rI   r‘     s#   ø€ ÷ 0ñ 0‘5—<‘<ð 0¡E§L¡Lñ 0r"   c                óø  € V P                   pV^,          pV P                  VP                  VP                  4      w  rEp\	        V RR4      p\
        P                  ! WaP                  VP                  R7      pWG,          ^V,
          V,          ,           p\	        V RVP                  4       ^8„  4      p	V	'       d   VP                  ^R4      pVP                  Ev r«VP                  ! . V
OVNVN5!  p\
        P                  ! RWL4      pVP                  ! . V
OVN5!  pVe?   VP                  VP                  VP                  R7      pWãP                  R4      ,          pV	'       d   VP                  ^R4      pV# )zÇ
Output transformation for OFT: applies orthogonal rotation to output.

OFT transforms the output channels using block-diagonal orthogonal matrices.

Reference: LyCORIS DiagOFTModule._bypass_forward
rS   rT   r=   rU   rV   rW   )r   r>   r$   r&   rX   r   r(   rY   r)   r   r[   r.   r,   )r   rQ   rª   r   r6   r   r   rS   r2   rU   r\   r]   r^   r_   r`   s   &&             r    ra   ÚOFTAdapter.g  sB  € ð �L‰LˆØ�A•$ˆà#'×#>Ñ#>¸q¿x¹xÈÏÉÓ#QÑ ˆ�jô ˜T <°Ó5ˆ
Ü�IŠI�j¯©¸¿¹ÔAˆØ�N˜a *�n°Õ1Õ1ˆô ˜$ 	¨1¯5©5«7°Q©;Ó7ˆçà—‘˜A˜rÓ"ˆAð &'§W¡WÑ"ˆð —F’FÐ?˜KÐ?¨Ð?°JÓ?ˆ	ô —l’lÐ#>ÀÓMˆð ×ÒÐ: Ð:¨\Ó:ˆð ÒØ—j‘j¨¯©¸¿¹�jÓ@ˆGØŸ™ RÓ(Õ(ˆCçà—-‘-  2Ó&ˆCàˆ
r"   r|   )é   rT   re   )rq   rr   rs   rt   r¢   r   Úclassmethodr†   r‰   r˜   r   r�   r®   r>   ra   ru   rv   )rH   s   @r    ry   ry   ‰   sd   ø‡ € Ø€Dòð ó3ó ð3ò%ð ÷)ñ )ó ð)ðN !Ÿ=™=Øô@òD(÷20ö 0r"   ry   )r    Útypingr   r   Úcomfy.model_managementr›   Úbaser   r   r   r   r	   ry   © r"   r    Ú<module>r»      s<   ðÛ Ý ã Û ÷ó ôxXÐ$ô xXôv~Ð"ö ~r"   