+
    �åûjg1  ã                   ó  € R t ^ RIt^RIHt ^@tRtRtR tRR R lltR tR R	 lt	R
 t
R tR R ltRR R lltRR R llt]P                  P!                  RRR7      RR R ll4       t]P$                  RR l4       tR# )aê  Sol-Attn: training-free sparse attention for video diffusion (arXiv 2607.24027).

Reference implementation: each query block attends a routed subset of key
blocks exactly; every other block contributes one pooled term (summary key,
summed values), so the whole sequence stays in the softmax denominator.

Defines the algorithm, not the CUDA kernel's arithmetic (full precision here
vs INT8 there -- expect cos ~0.999, not bitwise), and materialises the scores
densely, so memory is O(T^2).
N)Úregistrygþ‚+eG÷?c                ó  € V P                   V8w  d   \        RV RV P                    24      hV P                  4       ^8X  d\   V P                  ^,          ^8w  g   V P                  ^,          ^8w  d#   \        R\	        V P                  4       R24      hV R
,          p V P                  4       ^8X  d   V P                  ^ 4      p V P                  4       ^8w  g3   V P                  R,          V8w  g   V P                  ^ ,          ^V39  d(   \        R\	        V P                  4       RV RV 24      hV P                  \        P                  8X  d"   \        P                  ! V R\        R	4      4      p V P                  4       # )zÒReduce SDPA-mask-like forms -- (T,), (B, T), (B|1, 1, 1, T), bool or
float -- to (B, T) float log-bias. Rejects head/query-varying masks and
wrong-device tensors (a host pointer would poison the CUDA context).
zsol_attn: key_bias must be on z, got zKsol_attn: key_bias must be key-only; a mask varying over heads or queries (zI) cannot be expressed by this op -- use a dense attention for those callsz=sol_attn: key_bias must be (T,), (B, T) or (B, 1, 1, T), got z for T=z, B=ç        ú-inf)ºNNNé    r   r   éÿÿÿÿ)ÚdeviceÚ
ValueErrorÚdimÚshapeÚtupleÚ	unsqueezeÚdtypeÚtorchÚboolÚwhereÚfloat)Úkey_biasÚbatchÚtr	   s   &&&&ÚaC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\backends\eager\sol_attn.pyÚ_normalize_key_biasr      sV  € ð
 ‡�˜&Ô ÜØ,¨V¨H°F¸8¿?¹?Ð:KÐLóNð 	Nà‡|�|ƒ~˜ÔØ�>‰>˜!Õ Ô! X§^¡^°AÕ%6¸!Ô%;Üð%Ü%*¨8¯>©>Ó%:Ð$;ð <PðPóQð Qð ˜JÕ'ˆØ‡|�|ƒ~˜ÔØ×%Ñ% aÓ(ˆØ‡|�|ƒ~˜Ô˜hŸn™n¨RÕ0°AÔ5¸¿¹ÈÕ9JÐSTÐV[ÐR\Ô9\ÜØKÜ�X—^‘^Ó$Ð% W¨Q¨C¨t°E°7ð<ó=ð 	=ð ‡~�~œŸ™Ô#Ü—;’;˜x¨¬e°F«mÓ<ˆØ�>‰>ÓÐó    c                óP   € V ^8„  d   QhR\         R\         R\        P                  /# )é   r   ÚnÚreturn©Úintr   ÚTensor)Úformats   "r   Ú__annotate__r"   1   s%   € ÷ ñ ”cð œcð ¼e¿l¹lñ r   c                óž  € W^,
          \         ,          ,
          pVeo   VP                  V\        P                  R7      P	                  ^\         4      pV\         8  d-   VP                  4       pVR,          P	                  VR7      VR&   V# \        P                  ! V3\        \         4      VR7      pV\         8  d   \        V4      VR&   V# )zjLive tokens per block: the caller's table (zero-padded tiles) or 64
everywhere except a ragged last block.©r	   r   )Úmax©r	   r   )ÚBLOCKÚtor   Úfloat32ÚclampÚcloneÚfullr   )r   r   r	   Ú	block_lenÚtailÚlengthss   &&&&  r   Ú_block_lengthsr0   1   s    € ð �A•œ�Õ€DØÒØ—,‘, f´E·M±M�,ÓB×HÑHÈÌEÓRˆØ”%Œ<Ø—m‘m“oˆGØ! "�+×+Ñ+°Ð+Ó5ˆG�B‰KØˆÜ�jŠj˜!˜œu¤U›|°FÔ;€GØŒe„|Ü˜D“kˆ�‰Ø€Nr   c                óL   € \        ^ \        W 4      \        W4      ,
          4      # )z]Sink blocks inside [0, n): they are always exact, so the top-k budget
counts the others only.)r%   Úmin)r   Ús0Ús1s   &&&r   Ú_sink_countr5   A   s   € ô ˆq”#�b“*œs 2›zÕ)Ó*Ð*r   c                ód   € V ^8„  d   QhR\         R\        P                  R\        P                  /# )r   r   r/   r   r   )r!   s   "r   r"   r"   G   s)   € ÷ 1ñ 1”3ð 1¤§¡ð 1´%·,±,ñ 1r   c                ó†   € \         P                  ! WP                  R7      pV\        ,          W\        ,          ,          8  # )zJ(T,) bool: token t is live iff it sits in the first len rows of its block.r&   )r   Úaranger	   r'   )r   r/   Úposs   && r   Ú_valid_rowsr:   G   s*   € ä
�,Š,�q§¡Ô
0€CØ”%�K˜7¬%¥<Õ0Ñ0Ð0r   c                ó¶   € \         P                  ! WP                  ^^4      4      V,          p\         P                  ! \         P                  ! VRR7      V4      # )zwVSA coarse branch: dense attention over the block means. qm/km/vm are
``[BH, N, D]`` fp32; returns ``[BH, N, D]`` fp32.©r   r   )r   ÚbmmÚ	transposeÚsoftmax)ÚqmÚkmÚvmÚscaleÚss   &&&& r   Úcoarse_outputrE   M   s>   € ô 	�	Š	�"—l‘l 1 aÓ(Ó)¨EÕ1€AÜ�9Š9”U—]’] 1¨"Ô-¨rÓ2Ð2r   c           	     ód  € V P                   w  r4rVVP                  W5RV4      P                  ^ ^^^4      pVP                  4       pV\        ,          pV'       du   V RRV\        ,          13,          P                  W7\        WV4      P                  VRRV\        ,          13,          P                  W7\        WV4      VRRV1R3,          4       V\        ,          '       dP   V RV\        ,          R13,          P                  VRV\        ,          R13,          VRWw^,           13,          4       V # )z…out += gate * oc[block(t)], in place (one rounding: addcmul_ promotes
internally). out/gate ``(B, T, H, D)``; oc ``[BH, N, D]`` fp32.r   Nr   )r   ÚviewÚpermuteÚ
contiguousr'   Úaddcmul_)ÚoutÚocÚgateÚbr   ÚhÚdÚnfulls   &&&     r   Úadd_coarse_rR   T   s   € ð —‘�J€Aˆ!Ø	�‰��r˜1Ó	×	%Ñ	% a¨¨A¨qÓ	1€BØ�?‰?Ó€DØ”�J€EßØˆAˆ~�œ•ˆ~ÐÕ×#Ñ# A¬e°QÓ:×CÑCØ��N�UœU•]�NÐ"Õ#×(Ñ(¨´5¸!Ó?ÀÀAÀvÈÀvÈtÀOÕATô	VàŒ5‡y„yØˆAˆu”u�}‰~ÐÕ×'Ñ'¨¨Q°¼µ±Ð->Õ(?ÀÀAÀuÐUVÍYÀÐDVÕAWÔXØ€Jr   c                ó<   € V ^8„  d   QhR\         R\        R\         /# )r   r   Úratior   )r   r   )r!   s   "r   r"   r"   c   s!   € ÷ 8ñ 8”3ð 8œuð 8¬ñ 8r   c                ón   € \        ^ \        V ^,
          \        ^\        W,          4      4      4      4      # )z‡Key blocks a query block keeps under top-k: ratio * n, clamped to
[1, n-1]; 0 when n <= 1 (nothing to select beyond the forced blocks).)r%   r2   Úround)r   rT   s   &&r   Ú_topk_countrW   c   s*   € ô ˆq”#�a˜!•eœS ¤E¨%­)Ó$4Ó5Ó6Ó7Ð7r   c                óp   € V ^8„  d   QhR\         P                  R\        R\        R\         P                  /# )r   ÚxÚn_blocksÚreducer   )r   r    r   Ústr)r!   s   "r   r"   r"   i   s0   € ÷ 9ñ 9ŒU�\‰\ð 9¤Sð 9´#ð 9ÌÏÉñ 9r   c           	     óÜ  € V P                   w  rErgVf   \        WQV P                  4      pV \        WS4      P	                  ^R^^4      ,          p V\
        ,          V,
          pV'       d*   \        P                  ! W P                  WHWg4      .^R7      p V P                  WA\
        Wg4      p	VR8X  d   V	P                  ^R7      # V	P                  ^R7      VP	                  ^R^^4      ,          # )zC(B, T, H, D) -> (B, N, H, D), block mean or sum over the live rows.r<   Úsumr   )r   r0   r	   r:   rG   r'   r   ÚcatÚ	new_zerosÚreshaper^   )
rY   rZ   r[   r/   rN   r   rO   rP   ÚpadÚblockss
   &&&&      r   Ú_poolrd   i   sÃ   € à—‘�J€Aˆ!Ø‚Ü  ¨a¯h©hÓ7ˆØ	ŒK˜Ó#×(Ñ(¨¨B°°1Ó5Õ5€AØ
”UÕ
˜QÕ
€Cß
Ü�IŠI�qŸ+™+ a¨aÓ3Ð4¸!Ô<ˆØ�Y‰Y�q¤E¨1Ó0€FØ�„Ø�z‰z˜aˆzÓ Ð Ø�:‰:˜!ˆ:Ó˜wŸ|™|¨A¨r°1°aÓ8Õ8Ð8r   c                óÌ  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R,          R\        \        ,          R,          R\        \        ,          R,          R	\         P                  R,          R
\        R\
        R\         P                  R,          R\         P                  R,          R\        R\         P                  /# ©r   ÚqÚkÚvÚtaurC   NÚsink_blocksÚsink_qr   Ú
topk_ratior.   r-   Úcoarse_gateÚ	token_augr   ©r   r    r   Úlistr   r   )r!   s   "r   r"   r"   x   så   € ÷ }ñ }Ü‡|�|ð}ä‡|�|ð}ô ‡|�|ð}ô 
ð	}ô
 �4�<ð}ô ”c•˜TÕ!ð}ô ”�I˜Õð}ô �l‰l˜TÕ!ð}ô ð}ô ð}ô �|‰|˜dÕ"ð}ô —‘ Õ$ð}ô ð}ô ‡\�\ñ}r   c                ó†  a9a:a;a<€ V P                   w  o9po;o:V\        ,           ^,
          \        ,          o<Vf
   S:R,          pV\        ,          pT;'       g    ^ ^ .R,          w  ppT;'       g    ^ ^ .R,          w  ppS9S;,          V,          V,          ^,          pV\        8”  dE   \	        RVR,          R RS9 RS; RV RV P
                   R	V P                  P                   R
24      hV P                  4       VP                  4       VP                  4       ppp\        VS<V P                  V
4      p\        VS<RV4      p\        VS<RV4      pVP                  ^RR7      pVV,
          pVP                  ^4      P                  ^R7      p\        VS<RV4      pVP                  ^ ^^^4      pVV,
          P                  ^ ^^^4      pVP                  ^ ^^^4      p VP                  ^ ^^^4      p!VP                  ^ ^^^4      p"VVP                  RR4      ,          V,          p#V
eX   V#P                  \!        VV4      P#                  ^^^V4      ( \$        P&                  ! V#P
                  4      P(                  4      p#Ve>   \+        VS9WÐP                  4      p$V#V$\        ,          P-                  R^^V4      ,           p#VV!P                  RR4      ,          V,          p%\$        P.                  ! WÐP                  R7      \        ,          p&V
e&   V%\!        VV4      P#                  ^^V^4      ,          p%\$        P0                  ! S9S;S<S<V P                  V%P
                  R7      p'V'P3                  ^V&P#                  ^^V^4      P5                  S9S;VS<4      V%4       V'VP#                  ^^S<^4      ,          p'\$        P.                  ! S<V P                  R7      p(V'       dž   V'P7                  4       p)\        R4      V)RVV13&   \9        S<\;        S<VV4      ,
          V4      p*V*'       d/   V)P=                  V*RR7      P>                  RRR13,          p+V)V+8¬  p,M´\$        P@                  ! V)\$        PB                  R7      p,M�VP                  ^4      VPE                  ^4      ,          PG                  R4      p-V\$        PH                  ! V-V,          V,          R,           4      ,          p.V'V.P                  ^ ^^4      PE                  R4      8„  p,V,V(P#                  ^R4      V(P#                  R^4      ,
          PK                  4       ^8*  P#                  ^^S<S<4      ,          p,V,V(V8¬  V(V8  ,          P#                  ^^^S<4      ,          p,V,V(V8¬  V(V8  ,          P#                  ^^S<^4      ,          p,V,PM                  ^V&P#                  ^^V^4      P5                  S9S;VS<4      4      p/V/PO                  \        RR7      RRV13,          p0\$        P&                  ! V#P
                  4      P(                  p1V#P                  V0( V14      p#V'PM                  ^V&P#                  ^^V^4      P5                  S9S;VS<4      4      p%V%P                  V/V14      p%V	'       g   \$        PP                  ! V%V14      p%\$        PR                  ! V#V%.RR7      p2\$        PT                  ! V2V2PW                  RRR7      ,
          4      p3V3P                  V2V18*  R4      p3V3RRV13,          V ,          V3RVR13,          V",          ,           p4V3RRV13,          PG                  R4      V3RVR13,          VP#                  ^^^S<4      ,          PG                  R4      ,           p5V4V5PY                  R4      PE                  R4      ,          P                  ^ ^^^4      P[                  VP
                  4      p6V6P]                  4       p6VeP   V9V:V;V<3R lp7\_        V7! V4      V7! V4      V7! VVP#                  ^R^^4      ,          4      V4      p8\a        V6V8V4       V6# )a:  Sol-Attn over ``(B, T, H, D)`` tensors. See the module docstring.

``topk_ratio`` > 0 selects SLA-style per-query-block top-k instead of the
tau threshold (sinks and diagonal still forced exact); tau is ignored.
``tail=False`` drops the pooled term (softmax over routed blocks only),
``block_len`` marks the live rows at the front of each 64-block (values
clamped to [1, rows in the block]; dead rows are never keys and their
output rows are unspecified), and ``coarse_gate`` adds VSA's gated coarse
branch: ``gate * softmax(q_mean k_mean^T * scale) v_mean`` per block.
N:Nr   Nz7sol_attn: the eager reference is O(T^2) and would need z.1fz  GiB for the score tensor at (B=z, H=z, T=zŒ). It was selected because no fused backend accepted these inputs -- the fused backends take bfloat16 or float16 with head_dim 128, and got z on Ú.Úmeanr^   T)r   Úkeepdimr<   r&   r$   r   .)r   g�íµ ÷Æ°>r   g ÂëþKH´9c                 ó\   <€ V P                  ^ ^^^4      P                  SS,          SS4      # ©r   )rH   ra   )ÚprN   rP   rO   r   s   &€€€€r   Ú<lambda>Úsol_attn.<locals>.<lambda>ò   s'   ø€ ˜Ÿ™ 1 a¨¨AÓ.×6Ñ6°q¸1µu¸aÀÔCr   g      à¿i   @r   éþÿÿÿ)1r   r'   Ú_LOG2EÚ_MAX_SCORE_BYTESÚRuntimeErrorr   r	   Útyper   r0   rd   rt   ÚpowrH   r>   Úmasked_fillr:   rG   r   Úfinfor2   r   ra   r8   ÚzerosÚscatter_add_Úexpandr+   rW   r5   ÚtopkÚvaluesÚ
zeros_liker   r   r^   ÚsqrtÚabsÚgatherÚrepeat_interleaveÚ	full_liker_   Úexp2ÚamaxÚ	clamp_minr(   rI   rE   rR   )=rg   rh   ri   rj   rC   rk   rl   r   rm   r.   r-   rn   ro   r   Úlog2sÚsink_kv0Úsink_kv1Úsink_q0Úsink_q1Úscore_bytesÚfqÚfkÚfvr/   ÚkcÚvcÚk_meanÚkccÚkc_varÚcentroidÚqhÚkhÚvhÚkchÚvchÚs_tokÚkbÚs_blkÚqblkÚcolmeanÚidxÚrankedÚkkÚrow_thrÚexactÚvarÚthrÚex_tokÚkeep_tokÚnegÚlogitsrx   ÚnumÚdenrK   ÚflatrL   rN   rP   rO   r   s=   &&&&&&&&&&&&&                                            @@@@r   Úsol_attnr¸   x   s  û€ ð4 —‘�J€A€qˆ!ˆQØ	
ŒU��Q�œ5Õ €AØ‚}Ø�T•	ˆØ”F�N€EØ%×/Ð/¨!¨Q¨°Õ4Ñ€HˆhØ×(Ð( 1 a &¨"Õ-Ñ€GˆWà�a•%˜!•)˜a•- !Õ#€KØÔ%Ô%ÜØEØ˜UÕ" 3Ð'ð (Ø��D˜˜˜4 ˜sð #*à*+¯'©'¨°$°q·x±x·}±}°oÀQð	Hó
ð 	
ð —‘“˜AŸG™G›I q§w¡w£yˆBˆ€BÜ˜Q  1§8¡8¨YÓ7€GÜ	ˆr�1�f˜gÓ	&€BÜ	ˆr�1�e˜WÓ	%€Bð �W‰W˜ DˆWÓ)€FØ
ˆv�+€CØ�W‰W�Q‹Z�_‰_ ˆ_Ó#€Fä�R˜˜F GÓ,€Hà	�‰�A�q˜!˜QÓ	€BØ
ˆv�+×	Ñ	˜q ! Q¨Ó	*€BØ	�‰�A�q˜!˜QÓ	€BØ
�+‰+�a˜˜A˜qÓ
!€CØ
�*‰*�Q˜˜1˜aÓ
 €Cà�"—,‘,˜r 2Ó&Õ&¨%Õ/€EØÒØ×!Ñ!¤;¨q°'Ó#:×#?Ñ#?ÀÀ1ÀaÈÓ#KÐ"KÜ"'§+¢+¨e¯k©kÓ":×">Ñ">ó@ˆàÒô ! ¨1¨a·±Ó:ˆØ˜œf�×-Ñ-¨b°!°Q¸Ó:Õ:ˆØ�#—-‘-  BÓ'Õ'¨5Õ0€Eô �<Š<˜§(¡(Ô+¬uÕ4€DØÒØœ A wÓ/×4Ñ4°Q¸¸1¸aÓ@Õ@ˆÜ�kŠk˜!˜Q  1¨Q¯X©X¸U¿[¹[ÔI€GØ×Ñ˜˜DŸI™I a¨¨A¨qÓ1×8Ñ8¸¸A¸qÀ!ÓDÀeÔLØ˜Ÿ™ Q¨¨1¨aÓ0Õ0€GÜ
�,Š,�q §¡Ô
*€Cßà—‘“ˆÜ).¨v«ˆˆs�H˜XÐ%Ð%Ñ&Ü˜œ[¨¨H°hÓ?Õ?ÀÓLˆçØ—k‘k "¨"�kÓ-×4Ñ4°S¸"¹#°XÕ>ˆGØ˜gÑ%‰Eä×$Ò$ V´5·:±:Ô>‰Eð �|‰|˜A‹ ×!1Ñ!1°!Ó!4Õ4×9Ñ9¸"Ó=ˆØ”E—J’J˜s U�{¨UÕ2°TÕ9Ó:Õ:ˆØ˜#Ÿ+™+ a¨¨AÓ.×8Ñ8¸Ó<Ñ<ˆØ	ˆs�x‰x˜˜2‹ §¡¨"¨a£Õ0×5Ñ5Ó7¸1Ñ<×BÑBÀ1ÀaÈÈAÓNÕN€EØ	ˆs�h‰ 3¨¡>Õ2×8Ñ8¸¸A¸qÀ!ÓDÕD€EØ	ˆs�g‰~ #¨¡-Õ0×6Ñ6°q¸!¸QÀÓBÕB€Eà�\‰\˜!˜TŸY™Y q¨!¨Q°Ó2×9Ñ9¸!¸QÀÀ1ÓEÓF€FØ×'Ñ'¬°2Ð'Ó6°s¸B¸Q¸B°wÕ?€HÜ
�+Š+�e—k‘kÓ
"×
&Ñ
&€CØ×Ñ˜x˜i¨Ó-€Eà�N‰N˜1˜dŸi™i¨¨1¨a°Ó3×:Ñ:¸1¸aÀÀAÓFÓG€EØ×Ñ˜f cÓ*€EßÜ—’  sÓ+ˆô �YŠY˜˜u�~¨2Ô.€FÜ�
Š
�6˜FŸK™K¨B¸˜KÓ=Õ=Ó>€AØ	�‰�f ‘m SÓ)€AØ
ˆC��!�ˆG�*�r�/˜A˜c 1¡2˜g�J¨Õ,Õ
,€CØ
ˆC��!�ˆG�*�.‰.˜Ó
  # q¡r '¥
¨W¯\©\¸!¸QÀÀ1Ó-EÕ E×JÑJÈ2ÓNÕ
N€CØ�—‘˜uÓ%×/Ñ/°Ó3Õ3×
<Ñ
<¸QÀÀ1ÀaÓ
H×
KÑ
KÈAÏGÉGÓ
T€Cà
�.‰.Ó
€CØÒßCˆÜ™4 ›>©4°«8±T¸"¸w¿|¹|ÈAÈrÐSTÐVWÓ?XÕ:XÓ5YÐ[`ÓaˆÜ�C˜˜[Ô)Ø€Jr   zcomfy_kitchen::sol_attn)Úmutates_argsc                ó°  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R,          R\        \        ,          R\        \        ,          R	\         P                  R,          R
\        R\
        R\         P                  R,          R\         P                  R,          R\        R\         P                  /# rf   rp   )r!   s   "r   r"   r"   ù   sË   € ÷ ñ Ü‡|�|ðä‡|�|ðô ‡|�|ðô 
ð	ô
 �4�<ðô ”c•ðô ”�Iðô �l‰l˜TÕ!ðô ðô ðô �|‰|˜dÕ"ðô —‘ Õ$ðô ðô ‡\�\ñr   c                 óz   € R V RVRVRVRVRVRVRVRVR	V	R
V
RVRV/p\         P                  ! RVR7      pV! R/ VB # )rg   rh   ri   rj   rC   rk   rl   r   rm   r.   r-   rn   ro   r¸   )Úkwargs© )r   Úget_implementation)rg   rh   ri   rj   rC   rk   rl   r   rm   r.   r-   rn   ro   r¼   Úimpls   &&&&&&&&&&&&&  r   Ú_op_sol_attnrÀ   ø   sd   € ð" 	ˆQ��Q˜˜Q  s¨G°UØ�{ H¨fØ�H˜l¨JØ��k 9¨m¸[Ø�Yð€Fô ×&Ò& z¸&ÔA€DÙ‰>�&‰>Ðr   c                 óp   € \         P                  ! VP                  VP                  VP                  R 7      # ))r   r	   )r   Úemptyr   r   r	   )rg   rh   ri   rj   rC   rk   rl   r   rm   r.   r-   rn   ro   s   &&&&&&&&&&&&&r   Ú_op_sol_attn_fakerÃ     s#   € ô �;Š;�q—w‘w a§g¡g°a·h±hÔ?Ð?r   l        )N)
g      ð?NNNNr   TNNr   r½   rw   )Ú__doc__r   r   r'   r|   r}   r   r0   r5   r:   rE   rR   rW   rd   r¸   ÚlibraryÚ	custom_oprÀ   Úregister_fakerÃ   r½   r   r   Ú<module>rÈ      s™   ðñ	ó å  à
€Ø	€ð Ð ò÷4ò +õ1ò3òõ8÷9÷}ð@ ‡�×ÑÐ2ÀÐÓDöó Eðð4 ×Ñó@ó ò@r   