+
    õäûjY1  ã                  ó  € ^ RI Ht ^ RIt^ RIt^RIHt ]! ]P                  RR4      '       d   ^RIH	t
 MRt
RtRR R lltR R	 ltR
 R ltR]P                  3R R lltR R ltR tRR R lltR tR R ltR R ltR# )é    )ÚannotationsN)ÚcudaÚhip)r   c               ó    € V ^8„  d   QhRRRR/# )é   Údeviceztorch.device | NoneÚreturnÚbool© )Úformats   "ÚYC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\flash_attention.pyÚ__annotate__r      s   € ÷ Kñ KÐ,ð K¸ñ Kó    c                ó–  € V e   V P                   R8w  d   R# \        P                  P                  4       '       g   R# \        e   \        P
                  ! 4       # \        P                  '       d3   \        P                  e!   \        \        P                  R4      '       g   R# \        P                  P                  V 4      \        8¬  # )zKReturn whether flash attention decode is available on the requested device.r   FÚflash_attention_decode)ÚtypeÚtorchr   Úis_availableÚ_hip_backendÚ#flash_attention_decode_is_availableÚ_cuda_backendÚ_EXT_AVAILABLEÚ_CÚhasattrÚget_device_capabilityÚ_MINIMUM_CAPABILITY©r   s   &r   r   r      s�   € àÒ˜fŸk™k¨VÔ3ÙÜ�:‰:×"Ñ"×$Ò$ÙÜÒô ×?Ò?ÓAÐAä×(×(Ð(Ü×ÑÒ#Ü”}×'Ñ'Ð)A×BÒBáÜ�:‰:×+Ñ+¨FÓ3Ô7JÑJÐJr   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   Úbatch_headsÚintÚkv_capacityÚmultiprocessorsÚblock_nr	   r   )r   s   "r   r   r   (   s1   € ÷ ñ ˜Sð ¨sð ÀSð ÐSVð Ð[^ñ r   c                ó   a
€ W,           ^,
          V,          p\        ^ V^,          V4      pRo
. p\        ^V^,           4       Fª  pV^8H  ;'       gA    \        P                  ! WG,          4      \        P                  ! WG^,
          ,          4      8g  pW,          V^,          ,          p	V'       d   V	\        P                  ! V	4      ,          MRp	VP	                  V	4       \        S
V	4      o
K¬  	  \        V
3R l\        V^4       4       4      # )é   ç        c              3  óL   <"  € T F  w  rVR S,          8¼  g   K  Vx € K  	  R# 5i)g333333ë?Nr   )Ú.0ÚsplitsÚ
efficiencyÚbests   &  €r   Ú	<genexpr>Ú_num_splits.<locals>.<genexpr>5   s)   øé € ð á"<ÑˆFØ˜ �Ñ$÷ 	ŠÛ"<ùs   ƒ$š
$)ÚminÚrangeÚmathÚceilÚappendÚmaxÚnextÚ	enumerate)r   r!   r"   r#   ÚblocksÚ
max_splitsÚefficienciesr)   Úeligibler*   r+   s   &&&&      @r   Ú_num_splitsr:   (   sê   ø€ ØÕ# aÕ'¨GÕ3€FÜ�R˜¨1Õ,¨fÓ5€JØ€DØ€LÜ˜˜:¨�>Ö*ˆØ˜Q‘;÷ 
ð 
¤$§)¢)¨F­OÓ"<ÄÇ	Â	Ø˜q•jÕ!óA
ñ #
ˆð !Õ)¨_¸qÕ-@ÕAˆ
ß;C�Z¤$§)¢)¨JÓ"7Ö7Èˆ
Ø×Ñ˜JÔ'Ü�4˜Ó$Šñ +ô ô ä"+¨L¸!Ô"<óó ð r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   Úqútorch.TensorÚkÚvÚ
kv_lengthsr	   r   )r   s   "r   r   r   <   s;   € ÷ >Xñ >XØð>XØ$ð>XØ)5ð>XØCOð>Xàñ>Xr   c                ó   € V P                   w  rErgVP                   w  rXr•\        V P                  4      '       g   \        \        e   R4      hR4      hWi,          p
V P                  WIW§4      P                  ^^4      P                  WJ,          W—4      p\        P                  ! V4      p\        e   ^€M^@R^€^€R^@/P                  V^€4      p\        WI,          V\        P                  P                  V P                  4      P                  V4      p\        P                  ! WI,          V
,          \        P                  V P                  R7      pV^8”  di   \        P                  ! WäWš\        P                  V P                  R7      p\        P                  ! VVV	V
V\        P                  V P                  R7      pMVR,          ;pp\        e   \        P                   ! W±W#WÏVVV4	       Ms\"        P$                  P&                  ! . \)        \"        P*                  W±W#WÏVV34      OVN\        P                  P-                  V P                  4      P.                  N5!   VP1                  WJW—4      P                  ^^4      P3                  V 4      # )z~Decode BF16 [batch, 1, heads, head_dim] queries over a fixed KV cache.

CUDA and HIP support head dimensions 64, 128 and 256.
zeflash_attention_decode requires the HIP extension on an AMD device with bf16 support (RDNA3 or newer)zCflash_attention_decode requires the CUDA extension on SM80 or neweré   ©Údtyper   ºNr   N)Úshaper   r   ÚRuntimeErrorr   ÚreshapeÚ	transposer   Ú
empty_likeÚgetr:   r   Úget_device_propertiesÚmulti_processor_countÚemptyÚfloat32Úflash_decoder   r   r   ÚmapÚ_wrap_for_dlpackÚcurrent_streamÚcuda_streamÚviewÚ
reshape_as)r<   r>   r?   r@   ÚbatchÚ_Úquery_headsÚhead_dimr!   Úkv_headsÚgroupsÚqueryÚoutputr#   Ú
num_splitsÚsoftmax_lseÚsoftmax_lse_accumÚoutput_accums   &&&&              r   r   r   <   s?  € ð '(§g¡gÑ#€EˆkØ"#§'¡'Ñ€A�HÜ˜Ÿ™×!Ò!Üô Ò'ð1ó
ð 	
ð Wó	
ð 	
ð Õ$€Fà	�	‰	�% 6Ó4ß	‰�1�a‹ß	‰�• Ó	4ð 
ô
 ×Ò˜eÓ$€Fä!Ò-‰c°B¸¸SÀ#ÀsÈBÐ3O×3SÑ3SÐT\Ð^aÓ3b€GÜØÕØÜ�
‰
×(Ñ(¨¯©Ó2×HÑHØó	€Jô —+’+˜eÕ.°Õ7¼u¿}¹}ÐUV×U]ÑU]Ô^€KØ�A„~Ü!ŸKšKØ˜x´u·}±}ÈQÏXÉXô
Ðô —{’{ØØØØØÜ—-‘-Ø—8‘8ô
‰ð ,7°r­?Ð:Ð˜LÜÒÜ×!Ò!Ø�a VÐ:KÈ\Øõ	
ô
 	×Ñ×/Ò/ð 	
ÜÜ×.Ñ.Ø˜1¨&Ð?PÐR^Ð_óð	
ð
 ð	
ô �J‰J×%Ñ% a§h¡hÓ/×;Ñ;ô	
ð �;‰;�u hÓ9×CÑCÀAÀqÓI×TÑTÐUVÓWÐWr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r   ztorch.device | int | NonerD   ztorch.dtyper	   r
   r   )r   s   "r   r   r   }   s)   € ÷ Cñ CØ%ðCØ5@ðCà	ñCr   c                ó  € V\         P                  8w  d   R# \        V \        4      '       d   \         P                  ! RV 4      p \        V 4      '       g   R# \        e   \        P                  ! 4       # \        \        P                  R4      # )a&  Return whether flash_attention_decode_gqa and flash_attention_decode_tree_merge run
their native kernel on ``device`` for ``dtype`` inputs: BF16, head_dim 256, the CUDA
extension on SM80 or newer or the HIP extension on RDNA3 or newer. Both functions
otherwise compute the same result in torch.Fr   Úflash_attention_decode_gqa)r   Úbfloat16Ú
isinstancer    r   r   r   Ú'flash_attention_decode_gqa_is_availabler   r   r   ©r   rD   s   &&r   rh   rh   }   sg   € ð ”—‘ÔÙÜ�&œ#×ÒÜ—’˜f fÓ-ˆÜ˜×ÒÙÜÒÜ×CÒCÓEÐEÜ”=×#Ñ#Ð%AÓBÐBr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úrowsr    rZ   r	   r   )r   s   "r   r   r   �   s!   € ÷ ñ ˜sð ¨cð °cñ r   c                óf   € ^pY!R8X  d   ^M^8  d    W^,          ,          ^ 8X  d   V^,          pK.  V# )r%   rB   r   )rk   rZ   Úgpps   && r   Ú_hip_groups_per_passrn   �   s3   € ð €CØ
 #”o‘¨1Ô
-°$À½'Õ2BÀaÔ2GØˆq�ŠØ€Jr   c           	     óÀ  € V P                   w  rVrxVP                   w  ršr¹Wj,          p\        P                  ! V P                  4       P	                  WZWÇ,          V4      VP                  4       P                  RR4      4      VR,          ,          pVP                  \        P                  4      P                  ^ V4      P                  V^^4      pV'       d6   Wç,
          ^,           \        P                  ! WpP                  R7      ,           pVP                  V^V4      P                  ^^V4      P                  R4      pVP                  \        P                  ! W°P                  R7      V8¬  \        R4      4       VP!                  R4      p\        P"                  ! WßP%                  V\        R4      8H  R4      P                  R4      ,
          4      p\        P                  ! VVP                  4       4      P                  WZWÇV4      pVP'                  ^ ^^^^4      P	                  WWWh,          4      P                  V P(                  4      VP                  WVV4      3# )r%   r   ú-infr&   éÿÿÿÿéþÿÿÿç      à¿)rF   r   ÚmatmulÚfloatrH   rI   ÚtoÚint64ÚclamprU   Úaranger   ÚexpandÚrepeatÚ	unsqueezeÚmasked_fill_Ú	logsumexpÚexpÚmasked_fillÚpermuterD   )r<   r>   r?   r@   ÚcausalrW   ÚheadsÚquery_lengthrZ   rX   r[   r!   r\   ÚscoresÚlimitÚlseÚprobsÚouts   &&&&&             r   Ú_decode_gqa_torchrŠ   ˜   sÎ  € Ø+,¯7©7Ñ(€E�,Ø"#§'¡'Ñ€A�ØÕ€FÜ�\Š\˜!Ÿ'™'›)×+Ñ+¨E¸VÕ=RÐT\Ó]ØŸ'™'›)×-Ñ-¨b°"Ó5ó7Ø9AÀTÕ9IõJ€Fð �M‰Mœ%Ÿ+™+Ó&×,Ñ,¨Q°Ó<×AÑAÀ%ÈÈAÓN€EßØÕ$ qÕ(¬5¯<ª<¸ÏXÉXÔ+VÕVˆØ�L‰L˜  <Ó0×7Ñ7¸¸1¸fÓE×OÑOÐPRÓS€EØ
×ÑœŸš [¿¹ÔBÀeÑKÌUÐSYË]Ô[Ø
×
Ñ
˜2Ó
€Cä�IŠI�fŸ™¨s´e¸F³mÑ/CÀSÓI×SÑSÐTVÓWÕWÓX€EÜ
�,Š,�u˜aŸg™g›iÓ
(×
-Ñ
-¨e¸vÐU]Ó
^€CØ�K‰K˜˜1˜a  AÓ&×.Ñ.¨uÀEÕDTÓU×XÑXÐYZ×Y`ÑY`ÓaØ�H‰H�U <Ó0ð2ð 2r   c               ó4   € V ^8„  d   QhRRRRRRRRRRRRR	R/# )
r   r<   r=   r>   r?   r@   Ú
return_lser
   r‚   r	   r   )r   s   "r   r   r   ¬   sF   € ÷ 8ñ 8Øð8Ø$ð8Ø)5ð8ØCOð8Ø]að8àð8ð ñ8r   c                ó`  € V P                   w  rgr‰VP                   w  r«rÊV	R8w  g'   \        V P                  V P                  4      '       g   \	        WW#V4      w  rÞV'       d   WÞ3# V# \
        P                  ! WhWy,          3V P                  V P                  R7      p\
        P                  P                  V P                  4      P                  pWg,          V,          p\
        P                  ! V\
        P                  V P                  R7      p\        e<   W{,          V,          p\        Wk,          V\        VV	4      ,          ,          WÏ^€4      pM\        Yh^8X  d   TMT,          WÏ^@4      pV^8”  dy   \
        P                  ! VV,          \
        P                  V P                  R7      p\
        P                  ! VV,          V	,          \
        P                  V P                  R7      pMVR,          ;pp\        e   \        P                  ! WW#WÞVVVV4
       Mu\        P                   P"                  ! . \%        \        P&                  WW#WÞVV34      OVNVN\
        P                  P)                  V P                  4      P*                  N5!   V'       d   WÞP-                  WgV4      3# V# )a	  GQA decode attention for q [B, H, S, D] over k/v [B, Hk, capacity, D].

Causal: query row j of batch b attends cache slots < kv_lengths[b] - S + j + 1 (the
speculative verify staircase; S == 1 is plain decode). Otherwise every row attends slots <
kv_lengths[b], which is how a verify tree reads its committed prefix. Returns
[B, S, H*D], with the fp32 log-sum-exp of the scaled scores [B, H, S] as a second value
when return_lse. BF16 head_dim 256 runs the CUDA or HIP kernel; anything else the same
computation in torch.rB   rC   rE   )rF   rh   r   rD   rŠ   r   rN   r   rL   rM   rO   r   r:   rn   Úflash_decode_gqar   r   re   rQ   rR   rS   rT   rU   )r<   r>   r?   r@   rŒ   r‚   rW   rƒ   r„   rZ   rX   r[   r!   r^   r`   r"   rk   Úper_headr_   ra   rb   s   &&&&&&               r   re   re   ¬   s?  € ð ,-¯7©7Ñ(€E�,Ø"#§'¡'Ñ€A�Ø�3„ÔEÀaÇhÁhÐPQ×PWÑPW×XÒXÜ/°°aÀVÓLÑˆßØÐ&Ð&ØˆÜ�[Š[˜%¨uÕ/?Ð@ÈÏÉÐXY×X`ÑX`Ôa€FÜ—j‘j×6Ñ6°q·x±xÓ@×VÑV€OØ�=˜<Õ'€DÜ—+’+˜d¬%¯-©-ÀÇÁÔI€KÜÒð Õ$ |Õ3ˆÜ ØÕ Ô,@ÀÈ8Ó,TÕ TÕUØ¨#ó/‰
ô !Ø°Ô!2‘X¸Õ>ÀÐ^`óbˆ
à�A„~Ü!ŸKšK¨
°TÕ(9ÄÇÁÐWX×W_ÑW_Ô`ÐÜ—{’{ :°Õ#4°xÕ#?ÄuÇ}Á}Ð]^×]eÑ]eÔf‰à+6°r­?Ð:Ð˜LÜÒÜ×%Ò%Ø�! Ð6GÈÐWaØõ	ô 	×Ñ×3Ò3ð 	
ÜÜ×.Ñ.Ø�q fÐ;LÈlÐ[óð	
ð
 ð	
ð ð	
ô �J‰J×%Ñ% a§h¡hÓ/×;Ñ;ô	
÷ Ø×'Ñ'¨°lÓCÐCÐCØ€Mr   c                óˆ  € VP                   w  rxršWƒP                   ^,          ,          pVP                  4       P                  V^R7      pVP                  4       P                  V^R7      p\        P                  ! VP                  4       VP                  RR4      4      V
R,          ,          pVP                  V	^4      \        P                  ! W’P                  R7      ,	          ^,          pVP                  V^ 8H  \        R4      4       \        P                  ! WP                  R4      4      p\        P                  ! VV,
          4      p\        P                  ! VVP                  R4      ,
          4      pVVP                  R4      ,           pV P                  WyWŠ4      P                  ^^4      P                  4       VP                  R4      ,          pV\        P                  ! VV4      ,           pVP                  VVP                  R4      ,          P                  ^^4      P!                  WyWŠ,          4      4       V# )r%   )Údimr   rp   rq   rr   rs   )rF   ru   Úrepeat_interleaver   rt   rI   rU   ry   r   r}   ÚmaximumÚamaxr   r|   ÚsumÚcopy_rH   )r‰   r‡   r<   r>   r?   ÚmaskÚmergedrW   rƒ   rk   rZ   r\   ÚkfÚvfr…   ÚallowedÚmÚw_flashÚwÚdenomÚaccs   &&&&&&&              r   Ú_tree_merge_torchr¡   ç   sž  € Ø#$§7¡7Ñ €E�$Ø—g‘g˜a•jÕ €FØ	
�‰‹×	$Ñ	$ V°Ð	$Ó	3€BØ	
�‰‹×	$Ñ	$ V°Ð	$Ó	3€BÜ�\Š\˜!Ÿ'™'›) R§\¡\°"°bÓ%9Ó:¸XÈÕ=MÕM€FØ�y‰y˜˜qÓ!¤U§\¢\°$¿x¹xÔ%HÕHÈAÕM€GØ
×Ñ˜ 1™¤e¨F£mÔ4Ü�Š�cŸ;™; r›?Ó+€AÜ�iŠi˜˜a�Ó €GÜ�	Š	�&˜1Ÿ;™; r›?Õ*Ó+€AØ�a—e‘e˜B“iÕ€EØ
�(‰(�5 Ó
0×
:Ñ
:¸1¸aÓ
@×
FÑ
FÓ
HÈ7×K\ÑK\Ð]_ÓK`Õ
`€CØ
”—’˜Q Ó#Õ
#€CØ
‡L�L�#˜Ÿ™¨Ó+Õ+×6Ñ6°q¸!Ó<×DÑDÀUÐRWÕRbÓcÔdØ€Mr   c               ó8   € V ^8„  d   QhRRRRRRRRRRRRRRR	R/# )
r   r‰   r=   r‡   r<   r>   r?   r—   r˜   r	   r   )r   s   "r   r   r   ù   sN   € ÷ ñ Ø	ðØ(ðØ-9ðØ>JðØO[ðà
ðà ,ðð ñr   c                óÄ  € VP                   R,          R8w  g'   \        VP                  VP                  4      '       g   \	        WW#WEV4      # \
        e   \
        P                  ! WW#WEV4       V# \        P                  P                  ! . \        \        P                  WW#WEV34      O\        P                  P                  VP                  4      P                  N5!   V# )až  Fold a verify tree's own rows into a prefix-only decode result.

``out`` [B, S, H*D] and ``lse`` [B, H, S] come from flash_attention_decode_gqa over the
committed prefix with causal=False; q [B, H, S, D] and k/v [B, Hk, S, D] are the verify
rows' own rotated query/key/value, and mask [S] int32 the rows each row attends inside the
step (bit t = row t, including itself). Writes and returns ``merged`` [B, S, H*D].rB   rq   )rF   rh   r   rD   r¡   r   Úflash_decode_tree_merger   r   Ú!flash_attention_decode_tree_mergerQ   rR   r   r   rS   rT   )r‰   r‡   r<   r>   r?   r—   r˜   s   &&&&&&&r   r¥   r¥   ù   s¶   € ð 	‡w�wˆr…{�cÔÔ!HÈÏÉÐST×SZÑSZ×![Ò![Ü  ¨1°¸&ÓAÐAÜÒÜ×,Ò,¨S°q¸QÀfÔMØˆÜ×Ñ×6Ò6ð Ü	Œ]×+Ñ+¨c¸¸aÀvÐ-NÓ	Oðä�
‰
×!Ñ! !§(¡(Ó+×7Ñ7ôð €Mr   c               ó4   € V ^8„  d   QhRRRRRRRRRRRRRR/# )	r   r‰   r=   r‡   r<   r>   r?   r˜   r	   r   )r   s   "r   r   r     sO   € ÷ Nñ NØ	ðNØ(ðNØ-9ðNØ>JðNØO[ðNàðNð ñNr   c           	     óÎ   € ^\         P                  ! ^VP                  ^,          ^,           VP                  \         P                  R7      ,          ^,
          p\        WW#WFV4      # )zGFold the current step's rows causally into a prefix-only decode result.ri   )r   ry   rF   r   Úint32r¥   )r‰   r‡   r<   r>   r?   r˜   r—   s   &&&&&& r   Ú!flash_attention_decode_step_merger©     sH   € ð
 ”—’˜a §¡¨¥¨a¥¸¿¹ÌÏÉÔTÕTÐXYÕY€DÜ,¨S°q¸QÀfÓMÐMr   )é   r   )N)FT)Ú
__future__r   r0   r   Úbackendsr   r   ÚgetattrÚversionr   r   r   r   r:   r   rf   rh   rn   rŠ   re   r¡   r¥   r©   r   r   r   Ú<module>r¯      su   ðÝ "ã ã å +á
ˆ5�=‰=˜% ×&Ò&Þ-à€LàÐ ÷Kõ.õ(>XðD )-À5Ç>Á>÷Cõ$ò2÷(8òvõ$÷,Nr   