+
    Ðåûj|U  ã                  ó¤  a € 0 t $ R t^ RIHt ^ RIt^ RIt^ RIHt ]'       d   ^ RIH	t	 ^ RI
Ht ^ RIHt ^ RIHtHt ^ RIt^ RIHt ^R	IHt R
.tRsR]R&   RsR]R&   ] ! R R4      4       t]R R l4       tRAR R lltR R ltR R ltR R ltR R lt R R lt!]! R4      t"R R  lt#R! R" lt$RBR# R$ llt%RCR% R& llt&RDR'RR(RER)RER*RR+RR,RR-RR.R/R0R/	R1 R2 lllt'R'RR(RER)RER*RR+RR-RR0R/R3 R4 llt(R'RR(RER)RER*RR+RR-RR,RR0R/R5 R6 llt)R'RR(RR)R/R7 R8 llt*RFR'R/R9 R: lllt+RGR'R/R; R< lllt,R'R/R= R> llt-]P\                  ! R?]R@7       R# )Hz�
PROTOTYPE!
Flash Attention 3 implementation.
For fp8: only supports forward pass right now.
For fp16/bf16: supports forward and backward pass.
)ÚannotationsN)ÚTYPE_CHECKING)ÚCallable)Ú	dataclass)Úcache)ÚTypeVarTupleÚUnpack)ÚLibrary)Ú	_registryÚregister_flash_attention_fa3zCallable | NoneÚ_FA3_CUDA_FWDÚ_FA3_CUDA_BWDc                  ó.   € ] tR t^&t$ R]R&   R R ltRtR# )Ú
_FA3HandlezLibrary | NoneÚlibraryc               ó   € V ^8„  d   QhRR/# )é   ÚreturnÚNone© )Úformats   "ÚSC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\torch\nn\attention\_fa3.pyÚ__annotate__Ú_FA3Handle.__annotate__*   s   € ÷ )ñ )˜ñ )ó    c                	óR   € R V n         \        P                  P                  R4       R # )NF)r   ÚtorchÚ_CÚ_set_sdp_use_fa3)Úselfs   &r   ÚremoveÚ_FA3Handle.remove*   s   € ØˆŒä�‰×!Ñ! %Ö(r   )r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__annotations__r    Ú__static_attributes__r   r   r   r   r   &   s   ‡ àÓ÷)ñ )r   r   c               ó    € V ^8„  d   QhRRRR/# )r   Údeviceztorch.devicer   Úintr   )r   s   "r   r   r   1   s   € ÷ ñ ˜lð ¨sñ r   c                óH   € \         P                  P                  V 4      w  rV# ©N)r   ÚcudaÚget_device_capability)r)   ÚmajorÚ_s   &  r   Ú_get_device_majorr1   0   s   € ä�z‰z×/Ñ/°Ó7�H€EØ€Lr   c               ó    € V ^8„  d   QhRRRR/# )r   Úmodule_pathÚstrr   r   r   )r   s   "r   r   r   6   s   € ÷ /ñ /Øð/àñ/r   c                ó|   € \        V 4       \        P                  P                  R4       \	        \        4       4      # )z‰
Register FA3 flash attention kernels with the PyTorch dispatcher.

Args:
    module_path: Python module path to the FA3 implementation.
T)Ú_fa3_import_moduler   r   r   r   Ú_fa3_register_kernels©r3   s   &r   r   r   6   s/   € ô �{Ô#ô 
‡H�H×Ñ˜dÔ#äÔ+Ó-Ó.Ð.r   c               ó    € V ^8„  d   QhRRRR/# )r   r3   r4   r   r   r   )r   s   "r   r   r   G   s   € ÷ /ñ / Cð /¨Dñ /r   c                ó  € \         P                  ! V 4       \        \        P                  R 4      '       g   \        RV  R24      h\        \        P                  P                  R4      '       g   \        RV  R24      h\        \        P                  P                  R4      '       g   \        RV  R24      h\        P                  P                  P                  s\        P                  P                  P                  s
R# )Úflash_attn_3zModule 'z' does not expose FA3 kernelsÚfwdz%' does not expose FA3 forward kernelsÚbwdz&' does not expose FA3 backward kernelsN)Ú	importlibÚimport_moduleÚhasattrr   ÚopsÚRuntimeErrorr;   r<   r   r=   r   r8   s   &r   r6   r6   G   sÆ   € Ü×Ò˜KÔ(Ü”5—9‘9˜n×-Ò-Ü˜X k ]Ð2OÐPÓQÐQÜ”5—9‘9×)Ñ)¨5×1Ò1ÜØ�{�mÐ#HÐIó
ð 	
ô ”5—9‘9×)Ñ)¨5×1Ò1ÜØ�{�mÐ#IÐJó
ð 	
ô —I‘I×*Ñ*×.Ñ.€MÜ—I‘I×*Ñ*×.Ñ.‚Mr   c               ó   € V ^8„  d   QhRR/# )r   r   r	   r   )r   s   "r   r   r   X   s   € ÷ ñ ˜wñ r   c                 ób  € \        R RR4      p V P                  R\        R4       V P                  R\        R4       V P                  R\        R4       V P                  R\
        R4       V P                  R\        R4       V P                  R\        R4       V P                  R	\        R4       V # )
ÚatenÚIMPLÚCUDAz"_flash_attention_forward.quantizedz-_scaled_dot_product_flash_attention.quantizedÚ_flash_attention_forwardÚ+_flash_attention_forward_no_dropout_inplaceÚ#_scaled_dot_product_flash_attentionÚ_flash_attention_backwardÚ,_scaled_dot_product_flash_attention_backward)	r	   ÚimplÚ!_fa3_flash_attention_forward_implÚ4_fa3_scaled_dot_product_flash_attention_forward_implÚ)_fa3_flash_attention_forward_impl_defaultÚ4_fa3_flash_attention_forward_no_dropout_inplace_implÚ<_fa3_scaled_dot_product_flash_attention_forward_impl_defaultÚ"_fa3_flash_attention_backward_implÚ5_fa3_scaled_dot_product_flash_attention_backward_impl)Úlibs    r   r7   r7   X   s°   € Ü
�&˜& &Ó
)€CØ‡H�HØ,Ô.OÐQWôð ‡H�HØ7Ü<Øôð
 ‡H�HØ"Ô$MÈvôð ‡H�HØ5Ü<Øôð
 ‡H�HØ-ÜDØôð ‡H�HÐ(Ô*LÈfÔUØ‡H�HØ6Ü=Øôð
 €Jr   c               ó8   € V ^8„  d   QhRRRRRRRRR	RR
RRRRR/# )r   Úqueryútorch.TensorÚtensorsztuple[torch.Tensor, ...]Ú	dropout_pÚfloatÚ	cum_seq_qútorch.Tensor | NoneÚ	q_descaleÚ	k_descaleÚ	v_descaler   ú
str | Noner   )r   s   "r   r   r   y   sX   € ÷ "ñ "Øð"à%ð"ð ð"ð #ð	"ð
 #ð"ð #ð"ð #ð"ð ñ"r   c                ól  € VR 8w  d   R# \         ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   R# \        V Uu0 uF  qwP                  kK  	  up4      ^8w  d   R# V P                  \        P
                  8X  d(   Ve	   Ve   Vf   \        P                  ! R\        4       Vf   V P                  4       ^8w  d   R	# Ve   V P                  4       ^8w  d   R
# \        P                  P                  4       '       g   R# \        V P                  4      ^	8w  d   R# R# u upi )ç        zdropout_p must be 0c              3  ó8   "  € T F  qP                   x € K  	  R # 5ir,   )Úis_cuda©Ú.0Úts   & r   Ú	<genexpr>Ú,_fa3_common_support_error.<locals>.<genexpr>…   s   é € Ð*¡'˜Q�yŽy£'ùs   ‚FTzinputs must be CUDA tensorsúinputs must share deviceNz¯When using SDPA with fp8, descale tensor should always be used for accurate dequantization. Please use _scaled_dot_product_attention_quantized and provide the descale tensors.zdense query must be 4Dzragged query must be 3DzCUDA not availablez#FA3 requires compute capability 9.0)ÚallÚlenr)   Údtyper   Úfloat8_e4m3fnÚwarningsÚwarnÚUserWarningÚdimr-   Úis_availabler1   )rW   rY   rZ   r\   r^   r_   r`   rh   s   &&&&&&& r   Ú_fa3_common_support_errorru   y   sì   € ð �CÔÙ$ç‹3Ñ*¡'Ó*�3�3Š3Ñ*¡'Ó*×*Ò*Ù,Ü
™gÓ&™g˜�HŒH™gÑ&Ó'¨1Ô,Ù)Ø‡{�{”e×)Ñ)Ô)ØÒ˜YÒ.°)Ò2Cä�Šð+ô ô	
ð Ò˜UŸY™Y›[¨AÔ-Ù'ØÒ §¡£°Ô!1Ù(Ü�:‰:×"Ñ"×$Ò$Ù#Ü˜Ÿ™Ó&¨!Ô+Ù4Ùùò) 's   ÁD1c               óH   € V ^8„  d   QhRRRRRRRRRRR	R
RR
RR
RR
RR
RR
RR/# )r   rW   rX   ÚkeyÚvaluerZ   r[   Úreturn_debug_maskÚboolÚalibi_slopesr]   Ú	seqused_kr\   r^   r_   r`   r   ra   r   )r   s   "r   r   r   ž   s€   € ÷ (ñ (Øð(à	ð(ð ð(ð ð	(ð
 ð(ð &ð(ð #ð(ð #ð(ð #ð(ð #ð(ð #ð(ð ñ(r   c           	     ó2  a€ V'       d   R # Ve   R# Ve6   VP                   \        P                  8w  d   R# VP                  '       g   R# \        P                  \        P
                  \        P                  3o\        ;QJ d!    V3R lWV0 4       F  '       d   K   RM	  RM! V3R lWV0 4       4      '       g   RS 2# \        WV0 Uu0 uF  q»P                   kK  	  up4      ^8w  d   R	# \        V WV3VVVV	V
4      pVe   VR
8X  d   R# V# R# u upi )zreturn_debug_mask must be FalseNzalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDAc              3  ó@   <"  € T F  qP                   S9   x € K  	  R # 5ir,   ©rn   ©rg   rh   Úsupported_dtypess   & €r   ri   Ú-_fa3_forward_support_error.<locals>.<genexpr>µ   s   øé € ÐHÑ4G¨q�w‰wÐ*Ö*Ó4Gùó   ƒFTúinputs must be one of ú#all inputs must have the same dtyperk   z(query, key, value must be on same device)
rn   r   Úint32re   ro   Úfloat16Úbfloat16rl   rm   ru   )rW   rw   rx   rZ   ry   r{   r|   r\   r^   r_   r`   rh   Úerrorr�   s   &&&&&&&&&&&  @r   Ú_fa3_forward_support_errorrŠ   ž   sþ   ø€ ÷ Ù0ØÒÙ+ØÒØ�?‰?œeŸk™kÔ)Ù,Ø× × Ð Ù+Ü×+Ñ+¬U¯]©]¼E¿N¹NÐKÐß‹3ÔH°UÀÑ4GÓH�3�3Š3ÔH°UÀÑ4GÓH×HÒHØ'Ð(8Ð'9Ð:Ð:Ü
˜e¨%Ñ0Ó1Ñ0˜�GŒGÑ0Ñ1Ó2°aÔ7Ù4Ü%ØØ	�UÐØØØØØó€Eð ÒØÐ.Ô.Ù=ØˆÙùò 2s   ÃDc               óD   € V ^8„  d   QhRRRRRRRRRRRRRR	R
RRRRRRR/# )r   Úgrad_outrX   rW   rw   rx   ÚoutÚ	logsumexprZ   r[   r\   r]   Úwindow_size_leftú
int | NoneÚwindow_size_rightr   ra   r   )r   s   "r   r   r   É   sv   € ÷ #ñ #Øð#àð#ð 
ð#ð ð	#ð
 
ð#ð ð#ð ð#ð #ð#ð !ð#ð "ð#ð ñ#r   c
           	     óþ  a€ VP                   \        P                  8X  d    R # VP                   \        P                  8w  d   R# \        P                  \        P
                  3o\        ;QJ d"    V3R lWW#V0 4       F  '       d   K   RM	  RM! V3R lWW#V0 4       4      '       g   RS 2# \        WW#V0 U
u0 uF  qªP                   kK  	  up
4      ^8w  d   R# \        VWW#WE3VVRRR4      pVe   V# R# u up
i )zHFA3 backward does not support fp8 - use inference only (torch.no_grad())zlogsumexp dtype must be float32c              3  ó@   <"  € T F  qP                   S9   x € K  	  R # 5ir,   r   r€   s   & €r   ri   Ú._fa3_backward_support_error.<locals>.<genexpr>Ý   s   øé € ÐWÑ4V¨q�w‰wÐ*Ö*Ó4Vùrƒ   FTr„   r…   N)	rn   r   ro   Úfloat32r‡   rˆ   rl   rm   ru   )rŒ   rW   rw   rx   r�   rŽ   rZ   r\   r�   r‘   rh   r‰   r�   s   &&&&&&&&&&  @r   Ú_fa3_backward_support_errorr–   É   sí   ø€ ð ‡{�{”e×)Ñ)Ô)àVñ	
ð ‡�œ%Ÿ-™-Ô'Ù0ÜŸ™¤u§~¡~Ð6Ðß‹3ÔW°XÀcÐRUÑ4VÓW�3�3Š3ÔW°XÀcÐRUÑ4VÓW×WÒWØ'Ð(8Ð'9Ð:Ð:Ü
˜h¨s¸3Ñ?Ó@Ñ?˜�GŒGÑ?Ñ@ÓAÀQÔFÙ4Ü%ØØ	˜# cÐ5ØØØØØó€Eð ÒØˆÙùò As   Â<C:ÚTsc               ó    € V ^8„  d   QhRRRR/# )r   rY   z
Unpack[Ts]r   ztuple[Unpack[Ts]]r   )r   s   "r   r   r   ò   s   € ÷ 5ñ 5˜zð 5Ð.?ñ 5r   c                 óZ   € \         ;QJ d    . R  V  4       F  NK  	  5# ! R  V  4       4      # )c              3  óD   "  € T F  qP                  ^^4      x € K  	  R# 5i)é   N)Ú	transposerf   s   & r   ri   Ú#_transpose_dense.<locals>.<genexpr>ó   s   é € Ð4©G q—‘˜Q ×"Ð"«Gùs   ‚ )Útuple)rY   s   *r   Ú_transpose_denserŸ   ò   s$   € ßŒ5Ñ4©GÓ4�5Ð4ˆ5Ñ4©GÓ4Ó4Ð4r   c               ó    € V ^8„  d   QhRRRR/# )r   Úxr]   r   r   )r   s   "r   r   r   ö   s    € ÷ Hñ HÐ,ð HÐ1Dñ Hr   c                óZ   € V e'   V P                  R4      ^8w  d   V P                  4       # T # )z2Ensure tensor is contiguous in the last dimension.éÿÿÿÿ)ÚstrideÚ
contiguous)r¡   s   &r   Ú_maybe_contiguousr¦   ö   s&   € àš]¨q¯x©x¸«|¸qÔ/@ˆ1�<‰<‹>ÐGÀaÐGr   c          &     ód   € V ^8„  d   QhRRRRRRRRRRRR	R
R	RRRRRRRRRRRRRRRRRRRRRRRR/# )r   rW   rX   rw   rx   Úcu_seq_qr]   Úcu_seq_kÚmax_qr*   Úmax_kÚscaleúfloat | NoneÚ	is_causalrz   r�   r�   r‘   r|   r�   r^   r_   r`   Úblock_tableÚ
num_splitsr   z!tuple[torch.Tensor, torch.Tensor]r   )r   s   "r   r   r   û   sÛ   € ÷ N)ñ N)ØðN)à	ðN)ð ðN)ð "ð	N)ð
 "ðN)ð ðN)ð ðN)ð ðN)ð ðN)ð !ðN)ð "ðN)ð #ðN)ð 
ðN)ð #ðN)ð #ðN)ð  #ð!N)ð" %ð#N)ð$ ð%N)ð& 'ñ'N)r   c                óô  € \         f   \        R4      h\        V 4      p\        V4      pVP                  \        P
                  8X  d=   VP                  R4      ^8w  d'   VP                  R4      ^8w  d   VP                  4       M
\        V4      p\        V4      p\        V4      p\        V4      p\        V4      p\        . VNVNVNRNRNRNVNVNVNRNRNVNVNVNVNRNRNRNRNRNVNVNVNVNVNV	e   T	MRNV
e   T
MRN^ NRNRNRNT;'       g     \        P                  ! 4       '       d   ^M^ NRN\        P                  P                  4       ;'       g    ^ N5!  w  ppppVVP                  4       3# )z>
Run the FA3 forward pass by calling the C++ kernel directly.
NúFA3 not registeredrc   Tr£   éýÿÿÿ)r   rB   r¦   rn   r   ro   r¤   r¥   Ú$are_deterministic_algorithms_enabledr   Ú_get_sm_carveout_experimental)rW   rw   rx   r¨   r©   rª   r«   r¬   r®   r�   r‘   r|   r�   r^   r_   r`   r¯   r°   ÚqÚkÚvÚcu_seqlens_qÚcu_seqlens_kÚsoftmax_lseÚ	out_accumÚsoftmax_lse_accums   &&&&&&&&&&&&&&&&&&        r   Ú_fa3_run_forwardr¾   û   sl  € ô. ÒÜÐ/Ó0Ð0ä˜%Ó €AÜ˜#Ó€Að �;‰;œ%×-Ñ-Ô-Ø�L‰L˜Ó Ô!Ø�L‰L˜Ó Ô!ð 	×ÑÔô ˜uÓ%ð ô % XÓ.€LÜ$ XÓ.€LÜ! )Ó,€IÜ# KÓ0€Kä5Bð $6Ø	ð$6à	ð$6ð 	
ð$6ð 	ð	$6ð
 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð 	ð$6ð  	ð!$6ð" 	ð#$6ð$ 	ð%$6ð& 	ð'$6ð( 	ð)$6ð* 	ð+$6ð, 	ð-$6ð. 	ð/$6ð0 	ð1$6ð2 	ð3$6ð4 -Ò8Ñ¸bð5$6ð6 /Ò:ÑÀð7$6ð8 	
ð9$6ð: 	ð;$6ð< 	ð=$6ð> 	ð?$6ð@ 	÷ 	Fð 	FÜ×;Ò;×=Ò=‰AÀ1ðC$6ðD 	ðE$6ôF 	�‰×.Ñ.Ó0×5Ð5°AóG$6Ñ2€Cˆ�iÐ!2ðJ �×&Ñ&Ó(Ð(Ð(r   c                óX   € V ^8„  d   QhRRRRRRRRRRRRRR	R
R	RRRRRRRRRRRRRRRR/# )r   rŒ   rX   rW   rw   rx   r�   rŽ   r¨   r]   r©   Úmax_seqlen_qr�   Úmax_seqlen_kr¬   r­   r®   rz   r�   r*   r‘   Údeterministicr   z/tuple[torch.Tensor, torch.Tensor, torch.Tensor]r   )r   s   "r   r   r   L  s¨   € ÷ 8ñ 8Øð8àð8ð 
ð8ð ð	8ð
 
ð8ð ð8ð "ð8ð "ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð  5ñ!8r   c                ót  € \         f   \        R4      h\        V 4      pVP                  R4      ^8w  d   VP	                  4       MTpVP                  R4      ^8w  d   VP	                  4       MTpVP                  R4      ^8w  d   VP	                  4       MTp\        V4      p\        V4      p\
        P                  ! V4      p\
        P                  ! V4      p\
        P                  ! V4      p\        TTTTTTTTTTTR R TT	T
TTTRT\
        P                  P                  4       ;'       g    ^ 4       VVV3# )Nr²   rc   r£   )	r   rB   r¦   r¤   r¥   r   Ú
empty_liker   rµ   )rŒ   rW   rw   rx   r�   rŽ   r¨   r©   rÀ   rÁ   r¬   r®   r�   r‘   rÂ   Údoutr¶   r·   r¸   ÚoÚlseÚdqÚdkÚdvs   &&&&&&&&&&&&&&&         r   Ú_fa3_run_backwardrË   L  s2  € ô" ÒÜÐ/Ó0Ð0ô ˜XÓ&€DØ#Ÿl™l¨2Ó.°!Ô3ˆ×ÑÔ¸€AØŸJ™J r›N¨aÔ/ˆ�‰Ô°S€AØ#Ÿl™l¨2Ó.°!Ô3ˆ×ÑÔ¸€AÜ˜#Ó€AÜ
˜IÓ
&€Cô 
×	Ò	˜!Ó	€BÜ	×	Ò	˜!Ó	€BÜ	×	Ò	˜!Ó	€BÜØØ	Ø	Ø	Ø	ØØ
Ø
Ø
ØØØØØØØØØØØØÜ�‰×.Ñ.Ó0×5Ð5°Aô-ð0 ˆr�2ˆ:Ðr   r¬   r�   r‘   r|   r{   r�   r¯   Úcompute_auxiliaryTr°   c          ,     óp   € V ^8„  d   QhRRRRRRRRRRRR	R
R	RRRRRRRRRRRRRRRR	RR	RRRRRRRRRRRR/# )r   rW   rX   rw   rx   r\   r]   Ú	cum_seq_krª   r*   r«   rZ   r[   r®   rz   ry   r^   r_   r`   r¬   r­   r�   r‘   r|   r{   r�   r¯   rÌ   r°   r�   r   )r   s   "r   r   r   ‡  sü   € ÷ D:ñ D:ØðD:à	ðD:ð ðD:ð #ð	D:ð
 #ðD:ð ðD:ð ðD:ð ðD:ð ðD:ð ðD:ð #ðD:ð #ðD:ð #ðD:ð ðD:ð  ð!D:ð" ð#D:ð$ #ð%D:ð& &ð'D:ð( 
ð)D:ð* %ð+D:ð, ð-D:ð. ñ/D:r   c       	        óØ  € \        V VVVV	VVVV
VV4      pVe   \        RV 24      h\        V VVVVVVVVVVVVV
VVVV4      w  ppV'       d‘   \        P                  ! R\        P
                  V P                  R7      p\        P                  ! R\        P
                  V P                  R7      p\        P                  ! ^ V P                  V P                  R7      pMR pR pR pVVVVV3# )Nz)FA3 flash_attention forward unsupported: )rn   r)   )r   r   )	rŠ   rB   r¾   r   ÚzerosÚuint64r)   Úemptyrn   )rW   rw   rx   r\   rÎ   rª   r«   rZ   r®   ry   r^   r_   r`   r¬   r�   r‘   r|   r{   r�   r¯   rÌ   r°   r‰   rÇ   Ú	rng_stateÚphilox_offsetÚ
debug_masks   &&&&&&&&&&&&&$$$$$$$$$     r   rN   rN   ‡  s  € ô2 'ØØØØØØØØØØØó€Eð ÒÜÐFÀuÀgÐNÓOÐOÜØØØØØØØØØØØØØØØØØØó%�H€Cˆ÷( Ü—K’K ¬E¯L©LÀÇÁÔNˆ	ÜŸš B¬e¯l©lÀ5Ç<Á<ÔPˆÜ—[’[ ¨%¯+©+¸e¿l¹lÔK‰
àˆ	ØˆØˆ
Ø��Y ¨zÐ9Ð9r   c          $     ó`   € V ^8„  d   QhRRRRRRRRRRRRR	R
RR
RRRRRRRRRR
RR
RRRRRRRR/# )r   r�   rX   rW   rw   rx   r\   r]   rÎ   rª   r*   r«   rZ   r[   r®   rz   ry   r¬   r­   r�   r‘   r|   r{   r¯   r°   r�   r   )r   s   "r   r   r   Î  s¼   € ÷ -ñ -Ø	ð-àð-ð 
ð-ð ð	-ð
 #ð-ð #ð-ð ð-ð ð-ð ð-ð ð-ð ð-ð ð-ð ð-ð ð-ð  #ð!-ð" &ð#-ð$ %ð%-ð& ñ'-r   c               ób   € \        VVVVVVVVV	V
R R R 3RVRVRVRVRVRV RVRR	R
V/	 w  pp  pV# )Nr¬   r�   r‘   r|   r{   r�   r¯   rÌ   Fr°   ©rN   )r�   rW   rw   rx   r\   rÎ   rª   r«   rZ   r®   ry   r¬   r�   r‘   r|   r{   r¯   r°   r0   rÇ   s   &&&&&&&&&&&$$$$$$$  r   rQ   rQ   Î  s    € ô* 8ØØØØØØØØØØØØØñð ðð *ðð  ,ð!ð" ð#ð$ "ð%ð& ð'ð(  ð)ð*  ð+ð, ñ-�O€A€sˆAˆq�!ð0 €Jr   c          $     ó`   € V ^8„  d   QhRRRRRRRRRRRR	R
R	RRRRRRRRRR	RR	RRRRRRRRRR/# )r   rW   rX   rw   rx   r\   r]   rÎ   rª   r*   r«   rZ   r[   r®   rz   ry   r¬   r­   r�   r‘   r|   r{   r¯   r�   r°   r�   r   )r   s   "r   r   r   þ  s¼   € ÷ +ñ +Øð+à	ð+ð ð+ð #ð	+ð
 #ð+ð ð+ð ð+ð ð+ð ð+ð ð+ð ð+ð ð+ð ð+ð #ð+ð  &ð!+ð" %ð#+ð$ 
ð%+ð& ñ'+r   c
               óB   € \        V VVVVVVVVV	R R R V
VVVVVVVR7      # )N)r¬   r�   r‘   r|   r{   r�   r¯   r°   rØ   )rW   rw   rx   r\   rÎ   rª   r«   rZ   r®   ry   r¬   r�   r‘   r|   r{   r¯   r�   r°   s   &&&&&&&&&&$$$$$$$$r   rP   rP   þ  sP   € ô* -ØØØØØØØØØØØØØØØ)Ø+ØØ!ØØØô+ð r   c          "     ó\   € V ^8„  d   QhRRRRRRRRRRRRRR	R
R	RRRRRRRRRRRRRRRRRR/# )r   rŒ   rX   rW   rw   rx   r�   rŽ   r\   r]   rÎ   rª   r*   r«   rZ   r[   r®   rz   rÓ   Úunusedr¬   r­   r�   r�   r‘   r   )r   s   "r   r   r   ,  s²   € ÷ 8ñ 8Øð8àð8ð 
ð8ð ð	8ð
 
ð8ð ð8ð #ð8ð #ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð ð8ð  ð!8ð" !ð#8ð$ "ñ%8r   c               óÔ   € \        V VVVVVV
VVV4
      pVe   \        RV 24      h\        P                  ! 4       p\	        T TTTTTTTTT	TTVe   TMRVe   TMRV4      w  pppVVV3# )z0FA3 implementation of _flash_attention_backward.z*FA3 flash_attention backward unsupported: r£   )r–   rB   r   r´   rË   )rŒ   rW   rw   rx   r�   rŽ   r\   rÎ   rª   r«   rZ   r®   rÓ   rÜ   r¬   r�   r‘   r‰   rÂ   rÈ   rÉ   rÊ   s   &&&&&&&&&&&&&&$$$     r   rS   rS   ,  s°   € ô* (ØØØØØØØØØØó€Eð ÒÜÐGÈÀwÐOÓPÐPä×>Ò>Ó@€Mä"ØØØØØØØØØØØØØ,Ò8Ñ¸bØ.Ò:ÑÀØó�J€BˆˆBð" ˆr�2ˆ:Ðr   c               ó@   € V ^8„  d   QhRRRRRRRRRRRRR	R
RRRRRR/
# )r   rW   rX   rw   rx   r^   r]   r_   r`   rZ   r[   r®   rz   ry   r¬   r­   r   )r   s   "r   r   r   g  sx   € ÷ Dñ DØðDà	ðDð ðDð #ð	Dð
 #ðDð #ðDð ðDð ðDð ðDð ñDr   c	               ó  € \        V VVVVR R R VVV4      p
V
e   \        RV
 24      h\        WV4      w  r¼pV P                  \        P
                  8X  d   \        P                  MV P                  p\        P                  ! WR7      pVP                  ^^4      pVP                  ^4      pVP                  ^4      p\        VVVR R VVVVVV	VVVVR7      w  pppppV P                  ^4      pVP                  ^4      pVVR R VVVVV3	# )NzFA3 SDPA forward unsupported: r   )r¬   r�   r^   r_   r`   )rŠ   rB   rŸ   rn   r   ro   rˆ   rÄ   rœ   ÚsizerN   )rW   rw   rx   r^   r_   r`   rZ   r®   ry   r¬   r‰   r¶   r·   r¸   Ú	out_dtypeÚout_bhsdÚout_bshdÚmax_q_flashÚmax_k_flashr0   rÇ   rÓ   rÔ   rÕ   rª   r«   s   &&&&&&&&&$                r   rO   rO   g  s;  € ô 'ØØØØØØØØØØØó€Eð ÒÜÐ;¸E¸7ÐCÓDÐDÜ˜u¨5Ó1�G€Aˆ!ð
 #(§+¡+´×1DÑ1DÔ"D”—’È%Ï+É+€IÜ×Ò Ô7€HØ×!Ñ! ! QÓ'€Hà—&‘&˜“)€KØ—&‘&˜“)€KÜ3TØ	Ø	Ø	ØØØØØØØØØØØØô4Ñ0€A€sˆI�} jð" �J‰J�q‹M€EØ�H‰H�Q‹K€EàØØØØØØØØð
ð 
r   c               ó4   € V ^8„  d   QhRRRRRRRRRRR	RR
R/# )r   rW   rX   rw   rx   rZ   r[   r®   rz   ry   r¬   r­   r   )r   s   "r   r   r   ®  sN   € ÷ ñ Øðà	ðð ðð ð	ð
 ðð ðð ñr   c               ó,   € \        V VVR R R VVVVR7
      # )N©r¬   )rO   )rW   rw   rx   rZ   r®   ry   r¬   s   &&&&&&$r   rR   rR   ®  s0   € ô @ØØØØØØØØØØôð r   c               óT   € V ^8„  d   QhRRRRRRRRRRRRRR	R
R	RRRRRRRRRRRRRR/# )r   rŒ   rX   rW   rw   rx   r�   rŽ   r\   r]   rÎ   rª   r*   r«   rZ   r[   r®   rz   Úphilox_seedrÔ   r¬   r­   r   )r   s   "r   r   r   Æ  sž   € ÷ 2"ñ 2"Øð2"àð2"ð 
ð2"ð ð	2"ð
 
ð2"ð ð2"ð #ð2"ð #ð2"ð ð2"ð ð2"ð ð2"ð ð2"ð ð2"ð  ð2"ð  ñ!2"r   c               óÖ   € \        WW#WEV
RRR4
      pVe   \        RV 24      h\        WW#V4      w  ppppp\        VVVVVVRRVV	V
VVVVR7      w  ppp\        VVV4      w  pppVVV3# )zCFA3 implementation of _scaled_dot_product_flash_attention_backward.NzFA3 SDPA backward unsupported: rè   )r–   rB   rŸ   rS   )rŒ   rW   rw   rx   r�   rŽ   r\   rÎ   rª   r«   rZ   r®   rê   rÔ   r¬   r‰   Ú
grad_out_tÚq_tÚk_tÚv_tÚout_trÈ   rÉ   rÊ   Údq_outÚdk_outÚdv_outs   &&&&&&&&&&&&&&$            r   rT   rT   Æ  s¹   € ô& (Ø˜ S°YÀÀdÈDó€Eð ÒÜÐ<¸U¸GÐDÓEÐEô (8Ø˜ Só(Ñ$€J��S˜#˜uô 4ØØØØØØØØØØØØØØØô�J€BˆˆBô& .¨b°"°bÓ9Ñ€FˆF�FØ�6˜6Ð!Ð!r   ÚFA3)Úregister_fn)Úflash_attn_interface)NNNNNN)F)NNNr£   )NNNrc   FF)rc   FF)/Ú__conditional_annotations__Ú__doc__Ú
__future__r   r>   rp   Útypingr   Úcollections.abcr   Údataclassesr   Ú	functoolsr   Útyping_extensionsr   r   r   Útorch.libraryr	   Ú r
   Ú__all__r   r&   r   r   r1   r   r6   r7   ru   rŠ   r–   r—   rŸ   r¦   r¾   rË   rN   rQ   rP   rS   rO   rR   rT   Úregister_flash_attention_impl)r÷   s   @r   Ú<module>r     sM  øðôõ #ã Û Ý  ÷ Ý(å !Ý ß 2ã Ý !å ð #ð€ð
 "&€ˆÓ %Ø!%€ˆÓ %ð ÷)ð )ó ð)ð ôó ð÷
/õ"/õ"õB"õJ(õV#ñL �$Ó€õ5õH÷
N)÷b8ñvD:ð ðD:ð  ð!D:ð"  ð#D:ð$ &*ð%D:ð& )-ð'D:ð(  $ð)D:ð* (,ð+D:ð, #ð-D:ð. "÷/D:ð D:ðN-ð ð-ð ð-ð  ð-ð  &*ð!-ð" )-ð#-ð$ (,ð%-ð& "÷'-ð`+ð ð+ð ð+ð  ð+ð &*ð+ð  )-ð!+ð" (,ð#+ð$  $ð%+ð& "÷'+ð\8ð  ð!8ð" $(ð#8ð$ %)÷%8ñvDð ÷Dð DñNð ÷ð ð02"ð  ÷!2"ðj 
× 'Ò '¨Ð;W× Xr   