+
    �åûj¶n ã                   óp  a € ER­ EtL0 t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIH	t	 ^ RI
t
^ RIHtHtHtHt ^ RIHt . RNRNRNRNR	NR
NRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNR NR!NR"NR#NR$NR%NR&NR'NR(NR)NR*NR+NR,NR-NR.NR/NR0NR1NR2NR3NR4NR5NR6NR7NR8NR9NR:NR;NR<NR=NR>NR?NR@NRANRBNRCNRDNRENRFNRGNRHNtRt  ^ RIt]P,                  P.                  ^ ,          tRI t]P6                  RJ8X  d%   ]! ]RK4      t]'       d   ]P:                  ! ]4      tM—]! ]RL4      t]'       d‡   ]P<                  ! ]4       Fp  tRM]9   g   K  RN]9   g   K  ]P@                  ! ]4      ;_uu_ 4        ]PB                  ! ]PD                  PG                  ]]4      ]PH                  RO7       RRR4       Kr  	   Rt%]PD                  PG                  ]PD                  PM                  ]'4      ]P6                  RJ8X  d   RPMRQ4      t(]PD                  PS                  ](4      '       g˜   ]P6                  RJ8X  d   RRMRNt*]PD                  PM                  ]'4      t+]P<                  ! ]+4       FQ  t]PY                  RS4      '       g   K  ]P[                  ]*4      '       g   K5  ]PD                  PG                  ]+]4      t(KS  	  ]PD                  PS                  ](4      '       d‰   ]P\                  P_                  RT](4      t0]0'       d]   ]0Pb                  '       dK   ]P\                  Pe                  ]04      t%]%]Pf                  RT&   ]0Pb                  Pi                  ]%4       RUt5Rt6MRVt5RW]( 2t6MRVt5RX]( 2t6 ^ RZI:H;t< ^ R[I:H=t> ^ R\I:H?t@ ^ R]I:HAtB ^ R^ICHDtD ^ R_IEHFtG ^ R`IHHItJHKtL ^ RaIMHNtNHOtO ^ RbIMHPtQ ^ RcIMHRtS ^ RdITHUtUHVtVHWtWHXtXHYtYHZtZH[t[H\t\ ^ ReI]H^t^H_t_ ^ RfI`HataHbtbHctcHdtdHeteHftfHgtg ^ RgIhHitiHjtjHktk ^ RhI`Hltm ^ RiInHotoHptpHqtqHrtrHstsHtttHutuHvtvHwtwHxtxHyty ^ RjIzH{t{ ^ RkI|H}t} ^ RlI~HtH€t€H�t� ]5;'       d    ]‚! ]%RmRV4      tƒ/ t„] ^ k / t…] ^k / t†] ^k / t‡] ^k / tˆ] ^k / t‰] ^k / tŠ] ^k / t‹] ^k ER®tŒ]EP                  EP                  RnRo4      Rp8H  t�^t�]‘! ^]’! ]EP                  EP                  RqRr4      4      4      t“]EP                  EP                  RsRp4      Ro8g  t”/ t•] ^k Rt Ru lt–ER¯t—Rv Rw lt˜Rx Ry lt™Rz R{ ltšR| R} lt›R~ R ltœR€ R� lt�R‚ Rƒ ltžR„ R… ltŸR† R‡ lt Rˆ R‰ lt¡RŠ R‹ lt¢RŒ R� lt£RŽ R� lt¤R� R‘ lt¥R’ R“ lt¦R” R• lt§/ t¨] ^	k R– R— lt©R˜ R™ ltªRš R› lt«Rœ R� lt¬Rž RŸ lt­R  R¡ lt®R¢ R£ lt¯R¤ R¥ lt°R¦ R§ lt±R¨ R© lt²Rª R« lt³R¬ R­ lt´R® R¯ ltµR° R± lt¶R² R³ lt·]
EPp                  3R´ Rµ llt¹]
EPt                  3R¶ R· llt»]
EPp                  3R¸ R¹ llt¼ER°R» R¼ llt½]
EPt                  RU3R½ R¾ llt¾ER±R¿ RÀ llt¿ER±RÁ RÂ lltÀER±RÃ RÄ lltÁER±RÅ RÆ lltÂRÇ RÈ ltÃRÉ RÊ ltÄRË RÌ ltÅRÍ RÎ ltÆRÏ RÐ ltÇRÑ RÒ ltÈRÓ RÔ ltÉER²RÕ RÖ lltÊR× RØ ltËR]
EPt                  3RÙ RÚ lltÌRÛ RÜ ltÍRÝ]^^ 3RÞ Rß lltÎRÝ]^]
EPž                  3Rà Rá lltÐRRÝ]^Râ3Rã Rä lltÑER±Rå Ræ lltÒRç Rè ltÓ]Ô! ]%Ré4      tÕER³tÖRê Rë lt×Rì Rí ltØ]Ô! ]%Rî4      tÙRï tÚ^RÝRU]
EPp                  RUR^ ^RU3	Rð Rñ lltÛER±Rò Ró lltÜER´Rô Rõ lltÝRötÞ]EP                  EP                  R÷Ro4      Rp8H  tßERµRø Rù lltRRú Rû ltPRü Rý ltàER±Rþ Rÿ lltáER±ER  ER lltâER ER ltãER ER ltäER ER ltåER¶ER ER	 lltæER·ER
 ER lltçER·ER ER lltèER ER ltéER têER ER ltëER¸ER ER lltìER ER ltíRR^RÝ]
EPt                  3ER ER lltîER ER ltïER ER ltðRRR^RÝ]
EPt                  RRRºRR3ER ER lltlRRR^RÝ]
EPt                  RRRºRR3ER ER  lltñER! ER" ltòER# ER$ ltóER¹ER% ltôERºER& ER' lltõER»töER²ER( ER) llt÷^€tøER*tùER¼túER½tûER+ tüERºER, ltýER- tþER. tÿER/ Et / EtER0 EtER1 EtER2 EtER¾ER3 ER4 llEtER¿ER5 ER6 llEtER7 EtER8 EtER9 Et	ER: Et
ER; ER< lEtERÀER= ER> llEtERÀER? ER@ llEtERÁEtRÝEtERÂEtERA ERB lEt ! ERC ERD]	4      EtERE ERF lEtERG ERH lEtE]! ERI4      3ERJ ERK llEtRºE]! ERI4      RV3ERL ERM llEtERN ERO lEtERP ERQ lEtER±ERR ERS llEtERT ERU lEtERÃERV ERW llEtERÄERX ERY llEt/ Et] ^
k ]
EP>                  3ERZ ER[ llEt ER\ ER] lEt!ERÅER^ lEt"ER_ ER` lEt#ERa ERb lEt$ERÆERc ERd lltIERe ERf lEt%ERg ERh ltKERi ERj lEt&ER²ERk ERl llEt'ERm ERn lEt(ERo ERp lEt)ERq ERr lEt*ERs ERt lEt+ERu ERv lEt,ERw ERx lEt-ERy ERz lEt.ER{ ER| lEt/ER}^ /ER~ ER llEt0ER€ ER� lEt1ER}^ /ER‚ ERƒ llEt2ERÀER„ ER… llEt3ERÀER† ER‡ llEt4ERÇERˆ ER‰ llEt5ERÇERŠ ER‹ llEt6ERÀERŒ ER� llEt7ERÀERŽ ER� llEt8ERÈER� ER‘ llEt9ERÈER’ ER“ llEt:ER” ER• lEt;ER– ER— lEt<ER˜ ER™ lEt=ERš ER› lEt>^@Et?^€Et@/ EtA] ^k ERœ ER� lEtBERž ERŸ lEtCER  ER¡ lEtDERÉER¢ ER£ llEtEERÊER¤ ER¥ llEtFRÝEtGER¦ ER§ lEtHERËER¨ ER© llEtIERª ER« lEtJER¬ EtKE]K! 4        R#   ] d    ]
P.                  ^ ,          t E
Lki ; i  + '       g   i     E
K&  ; i  ] d     E	L¿i ; i  ]7 d   t8RVt5]9! ]84      t6Rt% Rt8A8EL(Rt8A8i] d   t8RVt5RY]8 2t6Rt% Rt8A8ELARt8A8ii ; i(Ì  é    N)Ú
NamedTuple)Ú_tensors_overlapÚcheck_rope_inplaceÚdetect_rms_rope_bnhdÚtrim_rope_freqs)Úallocation_contextÚna3dÚwindow_attentionÚ
delaunay3dÚmesh_bvhÚclosest_point_on_meshÚclosest_hit_on_meshÚconnected_componentsÚedge_collapse_checksÚmin_cutÚsol_attnÚsol_attn_chunkedÚadalnÚfp16_conv3dÚfp16_conv3d_outÚgroup_norm_silu_pad3dÚgroup_norm_silu_pad3d_outÚgroup_norm_stats3dÚ	rms_adalnÚ
apply_ropeÚapply_rope_Úapply_rope1Úapply_rope1_Úapply_rope_split_halfÚapply_rope_split_half_Úapply_rope_split_half1Úapply_rope_split_half1_Úrms_ropeÚ	rms_rope_Ú	rms_rope1Ú
rms_rope1_Úrms_rope_split_halfÚrms_rope_split_half_Úrms_rope_split_half1Úrms_rope_split_half1_Údequantize_nvfp4Údequantize_per_tensor_fp8Údequantize_int8_simpleÚdequantize_int8_simple_dtypeÚdequantize_int8_convrot_weightÚ$dequantize_int8_convrot_weight_dtypeÚdequantize_convrot_w4a4_weightÚdequantize_w4a8_int8_weightÚfp16_linearÚint8_linearÚw4a8_int8_linearÚw4a8_packed_linearÚint4_linearÚconvrot_w4a4_linearÚ#prepare_int4_weight_for_int8_linearÚquantize_int8_tensorwiseÚquantize_int8_rowwiseÚquantize_int4_rowwiseÚquantize_int4_rowwise_convrot64Ú'quantize_int4_rowwise_convrot64_to_int8Úquantize_convrot_w4a4_weightÚquantize_w4a8_int8_weightÚquantize_int8_convrot_weightÚrotate_int8_convrot_weightÚquantize_int8_rowwise_convrot64Úquantize_and_rotate_rowwiseÚgemv_awq_w4a16Úquantize_mxfp8Úquantize_nvfp4Úquantize_per_tensor_fp8Úquantize_svdquant_w4a4Úscaled_mm_nvfp4Úscaled_mm_svdquant_w4a4Ústochastic_rounding_fp8c                 óp   € \         P                  ! V 4       F  w  r#pV F  pW9   g   K  Vu u # 	  K  	  R # ©N)ÚosÚwalk)Ú	start_dirÚlib_patternÚrootÚ_dirsÚfilesÚfiles   &&    Ú`C:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\backends\cuda\__init__.pyÚfind_lib_dirrX   q   s4   € Ü"$§'¢'¨)Ö"4ÑˆD˜Û�ØÖ&Ø”Kó ñ #5ñ ó    Úwin32Ú
cublasLt64zlibcublasLt.soÚcublasLtz.so)Úmodez_C.abi3.pydz
_C.abi3.soz.pydz_C.zcomfy_kitchen.backends.cuda._CTFz!Could not create module spec for zExtension file not found: zFailed to load extension: )Úapply_input_act)Úapply_residual)Úinput_act_code)Úinput_act_width)Úadaln_prep_modulation)Úrope)r   r   )ÚDTYPE_CODE_TO_DTYPEÚDTYPE_TO_CODE)r-   )r:   )Ú_LOG2EÚ_block_lengthsÚ_normalize_key_biasÚ_sink_countÚ_topk_countÚ_valid_rowsÚadd_coarse_Úcoarse_output)Ú_INT4_GROUP_SIZEÚ_unpack_int4_row_major)Ú_QUANT_ROW_ELEM_BUDGETÚ_decide_codebookÚ&_dequantize_w4a8_int8_weight_from_int8Ú_quantize_w4a8_chunkedÚ_w4a8_geometryÚvalidate_w4a8_operandsÚvalidate_w4a8_weight_shape)Ú
_FIXED_LUTÚ	_mma_bitsÚw4a8_mma_record_bytes)r5   )ÚDivisibleByÚ	ExactDimsÚFunctionConstraintsÚMinDimsÚParamConstraintÚValidationResultÚna3d_common_call_ruleÚsol_attn_common_call_ruleÚ!window_attention_common_call_ruleÚwith_out_paramÚwith_w4a8_packed_storage)Úroundup)Úregistry)Ú_build_hadamardÚ_rotate_activationÚ_rotate_weightÚHAS_CUBLASLTÚ&COMFY_KITCHEN_FORCE_INT4_INT8_FALLBACKÚ0Ú1Ú&COMFY_KITCHEN_INT4_INT8_WEIGHT_CHUNK_NÚ4096ÚCOMFY_KITCHEN_W4A8_CHUNKEDc                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )é   ÚdeviceÚreturn)Útorchr“   ÚTensor)Úformats   "rW   Ú__annotate__r˜   
  s"   € ÷ ñ ¤5§<¡<ð ´E·L±Lñ rY   c                 ó„  € \         P                  V 4      ;pEf   \        P                  ! \        \        P
                  R7      p\        P                  ! R\        P                  R7      pVP                  \        P                  4      P                  4       pVR,          VR,          ,          P                  4       P                  R^4      P                  \        P                  4      P                  4       p\!        4       ;_uu_ 4        VP                  V 4      pR R R 4       V\         V &   V#   + '       g   i     L; i)N©Údtypeé   )ºNNNN)Nr�   é�ÿÿÿ)Ú_W4A8_FIXED_DECODE_LUTSÚgetr•   Útensorrw   Úfloat32ÚarangeÚuint8ÚviewÚfloat8_e4m3fnÚfloatÚround_Úclamp_ÚtoÚint8Ú
contiguousr   )r“   Ú
decode_lutÚcodebookÚscalesÚtables   &    rW   Ú_w4a8_fixed_decode_lutr±   
  sÒ   € Ü-×1Ñ1°&Ó9Ð9ˆ
ÓBÜ—<’<¤
´%·-±-Ô@ˆÜ—’˜c¬¯©Ô5ˆØ—‘œU×0Ñ0Ó1×7Ñ7Ó9ˆà�G�_˜x¨Õ0Õ0ß‰V‹Xß‰V�D˜#Óß‰R”—
‘
‹^ß‰Z‹\ð 	ô  ×!Õ!ØŸ™ &Ó)ˆJ÷ "à*4Ô Ñ'ØÐ÷ "×!ús   Ä
D/Ä/D?	c                óR   € V ^8„  d   QhR\         R\        \         \         3,          /# ©r’   Údevice_indexr”   )ÚintÚtuple)r—   s   "rW   r˜   r˜   .  s"   € ÷ ñ ¬#ð ´%¼¼S¸µ/ñ rY   c                 óˆ   € \         P                  V 4      pVf)   \        P                  P	                  V 4      pV\         V &   V# rN   )Ú_device_capability_cacher    r•   ÚcudaÚget_device_capability)r´   Ú
capabilitys   & rW   Ú_cuda_device_capabilityr¼   .  s=   € Ü)×-Ñ-¨lÓ;€JØÒÜ—Z‘Z×5Ñ5°lÓCˆ
Ø1;Ô  Ñ.ØÐrY   c                ó0   € V ^8„  d   QhR\         R\         /# r³   ©rµ   )r—   s   "rW   r˜   r˜   6  s   € ÷ ñ ´Cð ¼Cñ rY   c                 óœ   € \         P                  V 4      pVf3   \        P                  P	                  V 4      P
                  pV\         V &   V# rN   )Ú"_device_multiprocessor_count_cacher    r•   r¹   Úget_device_propertiesÚmulti_processor_count)r´   Úcounts   & rW   Ú!_cuda_device_multiprocessor_countrÄ   6  sA   € Ü.×2Ñ2°<Ó@€EØ‚}Ü—
‘
×0Ñ0°Ó>×TÑTˆØ;@Ô*¨<Ñ8Ø€LrY   c                ó0   € V ^8„  d   QhR\         R\         /# r³   r¾   )r—   s   "rW   r˜   r˜   >  s   € ÷ ñ ¬ð ´ñ rY   c                 ó¶   € \         P                  V 4      pVf@   \        P                  P	                  V 4      p\        VR^ 4      ;'       g    RpV\         V &   V# )NÚL2_cache_sizei   @)Ú_device_l2_bytes_cacher    r•   r¹   rÁ   Úgetattr)r´   Úl2Úpropss   &  rW   Ú_cuda_device_l2_bytesrÌ   >  sQ   € Ü	×	#Ñ	# LÓ	1€BØ	‚zÜ—
‘
×0Ñ0°Ó>ˆä�U˜O¨QÓ/×<Ð<°GˆØ/1Ô˜|Ñ,Ø€IrY   c                ó0   € V ^8„  d   QhR\         R\        /# r³   ©rµ   Úbool)r—   s   "rW   r˜   r˜   H  s   € ÷ ñ ¬ð ´ñ rY   c                 ó’   € \         P                  V 4      pVe   V# \        P                  P	                  V 4      R8H  pV\         V &   V# )N©é   é   )Ú_turing_device_cacher    r•   r¹   rº   )r´   ÚcachedÚ	is_turings   &  rW   Ú_cuda_device_is_turingr×   H  sG   € Ü!×%Ñ% lÓ3€FØÒØˆÜ—
‘
×0Ñ0°Ó>À&ÑH€IØ)2Ô˜Ñ&ØÐrY   c                ó0   € V ^8„  d   QhR\         R\        /# r³   rÎ   )r—   s   "rW   r˜   r˜   Q  s   € ÷ 	ñ 	´3ð 	¼4ñ 	rY   c                 ó8  a€ \         P                  V 4      pVe   V# Rp\        V 4      '       db   \        P                  P                  V 4      o\        ;QJ d#    V3R l\         4       F  '       g   K   RM	  RM! V3R l\         4       4      pV\         V &   V# )NFc              3   ó,   <"  € T F	  qS9   x € K  	  R # 5irN   © )Ú.0ÚmodelÚdevice_names   & €rW   Ú	<genexpr>Ú3_cuda_device_is_nvidia_16_series.<locals>.<genexpr>X  s   øé € ÐOÑ=N°E KÖ/Ó=Nùs   ƒT)Ú_nvidia_16_series_device_cacher    r×   r•   r¹   Úget_device_nameÚanyÚ_NVIDIA_16_SERIES)r´   rÕ   Úis_16_seriesrÞ   s   &  @rW   Ú _cuda_device_is_nvidia_16_seriesræ   Q  su   ø€ Ü+×/Ñ/°Ó=€FØÒØˆØ€LÜ˜l×+Ò+Ü—j‘j×0Ñ0°Ó>ˆß“sÔOÕ=NÓO—s—s’sÔOÕ=NÓOÓOˆØ3?Ô" <Ñ0ØÐrY   c                ó0   € V ^8„  d   QhR\         R\        /# r³   rÎ   )r—   s   "rW   r˜   r˜   ]  s   € ÷ ñ ¼ð Äñ rY   c                 óH   € \        V 4      ;'       d    \        V 4      '       * # rN   )r×   ræ   )r´   s   &rW   Ú&_cuda_device_should_use_turing_kernelsré   ]  s)   € Ü! ,Ó/÷ ð Ô8XØó9ô 5ð rY   c                óH   € V ^8„  d   QhR\         R\         R\         R\        /# )r’   ÚmÚnÚkr”   rÎ   )r—   s   "rW   r˜   r˜   c  s(   € ÷ "ñ "¤ð "¬ð "´ð "¼ñ "rY   c                ó4   € V ^€8*  ;'       g    V^V,          8¬  # )zFUse fused SM75 kernels for latency shapes and feed-forward expansions.rÛ   )rë   rì   rí   s   &&&rW   Ú_prefer_turing_fused_int8rï   c  s   € à�‰8×!Ð!�q˜A �E‘zÐ!rY   c                óD   € V ^8„  d   QhR\         P                  R\        /# ©r’   r¡   r”   ©r•   r–   rÏ   )r—   s   "rW   r˜   r˜   h  s   € ÷ 
ñ 
´u·|±|ð 
Ìñ 
rY   c                 óâ   € V P                   '       g   R # V P                  4       p\        P                  V4      pVe   V# \        P
                  P                  V4      w  r4V^8¬  pV\        V&   V# ©F)Úis_cudaÚ
get_deviceÚ_cutlass_int8_device_cacher    r•   r¹   rº   )r¡   r´   rÕ   ÚmajorÚ_minorÚ	supporteds   &     rW   Ú*_cuda_device_supports_cutlass_int8_dequantrû   h  sg   € Ø�>�>ˆ>ÙØ×$Ñ$Ó&€LÜ'×+Ñ+¨LÓ9€FØÒØˆÜ—J‘J×4Ñ4°\ÓB�M€EØ˜‘
€IØ/8Ô˜|Ñ,ØÐrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# rñ   rò   )r—   s   "rW   r˜   r˜   u  s   € ÷ ñ ´%·,±,ð Ä4ñ rY   c                 ó‚   € V P                   '       d   \        '       d   R # \        V P                  4       4      w  rV^8H  # rô   )rõ   Ú_FORCE_INT4_INT8_FALLBACKr¼   rö   )r¡   rø   rù   s   &  rW   Ú%_cuda_device_supports_native_int4_mmarÿ   u  s5   € Ø�>�>ˆ>×6Ó6ÙÜ+¨F×,=Ñ,=Ó,?Ó@�M€Eð �A‰:ÐrY   c          
      óT   € V ^8„  d   QhR\         R\         R\         R\         R\        /# ©r’   rë   rì   rí   r´   r”   rÎ   )r—   s   "rW   r˜   r˜     s:   € ÷ ñ Ü
ðä
ðô ðô ð	ô
 
ñrY   c                óV  € \        ^ ^V ^,
          P                  4       ,          4      pVR8:  d   RMRpV\        V4      ,          ^Ž,          pWA,          pVR8  ;'       dJ    Wv8  ;'       g>    VR8„  ;'       d1    Wv8H  ;'       d%    ^€T u;8*  ;'       d    Tu;8H  ;'       d    R8  # u # )z:Select the low-latency kernel for native SM8x INT4 shapes.é   i   i   i €  é   )ÚmaxÚ
bit_lengthrÄ   )rë   rì   rí   r´   Úm_tileÚbase_thresholdÚ	thresholdÚworkloads   &&&&    rW   Ú_prefer_legacy_int4_kernelr    s«   € ô ��Q˜1˜q�5×,Ñ,Ó.Õ.Ó/€FØ"# u¤*‘Y°'€NØÔ!BÀ<Ó!PÕPÐTWÕW€IØ�z€HØˆv‰:÷ ð ØÑ÷ 	Nð 	NØ�‰I×LÐL˜(Ñ/×LÐL°C¸1×4LÔ4LÀ×4LÔ4LÈÑ4Lðá4LðrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# rñ   rò   )r—   s   "rW   r˜   r˜   �  s   € ÷ ñ ¤E§L¡Lð ´Tñ rY   c                 óö   € V P                   ;'       dg    \        '       * ;'       dT    V P                  ^ ,          \        8„  ;'       d2    \	        V P                  4       4      ;'       d    \        \        R4      # )r   Úcutlass_turing_int4_dequant)rõ   rþ   ÚshapeÚ_INT4_PACKED_WEIGHT_SMALL_M_MAXré   rö   ÚhasattrÚ_C©r¡   s   &rW   Ú_should_use_turing_int4r  �  sm   € à�‰÷ 	7ð 	7Ü)Ô)÷	7ð 	7à�L‰L˜�OÔ=Ñ=÷	7ð 	7ô 3°6×3DÑ3DÓ3FÓG÷	7ð 	7ô ”BÐ5Ó6ðrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# rñ   ©r•   r–   rµ   )r—   s   "rW   r˜   r˜   š  s"   € ÷ Wñ W¤U§\¡\ð W´cñ WrY   c                 ól   € V P                   '       d"   \        V P                  4       4      '       d   ^ # ^# )é    )rõ   r×   rö   r  s   &rW   Ú_cublas_int8_n_alignmentr  š  s*   € à——�Ô$:¸6×;LÑ;LÓ;N×$OÒ$Oˆ2ÐVÐUVÐVrY   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r’   ÚvalueÚ	alignmentr”   r¾   )r—   s   "rW   r˜   r˜   Ÿ  s!   € ÷ >ñ >”Sð >¤Sð >¬Sñ >rY   c                 ó<   € W,           ^,
          V,          V,          # )é   rÛ   )r  r  s   &&rW   Ú	_round_upr  Ÿ  s   € ØÕ Õ" yÕ0°IÕ=Ð=rY   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r’   ÚxÚpadded_colsr”   r  )r—   s   "rW   r˜   r˜   £  ó)   € ÷ 7ñ 7”E—L‘Lð 7¬sð 7´u·|±|ñ 7rY   c                 ó2  € V P                   ^,          V8X  d   V # \        P                  ! V P                   ^ ,          WP                   ^,          ,
          3V P                  V P                  R7      p\        P
                  ! W3^R7      P                  4       # )r  ©r›   r“   ©Údim©r  r•   Úzerosr›   r“   Úcatr¬   )r!  r"  Úpaddings   && rW   Ú_pad_2d_colsr,  £  sh   € Ø‡w�wˆq…z�[Ô ØˆÜ�kŠk˜1Ÿ7™7 1�: {·W±W¸QµZÕ'?Ð@ÈÏÉÐXY×X`ÑX`Ôa€GÜ�9Š9�a�\ qÔ)×4Ñ4Ó6Ð6rY   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r’   r!  Úpadded_rowsr”   r  )r—   s   "rW   r˜   r˜   ª  r#  rY   c                 ó2  € V P                   ^ ,          V8X  d   V # \        P                  ! WP                   ^ ,          ,
          V P                   ^,          3V P                  V P                  R7      p\        P
                  ! W3^ R7      P                  4       # )r   r%  r&  r(  )r!  r.  r+  s   && rW   Ú_pad_2d_rowsr0  ª  sh   € Ø‡w�wˆq…z�[Ô ØˆÜ�kŠk˜;¯©°­Õ3°Q·W±W¸QµZÐ@ÈÏÉÐXY×X`ÑX`Ôa€GÜ�9Š9�a�\ qÔ)×4Ñ4Ó6Ð6rY   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r’   r!  Úpadded_sizer”   r  )r—   s   "rW   r˜   r˜   ±  s.   € ÷ Cñ CŒu�|‰|ð C¬#ð C´%·,±,ñ CrY   c                 ó"  € V P                  4       V8X  d   V # \        P                  ! WP                  4       ,
          3V P                  V P                  R 7      p\        P
                  ! V P                  R4      V3^ R7      P                  4       # )r%  r&  éÿÿÿÿ)Únumelr•   r)  r›   r“   r*  Úreshaper¬   )r!  r2  r+  s   && rW   Ú_pad_1dr7  ±  sb   € Ø‡w�wƒy�KÔØˆÜ�kŠk˜;¯©«Õ2Ð4¸A¿G¹GÈAÏHÉHÔU€GÜ�9Š9�a—i‘i “m WÐ-°1Ô5×@Ñ@ÓBÐBrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# ©r’   r!  r”   r  )r—   s   "rW   r˜   r˜   »  s   € ÷ ñ ¬E¯L©Lð ¼Sñ rY   c                 ó  € V P                  4       p\        P                  V4      pVe   V# \        P                  P                  V4      p\        V4      '       d   VP                  pM\        VRVP                  4      pV\        V&   V# )NÚshared_memory_per_block_optin)	rö   Ú_max_shared_memory_cacher    r•   r¹   rÁ   r×   Úshared_memory_per_blockrÉ   )r!  r´   rÕ   rË   Úlimits   &    rW   Ú$_max_dynamic_shared_memory_per_blockr?  »  sw   € Ø—<‘<“>€LÜ%×)Ñ)¨,Ó7€FØÒØˆÜ�J‰J×,Ñ,¨\Ó:€EÜ˜l×+Ò+Ø×-Ñ-‰ä˜Ð>À×@]Ñ@]Ó^ˆØ-2Ô˜\Ñ*Ø€LrY   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r’   rë   rí   r”   r¾   )r—   s   "rW   r˜   r˜   É  s!   € ÷ 0ñ 0¬sð 0´sð 0¼sñ 0rY   c                 ó¶   € V ^8X  d   RpM*VR8X  d   ^@pM VR8:  d   ^€pMVR8:  d   RpMVR8  d   RpMRpV^@,          pW^,          R,          ,           ^,          # )r  r  rœ   i   é   i 0  r  rÛ   )rë   rí   Úblock_threadsÚgroups_in_flights   &&  rW   Ú'_convrot_int8_fused_shared_memory_bytesrE  É  sf   € àˆA„vØ‰Ø	
ˆcŒØ‰Ø	
ˆdŒØ‰Ø	
ˆdŒØ‰Ø	
ˆUŒØ‰àˆØ$¨Õ*ÐØ 1Õ$ sÕ*Õ*¨aÕ/Ð/rY   c          
      óT   € V ^8„  d   QhR\         R\         R\         R\         R\         /# )r’   rë   rí   Ú
group_sizeÚ
dtype_sizer”   r¾   )r—   s   "rW   r˜   r˜   Û  s8   € ÷ Gñ G¬sð G´sð GÌð GÔY\ð GÔadñ GrY   c                 ó(  € VR9   d8   VR8”  d   RM^€pWB^,          ,          pW^,          V,          ,           V,          # V ^8w  d   VR8:  d   Rp^pMV ^8X  d   Rp^pMVR8X  d   Rp^pMRp^pV^@,          pWV,          R,          ,           V,          # )é   i   r  rœ   i <  i€  r  )rJ  é@   rÛ   )rë   rí   rG  rH  rC  rD  Úscratch_bufferss   &&&&   rW   Ú'_convrot_int4_fused_shared_memory_bytesrM  Û  s¢   € Ø�XÔØ  4œx™¨SˆØ(¸1­_Õ=ÐØ qÕ(¨:Õ5Õ5¸ÕCÐCàˆA„v�!�t”)ØˆØ‰Ø	
ˆaŒØˆØ‰Ø	
ˆeŒØˆØ‰àˆØˆØ$¨Õ*ÐØ ?Õ2°SÕ8Õ8¸JÕFÐFrY   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# ©r’   r!  rí   rG  r”   ©r•   r–   rµ   rÏ   )r—   s   "rW   r˜   r˜   ñ  s3   € ÷ Fñ F¬¯©ð F¼#ð FÌ3ð FÔSWñ FrY   c                 óŽ   € V P                   '       d   VR 8w  d   R# \        V P                  ^ ,          V4      pV\        V 4      8  # )rœ   T)rõ   rE  r  r?  ©r!  rí   rG  Úrequested_shareds   &&& rW   Ú!_convrot_fused_shared_memory_fitsrT  ñ  s;   € Ø�9�9ˆ9˜
 cÔ)ÙÜ>¸q¿w¹wÀq½zÈ1ÓMÐØÔBÀ1ÓEÑEÐErY   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# rO  rP  )r—   s   "rW   r˜   r˜   ø  s4   € ÷ Fñ F¬e¯l©lð F¼sð FÔPSð FÔX\ñ FrY   c                 ó¬   € V P                   '       d   VR9  d   R# \        V P                  ^ ,          WV P                  4       4      pV\	        V 4      8  # )rJ  T©rJ  rK  rœ   )rõ   rM  r  Úelement_sizer?  rR  s   &&& rW   Ú&_convrot_int4_fused_shared_memory_fitsrY  ø  sG   € Ø�9�9ˆ9˜
¨-Ô7ÙÜ>¸q¿w¹wÀq½zÈ1ÐZ[×ZhÑZhÓZjÓkÐØÔBÀ1ÓEÑEÐErY   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# rO  rP  )r—   s   "rW   r˜   r˜   ÿ  s-   € ÷ ñ ¬¯©ð ¼ð Ì#ð ÔRVñ rY   c                 ó²   € VR 8H  ;'       dL    VR ,          ^ 8H  ;'       d8    V\         8*  ;'       d'    VR8*  ;'       g    VR8¬  ;'       d    \        WV4      # )rœ   i   é    )Ú_CONVROT_FUSED_MAX_KrT  ©r!  rí   rG  s   &&&rW   Ú _should_use_convrot_fused_kernelr_  ÿ  st   € à�cÑ÷ 	@ð 	@Ø��G�q‰L÷	@ð 	@àÔ%Ñ%÷	@ð 	@ð �$‰Y×#Ð#˜!˜t™)÷	@ð 	@ô .¨a°JÓ?ðrY   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# rO  rP  )r—   s   "rW   r˜   r˜   	  s3   € ÷ Yñ Y¬%¯,©,ð Y¼3ð YÌCð YÔTXñ YrY   c                 óT   € VR9   ;'       d    W,          ^ 8H  ;'       d
    V\         8*  # )rK  )rK  rœ   )r]  r^  s   &&&rW   Ú"_should_use_convrot_dequant_kernelrb  	  s,   € ð ˜Ñ"×XÐX q¥~¸Ñ':×XÐX¸qÔDXÑ?XÐXrY   c                ó$   € V ^8„  d   QhR\         /# ©r’   r”   r¾   )r—   s   "rW   r˜   r˜     s   € ÷ ñ ¬ñ rY   c                 óž   € \         P                  P                  \         P                  P                  4       4      P                  ^	8¼  d   R# R# )zAReturn 32 MiB if using hopper, 4 MiB for all other architectures.i   i  @ )r•   r¹   rÁ   Úcurrent_devicerø   rÛ   rY   rW   Úget_cublas_workspace_size_bytesrg    s2   € ä‡z�z×'Ñ'¬¯
©
×(AÑ(AÓ(CÓD×JÑJÈaÔOÙÙrY   c                ó8   € V ^8„  d   QhR\         P                  /# rd  ©r•   r–   )r—   s   "rW   r˜   r˜     s   € ÷ ñ œeŸl™lñ rY   c                 óä   € \         P                  P                  4       p \        P	                  V 4      pVf9   \         P
                  ! \        4       \         P                  V R7      pV\        V &   V# )zReturns workspace for cublas.r%  )r•   r¹   rf  Ú_cublas_workspacesr    Úemptyrg  r¤   )r´   Ú	workspaces     rW   Úget_cublas_workspacern    s[   € ä—:‘:×,Ñ,Ó.€LÜ"×&Ñ& |Ó4€IØÒÜ—K’KÜ+Ó-Ü—+‘+Øô
ˆ	ð
 ,5Ô˜<Ñ(ØÐrY   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r’   r“   r›   r”   )r•   r“   r›   r–   )r—   s   "rW   r˜   r˜   $  s-   € ÷ ñ œuŸ|™|ð ´E·K±Kð ÄEÇLÁLñ rY   c                 ó¬   € V P                   V P                  V3p\        P                  V4      pVf"   \        P
                  ! ^ WR7      pV\        V&   V# )Nr%  )ÚtypeÚindexÚ_empty_cuda_tensorsr    r•   rl  )r“   r›   Úkeyrl  s   &&  rW   Ú_empty_cuda_tensorru  $  sK   € Ø�;‰;˜Ÿ™ eÐ
,€CÜ×#Ñ# CÓ(€EØ‚}Ü—’˜A UÔ:ˆØ#(Ô˜CÑ Ø€LrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# )r’   Útr”   rò   )r—   s   "rW   r˜   r˜   -  s   € ÷ "ñ "”%—,‘,ð "¤4ñ "rY   c                 ó6   € V P                  4       ^,          ^ 8H  # )rJ  )Údata_ptr)rw  s   &rW   Ú
_aligned16rz  -  s   € Ø�:‰:‹<˜"Õ Ñ!Ð!rY   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# )r’   Úvr“   r›   r”   )r•   r–   r“   r›   )r—   s   "rW   r˜   r˜   1  s;   € ÷ -ñ -œŸ™ð -¬e¯l©lð -Ä5Ç;Á;ð -ÔSX×S_ÑS_ñ -rY   c                ó  € V P                   V8X  d;   V P                  V8X  d*   V P                  4       '       d   \        V 4      '       d   V # V P	                  WR7      P                  4       p \        V 4      '       d   V # V P                  4       # )z�A per-channel vector as the CUTLASS epilogues read it: on device, in dtype,
contiguous and 16-byte aligned (the row-broadcast visitors load 16-byte vectors).©r“   r›   )r“   r›   Úis_contiguousrz  rª   r¬   Úclone)r|  r“   r›   s   &&&rW   Ú_gemm_vector_argr�  1  sg   € ð 	‡x�x�6Ô˜aŸg™g¨Ô.°1·?±?×3DÒ3DÌÐTUÏÊØˆØ	�‰�FˆÓ(×3Ñ3Ó5€AÜ˜1—’ˆ1Ð, 1§7¡7£9Ð,rY   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r’   Úweight_scaler“   r”   )r•   r–   r“   )r—   s   "rW   r˜   r˜   :  s5   € ÷ Xñ X¬¯©ð X¼u¿|¹|ð XÔPU×P\ÑP\ñ XrY   c                 ó  € V P                   V8X  d8   V P                  \        P                  8X  d   V P	                  4       '       d   V # V P                  V\        P                  R 7      P                  R4      P                  4       # )r~  r4  )r“   r›   r•   r¢   r  rª   r6  r¬   )rƒ  r“   s   &&rW   Ú_int8_weight_scale_argr…  :  sa   € Ø×Ñ˜fÔ$¨×);Ñ);¼u¿}¹}Ô)LÐQ]×QkÑQk×QmÒQmØÐØ�?‰? &´·±ˆ?Ó>×FÑFÀrÓJ×UÑUÓWÐWrY   c          
      óT   € V ^8„  d   QhR\         R\         R\         R\         R\        /# r  rÎ   )r—   s   "rW   r˜   r˜   @  s:   € ÷ ñ Ü
ðä
ðô ðô ð	ô
 
ñrY   c                 ó   € R # rô   rÛ   )rë   rì   rí   r´   s   &&&&rW   Ú_prefer_cublas_int8_fallbackrˆ  @  s   € ñ rY   c                ó8   € V ^8„  d   QhR\         P                  /# )r’   r¡   ri  )r—   s   "rW   r˜   r˜   I  s   € ÷ (ñ (œUŸ\™\ñ (rY   c                ój   € V P                   '       d   V P                  4       p V P                  RR7      # )a¶  Export tensor via DLPack without cross-stream sync.

Works around PyTorch issue where __dlpack__(stream=None) syncs with
the default stream, breaking CUDA graph capture on non-default streams.
See: https://github.com/pytorch/pytorch/pull/163242

Detaches first so nn.Parameter (requires_grad=True) inputs like bias
export without PyTorch's gradient-tracking refusal.

Returns a PyCapsule containing the DLTensor that nanobind can import.
)Ústreamr4  )Úrequires_gradÚdetachÚ
__dlpack__r  s   &rW   Ú_wrap_for_dlpackr�  I  s/   € ð ××ÐØ—‘“ˆØ×Ñ BÐÓ'Ð'rY   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# ©r’   r!  ÚscaleÚoutput_typer”   ©r•   r–   r›   )r—   s   "rW   r˜   r˜   [  s<   € ÷ *ñ *Ü‡|�|ð*Ü!ŸL™Lð*Ü7<·{±{ð*ä
‡\�\ñ*rY   c           	      ó   € \         V P                  ,          p\         V,          pV P                  4       '       g   V P                  4       p \        P
                  ! V P                  \        P                  V P                  R 7      pV P                  4       p\        P                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V4      VVVV4       VP!                  V4      # )r%  )re   r›   r  r¬   r•   rl  r  r¤   r“   r5  r¹   Úcurrent_streamÚcuda_streamr  rH   r�  r¥   )r!  r’  r“  Úinput_dtype_codeÚoutput_dtype_codeÚresult_uint8r5  Ú
stream_ptrs   &&&     rW   rH   rH   [  sÂ   € ô % Q§W¡WÕ-ÐÜ% kÕ2Ðà�?‰?×ÒØ�L‰L‹Nˆä—;’;˜qŸw™w¬e¯k©kÀ!Ç(Á(ÔK€Là�G‰G‹I€EÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×ÑÜ˜ÓÜ˜ÓÜ˜Ó&ØØØØôð ×Ñ˜[Ó)Ð)rY   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# r‘  r”  )r—   s   "rW   r˜   r˜   u  s<   € ÷ ñ Ü‡|�|ðÜ!ŸL™LðÜ7<·{±{ðä
‡\�\ñrY   c           	      ó
  € VP                  4       ^8X  g   Q R4       h\        V P                  ,          p\        V,          p\        P                  ! V P
                  W P                  R7      pV P                  4       p\        P                  P                  V P                  4      P                  p\        P                  \        V P                  \        P                  4      4      \        V4      \        V4      VVVV4       V# )r  zScale must be a scalar tensorr%  )r5  re   r›   r•   rl  r  r“   r¹   r–  r—  r  r,   r�  r¥   r¤   )r!  r’  r“  r˜  r™  Úresultr5  r›  s   &&&     rW   r,   r,   u  s¿   € ð �;‰;‹=˜AÔÐ>Ð>Ó>Ðä$ Q§W¡WÕ-ÐÜ% kÕ2Ðä�[Š[˜Ÿ™¨¿H¹HÔE€FØ�G‰G‹I€EÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€Jä× Ñ Ü˜Ÿ™¤§¡Ó,Ó-Ü˜ÓÜ˜Ó ØØØØôð €MrY   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# )r’   r!  Úrngr“  r”   r”  )r—   s   "rW   r˜   r˜   Ž  s@   € ÷ !ñ !Ü‡|�|ð!ä	�‰ð!ô —‘ð!ô ‡\�\ñ	!rY   c                 ó®  € \         V,          pV P                  4       '       g   V P                  4       p VP                  4       '       g   VP                  4       p\        P                  P                  V P                  4      P                  p\        P                  \        V4      \        V 4      VV P                  4       V4       VP                  V4      # rN   )re   r  r¬   r•   r¹   r–  r“   r—  r  Ústochastic_round_fp8r�  r5  r¥   )r!  r   r“  r™  r›  s   &&&  rW   rL   rL   Ž  sž   € ô
 & kÕ2Ðà�?‰?×ÒØ�L‰L‹NˆØ×Ñ×ÒØ�n‰nÓˆä—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×ÑÜ˜ÓÜ˜ÓØØ	�‰‹	Øôð �8‰8�KÓ Ð rY   ç        c                óÒ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        R\        \         P                  \         P                  3,          /# )r’   r!  Úper_tensor_scaleÚepsilonÚpad_16xÚhi_firstr”   )r•   r–   r§   rÏ   r¶   )r—   s   "rW   r˜   r˜   ¦  s^   € ÷ 2ñ 2Ü‡|�|ð2ä—l‘lð2ô ð2ô ð	2ô
 ð2ô Œ5�<‰<œŸ™Ð%Õ&ñ2rY   c           
      ó¦  € V P                  4       '       g   Q R 4       hV P                  w  rVV'       d   \        V^4      p\        V^4      pM2YVr‡V^,          ^ 8X  g   Q RV 24       hV^,          ^ 8X  g   Q RV 24       h\        P                  ! Wx^,          3V P
                  \        P                  \        P                  R7      p	\        V^€4      p
\        V^,          ^4      p\        P                  ! W«3V P
                  \        P                  R7      pVP                  4       ^ 8X  d   VP                  ^4      p\        P                  P                  V P
                  4      P                  p\        P                  \!        V 4      \!        V4      \!        V	4      \!        V4      VVVV4       VP#                  \        P$                  4      pWž3# )úInput tensor must be contiguousz&num_rows must be divisible by 16, got z&num_cols must be divisible by 16, got ©r“   r›   Úmemory_formatr~  )r  r  r…   r•   rl  r“   r¤   Úcontiguous_formatr)  r'  r6  r¹   r–  r—  r  rG   r�  r¥   r¦   )r!  r¥  r¦  r§  r¨  Ú	orig_rowsÚ	orig_colsÚnum_rowsÚnum_colsÚqxÚ
scale_rowsÚ
scale_colsÚsx_uint8r›  Úsxs   &&&&&          rW   rG   rG   ¦  sˆ  € ð �?‰?×ÒÐ?Ð?Ó?ÐàŸ7™7Ñ€IßÜ˜9 bÓ)ˆÜ˜9 bÓ)‰à&�(Ø˜"�} Ô!ÐVÐ%KÈHÈ:Ð#VÓVÐ!Ø˜"�} Ô!ÐVÐ%KÈHÈ:Ð#VÓVÐ!ô 
�Š�h¨A¥Ð.°q·x±xÄuÇ{Á{Ôbg×byÑbyÔ	z€Bô ˜ 3Ó'€JÜ˜ R�¨Ó+€JÜ�{Š{˜JÐ3¸A¿H¹HÌEÏKÉKÔX€Hð ×ÑÓ Ô"Ø+×3Ñ3°AÓ6Ðä—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×ÑÜ˜ÓÜÐ)Ó*Ü˜ÓÜ˜Ó"ØØØØô	ð 
�‰”u×*Ñ*Ó	+€Bàˆ6€MrY   c                óÄ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  /# )r’   r²  r¥  Úblock_scalesr“  r¨  r”   )r•   r–   r›   rÏ   )r—   s   "rW   r˜   r˜   Û  sX   € ÷  ñ  Ü�‰ð ä—l‘lð ô —,‘,ð ô —‘ð	 ô
 ð ô ‡\�\ñ rY   c           	      ó>  € V P                  4       '       g   Q R 4       hV P                  w  rVV^,          p\        P                  ! WW3V P                  VR7      pVP                  4       ^ 8X  d   VP                  ^4      pVP                  \        P                  4      p	\        V,          p
\        P                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V	4      \        V4      V
VV4       V# )rª  r~  )r  r  r•   rl  r“   r'  r6  r¥   r¤   re   r¹   r–  r—  r  r+   r�  )r²  r¥  r¸  r“  r¨  r°  Únum_cols_packedr±  ÚoutputÚblock_scales_uint8r™  r›  s   &&&&&       rW   r+   r+   Û  sð   € ð ×Ñ×ÒÐ@Ð@Ó@Ðà "§¡Ñ€HØ Õ"€Hä�[Š[˜(Ð-°b·i±iÀ{ÔS€Fð ×ÑÓ Ô"Ø+×3Ñ3°AÓ6Ðà%×*Ñ*¬5¯;©;Ó7ÐÜ% kÕ2ÐÜ—‘×*Ñ*¨2¯9©9Ó5×AÑA€Jä×ÑÜ˜ÓÜÐ)Ó*ÜÐ+Ó,Ü˜Ó ØØØôð €MrY   c                ó¨   € V ^8„  d   QhR\         P                  R\        R,          R\        \         P                  \         P                  3,          /# ©r’   r!  Ústochastic_roundingNr”   ©r•   r–   rµ   r¶   )r—   s   "rW   r˜   r˜   þ  sE   € ÷ Lñ LÜ‡|�|ðLä˜t�ðLô Œ5�<‰<œŸ™Ð%Õ&ñLrY   c           
     ó¶  € V P                   pV P                  RV P                   R,          4      P                  4       p\        P                  ! V\        P
                  R7      p\        P                  ! VP                   ^ ,          ^3\        P                  V P                  R7      p\        P                  P                  V P                  4      P                  p\        P                  \        V4      \        V4      \        V4      VRJ;'       d    V^ 8„  \        T;'       g    ^ 4      V4       VP                  V4      VP                  ! . VRR O^N5!  3# )z>Quantize tensor to INT8 with per-row scales (for activations).rš   r%  Nr4  )r  r6  r¬   r•   Ú
empty_liker«   rl  r¢   r“   r¹   r–  r—  r  r;   r�  rµ   ©r!  r¿  Ú
orig_shapeÚx_2dÚq_2dÚ	scales_2dr›  s   &&     rW   r;   r;   þ  s  € ð
 —‘€JØ�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DÜ×Ò˜D¬¯
©
Ô3€DÜ—’˜TŸZ™Z¨�]¨AÐ.´e·m±mÈAÏHÉHÔU€IÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€Jä×ÑÜ˜ÓÜ˜ÓÜ˜Ó#Ø 4Ð'×CÐCÐ,?À!Ñ,CÜÐ×$Ð$ 1Ó%Øôð �<‰<˜
Ó# Y×%6Ò%6Ð%K¸
À3ÀB¸Ð%KÈÓ%KÐKÐKrY   c                ó¨   € V ^8„  d   QhR\         P                  R\        R,          R\        \         P                  \         P                  3,          /# r¾  rÀ  )r—   s   "rW   r˜   r˜     sE   € ÷ gñ gÜ‡|�|ðgä˜t�ðgô Œ5�<‰<œŸ™Ð%Õ&ñgrY   c           
     óÈ  € V P                   pV P                  RV P                   R,          4      P                  4       pVP                   R,          ^@,          ^ 8w  d    \        RVP                   R,           24      h\        P
                  ! VP                   ^ ,          VP                   ^,          ^,          3\        P                  V P                  R7      p\        P
                  ! VP                   ^ ,          ^3\        P                  V P                  R7      p\        P                  P                  V P                  4      P                  p\        P                  \        V4      \        V4      \        V4      VRJ;'       d    V^ 8„  \        T;'       g    ^ 4      V4       VP                  ! . VRR OVR,          ^,          N5!  VP                  ! . VRR O^N5!  3# )zLQuantize a contiguous 2D tensor to signed int4 with one float scale per row.z:INT4 rowwise quantization requires K divisible by 64, got r%  Nr4  )r  r6  r¬   Ú
ValueErrorr•   rl  r«   r“   r¢   r¹   r–  r—  r  r<   r�  rµ   rÃ  s   &&     rW   r<   r<     s{  € ð
 —‘€JØ�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DØ‡z�z�"…~˜Õ˜aÔÜÐUÐVZ×V`ÑV`ÐacÕVdÐUeÐfÓgÐgÜ�;Š;˜Ÿ
™
 1� t§z¡z°!¥}¸Õ'9Ð:Ä%Ç*Á*ÐUV×U]ÑU]Ô^€DÜ—’˜TŸZ™Z¨�]¨AÐ.´e·m±mÈAÏHÉHÔU€IÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×ÑÜ˜ÓÜ˜ÓÜ˜Ó#Ø 4Ð'×CÐCÐ,?À!Ñ,CÜÐ×$Ð$ 1Ó%Øôð �<Š<Ð>˜ C R˜Ð>¨*°R­.¸AÕ*=Ó>À	×@QÒ@QÐ@fÐS]Ð^aÐ_aÐSbÐ@fÐdeÓ@fÐfÐfrY   c          
      ó´   € V ^8„  d   QhR\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# ©r’   r!  rG  r¿  Nr”   rÀ  )r—   s   "rW   r˜   r˜   ,  sP   € ÷ gñ gÜ‡|�|ðgäðgô ˜t�ðgô Œ5�<‰<œŸ™Ð%Õ&ñ	grY   c                óú  € V P                   pV P                  RV P                   R,          4      P                  4       pVR9  d   \        RV 24      hVP                   R,          V,          ^ 8w  d#   \        RV RVP                   R,           24      h\        P
                  ! VP                   ^ ,          VP                   ^,          ^,          3\        P                  V P                  R7      p\        P
                  ! VP                   ^ ,          ^3\        P                  V P                  R7      p\        P                  P                  V P                  4      P                  p\        P                  \        V4      \        V4      \        V4      TVRJ;'       d    V^ 8„  \        T;'       g    ^ 4      V4       VP                  ! . VRR OVR,          ^,          N5!  VP                  ! . VRR O^N5!  3# )zEFused regular ConvRot rotation plus rowwise signed int4 quantization.zHINT4 ConvRot fused quantization requires group_size 16, 64, or 256, got zCINT4 ConvRot fused quantization requires K divisible by group_size ú, got r%  Nr4  rW  )r  r6  r¬   rÊ  r•   rl  r«   r“   r¢   r¹   r–  r—  r  r=   r�  rµ   ©r!  rG  r¿  rÄ  rÅ  rÆ  rÇ  r›  s   &&&     rW   r=   r=   ,  s±  € ð —‘€JØ�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DØ˜Ô&ÜÐcÐdnÐcoÐpÓqÐqØ‡z�z�"…~˜
Õ" aÔ'ÜÐ^Ð_iÐ^jÐjpÐqu×q{Ñq{Ð|~Õqð  qAð  Bó  Cð  	CÜ�;Š;˜Ÿ
™
 1� t§z¡z°!¥}¸Õ'9Ð:Ä%Ç*Á*ÐUV×U]ÑU]Ô^€DÜ—’˜TŸZ™Z¨�]¨AÐ.´e·m±mÈAÏHÉHÔU€IÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×&Ñ&Ü˜ÓÜ˜ÓÜ˜Ó#ØØ 4Ð'×CÐCÐ,?À!Ñ,CÜÐ×$Ð$ 1Ó%Øôð �<Š<Ð>˜ C R˜Ð>¨*°R­.¸AÕ*=Ó>À	×@QÒ@QÐ@fÐS]Ð^aÐ_aÐSbÐ@fÐdeÓ@fÐfÐfrY   c          
      ó´   € V ^8„  d   QhR\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# rÌ  rÀ  )r—   s   "rW   r˜   r˜   G  sP   € ÷ Mñ MÜ‡|�|ðMäðMô ˜t�ðMô Œ5�<‰<œŸ™Ð%Õ&ñ	MrY   c                ób  € V P                   pV P                  RV P                   R,          4      P                  4       pVR8w  d   \        RV 24      hVP                   R,          V,          ^ 8w  d#   \        RV RVP                   R,           24      h\        P
                  ! V\        P                  R7      p\        P                  ! VP                   ^ ,          ^3\        P                  V P                  R7      p\        P                  P                  V P                  4      P                  p\        P                  \        V4      \        V4      \        V4      TVRJ;'       d    V^ 8„  \!        T;'       g    ^ 4      V4       VP                  ! V!  VP                  ! . VRR O^N5!  3# )	zJFused ConvRot-256 rotation plus int4-scale quantization into INT8 storage.rœ   zHINT4 fallback INT8 activation quantization requires group_size 256, got z;INT4 ConvRot fallback quantization requires K divisible by rÎ  rš   r%  Nr4  )r  r6  r¬   rÊ  r•   rÂ  r«   rl  r¢   r“   r¹   r–  r—  r  r>   r�  rµ   rÏ  s   &&&     rW   r>   r>   G  sg  € ð —‘€JØ�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DØ�SÔÜÐcÐdnÐcoÐpÓqÐqØ‡z�z�"…~˜
Õ" aÔ'ÜÐVÐWaÐVbÐbhÐim×isÑisÐtvÕiwÐhxÐyÓzÐzÜ×Ò˜D¬¯
©
Ô3€DÜ—’˜TŸZ™Z¨�]¨AÐ.´e·m±mÈAÏHÉHÔU€IÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×.Ñ.Ü˜ÓÜ˜ÓÜ˜Ó#ØØ 4Ð'×CÐCÐ,?À!Ñ,CÜÐ×$Ð$ 1Ó%Øôð �<Š<˜Ñ$ i×&7Ò&7Ð&L¸ÀCÀR¸Ð&LÈ!Ó&LÐLÐLrY   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r’   Úqdatar”   ri  )r—   s   "rW   r˜   r˜   b  s"   € ÷ ñ ¤U§\¡\ð ´e·l±lñ rY   c                 ó–  € V P                  4       p\        P                  ! VP                  ^ ,          VP                  ^,          ^,          3\        P                  VP
                  R7      p\        P                  P                  VP
                  4      P                  p\        P                  \        V4      \        V4      V4       V# )r   r%  )r¬   r•   rl  r  r«   r“   r¹   r–  r—  r  Úunpack_int4_to_int8r�  )rÓ  Úqdata_2dr»  r›  s   &   rW   Ú_unpack_int4_to_int8_cudar×  b  s‹   € Ø×ÑÓ!€HÜ�[Š[˜(Ÿ.™.¨Õ+¨X¯^©^¸AÕ->ÀÕ-BÐCÌ5Ï:É:Ð^f×^mÑ^mÔn€FÜ—‘×*Ñ*¨8¯?©?Ó;×GÑG€JÜ×ÑÔ+¨HÓ5Ô7GÈÓ7OÐQ[Ô\Ø€MrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  /# ©	r’   Úx_int8Úweight_packedÚx_scalerƒ  ÚbiasNÚ	out_dtyper”   r”  )r—   s   "rW   r˜   r˜   j  so   € ÷ %ñ %Ü�L‰Lð%ä—<‘<ð%ô �\‰\ð%ô —,‘,ð	%ô
 �,‰,˜Õ
ð%ô �{‰{ð%ô ‡\�\ñ%rY   c           
      óÌ  € V P                  4       ^8w  d   \        R4      hVP                  4       ^8w  g1   V P                  ^,          VP                  ^,          ^,          8w  d   \        R4      hV P                  ^ ,          pVP                  ^ ,          p\        P                  ! Wg3WPP
                  R7      pVP                  V P
                  \        P                  R7      P                  R^4      P                  4       p	V	P                  ^ ,          V8w  d#   \        RV RV	P                  ^ ,           24      hVP                  V P
                  \        P                  R7      P                  R4      P                  4       p
V
P                  4       V8w  d    \        RV RV
P                  4        24      hVe   TM\        V P
                  V4      pVen   VP
                  V P
                  8w  g(   VP                  V8w  g   VP                  4       '       g,   VP                  V P
                  VR7      P                  4       p\        P                  P                  V P
                  4      P                   p\"        P%                  \'        V 4      \'        VP                  4       4      \'        V	4      \'        V
4      \'        V4      \'        V4      \(        V,          V4       V# )	r’   z4packed INT4 weight GEMV expects a 2D INT8 activationz1packed INT4 weight GEMV K dimensions do not matchr%  r~  z*packed INT4 weight GEMV x_scale must have ú values, got z/packed INT4 weight GEMV weight_scale must have r4  )r'  rÊ  r  r•   rl  r“   rª   r¢   r6  r¬   r5  ru  r›   r  r¹   r–  r—  r  Ú!int4_weight_int8_act_gemv_dequantr�  re   )rÚ  rÛ  rÜ  rƒ  rÝ  rÞ  rë   rì   r»  Úx_scale_argÚweight_scale_argÚbias_argr›  s   &&&&&&       rW   Ú"_int4_weight_int8_act_gemv_dequantrå  j  s<  € ð ‡z�zƒ|�qÔÜÐOÓPÐPØ×ÑÓ˜aÔ 6§<¡<°¥?°m×6IÑ6IÈ!Õ6LÈqÕ6PÔ#PÜÐLÓMÐMà�‰�Q�€AØ×Ñ˜AÕ€AÜ�[Š[˜!˜ y¿¹ÔG€FØ—*‘* F§M¡M¼¿¹�*ÓG×OÑOÐPRÐTUÓV×aÑaÓc€KØ×Ñ˜Õ˜qÔ ÜÐEÀaÀSÈÐVa×VgÑVgÐhiÕVjÐUkÐlÓmÐmØ#—‘¨f¯m©mÄ5Ç=Á=�ÓQ×YÑYÐZ\Ó]×hÑhÓjÐØ×ÑÓ 1Ô$ÜÐJÈ1È#È]Ð[k×[qÑ[qÓ[sÐZtÐuÓvÐvØÒ'‰tÔ-?ÀÇÁÈyÓ-Y€HØÒ˜TŸ[™[¨F¯M©MÔ9¸T¿Z¹ZÈ9Ô=TÐ\`×\nÑ\n×\pÒ\pØ—7‘7 &§-¡-°y�7ÓA×LÑLÓNˆä—‘×*Ñ*¨6¯=©=Ó9×EÑE€JÜ×(Ñ(Ü˜Ó Ü˜×1Ñ1Ó3Ó4Ü˜Ó%ÜÐ)Ó*Ü˜Ó"Ü˜Ó Ü�iÕ Øô	ð €MrY   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r’   rë   rì   r”   r¾   )r—   s   "rW   r˜   r˜   ’  s!   € ÷ -ñ -¤Cð -¬Cð -´Cñ -rY   c                 ób   € VR 8:  d   V# V ^€8:  d   \        V\        4      # \        V\        4      # )é 
  )ÚminÚ_INT4_INT8_WEIGHT_CHUNK_N)rë   rì   s   &&rW   Ú_int4_int8_weight_chunk_colsrë  ’  s1   € ØˆD„yØˆØˆC„xÜ�1Ô/Ó0Ð0ÜˆqÔ+Ó,Ð,rY   c                óH   € V ^8„  d   QhR\         R\         R\         R\         /# )r’   rë   rì   r´   r”   r¾   )r—   s   "rW   r˜   r˜   š  s(   € ÷ .ñ .œsð .¤sð .¼#ð .Ä#ñ .rY   c                 ón   € \        V4      \        8  d   R \        P                  9  d   V# \	        W4      # )rŽ   )rÌ   Ú_SMALL_L2_BYTESrO   Úenvironrë  )rë   rì   r´   s   &&&rW   Ú_w4a8_weight_chunk_colsrð  š  s-   € ô 	˜lÓ+¬oÔ=Ø4¼B¿J¹JÔFàˆÜ'¨Ó-Ð-rY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  /# rÙ  r”  )r—   s   "rW   r˜   r˜   ¥  so   € ÷ /ñ /Ü�L‰Lð/ä—<‘<ð/ô �\‰\ð/ô —,‘,ð	/ô
 �,‰,˜Õ
ð/ô �{‰{ð/ô ‡\�\ñ/rY   c                 óÀ  € V P                  4       ^8w  d   \        R4      hVP                  4       ^8w  g1   V P                  ^,          VP                  ^,          ^,          8w  d   \        R4      hV P                  4       '       d   VP                  4       '       g   \        R4      hV P                  w  rgVP                  ^ ,          p\	        Wh4      p	\
        P                  ! Wh3WPP                  R7      p
\
        P                  ! W—3\
        P                  V P                  R7      p\
        P                  ! Wi3\
        P                  V P                  R7      pVP                  V P                  \
        P                  R7      P                  R	^4      P                  4       pVP                  ^ ,          V8w  d#   \        RV RVP                  ^ ,           24      hVP                  V P                  \
        P                  R7      P                  R	4      P                  4       pVP                  4       V8w  d    \        RV RVP                  4        24      hVe   \        W@P                  V4      M\!        V P                  V4      p\
        P"                  P%                  V P                  4      P&                  p\(        P+                  \-        V 4      \-        V4      \-        V4      \-        V4      \-        V4      \-        V
4      \-        V4      \-        V4      \-        \/        4       4      V	\1        V P3                  4       4      '       * \4        V,          V4       V
# )
r’   z5chunked INT4 weight GEMM expects a 2D INT8 activationz2chunked INT4 weight GEMM K dimensions do not matchzIchunked INT4 weight GEMM expects contiguous activation and weight tensorsr%  r~  z+chunked INT4 weight GEMM x_scale must have rà  z0chunked INT4 weight GEMM weight_scale must have r4  )r'  rÊ  r  r  rë  r•   rl  r“   r«   Úint32rª   r¢   r6  r¬   r5  r�  ru  r¹   r–  r—  r  Ú)int4_weight_int8_act_gemm_dequant_chunkedr�  rn  r×   rö   re   )rÚ  rÛ  rÜ  rƒ  rÝ  rÞ  rë   rí   rì   Ú
chunk_colsr»  Úweight_workspaceÚacc_workspacerâ  rã  rä  r›  s   &&&&&&           rW   Ú*_int4_weight_int8_act_gemm_dequant_chunkedrø  ¥  s”  € ð ‡z�zƒ|�qÔÜÐPÓQÐQØ×ÑÓ˜aÔ 6§<¡<°¥?°m×6IÑ6IÈ!Õ6LÈqÕ6PÔ#PÜÐMÓNÐNØ×Ñ×!Ò!¨×)DÑ)D×)FÒ)FÜÐdÓeÐeà�<‰<�D€AØ×Ñ˜AÕ€AÜ-¨aÓ3€JÜ�[Š[˜!˜ y¿¹ÔG€FÜ—{’{ J ?¼%¿*¹*ÈVÏ]É]Ô[ÐÜ—K’K  ´u·{±{È6Ï=É=ÔY€MØ—*‘* F§M¡M¼¿¹�*ÓG×OÑOÐPRÐTUÓV×aÑaÓc€KØ×Ñ˜Õ˜qÔ ÜÐFÀqÀcÈÐWb×WhÑWhÐijÕWkÐVlÐmÓnÐnØ#—‘¨f¯m©mÄ5Ç=Á=�ÓQ×YÑYÐZ\Ó]×hÑhÓjÐØ×ÑÓ 1Ô$ÜÐKÈAÈ3ÈmÐ\l×\rÑ\rÓ\tÐ[uÐvÓwÐwàDHÒDTÔ  §}¡}°iÔ@Ü'¨¯©°yÓAð ô —‘×*Ñ*¨6¯=©=Ó9×EÑE€JÜ×0Ñ0Ü˜Ó Ü˜Ó'Ü˜Ó%ÜÐ)Ó*Ü˜Ó"Ü˜Ó ÜÐ)Ó*Ü˜Ó'ÜÔ-Ó/Ó0ØÜ" 6×#4Ñ#4Ó#6Ó7Ô7Ü�iÕ Øôð €MrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  /# )	r’   rÚ  Úweight_int8rÜ  rƒ  rÝ  NrÞ  r”   r”  )r—   s   "rW   r˜   r˜   ×  sx   € ÷ nñ nÜ�L‰Lðnä—‘ðnô �\‰\ðnô —,‘,ð	nô
 �,‰,˜Õ
ðnô �{‰{ðnô ‡\�\ñnrY   c                 óÄ
  € V P                  4       ^8w  g   VP                  4       ^8w  d   \        R4      hV P                  ^,          VP                  ^,          8w  d   \        R4      hV P                  w  rgVP                  ^ ,          p\        P                  ! Wh3WPP
                  R7      p	VP                  V P
                  \        P                  R7      P                  R4      P                  4       p
VP                  V P
                  \        P                  R7      P                  R4      P                  4       pV
P                  4       V8w  d    \        RV RV
P                  4        24      hVP                  4       V8w  d    \        RV RVP                  4        24      hVe   \        W@P
                  V4      M\        V P
                  V4      p\        P                  P                  V P
                  4      P                  pV^8X  d”   V^,          ^ 8X  d†   \!        \"        R	4      '       dp   \"        P%                  \'        V 4      \'        V4      \'        V
P                  ^^4      4      \'        V4      \'        V4      \'        V	4      \(        V,          V4       V	# \+        WhV4      '       dR   \-        V P/                  4       4      '       d3   \!        \"        R
4      '       d   \1        T TT
TVe   TMRV4      pVe   V# Rp\3        VVVV P/                  4       4      pV'       gò   \4        '       gæ   \7        V 4      '       dÕ   VP                  4       V8X  d   TMVP9                  V4      P                  4       p\"        P;                  \'        V 4      \'        V4      \'        V
P                  V^4      4      \'        V4      \'        V4      \'        V	4      \(        V,          \        P                  P                  V P
                  4      P                  4      pV'       d   V	# \=        V P/                  4       4      '       dE   \?        V^4      p\?        V\A        V 4      4      p\C        V V4      p\E        \C        VV4      V4      pMTpT pTp\        P                  ! VV3\        PF                  V P
                  R7      p\"        PI                  \'        V4      \'        V4      \'        V4      \'        \K        4       4      V4       VV8w  d   VRRV13,          P                  4       p\"        PM                  VV
P                  V^4      VVV	\(        V,          V4       V	# )r’   z@INT4 fallback INT8 GEMM expects 2D activation and weight tensorsz1INT4 fallback INT8 GEMM K dimensions do not matchr%  r~  z INT4 fallback x_scale must have rà  z%INT4 fallback weight_scale must have NÚint8_gemv_dequantÚcutlass_turing_int8_dequantFr�   r4  )'r'  rÊ  r  r•   rl  r“   rª   r¢   r6  r¬   r5  r�  ru  r¹   r–  r—  r  r  rü  r�  re   rï   ré   rö   Ú_int8_linear_turing_quantizedrˆ  Ú_DISABLE_CUTLASS_INT8rû   ÚexpandÚcutlass_int8_dequantr×   r  r  r,  r0  ró  Úcublas_gemm_int8rn  Údequantize_int8_linear)rÚ  rú  rÜ  rƒ  rÝ  rÞ  rë   rí   rì   r»  râ  rã  rä  r›  Úturing_outputÚused_cutlassÚprefer_cublas_fallbackÚ
ws_cutlassÚpadded_kÚpadded_nÚcublas_xÚcublas_weightÚ	out_int32s   &&&&&&                 rW   Ú_int4_linear_via_int8_valuesr  ×  sh  € ð ‡z�zƒ|�qÔ˜KŸO™OÓ-°Ô2ÜÐ[Ó\Ð\Ø‡|�|�A…˜+×+Ñ+¨AÕ.Ô.ÜÐLÓMÐMØ�<‰<�D€AØ×Ñ˜!Õ€AÜ�[Š[˜!˜ y¿¹ÔG€Fà—*‘* F§M¡M¼¿¹�*ÓG×OÑOÐPRÓS×^Ñ^Ó`€KØ#—‘¨f¯m©mÄ5Ç=Á=�ÓQ×YÑYÐZ\Ó]×hÑhÓjÐØ×ÑÓ˜aÔÜÐ;¸A¸3¸mÈK×L]ÑL]ÓL_ÐK`ÐaÓbÐbØ×ÑÓ 1Ô$ÜÐ@ÀÀÀ=ÐQa×QgÑQgÓQiÐPjÐkÓlÐlØDHÒDTÔ  §}¡}°iÔ@Ü'¨¯©°yÓAð ô —‘×*Ñ*¨6¯=©=Ó9×EÑE€JØˆA„v�!�a•%˜1”*¤¬Ð-@×!AÒ!AÜ
×ÑÜ˜VÓ$Ü˜[Ó)Ü˜[×0Ñ0°°AÓ6Ó7ÜÐ-Ó.Ü˜XÓ&Ü˜VÓ$Ü˜)Õ$Øô		
ð ˆô 	" !¨×*Ò*Ü2°6×3DÑ3DÓ3F×GÒGÜ”BÐ5×6Ò6ä5ØØØØØÒ(‰H¨dØó
ˆð Ò$Ø Ð à€LÜ9Ø	Ø	Ø	Ø×ÑÓó	Ð÷ #ß%Ó%Ü6°v×>Ò>à)9×)?Ñ)?Ó)AÀQÔ)FÑ%ÐL\×LcÑLcÐdeÓLf×LqÑLqÓLsˆ
Ü×.Ñ.Ü˜VÓ$Ü˜[Ó)Ü˜[×0Ñ0°°AÓ6Ó7Ü˜ZÓ(Ü˜XÓ&Ü˜VÓ$Ü˜)Õ$Ü�J‰J×%Ñ% f§m¡mÓ4×@Ñ@ó	
ˆ÷ Øˆä˜f×/Ñ/Ó1×2Ò2Ü˜Q Ó#ˆÜ˜QÔ 8¸Ó @ÓAˆÜ ¨Ó1ˆÜ$¤\°+¸xÓ%HÈ(ÓS‰àˆØˆØ#ˆä—’˜Q ˜M´·±ÀVÇ]Á]ÔS€IÜ×ÑÜ˜Ó"Ü˜Ó'Ü˜Ó#ÜÔ-Ó/Ó0Øôð �1„}Ø˜a  ! ˜eÕ$×/Ñ/Ó1ˆ	Ü×ÑØØ×Ñ˜A˜qÓ!ØØØÜ�iÕ Øôð €MrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  R,          /# ©	r’   Úx_qdataÚweightrÜ  rƒ  rÝ  NrÞ  r”   r”  )r—   s   "rW   r˜   r˜   H  ót   € ÷ ñ Ü�\‰\ðä�L‰Lðô �\‰\ðô —,‘,ð	ô
 �,‰,˜Õ
ðô �{‰{ðô ‡\�\�DÕñrY   c                ó’  € V P                   ^ ,          pVP                   ^ ,          p\        P                  ! Wg3WPP                  R7      pVP	                  R4      p	Ve,   VP                  V P                  VR7      P                  4       M\        V P                  V4      p
\        P                  \        V 4      \        V4      \        VP	                  R4      4      \        V	4      \        V
4      \        V4      \        V,          \        P                  P                  V P                  4      P                  4      pV'       g   R# V# )zKRun row-wise INT8 GEMM with Turing tensor cores and a fused scale epilogue.r%  Nr~  r4  )r  r•   rl  r“   r6  rª   r¬   ru  r  rý  r�  re   r¹   r–  r—  )r  r  rÜ  rƒ  rÝ  rÞ  rë   rì   r»  rã  rä  Ú	used_int8s   &&&&&&      rW   rþ  rþ  H  sý   € ð 	�‰�aÕ€AØ�‰�Q�€AÜ�[Š[˜!˜ y¿¹ÔH€FØ#×+Ñ+¨BÓ/Ðð Òð 	�‰�w—~‘~¨YˆÓ7×BÑBÔDä §¡°	Ó:ð ô
 ×.Ñ.Ü˜Ó!Ü˜Ó Ü˜Ÿ™¨Ó,Ó-ÜÐ)Ó*Ü˜Ó"Ü˜Ó Ü�iÕ Ü�
‰
×!Ñ! '§.¡.Ó1×=Ñ=ó	€I÷ ÙØ€MrY   c                ó4  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  R,          R	\         P                  /# )
r’   r  r  rÜ  rƒ  rÝ  NrÞ  rú  r”   r”  )r—   s   "rW   r˜   r˜   i  sŒ   € ÷ eñ eÜ�\‰\ðeä�L‰Lðeô �\‰\ðeô —,‘,ð	eô
 �,‰,˜Õ
ðeô �{‰{ðeô —‘ Õ$ðeô ‡\�\ñerY   c                 óÔ  € V P                   w  rxVP                   ^ ,          p	V^,          p
\        V 4      pVP                   ^ ,          \        8:  d$   \        \        R4      '       d   \        W±W#WE4      # Vf   \        W±W#WE4      # VP                   Wš38w  g;   VP                  \        P                  8w  g   VP                  VP                  8w  d   \        R4      h\        W¶W#WE4      # )r   rá  zFprepared INT8 fallback weight has incompatible shape, dtype, or device)r  r×  r  r  r  rå  rø  r›   r•   r«   r“   rÊ  r  )r  r  rÜ  rƒ  rÝ  rÞ  rú  Ú_mÚk_halfrì   rí   rÚ  s   &&&&&&&     rW   Ú_int4_linear_via_int8r  i  sÄ   € ð —‘�J€BØ�‰�Q�€AØ��
€AÜ& wÓ/€FØ‡|�|�A…Ô9Ô9¼gÄbÐJm×>nÒ>nÜ1°&À'ÐY]ÓiÐiØÒÜ9¸&È'ÐaeÓqÐqØ	×	Ñ	˜q˜fÔ	$¨×(9Ñ(9¼U¿Z¹ZÔ(GÈ;×K]ÑK]Ðag×anÑanÔKnÜÐaÓbÐbÜ'¨¸WÐTXÓdÐdrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  R,          /# r  r”  )r—   s   "rW   r˜   r˜     r  rY   c           
     óŽ  € V P                   ^ ,          pVP                   ^ ,          p\        P                  ! Wg3WPP                  R7      p\        P                  P                  V P                  4      P                  p	Ve:   VP                  V P                  \        P                  R7      P                  4       M#\        V P                  \        P                  4      p
\        P                  \        V 4      \        V4      \        V4      \        V4      \        V
4      \        V4      \        V,          V	4      pV'       g   R# V# )z?Run packed W4A4 with Turing's native m8n8k32 INT4 tensor cores.r%  Nr~  )r  r•   rl  r“   r¹   r–  r—  rª   r¢   r¬   ru  r  r  r�  re   )r  r  rÜ  rƒ  rÝ  rÞ  rë   rì   r»  r›  rä  Ú	used_int4s   &&&&&&      rW   Ú_int4_linear_turingr    sï   € ð 	�‰�aÕ€AØ�‰�Q�€AÜ�[Š[˜!˜ y¿¹ÔH€FÜ—‘×*Ñ*¨7¯>©>Ó:×FÑF€Jð Òð 	�‰�w—~‘~¬U¯]©]ˆÓ;×FÑFÔHä §¡´·±Ó>ð ô
 ×.Ñ.Ü˜Ó!Ü˜Ó Ü˜Ó!Ü˜Ó&Ü˜Ó"Ü˜Ó Ü�iÕ Øó	€I÷ ÙØ€MrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\         P                  /# r  r”  )r—   s   "rW   r˜   r˜      sx   € ÷ Kñ KÜ�\‰\ðKä�L‰LðKô �\‰\ðKô —,‘,ð	Kô
 �,‰,˜Õ
ðKô �{‰{ðKô ‡\�\ñKrY   c           
     óì  € V P                  4       ^8w  g   VP                  4       ^8w  d   \        R4      hV P                  ^,          VP                  ^,          8w  d   \        R4      hV P                  w  rgVP                  ^ ,          pV^,          p	\        P                  ! Wh3WPP
                  R7      p
VP                  V P
                  \        P                  R7      P                  R4      P                  4       pVP                  V P
                  \        P                  R7      P                  R4      P                  4       pVP                  4       V8w  d    \        RV RVP                  4        24      hVP                  4       V8w  d    \        RV RVP                  4        24      hVe   \        W@P
                  V4      M\        V P
                  V4      p\        P                  P                  V P
                  4      P                  p\!        V 4      '       d9   \#        V P                  4       VP                  4       TTVe   TMRV4      pVe   V# \%        V 4      '       g8   \'        V P                  4       VP                  4       TTVe   VV4      # RV4      # R	p\(        '       gÁ   \+        WhW�P-                  4       4      '       g¡   \/        V 4      '       d�   \1        \2        R
4      '       dz   \2        P5                  \7        V P                  4       4      \7        VP                  4       4      \7        V4      \7        V4      \7        V4      \7        V
4      \8        V,          V4      pV'       d   V
# \2        P;                  \7        V P                  4       4      \7        VP                  4       4      \7        V4      \7        V4      \7        V4      \7        V
4      \8        V,          V4       V
# )zNSigned INT4 linear: output = (x_q @ weight.T) * x_scale * weight_scale + bias.z4INT4 linear expects 2D activation and weight tensorsz7INT4 linear activation/weight K dimensions do not matchr%  r~  zINT4 x_scale must have rà  zINT4 weight_scale must have NFÚcutlass_int4_dequantr4  )r'  rÊ  r  r•   rl  r“   rª   r¢   r6  r¬   r5  r�  ru  r¹   r–  r—  r  r  rÿ   r  rÿ  r  rö   rû   r  r  r   r�  re   r7   )r  r  rÜ  rƒ  rÝ  rÞ  rë   r  rì   rí   r»  râ  rã  rä  r›  r  r  s   &&&&&&           rW   r7   r7      s:  € ð ‡{�{ƒ}˜Ô˜VŸZ™Z›\¨QÔ.ÜÐOÓPÐPØ‡}�}�QÕ˜6Ÿ<™<¨�?Ô*ÜÐRÓSÐSØ—‘�I€AØ�‰�Q�€AØ��
€AÜ�[Š[˜!˜ y¿¹ÔH€FØ—*‘* G§N¡N¼%¿-¹-�*ÓH×PÑPÐQSÓT×_Ñ_Óa€KØ#—‘¨g¯n©nÄEÇMÁM�ÓR×ZÑZÐ[]Ó^×iÑiÓkÐØ×ÑÓ˜aÔÜÐ2°1°#°]À;×CTÑCTÓCVÐBWÐXÓYÐYØ×ÑÓ 1Ô$ÜÐ7¸°s¸-ÐHX×H^ÑH^ÓH`ÐGaÐbÓcÐcØEIÒEUÔ  §~¡~°yÔAÜ'¨¯©¸	ÓBð ä—‘×*Ñ*¨7¯>©>Ó:×FÑF€JÜ˜w×'Ò'Ü+Ø×ÑÓ Ø×ÑÓØØØÒ(‰H¨dØó
ˆð Ò$Ø Ð Ü0°×9Ò9Ü$Ø×ÑÓ Ø×ÑÓØØØÒ(ˆHØó
ð 	
ð
 /3Øó
ð 	
ð €Lç!Ó!Ü*¨1°×4FÑ4FÓ4H×IÒIÜ6°w×?Ò?Ü”BÐ.×/Ò/ä×.Ñ.Ü˜W×/Ñ/Ó1Ó2Ü˜V×.Ñ.Ó0Ó1Ü˜[Ó)ÜÐ-Ó.Ü˜XÓ&Ü˜VÓ$Ü˜)Õ$Øó	
ˆ÷ ØˆÜ‡N�NÜ˜×+Ñ+Ó-Ó.Ü˜×*Ñ*Ó,Ó-Ü˜Ó%ÜÐ)Ó*Ü˜Ó"Ü˜Ó Ü�iÕ Øô	ð €MrY   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r’   r  r”   ri  )r—   s   "rW   r˜   r˜   î  s"   € ÷ -ñ -´·±ð -ÄÇÁñ -rY   c                óX   € V P                  4       ^8w  d   \        R4      h\        V 4      # )zLPrepare packed signed INT4 weight as INT8 for non-native INT4 GEMM fallback.z1prepared INT4 fallback weight expects a 2D tensor)r'  rÊ  r×  )r  s   &rW   r9   r9   î  s'   € à‡z�zƒ|�qÔÜÐLÓMÐMÜ$ VÓ,Ð,rY   rœ   c                óÀ   € V ^8„  d   QhR\         P                  R\        R\        R\        R,          R\        \         P                  \         P                  3,          /# )r’   r  Úconvrot_groupsizeÚquant_group_sizer¿  Nr”   rÀ  )r—   s   "rW   r˜   r˜   õ  sT   € ÷ %ñ %Ü�L‰Lð%äð%ô ð%ô ˜t�ð	%ô
 Œ5�<‰<œŸ™Ð%Õ&ñ%rY   c                ó¤  € V\         8w  d   \        R\          24      hV P                  4       ^8w  d"   \        R\        V P                  4       24      hV P                  R,          V,          ^ 8w  d#   \        RV P                  R,           RV 24      hV P                  4       pVR	9   dK   \        \        R4      '       d5   \        WDP                  R,          V4      '       d   \        VVVR7      w  rVMJ\        WP                  V P                  R7      p\        WGV4      p\        VP                  4       VR7      w  rVWVP                  R4      3# )
zNOffline ConvRot weight rotation followed by row-wise signed INT4 quantization.ú*int4 MMA kernel requires quant_group_size z,ConvRot W4A4 expects a 2D tensor, got shape úin_features ú$ not divisible by convrot_groupsize r=   ©r¿  r~  r4  rW  )rn   rÊ  r'  r¶   r  r¬   r  r  rY  r=   r‡   r“   r›   r‰   r<   r6  )	r  r$  r%  r¿  Ú	weight_2drÓ  r¯   ÚhÚ
weight_rots	   &&&&     rW   r?   r?   õ  s)  € ð Ô+Ô+ÜÐEÔFVÐEWÐXÓYÐYØ‡z�zƒ|�qÔÜÐGÌÈfÏlÉlÓH[ÐG\Ð]Ó^Ð^Ø‡|�|�BÕÐ+Õ+¨qÔ0Ü˜<¨¯©°RÕ(8Ð'9Ð9]Ð^oÐ]pÐqÓrÐrØ×!Ñ!Ó#€Ià˜]Ô*Ü”BÐ9×:Ò:Ü2°9¿o¹oÈbÕ>QÐSd×eÒeä7ØØØ 3ô
‰ˆˆvô Ð-·m±mÈ6Ï<É<ÔXˆÜ# IÐ2CÓDˆ
Ü-¨j×.CÑ.CÓ.EÐ[nÔo‰ˆØ—.‘. Ó$Ð$Ð$rY   c                ó°   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\         P                  R\         P                  /# )r’   rÓ  r¯   r$  r%  Úoutput_dtyper”   ©r•   r–   rµ   r›   )r—   s   "rW   r˜   r˜     s\   € ÷ Pñ PÜ�<‰<ðPä�L‰LðPô ðPô ð	Pô
 —+‘+ðPô ‡\�\ñPrY   c                ó®  € V\         8w  d   \        R\          24      hV P                  4       ^8w  d"   \        R\        V P                  4       24      hV P                  R,          ^,          pWR,          ^ 8w  d   \        RV RV 24      hVR	9   Ed   \        \        R4      '       d÷   V P                  4       pVP                  V P                  \        P                  R7      P                  R4      P                  4       p\        P                  ! VP                  ^ ,          V3W@P                  R7      p\        P                  P                  V P                  4      P                   p	\        P#                  \%        V4      \%        V4      \%        V4      VV	4       V# \'        V 4      P                  \        P                  4      p
W¡P                  V P                  \        P                  R7      P                  R^4      ,          p\)        W P                  \        P                  R7      p\+        VP-                  4       WÂ4      P                  V4      # )
zIDequantize packed ConvRot W4A4 weights and rotate back to original basis.r'  z1ConvRot W4A4 dequant expects 2D qdata, got shape r(  r)  Údequantize_int4_convrot64r~  r%  r4  rW  )rn   rÊ  r'  r¶   r  r  r  r¬   rª   r“   r•   r¢   r6  rl  r¹   r–  r—  r2  r�  ro   r‡   r‰   r§   )rÓ  r¯   r$  r%  r/  rí   rÖ  Ú	scale_argr»  r›  Úw_intÚw_rotr,  s   &&&&&        rW   r1   r1     sÇ  € ð Ô+Ô+ÜÐEÔFVÐEWÐXÓYÐYØ‡y�yƒ{�aÔÜÐLÌUÐSX×S^ÑS^ÓM_ÐL`ÐaÓbÐbØ�‰�B�˜!Õ€AØÕ Ô!Ü˜<¨ sÐ*NÐO`ÐNaÐbÓcÐcØ˜MÕ)¬g´bÐ:U×.VÒ.VØ×#Ñ#Ó%ˆØ—I‘I U§\¡\¼¿¹�IÓG×OÑOÐPRÓS×^Ñ^Ó`ˆ	Ü—’˜hŸn™n¨QÕ/°Ð3¸<×P\ÑP\Ô]ˆÜ—Z‘Z×.Ñ.¨u¯|©|Ó<×HÑHˆ
Ü
×$Ñ$Ü˜XÓ&Ü˜YÓ'Ü˜VÓ$ØØô	
ð ˆÜ" 5Ó)×,Ñ,¬U¯]©]Ó;€EØ—I‘I U§\¡\¼¿¹�IÓG×OÑOÐPRÐTUÓVÕV€EÜÐ)·,±,ÄeÇmÁmÔT€AÜ˜%Ÿ+™+›-¨Ó>×AÑAÀ,ÓOÐOrY   Úint4c                óê   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R\        R	\         P                  /# )
r’   r!  ÚqweightÚwscalesrÝ  Nr$  r%  Úlinear_dtyper”   )r•   r–   rµ   Ústr)r—   s   "rW   r˜   r˜   6  s…   € ÷ TKñ TKÜ‡|�|ðTKä�\‰\ðTKô �\‰\ðTKô �,‰,˜Õ
ð	TKô
 ðTKô ðTKô ðTKô ‡\�\ñTKrY   c           	     ó   € VR9  d   \        RV: 24      hV\        8w  d   \        R\         24      hV P                  R,          VP                  R,          ^,          8w  d;   \        RV P                  R,           RVP                  R,          ^,           24      hV P                  R,          V,          ^ 8w  d#   \        RV P                  R,           RV 24      hV P                  pV P                  RVR,          4      P	                  4       pVR8X  g%   \        V4      '       Eg8   \        V4      '       Eg&   VR8X  dx   VP                  R,          R,          ^ 8X  dY   RVP                  R,          u;8:  d   \        8:  d5   M M1\        WˆP                  R,          V4      '       d   \        W„4      w  ršM`\        WˆP                  R,          V4      '       d   \        W„4      w  ršM/\        WHP                  VP                  R7      p\        W‹V4      w  rš\!        V	P#                  4       4      '       d]   \%        VP	                  4       4      p\'        V	VV
VVV P                  4      pVP                  ! . VR	R OVP                  ^ ,          N5!  # V	P                  ^ ,          \(        8:  dZ   \+        \,        R
4      '       dD   \/        V	VV
VVV P                  4      pVP                  ! . VR	R OVP                  ^ ,          N5!  # \1        V	VV
VVV P                  4      pVR	VP                  ^ ,           P                  ! . VR	R OVP                  ^ ,          N5!  # VR9   dH   \+        \,        R4      '       d2   \3        WˆP                  R,          V4      '       d   \5        W„4      w  rêMH\        WHP                  VP                  R7      p\7        W‹V4      P	                  4       p\9        V4      w  rê\;        VVV
VVV P                  R7      pVR	VP                  ^ ,           P                  ! . VR	R OVP                  ^ ,          N5!  # )z>Compute ``x @ W.T + bias`` using ConvRot W4A4 signed INT4 MMA.r«   z8ConvRot W4A4 linear_dtype must be 'int4' or 'int8', got r'  úInput K=z does not match qweight K=r)  rœ   r~  Nrá  r=   )rÝ  rÞ  >   r6  r«   r4  rW  )rÊ  rn   r  r6  r¬   rÿ   r  r]  rT  rC   r_  Úquantize_int8_rowwise_convrotr‡   r“   r›   rD   r×   rö   r9   r  r  r  r  rå  rø  rY  r=   rˆ   r<   r7   )r!  r8  r9  rÝ  r$  r%  r:  rÄ  Úx2dÚ	qact_int8rÜ  r,  Úqweight_int8ÚoutÚqactÚx_rots   &&&&&&&         rW   r8   r8   6  sª  € ð Ð+Ô+ÜÐSÐT`ÑScÐdÓeÐeØÔ+Ô+ÜÐEÔFVÐEWÐXÓYÐYØ‡w�wˆr…{�g—m‘m BÕ'¨!Õ+Ô+Ü˜8 A§G¡G¨B¥K =Ð0JÈ7Ï=É=ÐY[ÕK\Ð_`ÕK`ÐJaÐbÓcÐcØ‡w�wˆr…{Ð&Õ&¨!Ô+Ü˜8 A§G¡G¨B¥K =Ð0TÐUfÐTgÐhÓiÐià—‘€JØ
�)‰)�B˜
 2�Ó
'×
2Ñ
2Ó
4€CØ�vÔÜ-¨c×2Ó2Ô6MÈc×6RÓ6Rð  Ô$Ø—	‘	˜"• Õ# qÔ(Ø�s—y‘y •}Ö<Ô(<×<Ü1°#·y±yÀµ}ÐFW×XÒXä!@ÀÓ!XÑˆI�wÜ-¨c·9±9¸Rµ=ÐBS×TÒTÜ!>¸sÓ!VÑˆI�wäÐ 1¿*¹*ÈCÏIÉIÔVˆAÜ!<¸SÐEVÓ!WÑˆIÜ! )×"6Ñ"6Ó"8×9Ò9Ü>¸w×?QÑ?QÓ?SÓTˆLÜ.ØØØØØØ—‘óˆCð —;’;ÐB 
¨3¨B ÐB°·±¸qÕ1AÓBÐBØ�?‰?˜1ÕÔ!@Ô@ÄWÌRÐQt×EuÒEuÜ4ØØØØØØ—‘óˆCð —;’;ÐB 
¨3¨B ÐB°·±¸qÕ1AÓBÐBÜ8ØØØØØØ�G‰Gó
ˆð �>�S—Y‘Y˜q•\Ð"×*Ò*ÐN¨J°s¸¨OÐN¸W¿]¹]È1Õ=MÓNÐNà˜]Ô*Ü”BÐ9×:Ò:Ü2°3¿	¹	À"½ÐGX×YÒYä7¸ÓO‰ˆˆgäÐ-·j±jÈÏ	É	ÔRˆÜ" 3Ð+<Ó=×HÑHÓJˆÜ-¨eÓ4‰ˆÜ
ØØØØØØ—'‘'ô€Cð ˆ~�—‘˜1•Ð×&Ò&ÐJ¨
°3°B¨ÐJ¸¿¹ÀqÕ9IÓJÐJrY   c          
      ó´   € V ^8„  d   QhR\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# )r’   rÅ  rG  r¿  Nr”   rÀ  )r—   s   "rW   r˜   r˜   �  sJ   € ÷ ñ Ü
�,‰,ðäðô ˜t�ðô Œ5�<‰<œŸ™Ð%Õ&ñ	rY   c                óò  € \         P                  ! V \         P                  R7      p\         P                  ! V P                  ^ ,          ^3\         P
                  V P                  R7      p\         P                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V4      TVRJ;'       d    V^ 8„  \        T;'       g    ^ 4      V4       W43# )zßFused online ConvRot rotation + per-row INT8 quantization (single kernel).

Avoids materializing the rotated bf16 activation in global memory. Expects a
contiguous 2D [M, K] input with K divisible by group_size (256 only).
rš   r%  N)r•   rÂ  r«   rl  r  r¢   r“   r¹   r–  r—  r  r>  r�  rµ   )rÅ  rG  r¿  rÆ  rÇ  r›  s   &&&   rW   r>  r>  �  s¼   € ô ×Ò˜D¬¯
©
Ô3€DÜ—’˜TŸZ™Z¨�]¨AÐ.´e·m±mÈDÏKÉKÔX€IÜ—‘×*Ñ*¨4¯;©;Ó7×CÑC€Jä×$Ñ$Ü˜ÓÜ˜ÓÜ˜Ó#ØØ 4Ð'×CÐCÐ,?À!Ñ,CÜÐ×$Ð$ 1Ó%Øôð ˆ?ÐrY   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r’   r  rG  r”   r  )r—   s   "rW   r˜   r˜   ¨  s)   € ÷ 
ñ 
¤u§|¡|ð 
Äð 
ÌÏÉñ 
rY   c                óì   € \         P                  ! V 4      p\         P                  P                  V P                  4      P
                  p\        P                  \        V 4      \        V4      VV4       V# )z2ConvRot weight rotation using the CUDA FHT kernel.)	r•   rÂ  r¹   r–  r“   r—  r  rB   r�  )r  rG  r»  r›  s   &&  rW   rB   rB   ¨  s[   € ä×Ò˜fÓ%€FÜ—‘×*Ñ*¨6¯=©=Ó9×EÑE€JÜ×!Ñ!Ü˜Ó Ü˜Ó ØØô	ð €MrY   Úquantize_w4a8_convrotc                óÈ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\        RR/# )	r’   ÚrotatedÚcodebook_f32ÚpackedÚs_relÚ	s_channelr¿  r”   Nr  )r—   s   "rW   r˜   r˜   »  sb   € ÷ ñ Ü�\‰\ðä—,‘,ðô �L‰Lðô �<‰<ð	ô
 �|‰|ðô ðð 
ñrY   c                ól  € \         P                  P                  V P                  4      P                  p\
        P                  \        V P                  4       4      \        V4      \        V4      \        VP                  \         P                  4      4      \        V4      V^ 8„  \        V4      V4       R# )zZRun the fused kernel on ``rotated``, writing into the given (possibly row-sliced) outputs.N)r•   r¹   r–  r“   r—  r  rI  r�  r¬   r¥   r¤   rµ   )rK  rL  rM  rN  rO  r¿  r›  s   &&&&&& rW   Ú_quantize_w4a8_rotatedrQ  »  s‚   € ô —‘×*Ñ*¨7¯>©>Ó:×FÑF€JÜ×ÑÜ˜×+Ñ+Ó-Ó.Ü˜Ó&Ü˜Ó Ü˜Ÿ™¤E§K¡KÓ0Ó1Ü˜Ó#Ø˜aÑÜÐÓ Øö	rY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        \         P                  \         P                  \         P                  R\         P                  3,          /# )r’   r  r®   r$  r¿  r”   NrÀ  )r—   s   "rW   r˜   r˜   Ñ  sb   € ÷ .ñ .Ü�L‰Lð.ä�l‰lð.ô ð.ô ð	.ô
 Œ5�<‰<œŸ™¤u§|¡|°T¼5¿<¹<ÐGÕHñ.rY   c           
     óô  € V P                   w  rEV^,          pVP                  V P                  \        P                  R7      P                  4       p\        P                  ! WE^,          \        P                  V P                  R7      p\        P                  ! WF\        P                  V P                  R7      p	\        P                  ! V\        P                  V P                  R7      p
\        ^\        \        V^4      ,          4      p\        ^ WK4       FX  p\        WË,           V4      p\        WV P                  4       V4      pV^ 8”  d	   W<,           M^ p\        WçWŒV WœV W¬V V4       ?KZ  	  W‰V
RV3# )a  Rotate + fused requant (group_size 16, fp8 s_rel), row-chunked so peak memory stays
capped -- no full rotated copy. The kernel is per-row and s_rel/s_channel/packed are all
per-row, so row blocks write straight into their output slices (no cross-row reduction).r~  r%  N)r  rª   r“   r•   r¢   r¬   rl  r«   r¦   r  rp   Úrangeré  rB   rQ  )r  r®   r$  r¿  rì   rí   ÚgroupsÚcbrM  rN  rO  ÚblockÚr0Úr1ÚrotÚseeds   &&&&            rW   Ú_quantize_w4a8_stagedr\  Ñ  s%  € ð �<‰<�D€AØ�"�W€FØ	�‰˜FŸM™M´·±ˆÓ	?×	JÑ	JÓ	L€BÜ�[Š[˜ �F¬%¯*©*¸V¿]¹]ÔK€FÜ�KŠK˜¬×)<Ñ)<ÀVÇ]Á]ÔS€EÜ—’˜A¤U§]¡]¸6¿=¹=ÔI€IÜ�Ô)¬S°°A«YÕ6Ó7€EÜ�A�qÖ ˆÜ�•˜QÓˆÜ(¨°2¨×)AÑ)AÓ)CÐEVÓWˆà+>ÀÔ+BÐ"Ö'ÈˆÜ˜s¨°"¨°uÀ°|ÀYÐRTÐEUÐW[Ô\Úñ !ð ˜) T¨2Ð-Ð-rY   Úquantize_wxa8_convrot_fusedc                ó   € V P                   w  rg\        '       dR   V P                  \        P                  \        P
                  39  g#   VR8w  g   VR,          ^ 8w  g   Wr,          ^ 8w  d   R# V^8X  ds   VP                  V P                  \        P                  R7      P                  4       p\        P                  ! Wg^,          \        P                  V P                  R7      p	Mc\        V P                  \        P                  4      p\        P                  ! Wg^,          ^,          \        P                  V P                  R7      p	\        P                  ! WgV,          \        P                  V P                  R7      p
\        P                  ! V\        P                  V P                  R7      p\        P                  \!        V P                  4       4      \!        V4      \!        V	4      \!        V
P#                  \        P$                  4      4      \!        V4      VVV^ 8„  \'        V4      \        P(                  P+                  V P                  4      P,                  4
      pV'       g   R# YšTRV^8X  d   V3# R3# )z§One launch: ConvRot + requant from the raw weight (fp8 s_rel, no scale search). None when
the kernel declines the dtype or shape; the caller selects a staged fallback.rœ   Nr~  r%  )r  Ú_WXA8_FUSED_QUANTr›   r•   Úfloat16Úbfloat16rª   r“   r¢   r¬   rl  r«   ru  r¦   r  r]  r�  r¥   r¤   rµ   r¹   r–  r—  )r  ÚbitsrG  r$  r®   r¿  rì   rí   rV  rM  rN  rO  Úoks   &&&&&&       rW   Ú_fused_quantize_wxa8rd  î  s©  € ð �<‰<�D€AßÓ §¡´e·m±mÄUÇ^Á^Ð5TÔ!TØ  CÔ'¨1¨s­7°a¬<¸1½>ÈQÔ;NÙØˆq„yØ�[‰[ §¡´U·]±]ˆ[ÓC×NÑNÓPˆÜ—’˜Q Q¥¬e¯j©jÀÇÁÔO‰ä §¡¬u¯}©}Ó=ˆÜ—’˜Q A¥¨¥
´%·*±*ÀVÇ]Á]ÔSˆÜ�KŠK˜ 
�?´%×2EÑ2EÈfÏmÉmÔ\€EÜ—’˜A¤U§]¡]¸6¿=¹=ÔI€IÜ	×	'Ñ	'Ü˜×*Ñ*Ó,Ó-Ü˜ÓÜ˜Ó Ü˜Ÿ™¤E§K¡KÓ0Ó1Ü˜Ó#ØØØ˜aÑÜÐÓ Ü�
‰
×!Ñ! &§-¡-Ó0×<Ñ<ó
€B÷ ÙØ˜) T°$¸!´)¨BÐFÐFÀÐFÐFrY   c                ó¦  € V ^8„  d   QhR\         P                  R\        R\        R\        R\         P                  R\        R\         P                  R,          R	\        R
\        R\        R\
        \         P                  \         P                  \         P                  \         P                  R,          \         P                  R,          3,          /# )r’   r  rG  r$  Ú	symmetricÚscale_dtyper®   Úcodebook_tensorNr¿  rb  Úscale_searchr”   )r•   r–   rµ   rÏ   r›   r¶   )r—   s   "rW   r˜   r˜     s¾   € ÷ 3ñ 3Ü�L‰Lð3äð3ô ð3ô ð	3ô
 —‘ð3ô ð3ô —\‘\ DÕ(ð3ô ð3ô ð3ô ð3ô Ü	‡L�LÜ	‡L�LÜ	‡L�LÜ	‡L�L�4ÕÜ	‡L�L�4Õð	õñ3rY   c
                óš  € \        WW(4       V\        P                  8H  ;'       dP    T;'       dF    V^8H  ;'       d    T;'       d    V^8H  ;'       g"    V^8H  ;'       d    V	'       * ;'       d    VR9   p
V
'       dµ   RpV^8X  d   Ve   TM\        V \        W4      p\        WWW·4      pVe   V# V^8X  ds   \        '       dg   V P                  \        P                  \        P                  \        P                  39   d)   V P                  ^,          \        8:  d   \        WW'4      # Ve   TMTp\        V \        VVVVVVVVV	R7      # )a  Prepare W4A8/W6A8 weights: the fused CUDA requant for the default configs (4-bit symmetric
codebook g16, or 6-bit without the quantize-time scale search; fp8 scales). W4A8
uses staged CUDA requant when fusion is unavailable; other configs use chunked eager.N)	rG  r$  rf  rg  r®   Úcodebook_overrider¿  rb  ri  )rJ  r  rK  )rv   r•   r¦   rq   rB   rd  Ú_W4A8_STAGED_QUANTr›   r¢   r`  ra  r  Ú_W4A8_STAGED_MAX_Kr\  rs   )r  rG  r$  rf  rg  r®   rh  r¿  rb  ri  Úfused_okrV  rB  s   &&&&&&&&&&   rW   r@   r@     sG  € ô* ˜vÐ3DÔKØœe×1Ñ1Ñ1÷ ð °i÷ ð Ø	�‰×	4Ð	4�x×	4Ð	4 J°"Ñ$4÷ 	Kð 	KØ�A‰I×IÐI˜lÔ*×IÐI¨z¸\Ñ/Ið ÷ ØˆØ�1Œ9Ø%4Ò%@‘/Ü'¨Ô0JÈJÓjð ä" 6°ÐPRÓhˆØŠ?ØˆJØ�AŒI×,Ó,Ø—L‘L¤U§]¡]´E·M±MÄ5Ç>Á>Ð$RÔRØ—L‘L •OÔ'9Ô9Ü(¨Ð5FÓ\Ð\Ø "¢™"°OˆÜ!ØÜ"ØØ+ØØØØ)Ø/ØØ!ôð rY   c          
      ó´   € V ^8„  d   QhR\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# )r’   r+  rG  r¿  Nr”   rÀ  )r—   s   "rW   r˜   r˜   D  sJ   € ÷ ñ Ü�|‰|ðäðô ˜t�ðô Œ5�<‰<œŸ™Ð%Õ&ñ	rY   c                ó  € V P                   R,          V,          p\        P                  ! V 4      p\        P                  ! V P                   ^ ,          V3\        P                  V P
                  R7      p\        P                  ! V \        P                  R7      p\        P                  ! V P                   ^ ,          ^3\        P                  V P
                  R7      p\        P                  P                  V P
                  4      P                  p\        P                  \        V 4      \        V4      \        V4      \        V4      \        V4      TVRJ;'       d    V^ 8„  \        T;'       g    ^ 4      V4	       Wg3# )zCConvRot rotation with partial absmax followed by INT8 quantization.r%  rš   Nr4  )r  r•   rÂ  rl  r¢   r“   r«   r¹   r–  r—  r  Úquantize_int8_convrot_stagedr�  rµ   )	r+  rG  r¿  Ún_groupsrK  Úpartial_absmaxrÆ  rÇ  r›  s	   &&&      rW   rq  rq  D  s   € ð �‰˜rÕ" jÕ0€HÜ×Ò˜yÓ)€GÜ—[’[ )§/¡/°!Õ"4°hÐ!?ÄuÇ}Á}Ð]f×]mÑ]mÔn€NÜ×Ò˜I¬U¯Z©ZÔ8€DÜ—’˜YŸ_™_¨QÕ/°Ð3¼5¿=¹=ÐQZ×QaÑQaÔb€IÜ—‘×*Ñ*¨9×+;Ñ+;Ó<×HÑH€JÜ×#Ñ#Ü˜Ó#Ü˜Ó!Ü˜Ó(Ü˜ÓÜ˜Ó#ØØ 4Ð'×CÐCÐ,?À!Ñ,CÜÐ×$Ð$ 1Ó%Øô
ð ˆ?ÐrY   c                ó  € V ^8„  d   QhR\         P                  R\        R\        R,          R\        R,          R\         P                  R,          R\        R\
        \         P                  \         P                  3,          /# )	r’   r+  rG  r¿  NÚ	input_actÚinput_act_weightÚinput_act_epsr”   )r•   r–   rµ   r;  r§   r¶   )r—   s   "rW   r˜   r˜   ^  su   € ÷ #ñ #Ü�|‰|ð#äð#ô ˜t�ð#ô �T�zð	#ô
 —l‘l TÕ)ð#ô ð#ô Œ5�<‰<œŸ™Ð%Õ&ñ#rY   c                óî  € \         P                  ! V P                  ^ ,          V P                  ^,          \        V4      ,          3\         P                  V P
                  R7      p\         P                  ! V P                  ^ ,          ^3\         P                  V P
                  R7      p\         P                  P                  V P
                  4      P                  p\        W4V P
                  V P                  4      p	\        P                  \        V 4      \        V4      \        V4      TVRJ;'       d    V^ 8„  \        V4      \!        T;'       g    ^ 4      \        V	4      \#        V4      V4
       Wg3# )a   Fused ConvRot row-wise INT8 quantization using 64-lane groups.

input_act applies an activation on the way in, so an MLP's
`linear(act(proj(x)))` never materializes act's output in bf16. For the
gated pairs ("swiglu" on [gate | up], "situ" on [hidden | gate]) the quantized
rows are half as wide as the input rows. "rms_norm" normalizes each row with input_act_weight
(K elements) and input_act_eps before rotating.
r%  N)r•   rl  r  Ú_input_act_widthr«   r“   r¢   r¹   r–  r—  Ú_act_weight_argr›   r  rC   r�  Ú_input_act_coderµ   r§   )
r+  rG  r¿  ru  rv  rw  rÆ  rÇ  r›  Úact_weight_args
   &&&&&&    rW   rC   rC   ^  s$  € ô  �;Š;Ø	�‰˜Õ	˜YŸ_™_¨QÕ/Ô3CÀIÓ3NÕNÐOÜ�j‰j ×!1Ñ!1ô3€Dô —’˜YŸ_™_¨QÕ/°Ð3¼5¿=¹=ÐQZ×QaÑQaÔb€IÜ—‘×*Ñ*¨9×+;Ñ+;Ó<×HÑH€JÜ$Ø Y×%5Ñ%5°y·±óH€Nä×&Ñ&Ü˜Ó#Ü˜ÓÜ˜Ó#ØØ 4Ð'×CÐCÐ,?À!Ñ,CÜ˜	Ó"ÜÐ×$Ð$ 1Ó%Ü˜Ó(ÜˆmÓØôð ˆ?ÐrY   i @  ÚCOMFY_KITCHEN_DISABLE_CUTLASSc          
      ó  € V ^8„  d   QhR\         P                  R\         P                  \        ,          \        ,          R,          R\        R,          R\
        \         P                  \         P                  3,          /# )r’   r!  r’  Nr¿  r”   )r•   r–   r§   r;  rµ   r¶   )r—   s   "rW   r˜   r˜   �  sc   € ÷ cñ cÜ‡|�|ðcä�<‰<œ%Õ¤#Õ%¨Õ,ðcô ˜t�ðcô Œ5�<‰<œŸ™Ð%Õ&ñ	crY   c                ó   € \        WVR7      # )z8Quantize tensor to INT8 with a single tensor-wise scale.)r’  r¿  )Úeager_quantize_int8_tensorwise)r!  r’  r¿  s   &&&rW   r:   r:   �  s   € ô *¨!ÐNaÔbÐbrY   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r’   Úqr’  r”   ri  )r—   s   "rW   r˜   r˜   ™  s2   € ÷ Pñ PœeŸl™lð P´5·<±<ð PÄEÇLÁLñ PrY   c                óL   € \        W\        \        P                  ,          4      # )z"Dequantize INT8 tensor with scale.)r.   re   r•   r¢   )r‚  r’  s   &&rW   r-   r-   ™  s   € ä'¨´-ÄÇÁÕ2NÓOÐOrY   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# )r’   r‚  r’  r™  r”   r  )r—   s   "rW   r˜   r˜   ž  s8   € ÷ "ñ "¤E§L¡Lð "¼¿¹ð "ÔZ]ð "Ôbg×bnÑbnñ "rY   c                ón  € V P                  4       '       g   V P                  4       p RpV P                  4       ^ 8”  d   V P                  R,          M^pVP	                  4       ^8X  d   ^ pMµ\        VP                  4      \        V P                  4      8X  d   ^pM…V P                  4       ^ 8”  dq   VP                  4       V P                  4       8X  dN   \        VP                  RR 4      \        V P                  RR 4      8X  d   VP                  R,          ^8X  d   ^pV^ 8  d&   \        W4      P                  \        V,          4      # VP                  V P                  \        P                  R7      P                  4       p\        V,          p\        P                  ! V P                  WPP                  R7      p\        P                  P                  V P                  4      P                  p\         P#                  \%        V 4      \%        VP'                  R4      4      \%        V4      VVV4       V# )zKDequantize INT8 tensor with scale into the requested floating output dtype.Nr~  r%  r4  )r  r¬   r'  r  r5  r¶   Úeager_dequantize_int8_simplerª   rd   r“   r•   r¢   rl  r¹   r–  r—  r  r-   r�  r6  )r‚  r’  r™  Ú
scale_modeÚ	inner_dimr/  r»  r›  s   &&&     rW   r.   r.   ž  s�  € à�?‰?×ÒØ�L‰L‹Nˆà€JØ Ÿu™u›w¨œ{�—‘˜–°€IØ‡{�{ƒ}˜ÔØ‰
Ü	ˆu�{‰{Ó	œu Q§W¡W›~Ô	-Ø‰
à	�‰‹�!ŒØ�I‰I‹K˜1Ÿ5™5›7Ô"Ü�%—+‘+˜c˜rÐ"Ó#¤u¨Q¯W©W°S°b¨\Ó':Ô:Ø�K‰K˜�O˜qÔ àˆ
à�A„~Ü+¨AÓ5×8Ñ8Ô9LÐM^Õ9_Ó`Ð`à�H‰H˜AŸH™H¬E¯M©MˆHÓ:×EÑEÓG€EÜ&Ð'8Õ9€LÜ�[Š[˜Ÿ™¨¿X¹XÔF€FÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×ÑÜ˜ÓÜ˜Ÿ™ rÓ*Ó+Ü˜Ó ØØØôð €MrY   c                óÔ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# )r’   r!  r,  rG  r¿  Nr”   rÀ  )r—   s   "rW   r˜   r˜   Ã  s_   € ÷ Qñ QÜ‡|�|ðQä‡|�|ðQô ðQô ˜t�ð	Qô
 Œ5�<‰<œŸ™Ð%Õ&ñQrY   c                ó2   € \        WV4      p\        WCR7      # )zBOnline activation rotation followed by CUDA row-wise quantization.r*  )rˆ   r;   )r!  r,  rG  r¿  rD  s   &&&& rW   rD   rD   Ã  s   € ô ˜q ZÓ0€EÜ  ÔPÐPrY   c          
      ó´   € V ^8„  d   QhR\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# )r’   r  rG  r¿  Nr”   rÀ  )r—   s   "rW   r˜   r˜   Î  sP   € ÷ %tñ %tÜ�L‰Lð%täð%tô ˜t�ð%tô Œ5�<‰<œŸ™Ð%Õ&ñ	%trY   c                óv  € V P                  4       ^8w  d   \        R4      hV P                  4       pVP                  R,          pVR8X  dD   VR,          ^ 8X  d6   RTu;8:  d   \        8:  d#   M M\        W4V4      '       d   \        W1VR7      # VR8X  d.   VR,          ^ 8X  d    RTu;8:  d   R8  d   M M\        W1VR7      # \        W4V4      '       d   \        W1VR7      # VR8X  d   VR,          ^ 8X  d   \        W1VR7      # \        WP                  VP                  R7      p\        \        W5V4      VR7      # )zãOffline ConvRot weight rotation followed by row-wise INT8 quantization.

Uses the fused ConvRot CUDA kernel when it matches the linear path's launch
heuristics, otherwise falls back to explicit rotation plus row-wise quantize.
z;ConvRot INT8 weight quantization expects a 2D weight tensorrœ   r*  r  r\  r~  r4  )r'  rÊ  r¬   r  r]  rT  rC   rq  r_  r>  r‡   r“   r›   r;   r‰   )r  rG  r¿  r+  rí   r,  s   &&&   rW   rA   rA   Î  s  € ð ‡z�zƒ|�qÔÜÐVÓWÐWà×!Ñ!Ó#€IØ�‰˜Õ€Aà�cÔØ��G�qŒLØ�1Ö,Ô,×,Ü-¨i¸J×GÒGä.¨yÐZmÔnÐnð 	�cÔØ��G�qŒLØ�AÖ˜×ä+¨IÐWjÔkÐkä'¨	°j×AÒAÜ,¨YÐXkÔlÐlà�SÔ˜Q �W¨œ\Ü+¨IÐWjÔkÐkä˜
×+;Ñ+;À9Ç?Á?ÔS€AÜ ¤°	¸jÓ!IÐ_rÔsÐsrY   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# )r’   r‚  r’  rG  r”   r  )r—   s   "rW   r˜   r˜   ö  s>   € ÷ dñ d¤e§l¡lð d¼5¿<¹<ð dÔUXð dÔ]b×]iÑ]iñ drY   c                óN   € \        WV\        \        P                  ,          4      # )zKDequantize ConvRot INT8 weights and rotate them back to the original basis.)r0   re   r•   r¢   )r‚  r’  rG  s   &&&rW   r/   r/   ö  s   € ä/°¸*ÄmÔTY×TaÑTaÕFbÓcÐcrY   c          
      ó�   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\         P                  /# )r’   r‚  r’  rG  r™  r”   r  )r—   s   "rW   r˜   r˜   û  sG   € ÷ _ñ _Ü‡|�|ð_Ü!ŸL™Lð_Ü69ð_ÜNQð_ä
‡\�\ñ_rY   c                ó  € V P                  4       ^8w  d   \        R4      hV P                  4       pVP                  R,          p\        V,          p\        WEV4      '       dÞ   VP                  VP                  \        P                  R7      P                  R4      P                  4       p\        P                  ! VP                  WdP                  R7      p\        P                  P                  VP                  4      P                  p	\        P!                  \#        V4      \#        V4      \#        V4      VV	4       V# \%        W$P                  \        P                  R7      p
\'        \)        WA4      W¢4      P                  V4      # )zNDequantize ConvRot INT8 weights and rotate them back into the requested dtype.z8ConvRot INT8 weight dequantization expects a 2D q tensorr~  r%  r4  )r'  rÊ  r¬   r  rd   rb  rª   r“   r•   r¢   r6  rl  r¹   r–  r—  r  r/   r�  r‡   r‰   r-   )r‚  r’  rG  r™  rÆ  rí   r/  r3  r»  r›  r,  s   &&&&       rW   r0   r0   û  s  € ð 	‡u�uƒw�!„|ÜÐSÓTÐTà�<‰<‹>€DØ�
‰
�2�€AÜ&Ð'8Õ9€LÜ)¨$°:×>Ò>Ø—H‘H D§K¡K´u·}±}�HÓE×MÑMÈbÓQ×\Ñ\Ó^ˆ	Ü—’˜TŸZ™Z¨|ÇKÁKÔPˆÜ—Z‘Z×.Ñ.¨t¯{©{Ó;×GÑGˆ
Ü
×)Ñ)Ü˜TÓ"Ü˜YÓ'Ü˜VÓ$ØØô	
ð ˆä˜
¯;©;¼e¿m¹mÔL€AÜÔ0°Ó=¸qÓM×PÑPÐQ]Ó^Ð^rY   c                óø   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  /# )r’   r  r  rÜ  rƒ  rÝ  rÞ  r”   r”  )r—   s   "rW   r˜   r˜     sj   € ÷ ñ Ü�\‰\ðä�L‰Lðô �\‰\ðô —,‘,ð	ô
 �,‰,ðô �{‰{ðô ‡\�\ñrY   c           
     óª  € \         P                  ! ^VP                  ^ ,          3WPP                  R7      pVe   TM\	        V P                  V4      pVen   VP                  V P                  8w  g(   VP
                  V8w  g   VP                  4       '       g,   VP                  V P                  VR7      P                  4       p\         P                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V4      \        V4      \        V4      \        V4      \        V,          V4       V# )z/Single-row INT8 GEMV with fused dequantization.r%  r~  )r•   rl  r  r“   ru  r›   r  rª   r¬   r¹   r–  r—  r  rü  r�  re   )	r  r  rÜ  rƒ  rÝ  rÞ  rB  rä  r›  s	   &&&&&&   rW   rü  rü    sú   € ô �+Š+�q˜&Ÿ,™, q�/Ð*°)ÇNÁNÔ
S€CØÒ'‰tÔ-?ÀÇÁÐPYÓ-Z€HØÒ˜TŸ[™[¨G¯N©NÔ:¸d¿j¹jÈIÔ>UÐ]a×]oÑ]o×]qÒ]qØ—7‘7 '§.¡.¸	�7ÓB×MÑMÓOˆÜ—‘×*Ñ*¨7¯>©>Ó:×FÑF€JÜ×ÑÜ˜Ó!Ü˜Ó Ü˜Ó!Ü˜Ó&Ü˜Ó"Ü˜ÓÜ�iÕ Øô	ð €JrY   c                óš   € V ^8„  d   QhR\         P                  R\        R\        \         P                  \         P                  3,          /# )r’   r!  Úpad_32xr”   ©r•   r–   rÏ   r¶   )r—   s   "rW   r˜   r˜   1  s<   € ÷ !ñ !Ü‡|�|ð!äð!ô Œ5�<‰<œŸ™Ð%Õ&ñ!rY   c                 ó4  € V P                  4       '       g   Q R 4       hV P                  w  r#V'       d   \        V^ 4      p\        V^ 4      pM2Y#rTV^ ,          ^ 8X  g   Q RV 24       hV^ ,          ^ 8X  g   Q RV 24       h\        P                  ! WE3V P
                  \        P                  \        P                  R7      p\        V^€4      p\        V^ ,          ^4      p\        P                  ! Wx3V P
                  \        P                  R7      p	\        P                  P                  V P
                  4      P                  p
\        P                  \        V 4      \        V4      \        V	4      VV
4       V	P!                  \        P"                  4      pWk3# )rª  z&num_rows must be divisible by 32, got z&num_cols must be divisible by 32, got r«  r~  )r  r  r…   r•   rl  r“   r¦   r­  r)  r¤   r¹   r–  r—  r  rF   r�  r¥   Úfloat8_e8m0fnu)r!  r”  r®  r¯  r°  r±  r²  r³  r´  rµ  r›  r¶  s   &&          rW   rF   rF   1  sO  € ð �?‰?×ÒÐ?Ð?Ó?ÐàŸ7™7Ñ€IßÜ˜9 bÓ)ˆÜ˜9 bÓ)‰à&�(Ø˜"�} Ô!ÐVÐ%KÈHÈ:Ð#VÓVÐ!Ø˜"�} Ô!ÐVÐ%KÈHÈ:Ð#VÓVÐ!ä	�Š�hÐ)°!·(±(Ä%×BUÑBUÔej×e|Ñe|Ô	}€Bä˜ 3Ó'€JÜ˜ R�¨Ó+€JÜ�{Š{˜JÐ3¸A¿H¹HÌEÏKÉKÔX€Hä—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×ÑÜ˜ÓÜ˜ÓÜ˜Ó"ØØôð 
�‰”u×+Ñ+Ó	,€Bàˆ6€MrY   c                óX  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R	\         P                  R
\         P                  /
# )r’   ÚaÚbÚtensor_scale_aÚtensor_scale_bÚblock_scale_aÚblock_scale_brÝ  rÞ  Úalphar”   r”  )r—   s   "rW   r˜   r˜   U  s    € ÷ |ñ |Ü‡|�|ð|ä‡|�|ð|ô —L‘Lð|ô —L‘Lð	|ô
 —<‘<ð|ô —<‘<ð|ô �,‰,ð|ô �{‰{ð|ô �<‰<ð|ô ‡\�\ñ|rY   c	                 óP  € R p	^p
\        V\        P                  P                  4      '       d   VP                  p\        V\        P                  P                  4      '       d   VP                  pVf
   W#,          pM6\        V\        P                  P                  4      '       d   VP                  pVP
                  \        P                  8w  d    VP                  \        P                  4      pVP                  4       ^ 8X  d   VP                  ^4      p\        V \        P                  P                  4      '       d   V P                  p \        V\        P                  P                  4      '       d   VP                  p\        V\        P                  P                  4      '       d   VP                  p\        V\        P                  P                  4      '       d   VP                  pV P                  w  r¼VP                  w  rÞWÎ8X  g   Q R4       h^V,          p\        P                  ! W½WpP                  R7      pV^,          ^ 8X  g   Q R4       hVP
                  VP
                  8X  g   Q R4       hVP
                  \        P                  8X  d   V
^ 8X  g   Q R4       h\        R4      hVP
                  \        P                  8X  d]   V
^8X  g   Q R4       hVf   Q R4       hVP
                  \        P                  8X  g   Q R	4       hVP!                  4       ^8X  g   Q R
4       hM\        RVP
                   24      h\#        V^€4      p\#        V^€4      p\#        Wú,          ^4      pVP                  4       ^8X  g   Q R4       hVP%                  4       VV38X  g   Q R4       hVP                  4       ^8X  g   Q R4       hVP%                  4       VV38X  g   Q R4       hVf   \        P&                  ! 4       pM6VP
                  \        P(                  \        P*                  39   g   Q R4       hRR ppVP-                  \        P.                  4      pVP-                  \        P.                  4      p\0        V,          p\        P2                  P5                  V P                  4      P6                  pVe   VP!                  4       ^ 8X  d3   \        P                  ! ^ V P                  \        P(                  R7      pM6\        V\        P                  P                  4      '       d   VP                  p\8        P;                  \=        V4      \=        V4      \=        V 4      \=        V4      \=        V4      V\=        V4      \=        \?        4       4      V	\=        V4      V4       V# )FzMatrix dimensions do not matchr%  z-B tensor must have 8 alignment in N dimensionzA and B scale dtype must matchz#MXFP4 only supports block length 32z2MXFP4 is not supported yet for cuBLAS in CUDA 12.9z#NVFP4 only supports block length 16z alpha must be provided for NVFP4zalpha must be float32zalpha must be a scalarzUnsupported scale dtype: zInvalid A scale shapezInvalid B scale shapez*Only fp16 and bfloat16 bias are supported.Tr~  ) Ú
isinstancer•   ÚnnÚ	ParameterÚdatar›   r¢   rª   r'  r6  r  rl  r“   r—  rÊ  r¦   r5  r…   Úsizer–   r`  ra  r¥   r¤   re   r¹   r–  r—  r  Úcublas_gemm_blockwise_fp4r�  rn  )r™  rš  r›  rœ  r�  rž  rÝ  rÞ  rŸ  Ú
accumulateÚblock_lengthrë   Úk_arì   Úk_brí   rB  Ú	roundup_mÚ	roundup_nÚ
roundup_skÚ_transaÚ_transbÚblock_scale_b_uint8Úblock_scale_a_uint8Úout_dtype_coder›  s   &&&&&&&&&                 rW   rJ   rJ   U  s}  € ð €JØ€Lô �.¤%§(¡(×"4Ñ"4×5Ò5Ø'×,Ñ,ˆÜ�.¤%§(¡(×"4Ñ"4×5Ò5Ø'×,Ñ,ˆà‚}ØÕ/‰Ü	�Eœ5Ÿ8™8×-Ñ-×	.Ò	.Ø—
‘
ˆð ‡{�{”e—m‘mÔ#Ø—‘œŸ™Ó'ˆØ‡y�yƒ{�aÔØ—‘˜aÓ ˆô �!”U—X‘X×'Ñ'×(Ò(Ø�F‰FˆÜ�!”U—X‘X×'Ñ'×(Ò(Ø�F‰FˆÜ�-¤§¡×!3Ñ!3×4Ò4Ø%×*Ñ*ˆÜ�-¤§¡×!3Ñ!3×4Ò4Ø%×*Ñ*ˆà�W‰W�F€AØ�W‰W�F€AØŒ:Ð7Ð7Ó7ˆ:ð 	
ˆC�€Aä
�+Š+�a )·H±HÔ
=€Cð ˆq�5�AŒ:ÐFÐFÓFˆ:ð ×Ñ -×"5Ñ"5Ô5ÐWÐ7WÓWÐ5à×Ñœe×2Ñ2Ô2à˜rÔ!ÐHÐ#HÓHÐ!ÜÐMÓNÐNØ	×	Ñ	¤× 3Ñ 3Ô	3à˜rÔ!ÐHÐ#HÓHÐ!ØÒ ÐDÐ"DÓDÐ Ø�{‰{œeŸm™mÔ+ÐDÐ-DÓDÐ+Ø�{‰{‹} Ô!Ð;Ð#;Ó;Ñ!äÐ4°]×5HÑ5HÐ4IÐJÓKÐKä˜˜3“€IÜ˜˜3“€Iä˜Õ*¨AÓ.€Jà×ÑÓ !Ô#Ð<Ð%<Ó<Ð#Ø×ÑÓ I¨zÐ#:Ô:ÐSÐ<SÓSÐ:à×ÑÓ !Ô#Ð<Ð%<Ó<Ð#Ø×ÑÓ I¨zÐ#:Ô:ÐSÐ<SÓSÐ:à‚|Ü�|Š|‹~‰à�z‰zÜ�M‰MÜ�N‰Nð
ô 
ð 	8ð 8ó	8ð 
ð ˜UˆW€Gð (×,Ñ,¬U¯[©[Ó9ÐØ'×,Ñ,¬U¯[©[Ó9Ðä" 9Õ-€Nä—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€Jð ‚|�t—z‘z“| qÔ(Ü�{Š{˜1 Q§X¡X´U·]±]ÔC‰ô �dœEŸH™H×.Ñ.×/Ò/Ø—9‘9ˆDä× Ñ Ü˜ÓÜÐ,Ó-Ü˜ÓÜÐ,Ó-Ü˜ÓØÜ˜ÓÜÔ-Ó/Ó0ØÜ˜ÓØôð €JrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R\         P                  /# )r’   r!  r  rÝ  NÚresidualÚresidual_scaler”   ri  )r—   s   "rW   r˜   r˜   Ô  ss   € ÷ >Mñ >MÜ‡|�|ð>Mä�L‰Lð>Mô �,‰,˜Õ
ð>Mô �l‰l˜TÕ!ð	>Mô
 —L‘L 4Õ'ð>Mô ‡\�\ñ>MrY   c                óp  € Ve   Vf   \        R4      hV P                  pV P                  4       ^8X  d   V P                  4       '       d   T M0V P	                  RV P                  R,          4      P                  4       pVP                  ^ ,          pVP                  ^ ,          pV P                  \        P                  8H  ;'       Ed    VP                  \        P                  8H  ;'       dî    VP                  ;'       dÚ    VP                  V P                  8H  ;'       d¹    \        V4      ;'       d¦    \        V4      ;'       d“    VRJ ;'       g&    VP                  \        P                  8H  ;'       db    VRJ ;'       gV    VP                  \        P                  8H  ;'       d1    VP                  \        P                  8H  ;'       d    \        V4      p	V	'       g7   \        P                  P                  P                  WV4      p
\        W£V4      # VP                  4       '       g   VP                  4       p\        P                   P#                  V P                  4      P$                  p\        P&                  ! Wx3\        P                  V P                  R7      p
Ve%   \)        W P                  \        P                  4      M#\+        V P                  \        P                  4      pVf   RMVP	                  Wx4      P                  4       pVez   \,        P/                  \1        V4      \1        V4      \1        V4      \1        \)        W@P                  \        P                  4      4      \1        V4      \1        V
4      V4      '       g|   \,        P3                  \1        V4      \1        V4      \1        V4      \1        V
4      V4      pV'       g+   \        P                  P                  P                  WaV4      p
\        W­V4      p
\5        V4      ^8X  d   V
# V
P                  ! . VRR OVN5!  # )zfCUTLASS fp16-accumulate GEMM with fused bias/residual; cuBLAS when the
kernel cannot serve the shape.
Nz-fp16_linear: residual requires residual_scaler%  r4  )rÊ  r  r'  r  r6  r¬   r›   r•   r`  rõ   r“   rz  r¢  Ú
functionalÚlinearÚ_apply_residualr¹   r–  r—  rl  r�  ru  r  Úcutlass_fp16_linear_residualr�  Úcutlass_fp16_linearÚlen)r!  r  rÝ  r´  rµ  rÄ  rÅ  rë   rì   rú   rB  r›  rä  Úresid_2drc  s   &&&&&          rW   r3   r3   Ô  s@  € ð Ò Ò 6ÜÐHÓIÐIà—‘€JØ—‘“˜1” §¡×!2Ò!2‰1¸¿	¹	À"ÀaÇgÁgÈbÅkÓ8R×8]Ñ8]Ó8_€DØ�
‰
�1�€AØ�‰�Q�€Að 	
�‰”5—=‘=Ñ ÷ 
	=ñ 
	=Ø�L‰LœEŸM™MÑ)÷
	=ð 
	=à�N‰N÷
	=ð 
	=ð �M‰M˜QŸX™XÑ%÷
	=ð 
	=ô
 �tÓ÷
	=ð 
	=ô �vÓ÷
	=ð 
	=ð �Tˆ\×8Ð8˜TŸZ™Z¬5¯=©=Ñ8÷
	=ð 
	=ð ˜Ð÷ <ð < (§.¡.´E·M±MÑ"A÷ #;ð #;Ø&4×&:Ñ&:¼e¿m¹mÑ&K÷#;ð #;ä&0°Ó&:ð ÷ Ü�h‰h×!Ñ!×(Ñ(¨°DÓ9ˆÜ˜s¨nÓ=Ð=à×Ñ×!Ò!Ø×"Ñ"Ó$ˆÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ
�+Š+�q�f¤E§M¡M¸!¿(¹(Ô
C€CØCGÒCSÔ  §x¡x´·±Ô?Ü'¨¯©´%·-±-Ó@ð àÒ'‰t¨X×-=Ñ-=¸aÓ-C×-NÑ-NÓ-P€HØÒœr×>Ñ>Ü˜ÓÜ˜Ó Ü˜Ó"ÜÔ)¨.¿(¹(ÄEÇMÁMÓRÓSÜ˜Ó"Ü˜ÓØ÷ ò  ô ×#Ñ#Ü˜TÓ"Ü˜VÓ$Ü˜XÓ&Ü˜SÓ!Øó
ˆ÷ Ü—(‘(×%Ñ%×,Ñ,¨T¸4Ó@ˆCÜ˜c¨^Ó<ˆä�j“/ QÔ&ˆ3ÐL¨C¯KªKÐ,L¸ÀCÀR¸Ð,LÈ!Ó,LÐLrY   c                ój   € V ^8„  d   QhR\         R\        R,          R\        P                  R\        /# )r’   rí   ru  Nr“   r”   )rµ   r;  r•   r“   rÏ   )r—   s   "rW   r˜   r˜     s1   € ÷ 	ñ 	¤#ð 	´#¸µ*ð 	ÄeÇlÁlð 	ÔW[ñ 	rY   c                óÊ   € VP                   R8H  ;'       dN    VR9   ;'       dA    V R,          ^ 8H  ;'       d-    V R8*  ;'       d     \        VP                  4      ^ ,          ^	8¬  # )z�Whether the single-launch ConvRot quantizer of the streamed W4A8 decode (sm_90
clusters, at most four passes of 8 x 8 groups) covers act(x) for a K-wide row.r¹   rœ   )NÚswigluÚrms_normé   )rq  r¼   rr  )rí   ru  r“   s   &&&rW   Ú_w4a8_cluster_quantizer_okrÃ    ss   € ð 	�‰�vÑ÷ 	:ð 	:ØÐ5Ñ5÷	:ð 	:à��G�q‰L÷	:ð 	:ð �Ñ ÷	:ð 	:ô $ F§L¡LÓ1°!Õ4¸Ñ9ðrY   c                ó’  € VP                   w  pp\        P                  ! VV,          V^,          ,           \        P                  VP                  R7      pVRVV,           P                  VV4      VVV,          R pp\        VP                  4       VVP                   ^,          4      pV^8X  d   \        VP                  4      MRp\        P                  \        V 4      Ve   \        V4      MRVe   \        V4      MRVe   \        V4      MR\        V4      \        V4      \        V4      \        V4      Ve   \        V4      MR\        V	4      \        V4      V
e   \        V
4      MRVe   \        V4      MRVe   \        V4      MR\        V4      \        V4      \        V4      VVVV4      pV'       g   \        R4      hR# )z5Quantize x_2d and run the streamed MMA GEMM into out.r%  Nz4MMA-packed W4A8 weight is unsupported on this device)r  r•   r)  ró  r“   r¥   rx   r5  r±   r  Úw4a8_codebook_mma_linearr�  r§   r{  ÚRuntimeError)rÅ  rK  rs  Ú
act_weightrw  ru  ÚxqÚxsrÓ  rO  Ú
bias_floatr½  Úresid_scalerB  r$  rG  r™  r›  rë   rì   Úscratchrm  Úcountersrb  r­   Úuseds   &&&&&&&&&&&&&&&&&&        rW   Ú_w4a8_mma_linearrÏ  !  s†  € ð �9‰9�D€A€qä�kŠk˜!˜a�% ! r¥'�/´·±ÀSÇZÁZÔP€GØ! & 1 q¥5˜/×.Ñ.¨q°!Ó4°g¸aÀ!½e¸f°oˆx€IÜ�U—[‘[“] A r§x¡x°¥{Ó3€DØ9=À¼Ô'¨¯©Ô5È€JÜ×&Ñ&Ü˜ÓØ%,Ò%8Ô˜Ô!¸dØ,:Ò,FÔ˜Ô(ÈDØ(2Ò(>Ô˜Ô$ÀDÜˆmÓÜ˜	Ó"Ü˜ÓÜ˜ÓØ(2Ò(>Ô˜Ô$ÀDÜ˜Ó#Ü˜ÓØ(2Ò(>Ô˜Ô$ÀDØ&.Ò&:Ô˜Ô"ÀØ)4Ò)@Ô˜Ô%ÀdÜ˜Ó#Ü˜Ó"Ü˜ÓØØØØó+€D÷. ÜÐQÓRÐRñ rY   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r’   r›   r”   )r•   r›   r–   )r—   s   "rW   r˜   r˜   H  s%   € ÷ -ñ -ÄÇÁð -ÔPU×P\ÑP\ñ -rY   c                ón   € V R8X  d%   Ve!   VP                  W#R7      P                  4       # \        W#4      # )zsThe fused quantizer's act-weight operand: the norm weight for acts
that carry one, the empty placeholder otherwise.rÁ  r~  )rª   r¬   ru  )ru  rv  r“   r›   s   &&&&rW   rz  rz  H  s:   € ð �JÔÐ#3Ò#?Ø×"Ñ"¨&Ð"Ó>×IÑIÓKÐKÜ˜fÓ,Ð,rY   c                ó   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\        R\        R\
        R	,          R
\         P                  R	,          R\        R\         P                  R	,          R\         P                  R	,          R\         P                  /# )r’   r!  r  rƒ  rÝ  rÞ  Úconvrotr$  ru  Nrv  rw  r´  rµ  r”   )r•   r–   r›   rÏ   rµ   r;  r§   )r—   s   "rW   r˜   r˜   P  sÐ   € ÷ ]ñ ]Ü‡|�|ð]ä�L‰Lð]ô —,‘,ð]ô �,‰,ð	]ô
 �{‰{ð]ô ð]ô ð]ô �T�zð]ô —l‘l TÕ)ð]ô ð]ô �l‰l˜TÕ!ð]ô —L‘L 4Õ'ð]ô ‡\�\ñ]rY   c                 óÎ  a aa
aa+a,a-a.€ S
e   Sf   \        R4      hS P                  o.S P                  4       ^8X  d   S P                  4       '       d   S M0S P	                  RS P                  R,          4      P                  4       pVP                  4       '       g   VP                  4       p\        P                  P                  S P                  4      P                  pVP                  R,          \        V4      ,          pT;'       dP    VR8H  ;'       dC    VR,          ^ 8H  ;'       d/    RTu;8*  ;'       d
    \        8*  Mu ;'       d    \        WÎV4      pVR9  d   V'       g   \        WÇW‰4      pR pVP                  ^ ,          pTpVP                  w  o-pVV8X  g   Q R4       hS;'       g    S P                  o\         S,          p\#        S.4      ^8H  o,Ro+V+V-VV
VV 3R lpV+V,V-V.V
V3R lpV^8H  ;'       dL    T;'       dB    VP%                  4       ^,          ^ 8H  ;'       d     \'        VV4      V,           \)        V4      8  pV^8H  ;'       dC    V'       * ;'       d4    V^,          ^ 8H  ;'       d     VR8*  ;'       g    VR8H  ;'       d    S-^€8*  pV'       g   VR9   EdY   V'       EdP   V'       dk   \+        S P                  \        P,                  4      p\+        S P                  \        P.                  4      p\1        WxS P                  VP                  4      pM„\        P2                  ! ^V3\        P,                  S P                  R	7      p\        P2                  ! R\        P.                  S P                  R	7      p\+        S P                  VP                  4      p\5        VS P                  4      p\        P2                  ! ^S-3SS P                  R	7      pVe   TM\+        S P                  S4      pVen   VP                  S P                  8w  g(   VP                  S8w  g   VP                  4       '       g,   VP7                  S P                  SR
7      P                  4       pV! 4       w  pp\8        P;                  \=        V4      \=        V4      \=        V4      \=        V4      \=        V4      \=        V4      \=        V4      VVV\?        V4      \=        V4      \A        V	4      VVVR7       V! V4      # V'       Ed9   V'       dÝ   \        P2                  ! VV3\        P,                  S P                  R	7      p\        P2                  ! V^3\        P.                  S P                  R	7      p\1        WxS P                  VP                  4      p\8        PC                  \=        V4      \=        V4      \=        V4      VR\?        V4      ^ \=        V4      \A        V	4      V4
       Mï\E        VVV4      '       d   \G        WÆ4      w  ppMÍ\I        WlP                  VP                  R
7      p\K        VVV4      w  ppM›\        P2                  ! VV3\        P,                  S P                  R	7      p\        P2                  ! V^3\        P.                  S P                  R	7      p\8        PM                  \=        V4      \=        V4      \=        V4      R^ V4       V^8X  d   V^,          ^ 8X  g   V^8X  Eds   V^,          ^ 8X  Edd   V^8X  d"   \O        V4      '       g   VPQ                  4       p\5        VS P                  4      p\        P2                  ! VS-3SS P                  R	7      pVe   TM\+        S P                  S4      pVen   VP                  S P                  8w  g(   VP                  S8w  g   VP                  4       '       g,   VP7                  S P                  SR
7      P                  4       pV^8X  d   V! 4       w  ppMR ;pp\8        PS                  \=        V4      \=        V4      \=        V4      \=        V4      \=        V4      \=        V4      VVVVR7
       V! V4      # \        P2                  ! VS-3SS P                  R	7      p\5        VS P                  4      pVe   \U        VS P                  S4      M\+        S P                  S4      p\W        VS-V4      '       dX   \Y        VP[                  4       4      '       d9   \]        \8        R4      '       d#   \_        TTTTVe   TMR S4      p V e	   V! V 4      # Rp!\a        VS-VVP[                  4       4      p"V"'       EgŒ   \b        '       Eg   \e        V4      '       Edm   VPg                  4       S-8X  d   TMVPi                  S-4      P                  4       p#R p$S
eV   S
P                  S8X  dE   SP                  S8X  d4   S
P	                  VS-4      P                  4       p$\O        V$4      '       g   R p$V$e~   \8        Pk                  \=        V4      \=        V4      \=        V4      \=        V#4      \=        V4      \=        \U        SS P                  S4      4      \=        V$4      \=        V4      VV4
      p!V!o+V!'       gS   \8        Pm                  \=        V4      \=        V4      \=        V4      \=        V#4      \=        V4      \=        V4      VV4      p!V!'       Egb   VPn                  ;'       d    \q        VP[                  4       4      p%V%'       dE   \s        V^4      p&\s        S-\u        V4      4      p'\w        VV&4      p(\y        \w        VV&4      V'4      p)MS-p'Tp(Tp)\        P2                  ! VV'3\        Pz                  S P                  R	7      p*\8        P}                  \=        V(4      \=        V)4      \=        V*4      \=        \        4       4      V4       V'S-8w  d   V*RR S-13,          P                  4       p*\8        P�                  \=        V*4      \=        V4      \=        V4      \=        V4      \=        V4      VV4       V! V4      # )Nz-int8_linear: residual requires residual_scalerœ   z,Input and weight inner dimensions must matchFc                  ó>  <€ SeM   SP                   S8w  g<   SP                   S8w  g+   SP                  4       S8w  g   SP                  4       S8w  d   R# Ro \        SP                  ^S4      P	                  4       4      \        \        SSP                  S4      4      3# )NT©NN)r›   r5  r�  r6  r¬   r�  r“   )Úfused_residual_donerì   rÞ  r´  rµ  r!  s   €€€€€€rW   Ú_gemv_residual_argsÚ(int8_linear.<locals>._gemv_residual_args‚  s�   ø€ ð Ò §¡°)Ô ;¸~×?SÑ?SÐW`Ô?`Ø—>‘>Ó# qÔ(¨N×,@Ñ,@Ó,BÀaÔ,GØÐØ"ÐÜ  ×!1Ñ!1°!°QÓ!7×!BÑ!BÓ!DÓEÜ Ô!1°.À!Ç(Á(ÈIÓ!VÓWðYð 	YrY   c                 ó°   <€ S'       g-   \        T Sf   R MSP                  V P                  4      S4      p S'       d   V # V P                  ! . SR R OSN5!  # )Nr4  )r¹  r6  r  )Úor×  Úis_2d_outputrì   rÄ  r´  rµ  s   &€€€€€€rW   Ú_finishÚint8_linear.<locals>._finish�  sT   ø€ ß"Ü ¨8Ò+;¡4À×AQÑAQÐRS×RYÑRYÓAZØ .ó0ˆAç ˆqÐD a§i¢iÐ&D°¸C¸R°Ð&DÀ!Ó&DÐDrY   rè  rB  r%  r~  )r´  rµ  rý  r�   r4  ©NÚnone)r  r  )ArÊ  r  r'  r  r6  r¬   r•   r¹   r–  r“   r—  ry  r]  rT  Ú_apply_input_actr›   re   r¼  ry  rE  r?  ru  r«   r¢   rz  rl  r…  rª   r  Úint8_linear_m1r�  r{  r§   rC   r_  r>  r‡   rD   r;   rz  r€  rü  r�  rï   ré   rö   r  rþ  rˆ  rÿ  rû   r5  r   Úcutlass_int8_dequant_residualr  rõ   r×   r  r  r,  r0  ró  r  rn  r  )/r!  r  rƒ  rÝ  rÞ  rÓ  r$  ru  rv  rw  r´  rµ  rÅ  r›  Úk_actÚ_fused_convrot_okrë   rí   Úk_wr™  rØ  rÝ  Úconvrot_m1_supportedÚnonconvrot_m1_supportedr  rÜ  r|  rB  rä  Ú	resid_argÚresid_scale_argr,  Ú
turing_outr  r  r  r½  Úuse_turing_paddingr  r	  r
  r  r  r×  rÜ  rì   rÄ  s/   f&&&f&&&&&ff                               @@@@rW   r4   r4   P  sQ
  ÿ€ ð Ò Ò 6ÜÐHÓIÐIØ—‘€JØ—‘“˜1” §¡×!2Ò!2‰1¸¿	¹	À"ÀaÇgÁgÈbÅkÓ8R×8]Ñ8]Ó8_€DØ×Ñ×!Ò!Ø×"Ñ"Ó$ˆÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€Jð
 �J‰J�r�NÔ.¨yÓ9Õ9€Eà÷ 	Nð 	NØ Ñ$÷	Nð 	Nà�C�K˜1Ñ÷	Nð 	Nð �5×0Ô0Ô0Ô0÷	Nð 	Nô .¨dÐ;LÓMð ð ˜Ô&×/@Ü Ð1AÓQˆØˆ	à�
‰
�1�€AØ€AØ�\‰\�F€A€sØ�Œ8ÐCÐCÓCˆ8à×$Ð$˜QŸW™W€IÜ% iÕ0ÐÜ�z“? aÑ'€Lð  Ð÷	Yò 	Y÷Eò Eð 	
ˆQ‰÷ 	kð 	kØ÷	kð 	kà�O‰OÓ Õ" aÑ'÷	kð 	kô 4°A°qÓ9¸AÕ=Ô@dÐeiÓ@jÑjð	 ð 	
ˆQ‰÷ 	4ð 	4ØŒK÷	4ð 	4à��E�Q‰J÷	4ð 	4ð �$‰Y×2Ð2˜1 ™9×1Ð1¨¨c©ð	 ÷  	¨^Õ ;×@WÐ@Wßä(¨¯©´5·:±:Ó>ˆGÜ(¨¯©´5·=±=ÓAˆGÜ,¨YÈ!Ï(É(ÐTX×T^ÑT^Ó_‰Nä—k’k 1 a &´·
±
À1Ç8Á8ÔLˆGÜ—k’k &´·±ÀaÇhÁhÔOˆGÜ/°·±¸$¿*¹*ÓEˆNÜ-¨l¸A¿H¹HÓEˆÜ�kŠk˜1˜a˜&¨	¸!¿(¹(ÔCˆØÒ+‘4Ô1CÀAÇHÁHÈiÓ1XˆØÒ §¡°·±Ô!8¸D¿J¹JÈ)Ô<SÐ[_×[mÑ[m×[oÒ[oØ—w‘w a§h¡h°i�wÓ@×KÑKÓMˆHÙ%8Ó%:Ñ"ˆ	�?Ü
×ÑÜ˜TÓ"Ü˜WÓ%Ü˜WÓ%Ü˜VÓ$Ü˜\Ó*Ü˜XÓ&Ü˜SÓ!ØØØÜ˜IÓ&Ü˜^Ó,Ü�-Ó ØØØ*ð! 	ô 	
ñ$ �s‹|Ð÷ €w÷ Ü—k’k 1 a &´·
±
À1Ç8Á8ÔLˆGÜ—k’k 1 a &´·±ÀaÇhÁhÔOˆGÜ,Ø¨Q¯X©X°t·z±zóCˆNä×.Ñ.Ü  Ó&Ü  Ó)Ü  Ó)Ø!ØÜ 	Ó*ØÜ  Ó0Ü�mÓ$Øõô .¨d°AÐ7H×IÒIä<¸TÓUÑˆG‘Wô  Ð 1¿+¹+ÈTÏZÉZÔXˆAÜ:¸4ÀÐDUÓVÑˆG‘Wä—+’+˜q !˜f¬E¯J©J¸q¿x¹xÔHˆÜ—+’+˜q !˜f¬E¯M©MÀ!Ç(Á(ÔKˆÜ
× Ñ Ü˜TÓ"Ü˜WÓ%Ü˜WÓ%ØØØô	
ð 	
ˆQŒ�1�q•5˜A”: 1¨¥6¨a°"­f¸­kà�Œ6œ* V×,Ò,Ø—\‘\“^ˆFÜ-¨l¸A¿H¹HÓEˆÜ�kŠk˜1˜a˜&¨	¸!¿(¹(ÔCˆØÒ+‘4Ô1CÀAÇHÁHÈiÓ1XˆØÒ §¡°·±Ô!8¸D¿J¹JÈ)Ô<SÐ[_×[mÑ[m×[oÒ[oØ—w‘w a§h¡h°i�wÓ@×KÑKÓMˆHØ�Œ6Ù)<Ó)>Ñ&ˆI‘à*.Ð.ˆI˜Ü
×ÑÜ˜WÓ%Ü˜VÓ$Ü˜WÓ%Ü˜\Ó*Ü˜XÓ&Ü˜SÓ!ØØØØ*ð 	ô 	
ñ �s‹|Ðä
�+Š+�q˜!�f I°a·h±hÔ
?€CÜ)¨,¸¿¹ÓA€LØ?CÒ?OÔ   q§x¡x°Ô;Ü'¨¯©°)Ó<ð ô 	" ! Q¨×*Ò*Ü2°7×3EÑ3EÓ3G×HÒHÜ”BÐ5×6Ò6ä2ØØØØØÒ(‰H¨dØó
ˆ
ð Ò!Ù˜:Ó&Ð&à€LÜ9Ø	Ø	Ø	Ø×ÑÓó	Ð÷ #Ð"ß%Ô%Ü6°w×?Ó?à%1×%7Ñ%7Ó%9¸QÔ%>‘\ÀL×DWÑDWÐXYÓDZ×DeÑDeÓDgˆ
ð ˆØÒ H§N¡N°iÔ$?ÀN×DXÑDXÐ\eÔDeØ×'Ñ'¨¨1Ó-×8Ñ8Ó:ˆHÜ˜h×'Ò'Ø�ØÒÜ×;Ñ;Ü  Ó)Ü  Ó(Ü  Ó)Ü  Ó,Ü  Ó*Ü Ô!1°.À!Ç(Á(ÈIÓ!VÓWÜ  Ó*Ü  Ó%Ø!ØóˆLð #/ÐßÜ×2Ñ2Ü  Ó)Ü  Ó(Ü  Ó)Ü  Ó,Ü  Ó*Ü  Ó%Ø!Øó	ˆL÷ ˆ<à$Ÿ_™_×]Ð]Ô1GÈ×HZÑHZÓH\Ó1]ÐßÜ   BÓ'ˆHÜ  Ô$<¸WÓ$EÓFˆHÜ# G¨XÓ6ˆHÜ(¬°f¸hÓ)GÈÓR‰MàˆHØˆHØ"ˆMä—K’K  H ´U·[±[ÈÏÉÔRˆ	Ü
×ÑÜ˜XÓ&Ü˜]Ó+Ü˜YÓ'ÜÔ1Ó3Ó4Øô	
ð �qŒ=Ø! ! R a R %Õ(×3Ñ3Ó5ˆIÜ
×!Ñ!Ü˜YÓ'Ü˜WÓ%Ü˜\Ó*Ü˜XÓ&Ü˜SÓ!ØØô	
ñ �3‹<ÐrY   c          
      ó²   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\        R\         P                  /# )r’   rÓ  rN  r®   NrG  r”   r  )r—   s   "rW   r˜   r˜   p	  sO   € ÷  ñ  Ü�<‰<ð ä�<‰<ð ô �l‰l˜TÕ!ð ô ð	 ô
 ‡\�\ñ rY   c                óº  € V P                  4       p VP                  4       p\        WV4      w  rEp\        P                  ! WE\        P                  V P
                  R7      pVe   \        VP                  4       4      MRp\        P                  P                  V P
                  4      P                  p	VP                  \        P                  8X  dU   \        P                  \        V 4      \        VP                  \        P                  4      4      V\        V4      VV	4       V# \        P!                  \        V 4      \        V4      V\        V4      VV	4       V# )z£Native decode of packed INT4/INT6 storage to the grouped INT8 grid the GEMM consumes
(bit-exact with the eager decode). Shared by dequantize and the 2-pass linear.r%  N)r¬   rt   r•   rl  r«   r“   r�  r¹   r–  r—  r›   r¦   r  Ú!dequant_int4_grouped_to_int8_e4m3r¥   r¤   Údequant_int4_grouped_to_int8)
rÓ  rN  r®   rG  rì   rí   Ú_bitsrB  Úcodebook_argr›  s
   &&&&      rW   Ú_dequant_int4_grouped_to_int8ró  p	  s  € ð ×ÑÓ€EØ×ÑÓ€EÜ  ¨zÓ:�K€Aˆ%Ü
�+Š+�a¤%§*¡*°U·\±\Ô
B€CØ>FÒ>RÔ# H×$7Ñ$7Ó$9Ô:ÐX\€LÜ—‘×*Ñ*¨5¯<©<Ó8×DÑD€JØ‡{�{”e×)Ñ)Ô)Ü
×,Ñ,Ü˜UÓ#Ü˜UŸZ™Z¬¯©Ó4Ó5ØÜ˜SÓ!ØØô	
ð" €Jô 	×'Ñ'Ü˜UÓ#Ü˜UÓ#ØÜ˜SÓ!ØØô	
ð €JrY   c                ó,  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\        R\        R	\         P                  R
\         P                  /	# )r’   rÓ  rN  rO  r®   NÚ
correctionrG  r$  r/  r”   r0  )r—   s   "rW   r˜   r˜   “	  s“   € ÷ gñ gÜ�<‰<ðgä�<‰<ðgô �|‰|ðgô �l‰l˜TÕ!ð	gô
 —‘˜tÕ#ðgô ðgô ðgô —+‘+ðgô ‡\�\ñgrY   c           	     ó¬   € \        V VVVVVV4       \        WW54      p\        VVVVV4      p	\        V	P	                  4       V4      P                  V4      # )zGDequantize W4A8 weights with native CUDA decode and ConvRot operations.)ru   ró  rr   rB   r¬   rª   )
rÓ  rN  rO  r®   rõ  rG  r$  r/  Úint8_weightÚweight_rotateds
   &&&&&&&&  rW   r2   r2   “	  sm   € ô ØØØØØØØôô 0°¸hÓS€KÜ;ØØØØØó€Nô & n×&?Ñ&?Ó&AÐCTÓU×XÑXÐYeÓfÐfrY   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r’   ÚdestrN  ri  )r—   s   "rW   r˜   r˜   ²	  s"   € ÷ ñ ¤§¡ð ´U·\±\ñ rY   c                ó.  € VP                   w  r#V^,          pV P                  4       W$,          ^	,          ^,          8X  d   ^M^pVP                  \        P                  4      P                  4       p\        P                  ! ^V,          V,          \        P                  V P                  R7      p\        P                  \        V 4      \        V4      \        V4      \        P                  P                  V P                  4      P                  4       R# )zâCanonical codes at dest's start -> packed records in the same storage.

Scratch is the saved group scales plus one 16-row slab. All work runs on
the caller's stream; the output keeps slab/K order without a global permutation.
r%  N)r  r5  r¥   r•   r¤   r€  rl  r“   r  Úpack_w4a8_mma_inplacer�  r¹   r–  r—  )rú  rN  rì   rU  rí   rb  r¯   rÌ  s   &&      rW   Úpack_w4a8_mma_weight_rý  ²	  sµ   € ð —‘�I€AØ��€AØ—
‘
“ ¥¨¥	¨R¥Ô/‰1°Q€DØ�Z‰ZœŸ™Ó$×*Ñ*Ó,€FÜ�kŠk˜!˜a�% $�,¬e¯k©kÀ$Ç+Á+ÔN€GÜ×ÑÜ˜ÓÔ 0°Ó 8Ô:JÈ7Ó:SÜ�
‰
×!Ñ! $§+¡+Ó.×:Ñ:örY   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r’   ÚdstÚsrcri  )r—   s   "rW   r˜   r˜   Ã	  s"   € ÷ ñ ”%—,‘,ð ¤U§\¡\ñ rY   c                ó  € \         P                  ! V P                  R,          ^,           \         P                  V P                  R7      p\
        P                  \        V P                  \         P                  4      P                  R4      4      \        VP                  \         P                  4      P                  R4      4      \        V4      \         P                  P                  V P                  4      P                  4       R# )zFCopy src's bytes to dst on the current stream; the ranges may overlap.rÂ  r%  Nr4  )r•   r)  Únbytesró  r“   r  Úmemmover�  r¥   r¤   r¹   r–  r—  )rÿ  r   Úflagss   && rW   Úmemmove_r  Ã	  s§   € ä�KŠK˜Ÿ
™
 eÕ+¨aÕ/´u·{±{È3Ï:É:ÔV€EÜ‡J�JÜ˜Ÿ™¤%§+¡+Ó.×3Ñ3°BÓ7Ó8Ô:JÈ3Ï8É8ÔTY×T_ÑT_ÓK`×KeÑKeÐfhÓKiÓ:jÔl|ð  ~Có  mDÜ�
‰
×!Ñ! #§*¡*Ó-×9Ñ9örY   c                 ó*  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R	\        R
\        R\         P                  R\        R,          R\         P                  R,          R\
        R\         P                  R,          R\         P                  R,          R\         P                  /# ©r’   r!  rÓ  rN  rO  r®   Nrõ  rÝ  rG  r$  rÞ  ru  rv  rw  r´  rµ  r”   ©r•   r–   rµ   r›   r;  r§   )r—   s   "rW   r˜   r˜   Ì	  s  € ÷ O)ñ O)Ü‡|�|ðO)ä�<‰<ðO)ô �<‰<ðO)ô �|‰|ð	O)ô
 �l‰l˜TÕ!ðO)ô —‘˜tÕ#ðO)ô �,‰,˜Õ
ðO)ô ðO)ô ðO)ô �{‰{ðO)ô �T�zðO)ô —l‘l TÕ)ðO)ô ðO)ô �l‰l˜TÕ!ðO)ô —L‘L 4Õ'ðO)ô  ‡\�\ñ!O)rY   c                óº	  a€ Ve   \        \        WW#SWVVW‰W«V4      VV4      # \        ;'       d*    VRJ ;'       d    VP                  \        P
                  8H  pT;'       d    \        W
V4      pV'       g   \        W
W¼4      p Rp
\        VVVSVVV4      w  pppV P                  R	,          V\        V
4      ,          8w  d#   \        RV P                  R	,           RV 24      hVV,          pV P                  R	V P                  R	,          4      P                  4       pVP                  ^ ,          p\        V	,          p\        P                  P!                  V P"                  4      P$                  pV3R lp\        P&                  ! VV\        P(                  V P"                  R7      p\        P&                  ! V^\        P*                  V P"                  R7      p\        P&                  ! VVW�P"                  R7      pVe   \-        W`P"                  V	4      MRp\        '       dû   V^8:  dô   Vfð   VP                  \        P
                  8X  dÑ   V^8¼  dÊ   V^,          ^ 8X  d¼   \.        P1                  \3        V4      \3        V4      \3        V4      \3        V4      \3        VP5                  \        P6                  4      4      V! 4       \3        V4      Ve   \3        V4      MR\3        V4      VVVV4      pV'       d#   VP                  ! . V P                  RR	 OVN5!  # V'       dF   \9        VWVSVVVVW‡RW«V4      w  pppV'       d#   VP                  ! . V P                  RR	 OVN5!  # M6\.        P;                  \3        V4      \3        V4      \3        V4      VR^ V4       \=        WSV4      p\.        P?                  \3        V4      \3        V4      \3        V4      \3        V4      \3        Ve   TM\A        V P"                  V	4      4      \3        V4      VV4      pV'       g   \C        V VVVSVVVVV	V
VVR7      # Ve{   VP5                  VVV4      PE                  R	\        PF                  R7      PI                  V	4      p V VPI                  V	4      ,          p VPK                  V VPI                  V	4      4       VP                  ! . V P                  RR	 OVN5!  # )
zMCUDA W4A8/W6A8 linear using chunked INT4/INT6 decode and the tuned INT8 GEMM.Nr=  z does not match qdata K=c                  ó&   <€ S e   \        S 4      # R # rN   )r�  )r®   s   €rW   Úwrap_codebookÚ'w4a8_int8_linear.<locals>.wrap_codebook
  s   ø€ Ø-5Ò-AÔ Ó)ÐKÀtÐKrY   r%  F)	r®   rõ  rÝ  rG  r$  rÞ  ru  rv  rw  rš   r4  )&r¹  r5   Ú_W4A8_CHUNKEDr›   r•   r¦   Ú_w4a8_chunked_act_fusablerá  ru   r  ry  rÊ  r6  r¬   re   r¹   r–  r“   r—  rl  r«   r¢   r�  r  Úw4a8_codebook_gemvr�  r¥   r¤   Ú_w4a8_linear_chunkedr>  ró  r  ru  Úeager_w4a8_int8_linearÚsumró  rª   Úaddmm_)!r!  rÓ  rN  rO  r®   rõ  rÝ  rG  r$  rÞ  ru  rv  rw  r´  rµ  ÚchunkedÚfuse_actrì   rí   rñ  rU  rÅ  rë   r™  r›  r  rÈ  rÉ  rB  rä  rÎ  r÷  r¶  s!   &&&&f&&&&&&&&&&                  rW   r5   r5   Ì	  s_  ø€ ð$ ÒÜÜØ˜%¨H°jÈ
Ø!¨iØóð
 Øó
ð 	
ô 	÷ 	/ð 	/Ø˜$Ð÷	/ð 	/à�K‰Kœ5×.Ñ.Ñ.ð ð
 ×UÐUÔ4°QÐCTÓU€HßÜ˜QÐ+;ÓKˆØˆ	Ü(ØØØØØØØó�K€A€qˆ%ð 	‡w�wˆr…{�aÔ*¨9Ó5Õ5Ô5Ü˜8 A§G¡G¨B¥K =Ð0HÈÈÐLÓMÐMØ�*�_€FØ�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DØ�
‰
�1�€AÜ% iÕ0ÐÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JõLô 
�Š�Q˜¤§¡°A·H±HÔ	=€BÜ	�Š�Q˜¤§¡°q·x±xÔ	@€BÜ
�+Š+�a˜ )·H±HÔ
=€Cà>BÒ>NÔ §h¡h°	Ô:ÐTX€H÷
 	‹Ø�ŒFØÒØ�K‰Kœ5×.Ñ.Ô.Ø˜"ÔØ˜�O˜qÔ ä×$Ñ$Ü˜TÓ"Ü˜RÓ Ü˜RÓ Ü˜UÓ#Ü˜UŸZ™Z¬¯©Ó4Ó5Ù‹OÜ˜YÓ'Ø*2Ò*>Ô˜XÔ&ÀDÜ˜SÓ!ØØØØó
ˆ÷ Ø—;’;Ð0 §¡¨¨ Ð0¨aÓ0Ð0çÜ+Ø�% 	¨8°X¸rÀ2ÀsØ¨5Ø¨ó
‰ˆˆb�"÷
 Ø—;’;Ð0 §¡¨¨ Ð0¨aÓ0Ð0ð ô 	×(Ñ(Ü˜TÓ"Ü˜RÓ Ü˜RÓ ØØØØô	
ô 0°¸hÈ
ÓS€Kä×"Ñ"Ü˜ÓÜ˜Ó%Ü˜ÓÜ˜Ó#Ü XÒ%9™Ô?QÐRS×RZÑRZÐ\eÓ?fÓgÜ˜ÓØØó	€D÷ Ü%ØØØØØØ!ØØ!Ø/ØØØ-Ø'ô
ð 	
ð  ÒØ�W‰W�Q˜ 
Ó+×/Ñ/°¼%¿+¹+Ð/ÓF×IÑIÈ)ÓTˆØ�"—%‘%˜	Ó"Õ"ˆØ�
‰
�2�z—}‘} YÓ/Ô0Ø�;Š;Ð(˜Ÿ™  ˜Ð( aÓ(Ð(rY   c                 ó*  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R	\        R
\        R\         P                  R\        R,          R\         P                  R,          R\
        R\         P                  R,          R\         P                  R,          R\         P                  /# r  r  )r—   s   "rW   r˜   r˜   ^
  s  € ÷ ])ñ ])Ü‡|�|ð])ä�<‰<ð])ô �<‰<ð])ô �|‰|ð	])ô
 �l‰l˜TÕ!ð])ô —‘˜tÕ#ð])ô �,‰,˜Õ
ð])ô ð])ô ð])ô �{‰{ð])ô �T�zð])ô —l‘l TÕ)ð])ô ð])ô �l‰l˜TÕ!ð])ô —L‘L 4Õ'ð])ô  ‡\�\ñ!])rY   c                óš	  € V
R	9   d   Rp
Ve   Vf   \        R4      hVP                  4       pV P                  R
,          \        V
4      ,          pV
RJ;'       dR    V P                  4       ^V P                  R
,          ,          8*  ;'       d    \	        VW P
                  4      ;'       gL    V
RJ;'       d@    V P                  4       ^V P                  R
,          ,          8„  ;'       d    \        W
V4      pV
e   V'       g   \        W
W¼4      p Rp
VP                  4       ^8w  g¢   VP                  \        P                  8w  gƒ   V^,          '       gt   VP                  4       VV,          \        ^4      ,          R,          VV,          \        ^4      ,          R,          39  g!   VP                  4       ^ 8w  g   V^8w  g   Ve   \        R4      hV P                  R
V P                  R
,          4      P                  4       pVP                  ^ ,          pVeU   V^8:  d2   VP                  4       VV,          8X  d   VP                  4       V8X  g   \        \!        WW#WEWgW‰W«V4      VV4      # \        P"                  ! VV\        P                  V P
                  R7      p\        P"                  ! V^\        P$                  V P
                  R7      p\        P"                  ! VWùV P
                  R7      pV^8:  Edf   R;p;ppV
fm   \	        VRV P
                  4      '       gP   \        P&                  ! V4      p\        P"                  ! VVV,          \        P$                  V P
                  R7      pV
R8X  d"   \)        W«V P
                  VP                  4      pR;ppVeF   VP                  VV4      P+                  V	4      P                  4       p\-        WàP
                  V	4      pVe   VP/                  4       P                  4       MRp\1        VVVVWÊVVWVVVVW‡\2        V	,          \        P4                  P7                  V P
                  4      P8                  4       MHVe   \-        W`P
                  V	4      MRp\;        VWW4VVVVW‡RW«V4      w  p pV'       g   \=        R4      hVP                  ! . V P                  RR
 OVN5!  # )zSResident packed W4A8/W6A8: small-M MMA or direct packed dequantization for prefill.Nz2w4a8_int8_linear: residual requires residual_scaler  z invalid MMA-packed W4A8 operandsr%  rÁ  Tz3Packed W4A8 prefill is unsupported by the INT8 GEMMrß  r4  )rÊ  r5  r  ry  rÃ  r“   r  rá  r'  r›   r•   r«   ry   r6  r¬   r¹  r6   rl  r¢   rÂ  rz  rª   r�  r§   rÏ  re   r¹   r–  r—  r  rÆ  ) r!  rÓ  rN  rO  r®   rõ  rÝ  rG  r$  rÞ  ru  rv  rw  r´  rµ  rì   rí   r  rÅ  rë   rÈ  rÉ  rB  rK  rs  rÇ  r½  rË  rÊ  rä  rÎ  Ú_s    &&&&&&&&&&&&&&&                 rW   r6   r6   ^
  s  € ð$ �NÔ"Øˆ	ØÒ Ò 6ÜÐMÓNÐNØ�‰Ó€AØ	�‰��Ô'¨	Ó2Õ2€Aà˜Ð÷ 	?ð 	?Ø�G‰G‹I˜˜QŸW™W R�[�Ñ(÷	?ð 	?ä& q¨)·X±XÓ>÷ð ð
 	˜Ð÷ 	Gð 	GØ�G‰G‹I˜˜AŸG™G B�K�Ñ'÷	Gð 	Gä% aÐ4EÓFð ð Ò§XÜ˜QÐ+;ÓKˆØˆ	à�	‰	‹�qÔØ�;‰;œ%Ÿ*™*Ô$Øˆr�6Œ6Ø�;‰;‹=  Q¥Ô)>¸qÓ)AÕ!AÀSÕ!HÈ!ÈaÍ%ÔRgÐhiÓRjÕJjÐnqÕJqÐ rÔrØ�;‰;‹=˜AÔØ˜ÔØÒ!äÐ;Ó<Ð<Ø�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DØ�
‰
�1�€AØÒØ	ˆQŒ�8—>‘>Ó# q¨1¥uÔ,°×1EÑ1EÓ1GÈ1Ô1LäÜØ˜%¨HÀ$Ø!¨iÈ=óð Øó
ð 	
ô 
�Š�Q˜¤§¡°A·H±HÔ	=€BÜ	�Š�Q˜¤§¡°q·x±xÔ	@€BÜ
�+Š+�a˜°A·H±HÔ
=€CØˆA…vð 15Ð4ˆÐ4�. :ØÒÔ%?ÀÀ4ÈÏÉ×%RÒ%RÜ×&Ò& tÓ,ˆGÜ"Ÿ[š[Ø�1Ð)Õ)´·±ÀqÇxÁxôˆNð ˜
Ô"Ü(¨ÀaÇhÁhÐPT×PZÑPZÓ[ˆJØ!%Ð%ˆ�;ØÒØ×'Ñ'¨¨1Ó-×0Ñ0°Ó;×FÑFÓHˆHÜ*¨>¿8¹8ÀYÓOˆKØ26Ò2B�T—Z‘Z“\×,Ñ,Ô.Èˆ
ÜØ�'˜>¨:°}ÐQSÐUWØ˜j¨(°KÀØ¬=¸Õ+CÜ�J‰J×%Ñ% a§h¡hÓ/×;Ñ;õ		
ð CGÒBRÔ# D¯(©(°IÔ>ÐX\ˆÜ)Ø�% 	°X¸rÀ2ÀsØ¨4Ø¨ó
‰
ˆˆa�÷
 ÜÐTÓUÐUØ�;Š;Ð(˜Ÿ™  ˜Ð( aÓ(Ð(rY   c          
      óh   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        /# )r’   r!  rë   rí   r$  r”   rP  )r—   s   "rW   r˜   r˜   ¾
  s4   € ÷ ñ œŸ™ð ¬#ð ´#ð Ì#ð ÔRVñ rY   c                ó  € VR8¬  ;'       dt    VR8H  ;'       dg    VR,          ^ 8H  ;'       dS    RTu;8*  ;'       d
    \         8*  Mu ;'       d1    V P                  '       * ;'       g    \        W4      \        V 4      8  # )zúWhether the chunked prefill path quantizes with the 64-lane ConvRot kernel (the one
that fuses input activations): large M and a K-wide activated row it can hold in
shared memory. x only supplies the device; m and k are the activated row count/width.r  rœ   )r]  rõ   rE  r?  )r!  rë   rí   r$  s   &&&&rW   Ú_w4a8_fast_act_okr  ¾
  s‘   € ð
 	
ˆS‰÷ 	ið 	iØ Ñ$÷	ið 	ià��G�q‰L÷	ið 	ið �1×,Ô,Ô,Ô,÷	ið 	ið —‘Œ]÷ hð hÜ7¸Ó=Ô@dÐefÓ@gÑgðrY   c                ój   € V ^8„  d   QhR\         P                  R\        R,          R\        R\        /# )r’   r!  ru  Nr$  r”   ©r•   r–   r;  rµ   rÏ   )r—   s   "rW   r˜   r˜   Ì
  s8   € ÷ yñ y¤§¡ð y¼#À½*ð yÔY\ð yÔaeñ yrY   c                ó¼   € VR9   d   R# \        W P                  4       V P                  R,          ,          V P                  R,          \        V4      ,          V4      # )zeWhether the chunked prefill path absorbs input_act; x is the raw input (2K wide for the gated pairs).Frß  r4  )r  r5  r  ry  )r!  ru  r$  s   &&&rW   r  r  Ì
  sD   € à�NÔ"ÙÜ˜Q§¡£	¨Q¯W©W°R­[Õ 8¸!¿'¹'À"½+ÔIYÐZcÓIdÕ:dÐfwÓxÐxrY   c                ó~  € VR9   d   RpV P                   w  ppV\        V4      ,          pVP                  4       p\        VP                  ,          p\
        P                  P                  V P                  4      P                  p\        VVV P                  4       4      p\
        P                  ! \        VV4      V\
        P                  V P                  R7      p\        WVV	4      pVe   V'       g   \!        R4      hV'       d   \#        W	VWÞR7      w  rgM÷\%        \&        R4      '       d¬   \&        P)                  \+        V 4      \+        V4      \+        V4      \+        V4      \+        VP-                  \
        P.                  4      4      Ve   \+        V4      MR\+        V4      Ve   \+        V4      MR\+        V4      \+        V4      V	V
VVVV4      pVWg3# \&        P1                  \+        V 4      \+        V4      \+        V4      V	R^ V4       \&        P3                  \+        V4      \+        V4      \+        VP-                  \
        P.                  4      4      Ve   \+        V4      MR\+        V4      \+        VP5                  V4      4      Ve   \+        V4      MR\+        V4      \+        V4      V
VVVV4      pVWg3# )a   Shared bounded INT8 GEMM; the native dequantizer reads the selected storage format.

With input_act set, x_2d is the raw row (two K-wide halves for the gated pairs) and
the 64-lane quantizer applies the activation; callers check _w4a8_chunked_act_fusable.
Nr%  zGw4a8 chunked linear: fused input activation needs the 64-lane quantizer)ru  rv  rw  Úw4a8_codebook_linear_chunkedFrß  )r  ry  r5  re   r›   r•   r¹   r–  r“   r—  rð  rö   rl  ré  r«   r  rÆ  rC   r  r  r   r�  r¥   r¤   r>  Úw4a8_codebook_gemm_chunkedr6  )rÅ  rÓ  rN  rO  r®   rä  rÈ  rÉ  rB  r$  rG  Ú
mma_packedru  rv  rw  rë   rí   rì   r™  r›  rõ  rm  Úfast_actrÎ  s   &&&&&&&&&&&&&&&         rW   r  r  Ó
  sS  € ð �NÔ"Øˆ	Ø�:‰:�D€A€qØÔ
˜9Ó
%Õ%€AØ�‰Ó€AÜ% c§i¡iÕ0ÐÜ—‘×*Ñ*¨4¯;©;Ó7×CÑC€JÜ(¨¨A¨t¯©Ó/@ÓA€JÜ—’œC 
¨AÓ.°¼¿¹ÈDÏKÉKÔX€Iä  ¨!Ð->Ó?€HØÒ§XÜÐdÓeÐeßÜ0Ø¨yØ-ôL‰ˆˆBô 
”Ð3×	4Ò	4Ü×.Ñ.Ü˜TÓ"Ü˜RÓ Ü˜RÓ Ü˜UÓ#Ü˜UŸZ™Z¬¯©Ó4Ó5Ø*2Ò*>Ô˜XÔ&ÀDÜ˜YÓ'Ø*2Ò*>Ô˜XÔ&ÀDÜ˜YÓ'Ü˜SÓ!ØØØØØØó!
ˆð$ �Rˆ|Ðä
×(Ñ(Ü˜TÓ"Ü˜RÓ Ü˜RÓ ØØØØô	
ô ×(Ñ(Ü˜ÓÜ˜ÓÜ˜Ÿ™¤E§K¡KÓ0Ó1Ø&.Ò&:Ô˜Ô"ÀÜ˜Ó#Ü˜Ÿ™ A›Ó'Ø&.Ò&:Ô˜Ô"ÀÜ˜Ó#Ü˜ÓØØØØØó€Dð  �ˆ<ÐrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \        ,          R\        \        ,          R,          R\
        R,          R\         P                  /# )	r’   r‚  rí   r|  Úkernel_sizeÚ	is_causalNr’  r”   ©r•   r–   Úlistrµ   rÏ   r§   )r—   s   "rW   r˜   r˜     so   € ÷ ñ Ü‡|�|ðä‡|�|ðô ‡|�|ðô ”c•ð	ô
 ”D�z˜DÕ ðô �4�<ðô ‡\�\ñrY   c                óä  € Vf   . ROM
\        V4      pV P                  w  rxršr¼Vf
   VR,          pV P                  4       p VP                  4       pVP                  4       p\        P                  ! V 4      p\        P
                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V4      \        V4      WxWšW¼\        V^ ,          4      \        V^,          4      \        V^,          4      \        V^ ,          4      \        V^,          4      \        V^,          4      \        V4      \        V P                  ,          V4       V# )zsFused 3D neighborhood attention (NATTEN ``na3d`` semantics) over
``(B, T, H, W, NH, HD)`` tensors. See ops/na3d.cu.)FFFç      à¿)r(  r  r¬   r•   rÂ  r¹   r–  r“   r—  r  r	   r�  rµ   r§   re   r›   )r‚  rí   r|  r%  r&  r’  ÚcausalÚbatchrw  r,  ÚwÚnhÚhdrB  r›  s   &&&&&&         rW   r	   r	     s  € ð '0Ò&7Ó"¼TÀ)»_€FØŸW™WÑ€Eˆa�BØ‚}Ø�d•
ˆØ	�‰‹€AØ	�‰‹€AØ	�‰‹€AÜ
×
Ò
˜1Ó
€CÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ‡G�GÜ˜ÓÜ˜ÓÜ˜ÓÜ˜ÓØ�!˜ÜˆK˜�NÓœS ¨Q¥Ó0´#°kÀ!µnÓ2EÜˆF�1�I‹œ˜F 1�I›¬¨F°1­I«Üˆe‹Ü�a—g‘gÕØôð €JrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \        ,          R\        \        ,          R\        R\        R\
        R	,          R
\         P                  /	# )r’   r‚  rí   r|  ÚkhÚkwÚtemporal_windowÚclampr’  Nr”   r'  )r—   s   "rW   r˜   r˜   D  s~   € ÷  ñ  Ü‡|�|ð ä‡|�|ð ô ‡|�|ð ô 	ŒS�	ð	 ô
 	ŒS�	ð ô ð ô ð ô �4�<ð ô ‡\�\ñ rY   c                óÞ  € V P                   w  r‰r«rÍVf
   VR,          pV P                  4       p VP                  4       pVP                  4       p\        P                  ! V 4      p\        P                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V4      \        V4      Y‰VP                   ^,          Y«YÁP                   ^,          TV Uu. uF  p\        V4      NK  	  upV Uu. uF  p\        V4      NK  	  up\        V4      \        V4      \        V4      \        V P                  ,          V4       V# u upi u upi )z…Window attention over stacked key planes, ``(B, Tq, H, W, NH, HD)``
queries against ``(B, Tk, H, W, KVH, HD)`` keys. See ops/na3d.cu.r*  )r  r¬   r•   rÂ  r¹   r–  r“   r—  r  r
   r�  rµ   r§   re   r›   )r‚  rí   r|  r1  r2  r3  r4  r’  r,  Útqr,  r-  r.  r/  rB  r›  r!  s   &&&&&&&&         rW   r
   r
   D  s  € ð  Ÿg™gÑ€Eˆq�RØ‚}Ø�d•
ˆØ	�‰‹€AØ	�‰‹€AØ	�‰‹€AÜ
×
Ò
˜1Ó
€CÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×ÑÜ˜ÓÜ˜ÓÜ˜ÓÜ˜ÓØ�1—7‘7˜1•:˜q R¯©°­°RÙÓ™�AŒˆQŽ™Ñ©rÓ2©r¨!œs 1žv©rÑ2ÜˆOÓœc %›jÜˆe‹Ü�a—g‘gÕØôð €Jùò 	ùÒ2s   Ã9E% ÄE*"gš™™™™™ñ?c                ó  € VP                   ^,          pVP                  4       P                  RR4      R,          P                  R4      p\        P
                  ! W',          4      P                  R^4      pVw  rš\        V\        WiV
4      ,
          V4      pV^ 8X  dE   \        P                  ! V P                   RR \        R4      V P                  V P                  R7      # \        ^\        Wf,          V P                  4       ,          ,          4      pW,          P!                  R4      pVP#                  R4      P!                  R4      p\        P$                  ! V P                   RR V P                  V P                  R7      p\'        ^ V P                   ^ ,          V4       F°  pVV,           p\        P(                  ! V VV VVV P+                  ^^4      4      pVP-                  VVV 4      P-                  VVV 4       W©8”  d   \        R4      VRWš13&   VP/                  VRR	R
7      P0                  P3                  R4      P0                  VVV% K²  	  WÿP                  4       R,          ,
          P5                  4       # )aÞ  Per-query-block top-k as a threshold: the k-th largest pooled score
(the kernel keeps score >= threshold, so ties over-select).
``c8``/``csc`` are the quantized centroids ``[BH, N, D]``/``[BH, N]``, ``kc``
the centred pooled keys. Scores are computed through the route kernel's own
int8 quantization and operation order; an fp32 threshold against int8
kernel scores inflates the kept budget ~15%. Sink blocks are always kept,
so they neither count toward nor consume the budget.Tç     À_@çê-�™—q=NÚinfr~  ú-inf.F)r'  Úsortedgñhãˆµøä>r4  rž   éþÿÿÿ)r  ÚabsÚamaxÚ	clamp_minr•   Úroundr©   rj   ri   Úfullr§   r“   r›   r  Ú_SOL_TOPK_SLICE_BYTESrX  Ú	unsqueezeÚsqueezerl  rT  ÚbmmÚ	transposeÚmul_ÚtopkÚvaluesré  r¬   )Úc8ÚcscÚkcÚ
topk_ratioÚlog2sÚsinksrì   ÚkscÚk8Ús0Ús1ÚkkÚstepÚ	row_scaleÚ	col_scaleÚkthr™  rš  Úss   &&&&&&             rW   Ú_topk_from_pooledr[  n  së  € ð 	�‰��€AØ�6‰6‹8�=‰=˜˜TÓ" UÕ*×
5Ñ
5°eÓ
<€CÜ	�Š�R•XÓ	×	%Ñ	% d¨CÓ	0€BØ�F€BÜ	�Qœ Q¨BÓ/Õ/°Ó	<€BØ	ˆQ„wÜ�zŠz˜"Ÿ(™( 3 B˜-¬¨u«¸b¿i¹iÈrÏxÉxÔXÐXäˆqÔ'¨A­E°B·O±OÓ4EÕ,EÕFÓG€DØ•×'Ñ'¨Ó+€IØ—‘˜B“×)Ñ)¨"Ó-€IÜ
�+Š+�b—h‘h˜s �m¨B¯I©I¸R¿X¹XÔ
F€CÜ�1�b—h‘h˜q•k 4Ö(ˆØ��HˆÜ�IŠI�b˜˜1�g˜r ! A˜w×0Ñ0°°AÓ6Ó7ˆØ	�‰ˆy˜˜1ˆ~Ó×#Ñ# I¨a° NÔ3ØŒ7Ü! &›MˆAˆc�2�5ˆj‰MØ—6‘6˜" "¨U�6Ó3×:Ñ:×>Ñ>¸rÓB×IÑIˆˆAˆaŠñ )ð —'‘'“)˜dÕ"Õ"×.Ñ.Ó0Ð0rY   c           
     óÄ  € V P                   w  r4rVV^@,          ^@,          pVf¡   V RRV13,          P                  VR^@WV4      P                  ^\        P                  R7      R,          pWt8w  dT   V RVR13,          P                  ^R\        P                  R7      WG,
          ,          p	\        P
                  ! W‰.^R7      pEMs. p
\        ^ V\        4       EF*  pV RW»\        ,           13,          pVeA   WÂW»\        ,            P                  ^R^^4      P                  V P                  4      ,          pVP                   ^,          ^@,          ^@,          pV'       dN   V
P                  VRRV13,          P                  VR^@WV4      P                  ^\        P                  R7      4       WÜP                   ^,          8w  g   Kî  V
P                  VRVR13,          P                  ^R\        P                  R7      4       EK-  	  \        P
                  ! V
^R7      VP                  ^R^^4      ,          pVP                  ^ ^^^4      P                  W5,          RV4      # )	a  (B, T, H, D) -> [BH, N, D] fp32 means over each 64-block's live rows,
with fp32 accumulation but no fp32 copy of x. ``lengths``/``valid`` only
for zero-padded tiles; the default divides by Python floats so the
routing threshold stays bit-identical run to run.Nr�   rš   g      P@T)Úkeepdimr›   r&  r4  )r  r¥   r  r•   r¢   r*  rT  Ú_SOL_MEAN_SLICE_ROWSrª   r›   ÚappendÚpermuter6  )r!  ÚlengthsÚvalidrš  rw  r,  ÚdrB  rë   ÚtailÚpartsr™  rÉ  Úfs   &&&           rW   Ú_block_meansrg  �  sæ  € ð
 —‘�J€Aˆ!Ø��G�r�>€DØ‚Øˆa��$�ˆh�K×Ñ˜Q  B¨Ó-×1Ñ1°!¼5¿=¹=Ð1ÓIÈDÕPˆØŒ9Ø�Q˜™�X•;—?‘? 1¨d¼%¿-¹-�?ÓHÈAÍHÕUˆDÜ—	’	˜1˜)¨Ô+ˆAùð ˆÜ�q˜!Ô1×2ˆAØ�1�aÔ0Õ0Ð0Ð0Õ1ˆBØÒ Ø Ô&:Õ":Ð;×@Ñ@ÀÀBÈÈ1ÓM×PÑPÐQR×QXÑQXÓYÕY�Ø—‘˜!• Õ" bÕ(ˆAßØ—‘˜R  2 A 2 �YŸ^™^¨A¨r°2°qÓ<×@Ñ@ÀÌ%Ï-É-Ð@ÓXÔYØ—H‘H˜Q•KÖØ—‘˜R  1¡2 �YŸ]™]¨1°dÄ%Ç-Á-˜]ÓP×Qñ 3ô �IŠI�e Ô# g§l¡l°1°b¸!¸QÓ&?Õ?ˆØ�9‰9�Q˜˜1˜aÓ ×(Ñ(¨­°°AÓ6Ð6rY   c           
      óŽ  € V^?,           ^@,          pV P                   \        P                  8w  g+   V P                  4       ^8w  g   V P	                  4       V8w  d2   \        RV RV P                    R\        V P                  4       24      hV P                  V8w  d   \        RV RV P                   24      hV P                  4       # )é?   z,sol_attn: block_len must be int32 of shape (z,), got Ú zsol_attn: block_len must be on rÎ  )
r›   r•   ró  r'  r5  rÊ  r¶   r  r“   r¬   )Ú	block_lenrw  r“   rì   s   &&& rW   Ú_check_block_lenrl  ©  s°   € Ø	
ˆR��B�€AØ‡�œ%Ÿ+™+Ô%¨¯©«¸AÔ)=ÀÇÁÓARÐVWÔAWÜØ:¸1¸#¸XÀiÇoÁoÐEVÐVWÜ�Y—_‘_Ó%Ð&ð(ó)ð 	)ð ×Ñ˜6Ô!ÜÐ:¸6¸(À&È×IYÑIYÐHZÐ[Ó\Ð\Ø×ÑÓ!Ð!rY   c           
      óè   € \        V P                  4      \        V4      8w  g   V P                  V8w  d>   \        R \        V4       RV R\        V P                  4       RV P                   24      hV # )zsol_attn: coarse_gate must be ú on rÎ  )r¶   r  r“   rÊ  )Úcoarse_gater  r“   s   &&&rW   Ú_check_coarse_gaterp  ´  st   € Üˆ[×ÑÓ¤5¨£<Ô/°;×3EÑ3EÈÔ3OÜØ,¬U°5«\¨N¸$¸v¸hÀfÜ�[×&Ñ&Ó'Ð(¨¨[×-?Ñ-?Ð,@ðBóCð 	Cð ÐrY   c                óx  € \        WV4      p\        WV4      pWˆP                  ^RR7      ,
          pVP                  4       P                  RR4      R,          P	                  R4      p	\
        P                  ! Wy,          4      P                  R^4      p
\        W©P                  R4      W‚V\        ,          V4      # )zETop-k threshold from post-rope q/k, pooled and quantized like prep_q.T©r'  r]  r8  ç:Œ0âŽyE>r4  rž   )rg  Úmeanr>  r?  r@  r•   rA  r©   r[  rE  rf   )r‚  rí   rN  r’  ra  rb  rP  ÚcenrM  rL  rK  s   &&&&&&&    rW   Ú_topk_thresholdrv  ¼  s�   € ä
�q 5Ó
)€CÜ	�a %Ó	(€BØ	�g‰g˜! TˆgÓ*Õ	*€BØ�7‰7‹9�>‰>˜"˜dÓ# eÕ+×
6Ñ
6°tÓ
<€CÜ	�Š�S•YÓ	×	&Ñ	& t¨SÓ	1€BÜ˜R§¡¨R£°"À%Ì&Å.ÐRWÓXÐXrY   c           
     ó  € \        V 4      W3p\        P                  V4      pVe   V^ ,          ! 4       V J d
   V^,          # V P                  RV^,          ^^4      pVP                  ^ ,          V8w  d#   \        RVP                  ^ ,           RV 24      h\        4       ;_uu_ 4        \        P                  ! W^V P                  \        P                  R7      pRRR4       VR,          XRRV^,          1^ 3&   VR,          VRRV^,          1^3&   VR	,          VRV^,          R1^ 3&   VR
,          VRV^,          R1^3&   \        P                  4        \        P                  ! V 4      V3\        V&   V#   + '       g   i     L¥; i)zÍ[..., T, 1, rot/2, 2, 2] -> [T, rot, 2] per-channel (self, partner)
coefficients. Cached per freqs tensor (H3 reuses one across a step's
layers); the weak ref validates an id() hit without retaining freqs.Nzsol_attn: rope_freqs covers ú tokens, T=r~  r�   r4  )r�   r�   r   r   )r�   r�   r   r  )r�   r�   r  r  )r�   r�   r  r   )ÚidÚ_ROPE_FAB_CACHEr    r6  r  rÊ  r   r•   rl  r“   r¢   ÚclearÚweakrefÚref)Úfreqsrw  rZ  rt  Úhitrf  Úfabs   &&&    rW   Ú_packed_rope_fabr�  É  sM  € ô ˆe‹9�aÐ
€CÜ
×
Ñ
˜cÓ
"€CØ
‚˜3˜qž6›8 uÓ,Ø�1�vˆØ�‰�b˜# �( A qÓ)€AØ‡w�wˆq…z�Q„ÜÐ7¸¿¹À½
°|À;ÈqÈcÐRÓSÐSÜ	×	Õ	Ü�kŠk˜! !¨E¯L©LÄÇÁÔNˆ÷ 
à˜Z�=€Cˆˆ9ˆC�1�Hˆ9�aˆÑØ˜Z�=€Cˆˆ9ˆC�1�Hˆ9�aˆÑØ˜Z�=€Cˆˆ3�!�8‰9�aˆÑØ˜Z�=€Cˆˆ3�!�8‰9�aˆÑÜ×ÑÔÜ#ŸKšK¨Ó.°Ð4„O�CÑØ€J÷ 
×	ús   Â)3E9Å9F		c                 óZ   € V f   ^ ^ .# \        V ^ ,          4      \        V ^,          4      .# rN   r¾   )r  s   &rW   Ú
_sink_pairrƒ  ß  s*   € Ø’]ˆAˆqˆ6ÐF¬¨U°1­X«¼¸EÀ!½H»Ð(FÐFrY   c                ó(  € \         P                  P                  V 4      pVR	8  d"   \        RV  RV^ ,           V^,           24      h\	        RVRVRV/VC4      pVP
                  '       g&   \        RVP                   RVP                   24      hR# )
zÐShared by both CUDA entries (callers may bypass the registry): the
registry rule plus a per-device sm_80 check -- the registry gate caches the
CURRENT device's capability, and sub-sm_80 cubins return garbage.z0sol_attn: requires sm_80+ (cp.async, INT8 MMA); z is sm_Úsink_blocksÚsink_qrN  ú
sol_attn: z: N©é   r   )	r•   r¹   rº   rÆ  r�   ÚsuccessrÊ  Úfailed_paramÚfailure_reason)r“   r…  r†  rN  ÚtensorsÚcapÚchecks   &&&&,  rW   Ú_check_sol_argsr�  ã  s¦   € ô �*‰*×
*Ñ
*¨6Ó
2€CØ
ˆV„|ÜØ>¸v¸hð GØ�a•&�˜#˜a�&˜ð#ó$ð 	$ô &Ø	˜ X¨v°|ÀZÐ[ÐSZÐ[ó]€Eà�=�=ˆ=Ü˜: e×&8Ñ&8Ð%9¸¸E×<PÑ<PÐ;QÐRÓSÐSñ rY   c                óÌ  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R,          R\        \        ,          R,          R\        \        ,          R,          R	\         P                  R,          R
\        R\
        R\         P                  R,          R\         P                  R,          R\        R\         P                  /# )r’   r‚  rí   r|  Útaur’  Nr…  r†  Úkey_biasrN  rd  rk  ro  Ú	token_augr”   )r•   r–   r§   r(  rµ   rÏ   )r—   s   "rW   r˜   r˜   ò  så   € ÷ Yñ YÜ‡|�|ðYä‡|�|ðYô ‡|�|ðYô 
ð	Yô
 �4�<ðYô ”c•˜TÕ!ðYô ”�I˜ÕðYô �l‰l˜TÕ!ðYô ðYô ðYô �|‰|˜dÕ"ðYô —‘ Õ$ðYô ðYô ‡\�\ñYrY   c                óò  € V P                   w  rÞppV P                  \        P                  \        P                  39  d   \        RV P                   24      h\        V P                  WVW€WW«R7	       Vf
   VR,          pV
e   V
P                  4       p
\        Wî^?,           ^@,          V P                  V
4      pV
e   \        VV4      MRpV
e   TMRpRV 3RV3RV33 Fî  w  ppVP                  R4      ^8w  d   \        RV R24      hVP                  4       ^,          '       d9   \        RV R	VP                  4        R
VP                  4       ^,           R24      h\        ^4       F_  pVP                   V,          ^8”  g   K  VP                  V4      ^,          '       g   K=  \        RV RV RVP                  V4       R24      h	  Kð  	  \        V4      \        V4      ppRpVeA   \!        W}WàP                  4      pV\"        ,          P%                  WÞ4      P                  4       p\        P&                  ! V P                   V P                  V P                  R7      p\(        P+                  WÞV\-        V4      R7      p\        P&                  ! VR,          \        P.                  V P                  R7      pV'       d   \1        WW„VVV4      MRp\(        P3                  \5        V 4      \5        V4      \5        V4      \5        V4      \5        V4      YÞTT\7        V4      \7        V4      V^ ,          V^,          V^ ,          V^,          \        P8                  P;                  V P                  4      P<                  Vf   RM
\5        V4      Vf   RM
\5        V4      V
f   RM
\5        V
4      \?        V	4      \-        V4      R7       Ve,   \A        V\C        . \E        VVWß,          V4      OVN5!  V4       V# )a  Sol-Attn sparse attention over ``(B, T, H, 128)`` bf16 or fp16 tensors.
See sage_attention/sol_attn.cu and the public docstring for ``tail``,
``block_len`` and ``coarse_gate``.

``topk_ratio`` > 0 switches selection from the tau threshold to SLA-style
per-query-block top-k: keep that fraction of key blocks per query block
(sinks and the diagonal still ride on top). tau is ignored then.

``token_aug`` (0, or a multiple of 64 up to 256): on top of the routed
blocks, every row attends up to ``token_aug`` unrouted tokens its query
block's centroid scores highest (whole histogram bins only, so the set
never depends on scheduling; a flat score profile may admit none), and the
remaining tail is exact for the centroid instead of pooled per block.
The eager reference ignores it.
z1sol_attn: q/k/v must be bfloat16 or float16, got )r‚  rí   r|  rk  ro  Nr‚  rí   r|  r‡  z  must have a contiguous last dimz) must be 16-byte aligned (storage_offset z leaves it at +z); call .contiguous() on itz stride(z) = zl elements is not a multiple of 8, so the 16-byte staging loads would be misaligned; call .contiguous() on itr%  ©r”  Útotal)r“  r	  rk  rd  r”  r*  r4  )#r  r›   r•   ra  r`  rÊ  r�  r“   r¬   rg   rk   Ústridery  Ústorage_offsetrT  rƒ  rh   rf   r   rl  r  Úsol_attn_planrµ   r¤   rv  r   r�  r§   r¹   r–  r—  rÏ   rl   rm   Ú_ws_block_means)r‚  rí   r|  r’  r’  r…  r†  r“  rN  rd  rk  ro  r”  r,  rw  r,  rc  ra  rb  Úmean_lengthsÚnamer!  r'  ÚsbÚsqÚkbrB  Úprm  Úthrs   &&&&&&&&&&&&&                 rW   r   r   ò  sq  € ð< —W‘W�N€Eˆa�Ø‡w�w”u—~‘~¤u§}¡}Ð5Ô5ÜÐLÈQÏWÉWÈIÐVÓWÐWÜ�A—H‘H˜k°:ÀaØ'õBà‚}Ø�T•	ˆØÒØ×(Ñ(Ó*ˆ	Ü˜Q R¥¨B¥°·±¸)ÓD€GØ'0Ò'<ŒK˜˜7Ô#À$€EØ'Ò3‘7¸€Lð ˜!�H˜s A˜h¨¨a¨Ó1‰ˆˆaØ�8‰8�B‹<˜1ÔÜ˜z¨$¨Ð/OÐPÓQÐQØ�:‰:‹<˜"×ÔÜØ˜T˜FÐ"KØ×#Ñ#Ó%Ð& o°a·j±j³lÀRÕ6GÐ5Hð I+ð,ó-ð -ô ˜–8ˆCØ�w‰w�s�|˜aÖ A§H¡H¨S£M°A×$5Ö$5Ü Ø    h¨s¨e°4¸¿¹À»°ð G>ð ?ó@ð @ó ñ 2ô ˜Ó$¤j°Ó&8ˆ€BØ	€BØÒä  °!·X±XÓ>ˆØ”6�k×!Ñ! %Ó+×6Ñ6Ó8ˆÜ
�+Š+�a—g‘g Q§W¡W°Q·X±XÔ
>€CÜ
×Ñ˜ 1´°I³ÐÓ?€AÜ—’˜A˜g�J¬e¯k©kÀ!Ç(Á(ÔK€I÷ ô ˜1 °LÀ%ÈÔLØ"ð ä‡K�KÜ˜ÓÜ˜ÓÜ˜ÓÜ˜ÓÜ˜Ó#Ø�!�QÜˆc‹
”E˜%“LØ
ˆ1�ˆr�!�u�b˜•e˜R �UÜ�
‰
×!Ñ! !§(¡(Ó+×7Ñ7Øš‘Ô)9¸"Ó)=Øš+‘$Ô+;¸CÓ+@Ø#Ò+‘$Ô1AÀ)Ó1LÜ�$‹Z¤3 y£>ð ô ð ÒÜ�CœÐa¬¸	À1ÀeÅiÐQXÓ(YÐaÐ[`ÓaØô	!à€JrY   c                 ó  € V ^8„  d   QhR\         R\         R\        P                  R\        \        P                  \        P                  3,          R\        P                  R,          R\        P                  R,          R\        R	\        R
\        R,          R\
        \         ,          R,          R\
        \         ,          R,          R\        R\        R\        P                  R,          R\        P                  R,          R\         /# )r’   rw  r,  Ú
rope_freqsÚqk_norm_weightsÚkmeanNÚvscaler’  rN  r’  r…  r†  Úrope_epsrd  rk  ro  r”  )rµ   r•   r–   r¶   r§   r(  rÏ   )r—   s   "rW   r˜   r˜   N  s  € ÷ g-ñ g-ä
ðg-ô ðg-ô —‘ð	g-ô
 œ5Ÿ<™<¬¯©Ð5Õ6ðg-ô �<‰<˜$Õðg-ô �L‰L˜4Õðg-ô 
ðg-ô ðg-ô �4�<ðg-ô ”c•˜TÕ!ðg-ô ”�I˜Õðg-ô ðg-ô ðg-ô �|‰|˜dÕ"ðg-ô  —‘ Õ$ð!g-ô" ñ#g-rY   c                óð  a aaaaaaaaaa a!a"a#a$€ \         pVP                  R,          ^,          o!S!^,          '       g   ^ S!u;8  d   V8:  g   M \        RV RS! 24      h\        VSS!4      oSP                  o\        SW«V4       V	f
   VR,          p	Se   \        SSS4      oVe   \        V^SSV3S4      p\        SS^?,           ^@,          SS4      pV3R lV 4       w  o o\        S 4      '       d   S MRoSf   Ve   Vf   \        S 4      o V 3R lo\        P                  ^SS\        S4      R7      p\        P                  ! VR,          \        P                   SR7      o$\        P"                  P%                  S4      P&                  o"^S,          V,          o#VVVVVVV V VV!V"VVV#V$3R	 lpR
 pVe   Vf´   V! \        P(                  ! SVSR7      \        P*                  ! SVSR7      4       \-        S$VS4      P/                  ^4      VP/                  4       ,          pV! S$VR,          VR,          SV,          ^,          ,            P1                  \        P2                  4      4      pVP5                  S\        P2                  R7      P7                  4       pVP5                  S\        P2                  R7      P9                  R4      P7                  4       pV! WV4       \;        V
4      \;        V4      pp\        P                  ! ^SSV\        P<                  SR7      p\        P                  ! SVS\        P2                  R7      p\        P                  ! SVS\        P2                  R7      pV'       d   \?        S$VSW‰VV4      MRp\        PA                  \C        S$4      \C        V4      \C        V4      \C        V4      \C        V4      ^SS\E        V4      \E        V	4      V^ ,          V^,          V^ ,          V^,          S"Vf   RM
\C        V4      Sf   RM
\C        S4      \G        V4      \        S4      R7       Ve&   \I        V\K        . \M        S$VSV4      OV	N5!  V4       VVV! V4      3# )a  Chunked-producer Sol-Attn over fused qkv projection chunks ([M, 3*H*128]
bf16, 64-aligned starts, B=1); full Q/K/V are never materialised.
``tail`` / ``block_len`` / ``coarse_gate`` / ``token_aug`` as in ``sol_attn``.

``qkv_chunks``: an iterable of chunks or a zero-arg callable returning one.
``kmean``/``vscale`` are LAST step's statistics ([H,128] f32); when None the
producer runs twice (measure, then quantize), so pass a callable to stream
on the first call. Returns ``(out[1,T,H,128] bf16, kmean_next, vscale_next)``.z9sol_attn_chunked: rot_dim must be a multiple of 8 in (0, z], got Nc              3   ó€   <"  € T F3  qP                  S\        P                  R 7      P                  4       x € K5  	  R# 5i)r~  N)rª   r•   ra  r¬   )rÜ   r-  Údevs   & €rW   rß   Ú#sol_attn_chunked.<locals>.<genexpr>x  s,   øé € Ð[É?Àa�d‰d˜#¤U§^¡^ˆdÓ4×?Ñ?×AÐAË?ùs   ƒ;>c                  ó   <€ \        S 4      # rN   )Úiter)Ú
qkv_chunkss   €rW   Ú<lambda>Ú"sol_attn_chunked.<locals>.<lambda>|  s
   ø€ œ$˜zÔ*rY   r–  r—  r%  c                 ót  <€ \         P                  \        S4      ^SS	S\        S4      R7       ^ pSe   S! 4       MS EF]  pVP                  R,          pV^@,          '       d   V'       d   \        R4      hVP                  R,          S8w  g1   VP                  \        P                  8w  g   VP                  S8w  dB   \        RS RS R\        VP                  4       RVP                   RVP                   2
4      h\         P                  \        S4      \        VP                  4       4      \        S4      \        S4      \        S
4      \        V 4      \        V4      \        S4      SY$^SS	SSf   RM
\        S4      \        S4      R	7       W$,          pEK`  	  VS8w  d   \        R
V RS 24      hR# )r  r–  Nz1sol_attn_chunked: chunk starts must be 64-alignedz%sol_attn_chunked: chunks must be [M, z] bfloat16 on rÎ  rj  rn  )rk  r”  zsol_attn_chunked: chunks cover rx  r=  r4  )r  Úsol_producer_beginr�  rµ   r  rÊ  r›   r•   ra  r“   r¶   Úsol_producer_chunkr¬   r§   )ÚkmÚvscÚt0Úchunkrë   rk  r«  r€  Úfactoryr,  r2  r¯  Úqwr¨  rZ  r‹  rw  r”  ÚwidthÚwss   &&   €€€€€€€€€€€€€€€rW   ÚproduceÚ!sol_attn_chunked.<locals>.produce‚  s‘  ø€ Ü
×ÑÔ.¨rÓ2°A°q¸!¸VÌsÐS\Ë~ÐÔ^ØˆØ#*Ò#6‘g”i¸JÓFˆEØ—‘˜B•ˆAØ�B�wŒwŸ1Ü Ð!TÓUÐUà�{‰{˜2� %Ô'¨5¯;©;¼%¿.¹.Ô+HÈEÏLÉLÐ\_ÔL_Ü Ø;¸E¸7À.ÐQTÐPUð VÜ  §¡Ó-Ð.¨a°·±¨}¸DÀÇÁÀðPóQð Qô ×!Ñ!Ü  Ó$Ô&6°u×7GÑ7GÓ7IÓ&JÜ  Ó%Ô'7¸Ó';Ô=MÈbÓ=QÜ  Ó$Ô&6°sÓ&;Ü�h“  b¨Q°°1°fØ"+Ò"3™$Ô9IÈ)Ó9TÜ˜i›.ð "ô *ð �G‹Bñ! Gð" �Œ7ÜÐ>¸r¸dÀ+ÈaÈSÐQÓRÐRñ rY   c                 óH   € V R ,          \         ,          P                  R4      # )r8  rs  )Ú_SOL_VSCALE_MARGINr@  )Úvamaxs   &rW   Ú	vscale_ofÚ#sol_attn_chunked.<locals>.vscale_of™  s   € Ø˜•Ô 2Õ2×=Ñ=¸dÓCÐCrY   ©r“   ÚstatsVr~  rs  )r	  rk  rd  r”  éýÿÿÿr*  )'Ú_SOL_HDr  rÊ  r�  r“   r�  rl  rp  rg   Úcallabler(  r  rš  rµ   r•   rl  r¤   r¹   r–  r—  r)  ÚonesÚ	_ws_ksumsr  r¥   r¢   rª   r¬   r@  rƒ  ra  Ú_topk_threshold_from_workspaceÚsol_attn_corer�  r§   rÏ   rl   rm   r›  )%r¯  rw  r,  r¤  r¥  r¦  r§  r’  rN  r’  r…  r†  r¨  rd  rk  ro  r”  rc  ra  r¡  r½  rÂ  rž  rŸ  rB  Ú
kmean_nextrÁ  r	  r«  r€  r¹  r2  rº  rZ  r‹  r»  r¼  s%   fff&&&&&&&&&f&f&f           @@@@@@@@@rW   r   r   N  s±  ÿþ€ ô6 	€AØ
×
Ñ
˜2Õ
 Õ
"€Cà
ˆQ‡w„w�a˜#–l –lÜÐTÐUVÐTWÐW^Ð_bÐ^cÐdÓeÐeÜ
˜: q¨#Ó
.€CØ
�*‰*€CÜ�C˜¨jÔ9Ø‚}Ø�T•	ˆØÒÜ$ Y°°3Ó7ˆ	ØÒÜ(¨°q¸!¸QÀ°lÀCÓHˆÜ˜Q  R¥¨B¥°°YÓ?€GÜ[É?Ó[�F€BˆÜ$ Z×0Ò0‰j°d€GØ‚˜EšM¨Vª^Ü˜*Ó%ˆ
Ü*ˆÜ
×Ñ˜˜A˜q¬C°	«NÐÓ;€AÜ	�Š�Q�w•Z¤u§{¡{¸3Ô	?€BÜ�Z‰Z×&Ñ& sÓ+×7Ñ7€FØ��E�A�I€E÷S÷ Só Sò.Dð ‚}˜šñ 	”—’˜A˜q¨Ô-¬u¯zªz¸!¸QÀsÔ/KÔLÜ˜"˜a Ó#×'Ñ'¨Ó*¨W¯[©[«]Õ:ˆÙ˜2˜a �k¨!¨H­+¸¸A½À½	Õ*AÐB×GÑGÌÏÉÓVÓWˆØ�H‰H˜C¤u§}¡}ˆHÓ5×@Ñ@ÓB€Eà�Y‰Y˜c¬¯©ˆYÓ7×AÑAÀ$ÓG×RÑRÓT€FÙˆEÔÜ˜Ó$¤j°Ó&8ˆ€BÜ
�+Š+�a˜˜A˜q¬¯©¸sÔ
C€CÜ—’˜Q ¨#´U·]±]ÔC€JÜ�KŠK˜˜1 S´·±Ô>€E÷ ô 0°°A°q¸*ÈWÐVXÔYØ$(ð ä×ÑÜ˜ÓÔ.¨sÓ3Ô5EÀfÓ5MÜ˜Ó$Ô&6°uÓ&=Ø	ˆ1ˆa”�s“œU 5›\¨2¨a­5°"°Qµ%¸¸A½ÀÀ1ÅÀvØ#Ò+‘$Ô1AÀ)Ó1LØ#Ò+‘$Ô1AÀ)Ó1LÔSWÐX\ÓS]ÔilÐmvÓiwð ô yð ÒÜ�CœÐR¬¸¸A¸qÀ'Ó(JÐRÈEÓRÐT_Ô`Ø�
™I eÓ,Ð,Ð,rY   c                 ó  € \         P                  ! ^€4      pV^,	          V^,          r2V^,	          V^,          rTV^ ,          ^V^,	          ,          ,           ^V,          ,           V^,          ,           P                  V 4      # )é€   )r•   r£   rª   )r“   rc  rM  Úremr,  Úr2s   &     rW   Ú_perm_d_indexrÒ  ¸  sb   € Ü�Š�SÓ€AØ�1�f�a˜"•fˆØ�1�H�c˜B•h€rØ��G�a˜2 �7•mÕ# a¨!¥eÕ+¨r°A­vÕ6×:Ñ:¸6ÓBÐBrY   c                ó  € \         pWR,          VR,          W!R,          ,          V,          ^,          ,            P                  \        P                  4      P                  W!R,          V4      RRVR,          13,          # )z�The producer's post-rope block K sums, [BH, NTB, 128] f32 view into
scratch (block MEANS on the direct path, and once sol_attn_core has run).rÌ  ÚNPADr�   NÚNTB)rÇ  r¥   r•   r¢   )r¼  r¡  Úbhrc  s   &&& rW   rÊ  rÊ  ¿  se   € ô 	€AØ�	�l˜1˜Y�<¨"°­y­.¸1Õ*<¸qÕ*@Õ@ÐAß	‰Œe�m‰mÓ	ŸT™T "¨¥i°Ó3°A°y¸¸%½°y°LõBð BrY   c                óP  € VR,          VR,          \         repWR,          VR,          W%,          V,          ^,          ,            P                  \        P                  4      P                  W%V4      RRV13,          pWR,          VR,          W&,          V,          ^,          ,            P                  \        P                  4      P                  W&V4      RRRV13,          pV\        WV4      VP                  ^^4      P                  4       VP                  ^R^4      ,          3# )z´(qm, km, vm) ``[BH, NTB, 128]`` fp32 block means over live rows, read
from what the kernels already left in the workspace: the qmean slot, K
means in scratch, V sums in vcT (bf16).rÕ  rÔ  Úqmeanr�   NÚvcTr4  )rÇ  r¥   r•   r¢   ra  rÊ  rG  r§   )	r¼  r¡  rÖ  ra  ÚntbÚnpadrc  ÚqmÚvms	   &&&&     rW   r›  r›  Ç  sù   € ð �U•8˜Q˜v�Y¬ˆq€CØ	ˆg�J�q˜•z B¥I°¥M°AÕ$5Õ5Ð	6×	;Ñ	;¼E¿M¹MÓ	J×	OÑ	OÐPRÐZ[Ó	\Ð]^Ð`dÐadÐ`dÐ]dÕ	e€BØ	ˆe�H�Q�u•X ¥¨¥°Õ 1Õ1Ð	2×	7Ñ	7¼¿¹Ó	G×	LÑ	LÈRÐTXÓ	YÐZ[Ð]^Ð`dÐadÐ`dÐZdÕ	e€BØŒy˜ Ó# R§\¡\°!°QÓ%7×%=Ñ%=Ó%?À'Ç,Á,ÈqÐRTÐVWÓBXÕ%XÐXÐXrY   c                ón  € VR,          \         r‡WR,          VR,          W',          V,          ,            P                  \        P                  4      P                  W'V4      P	                  4       p	WR,          VR,          W',          ^,          ,            P                  \        P
                  4      P                  W'4      p
\        WV4      VP                  ^R^4      ,          pW»P                  ^RR7      ,
          pV	RR\        V P                  4      3,          p	\        WšW³V\        ,          V4      # )zFProducer-path top-k threshold from the workspace's own pooled outputs.rÕ  Úcen8ÚcensTrr  r�   r4  )rÇ  r¥   r•   r«   r§   r¢   rÊ  rt  rÒ  r“   r[  rf   )r¼  r¡  r,  rN  r’  ra  rP  rÚ  rc  rß  rà  rM  s   &&&&&&&     rW   rË  rË  Ñ  sê   € àˆu�X”wˆØ��i˜˜&�	 A¥G¨a¥KÕ/Ð0×5Ñ5´e·j±jÓA×FÑFÀqÈqÓQ×WÑWÓY€DØ��i˜˜&�	 A¥G¨a¥KÕ/Ð0×5Ñ5´e·m±mÓD×IÑIÈ!ÓQ€DÜ	�2˜!Ó	˜wŸ|™|¨A¨r°1Ó5Õ	5€BØ	�g‰g˜! TˆgÓ*Õ	*€Bð ��1”m B§I¡IÓ.Ð.Õ/€DÜ˜T¨¸Ä½ÈÓOÐOrY   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        /# )r’   r!  r’  ÚshiftÚeps©r•   r–   r§   )r—   s   "rW   r˜   r˜   Þ  s5   € ÷ (ñ (œ5Ÿ<™<ð (´·±ð (ÄUÇ\Á\ð (ÔX]ñ (rY   c                 ób  € VP                   pVP                   R,          pVP                  4       V,          pVP                  Wv4      pVP                  4       '       g   VP	                  4       p\        W!Wv4      w  rš\        W1Wv4      w  r¼\        P                  ! V4      p\        VP                  ,          p\        P                  P                  VP                  4      P                  pV ! \        V4      \        V	4      \        V4      \        V4      VVV
VVVV4       VP                  V4      # )r  r4  )r  r5  r6  r  r¬   rb   r•   rÂ  re   r›   r¹   r–  r“   r—  r�  )Úkernelr!  r’  râ  rã  rÄ  rc  rì   Úx_flatÚ
scale_flatÚscale_groupÚ
shift_flatÚshift_groupÚout_flatÚ
dtype_coder›  s   &&&&&           rW   Ú_adaln_implrî  Þ  sû   € Ø—‘€JØ	�‰��€AØ	�‰‹	�Q�€Aà�Y‰Y�q‹_€FØ×Ñ×!Ò!Ø×"Ñ"Ó$ˆä3°E¸aÓCÑ€JÜ3°E¸aÓCÑ€Jä×Ò Ó'€HÜ˜qŸw™wÕ'€JÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€Já
Ü˜Ó Ü˜Ó$Ü˜Ó$Ü˜Ó"Ø	Ø	ØØØØØôð ×Ñ˜JÓ'Ð'rY   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# ©r’   r!  r’  râ  rã  r”   rä  )r—   s   "rW   r˜   r˜   ÿ  sB   € ÷ 7ñ 7ŒU�\‰\ð 7¤%§,¡,ð 7´u·|±|ð 7Ì%ð 7Ô[`×[gÑ[gñ 7rY   c                 ó8   € \        \        P                  WW#4      # rN   )rî  r  r   ©r!  r’  râ  rã  s   &&&&rW   r   r   ÿ  s   € Ü”r—x‘x ¨5Ó6Ð6rY   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rð  rä  )r—   s   "rW   r˜   r˜     sB   € ÷ ;ñ ;”—‘ð ;¤e§l¡lð ;¼5¿<¹<ð ;Ìeð ;Ô_d×_kÑ_kñ ;rY   c                 ó8   € \        \        P                  WW#4      # rN   )rî  r  r   rò  s   &&&&rW   r   r     s   € Ü”r—|‘| Q¨uÓ:Ð:rY   c                ó¬   € V ^8„  d   QhR\         P                  R\        \         P                  \         P                  \         P                  3,          /# )r’   Úpointsr”   ©r•   r–   r¶   )r—   s   "rW   r˜   r˜     s>   € ÷ yOñ yO”u—|‘|ð yO¬¬e¯l©l¼E¿L¹LÌ%Ï,É,Ð.VÕ(Wñ yOrY   c                ó`  aaaaaaaaaa a!a"a#a$€ \        RV R4       V P                  pV P                  ^ ,          oS^ 8X  d   \        R4      h\        P
                  P                  V4      P                  oR\        P                  RV/oR\        P                  RV/o"\        P                  ! S^,           ^3\        P                  VR7      o#V S#RS% S#RS P                  ^ 4      P                  S#RS P                  ^ 4      P                  r2\        P                  ! W2,
          P!                  4       ^,          S#RS P#                  4       P                  4       .4      P%                  4       w  rE\        P&                  ! VR7      P)                  ^ 4      pR\        WER	4      ,          pS#RS;;; \        P*                  ! S^3V\        P                  VR
7      ^,          ^,
          V,          ,          uuu% \        P,                  ! . RO. RO. RO. RO.\        P                  VR7      pW#,           ^,          V^\        V^
V,          4      ,          ,          ,           S#SR% \        ^S,          ^@,           \.        4      o\        P0                  ! S^3R3/ SB o!\        P,                  ! SS^,           S^,           S^,           .3/ SB S!^ &   \        P0                  ! S^3R3/ SB o \        P2                  ! S3/ S"B o^S^ &   \        P2                  ! ^S,          3/ S"B o\        P2                  ! S3/ SB o\        P4                  ! SWaR7      P7                  4       p	\        P8                  ! V	4      P7                  4       p
VVVVVV V!V"3R lo\:        o$^oVVVVVVVVVV V!V"V#V$3R lpVVVVV V!V$3R lp \        P0                  ! S3\<        3/ SB p\>        PA                  S$! S4      S$! V	4      SS$! V4      S4       V\<        8g  p\7        VPC                  4       4      pV^ 8X  d   EM¤\        PD                  ! V^ \        P                  R7      VP7                  4       ,
          pS! S^V,          ,           4       \        P2                  ! S3/ S"B p\>        PG                  SS$! V4      S$! V4      S$! V
4      S$! S!4      S$! S 4      S$! S4      S$! S4      S$! V4      S4
       \>        PI                  SS$! V4      S$! V4      SS$! S!4      S$! S#4      S$! S4      S4       S^V,          ,          oV! V4       \7        SRS PC                  4       4      p^V,          ^S,          8  g   EK±  V! S4      w  po!o SPK                  \        PL                  ! S^ 8¬  VSPO                  ^ R7      ,          S4      4       SPQ                  4        ^SRV% VoEK  V! SPS                  4       4       V! \7        SRS PC                  4       4      4      w  pppS#V\        PL                  ! V^ 8¬  V^,	          V4      3# )zrSee comfy_kitchen.delaunay3d. Rounds and flip passes loop here because each
launch is sized by the step before it.rö  Nz#delaunay3d needs at least one pointr›   r“   r%  rÄ  g»½×Ùß|Û=r9  )Ú	generatorr›   r“   )rù  r“   c                 óN  <€ V S8:  d   R # V \         8”  d   \        RS R\          R24      h\        \        V S^,          ^,          4      \         4      S,
          p\        P
                  ! S\        P                  ! V^3R3/ SB .4      o\        P
                  ! S\        P                  ! V^3R3/ SB .4      o\        P
                  ! S\        P                  ! V3/ S	B .4      o\        P
                  ! S\        P                  ! ^V,          3/ S	B .4      oSV,          oR # )Nzdelaunay3d: z points need more than z tetsr4  )Ú_DELAUNAY_MAX_TETSrÊ  ré  r  r•   r*  rB  r)  )
ÚneedÚextraÚaliverŽ  Úi32rì   ÚpendÚtet_oppÚtet_vÚu8s
   & €€€€€€€€rW   ÚgrowÚdelaunay3d.<locals>.grow0  sò   ø€ à�3Œ;ÙØÔ$Ô$Ü˜|¨A¨3Ð.EÔFXÐEYÐY^Ð_Ó`Ð`Ü”C˜˜c A�g¨�lÓ+Ô-?Ó@À3ÕFˆÜ—	’	˜5¤%§*¢*¨e°Q¨Z¸Ñ"C¸sÑ"CÐDÓEˆÜ—)’)˜W¤e§j¢j°%¸°¸RÑ&GÀ3Ñ&GÐHÓIˆÜ—	’	˜5¤%§+¢+¨eÑ":°rÑ":Ð;Ó<ˆÜ�yŠy˜$¤§¢¨A°­IÑ <¸Ñ <Ð=Ó>ˆØˆu�ŠrY   c                 ó&  <a€ \        \        4       EFú  pV R S SR S ,          pVP                  4       P                  ^4      P	                  4       pVP                  4       ^ 8X  d    R # ^VP                  4       ,          oVV3R l\        ^4       4       w  rEp\        P                  ! S3\        3/ SB p\        P                  ! S3/ SB p \        P                  VP                  4       S ! V4      S ! V4      S ! S4      S ! S4      S ! S4      S ! V4      S ! V4      S ! V4      S ! V4      S ! V 4      S4       \        P                  ! S3/ SB p\        P                  SS ! V4      S ! V4      S ! V4      S ! V4      S ! V4      S ! V4      S ! V 4      S4	       V^ 8g  p	V^,          ^8H  p
\        P                  ! V	P                  4       V
P                  4       .4      P!                  4       w  r¼V^ 8X  d    R # \        P"                  ! V
^ \        P$                  R7      V
P	                  4       ,
          p\        P"                  ! V	^ \        P$                  R7      V	P	                  4       ,
          pS! SV,           4       \        P&                  ! V \        P                  ! SV P                  4       ,
          3/ SB .4      p \        P                  ! S3/ SB p\        P                  ! ^S,          3/ SB p\        P                  ! ^V,          3/ SB p\        P)                  SS ! V4      S ! V4      S ! V4      S ! V4      S ! V4      S ! V4      SS ! S4      S ! S4      S ! S4      S ! S4      S ! V4      S ! V4      S ! S4      S ! V4      S ! V 4      S4       \        P+                  VS ! V4      S ! V4      S ! V4      S ! S4      S ! S4      S4       \        P-                  SS ! V4      S ! V4      S ! S4      S ! S4      S ! S4      S4       SV,          oEKý  	  R # )Nc              3   óR   <"  € T F  p\         P                  ! S3/ SB x € K  	  R # 5irN   )r•   rl  )rÜ   r  rÿ  Ún_cands   & €€rW   rß   Ú+delaunay3d.<locals>.flip.<locals>.<genexpr>I  s   øé € Ð(VÉXÈ¬¯ª°VÑ)C¸sÖ)CËXùs   ƒ$'rš   )rT  Ú_DELAUNAY_MAX_FLIP_PASSESÚnonzerorE  rµ   r5  r•   rB  Ú
_INT32_MAXr)  r  Údelaunay_flip_detectrl  Údelaunay_flip_decideÚstackr  ÚtolistÚcumsumró  r*  Údelaunay_flip_executeÚdelaunay_flip_fixupÚdelaunay_relocate_flip)!Úactiver  Úactive_flagÚactive_listÚ	cand_infoÚcand_nÚcand_mÚownerÚrunÚranÚis23Ún_runÚn23Ú	slot_rankÚrun_rankÚflippedÚredirectÚnew_tetsr  rþ  rŽ  r  rÿ  rì   Ún_tetsr   Úpt_tetr‹  r  r  r  Úvertsr-  s!   &                 @€€€€€€€€€€€€€€rW   ÚflipÚdelaunay3d.<locals>.flip@  sh  ù€ ô Ô0×1ˆAØ   &˜/¨E°'°6¨NÕ:ˆKØ%×-Ñ-Ó/×7Ñ7¸Ó:×>Ñ>Ó@ˆKØ× Ñ Ó" aÔ'ÚØ˜×*Ñ*Ó,Õ,ˆFÝ(VÌUÐSTÌXÓ(VÑ%ˆI˜vÜ—J’J ˜y¬*Ñ<¸Ñ<ˆEÜ—[’[ Ñ+¨Ñ+ˆFÜ×#Ñ# K×$5Ñ$5Ó$7¹¸;»ÉÈ;ËÑYZÐ[`ÓYaÑcdÐelÓcmÙ$% e£H©a°	«l¹A¸f»IÁqÈÃyÑRSÐTYÓRZÑ\]Ð^dÓ\eÐgmôoä—+’+˜fÑ,¨Ñ,ˆCÜ×#Ñ# F©A¨k«N¹A¸i»LÉ!ÈFË)ÑUVÐW]ÓU^Ñ`aÐbgÓ`hÑjkÐloÓjpÙ$% f£I¨vô7à˜‘(ˆCØ˜!•G ‘>ˆDÜŸš c§g¡g£i°·±³Ð%<Ó=×DÑDÓF‰JˆEØ˜ŒzÚÜŸš T¨1´E·K±KÔ@À4Ç8Á8Ã:ÕMˆIÜ—|’| C¨´%·+±+Ô>ÀÇÁÃÕJˆHÙ�˜#•ÔÜ—Y’Y ¬¯ª°C¸&¿,¹,».Õ4HÑ(OÈBÑ(OÐPÓQˆFÜ—k’k &Ñ0¨CÑ0ˆGÜ—{’{ 1 v¥:Ñ5°Ñ5ˆHÜ—{’{ 1 u¥9Ñ4°Ñ4ˆHÜ×$Ñ$ V©Q¨{«^¹Q¸s»VÁQÀvÃYÑPQÐRXÓPYÑ[\Ð]fÓ[gÑijÐksÓitØ%+©Q¨u«X±q¸³xÁÀ7ÃÉQÈuËXÑWXÐY`ÓWaÑcdÐemÓcnÙ%& t£W©a°«k¹1¸V»9ÀfôNô ×"Ñ" 5©!¨H«+±q¸³zÁ1ÀXÃ;ÑPQÐRYÓPZÑ\]Ð^bÓ\cÐekÔlÜ×%Ñ% a©¨7«±Q°x³[Á!ÀEÃ(ÉAÈeËHÑVWÐX^ÓV_ÐagÔhØ�c�M‹Fó? 2rY   c                 óV  <€ \         P                  ! SR S ^ \         P                  R7      ^,
          p\         P                  ! V ^3R3/ SB p\         P                  ! V ^3R3/ SB p\        P                  SS
! S4      S
! V4      S
! S	4      S
! S4      S
! V4      S
! V4      S4       WV3# )Nrš   r4  )r•   r  ró  rB  r  Údelaunay_compact)ÚrowsÚ	new_indexÚout_vÚout_orþ  rÿ  r&  r‹  r  r  r-  s   &   €€€€€€€rW   Ú	compactedÚdelaunay3d.<locals>.compactedd  sœ   ø€ ä—L’L  w¨ °¼%¿+¹+ÔFÈÕJˆ	Ü—
’
˜D !˜9 bÑ0¨CÑ0ˆÜ—
’
˜D !˜9 bÑ0¨CÑ0ˆÜ
×Ñ˜F¡A e£H©a°	«l¹A¸e»HÁaÈÃjÑRSÐTYÓRZÑ\]Ð^cÓ\dÐflÔmØ Ð&Ð&rY   rš   )ré  ©Né   )r  r  r  r4  )r  r4  r4  )r4  r  r4  )r4  r4  r  )*Ú_check_shaper“   r  rÊ  r•   r¹   r–  r—  ró  r¤   rl  Úfloat64ré  rJ  r  r  Únormr>  r  Ú	GeneratorÚmanual_seedÚrandr¡   rû  rB  r)  Úrandpermrµ   Úargsortr�  r  r  Údelaunay_voter  r  Údelaunay_splitÚdelaunay_relocate_splitÚcopy_Úwherer4  Úzero_r€  )%rö  r“   ÚloÚhiÚradiusÚ	magnitudeÚgenÚjogÚcornersÚprioÚ
prio_to_ptr)  r1  ÚvoteÚsplitÚn_splitÚ
split_rankr  Ún_aliver.  r  ÚtetsÚ	neighborsrþ  rŽ  r  rÿ  rì   r&  r   r'  r‹  r  r  r  r(  r-  s%   &                      @@@@@@@@@@@@@@rW   r   r     só  ÿý€ ô �˜6 9Ô-Ø�]‰]€FØ�‰�Q�€AØˆA„vÜÐ>Ó?Ð?Ü�Z‰Z×&Ñ& vÓ.×:Ñ:€FØ”E—K‘K ¨6Ð
2€CØ
”5—;‘; ¨&Ð	1€Bä�KŠK˜˜Q� ˜
¬%¯-©-ÀÔG€EØ€Eˆ"ˆ1€IØ�2�AˆY�]‰]˜1Ó×$Ñ$ e¨B¨Q i§m¡m°AÓ&6×&=Ñ&=ˆÜŸš b¥g§^¡^Ó%5¸Õ%9¸5ÀÀ!¸9¿=¹=»?×;NÑ;NÓ;PÐ$QÓR×YÑYÓ[Ñ€FÜ
�/Š/ Ô
(×
4Ñ
4°QÓ
7€Cð ”#�f¨Ó/Õ
/€CØ	ˆ"ˆ1ƒI”%—*’*˜a ˜V¨s¼%¿-¹-ÐPVÔWÐZ[Õ[Ð^_Õ_ÐcfÕfÕfƒIÜ�lŠlšI¢{²KÂÐMÔUZ×UbÑUbÐkqÔr€Gà•˜A• ¨2´°F¸BÀ½HÓ0EÕ+EÕ FÕF€Eˆ!ˆ"€Iä
ˆa�!�e�b�jÔ,Ó
-€CÜ�JŠJ˜˜Q�x Ñ+ sÑ+€EÜ�|Š|˜Q  A¥ q¨1¥u¨a°!­eÐ4Ñ<¸Ñ<€Eˆ!�HÜ�jŠj˜#˜q˜ 2Ñ-¨Ñ-€GÜ�KŠK˜Ñ"˜rÑ"€EØ€Eˆ!�HÜ�;Š;�q˜3•wÑ% "Ñ%€DÜ�[Š[˜Ñ"˜cÑ"€FÜ�>Š>˜! sÔ:×>Ñ>Ó@€DÜ—’˜tÓ$×(Ñ(Ó*€J÷ô ô 	€AØ€F÷"÷ "ò "÷H'ó 'ð Ü�zŠz˜6˜)¤ZÑ7°3Ñ7ˆÜ
×Ñ™˜6›¡A d£G¨Q±°$³¸Ô@Øœ
Ñ"ˆÜ�e—i‘i“kÓ"ˆØ�aŒ<ÙÜ—\’\ %¨´%·+±+Ô>ÀÇÁÃÕLˆ
ÙˆV�a˜'•kÕ!Ô"Ü—’˜SÑ' BÑ'ˆÜ
×Ñ˜&¡! D£'©1¨Z«=¹!¸J»-ÉÈ5ËÑSTÐU\ÓS]Ñ_`ÐafÓ_gÙ˜F›)¡Q v£Y°ô	8ä
×"Ñ" 1¡a¨£g©q°«}¸fÁaÈÃhÑPQÐRWÓPXÑZ[Ð\bÓZcÐekÔlØ�!�g•+ÕˆÙˆVŒÜ�e˜G˜V�n×(Ñ(Ó*Ó+ˆØˆw�;˜˜V�×#Ù(1°#«Ñ%ˆI�u˜gØ�L‰LœŸš V¨q¡[°)¸F¿L¹LÈQ¸LÓ<OÕ2PÐRXÓYÔZØ�K‰KŒMØˆE�(�7ˆOØ‹FÙˆ�‰‹Ôá"¤3 u¨W¨f ~×'9Ñ'9Ó';Ó#<Ó=Ñ€A€tˆYØ�$œŸš I°¡N°IÀµNÀIÓNÐNÐNrY   c                   ó,   a € ] tR tRt o V 3R ltRtV tR# )ÚMeshBVHiˆ  c                óš   <€ V ^8„  d   Qh/ S[ P                  ;R&   S[ P                  ;R&   S[ P                  ;R&   S[ P                  ;R&   # )r’   Ú	trianglesrr  ÚchildÚboxri  )r—   Ú__classdict__s   "€rW   r˜   ÚMeshBVH.__annotate__ˆ  sE   ø‡ ‚ Ù�|‰|Ññ á�<‰<Ññ ñ �<‰<Ññ ñ 
�‰Ñò	 rY   rÛ   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)rY  s   @rW   rT  rT  ˆ  s   ø‡ ‡ „ rY   rT  c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# r9  ri  )r—   s   "rW   r˜   r˜   �  s"   € ÷ -ñ -œŸ™ð -¬¯©ñ -rY   c                 óð   € V R ,          p W ^ ,          ,          R,          p W ^,          ,          R,          p W ^,          ,          R,          p W ^,          ,          R,          p W ^,          ,          R,          # )éÿÿ l   ÿ   ø l   ÿ  ~ ø l   pÀx  l   Ã0Ã0 l   IIII rÛ   )r!  s   &rW   Ú_morton_spread21re  �  sa   € à	ˆH�€AØ	
�"�W�Ð(Õ(€AØ	
�"�W�Ð(Õ(€AØ	
�!�V�Ð)Õ)€AØ	
�!�V�Ð)Õ)€AØ�Q•�JÐ,Õ,Ð,rY   c                óD   € V ^8„  d   QhR\         P                  R\        /# )r’   rV  r”   )r•   r–   rT  )r—   s   "rW   r˜   r˜   ™  s   € ÷ ;ñ ;œŸ™ð ;¬ñ ;rY   c                 ó  € \        R V R4       V P                  pV P                  ^ ,          pV^ 8X  d   \        R4      hV P	                  ^4      V P                  ^4      ,           R,          pVP                  4       p\        P                  ! WC\        R4      4      P	                  ^ 4      p\        P                  ! WC\        R4      4      P                  ^ 4      pW5,
          We,
          P                  R4      ,          R	,          P                  4       P                  ^ R	4      p\        VR
,          4      ^,          \        VR,          4      ^,          ,          \        VR,          4      ,          p\        P                  ! V4      w  r‰W	,          p
\        P                  ! \!        V^,
          ^4      ^3\        P"                  VR7      p\        P                  ! ^V,          ^,
          \        P"                  VR7      p\        P$                  ! \!        V^,
          ^4      \        P"                  VR7      p\        P                  ! ^V,          ^,
          ^3\        P&                  VR7      p\(        p\*        P-                  V! V4      V! V
4      W/! V4      V! V4      V! V4      V! V4      \        P.                  P1                  V4      P2                  4       \5        W©P7                  4       VRV^,
           V4      # )rV  Nz$mesh_bvh needs at least one triangleg      à?r:  r;  g ÂëþKH´9r%  )Nr4  r4  rd  )r�   r   )r�   r  )r�   r’   )r5  r“   r  rÊ  Úaminr?  Úisfiniter•   rA  r§   r@  Úlongr4  re  Úsortrl  r  ró  r)  r¢   r�  r  Úmesh_bvh_buildr¹   r–  r—  rT  rµ   )rV  r“   rì   ÚcentreÚfiniterC  rD  ÚcellÚcodesÚorderÚtrisrW  ÚparentÚvisitsrX  r-  s   &               rW   r   r   ™  s#  € Ü�˜i¨Ô6Ø×Ñ€FØ�‰˜Õ€AØˆA„vÜÐ?Ó@Ð@Ø�n‰n˜QÓ )§.¡.°Ó"3Õ3°sÕ:€Fà�_‰_Ó€FÜ	�Š�V¤U¨5£\Ó	2×	7Ñ	7¸Ó	:€BÜ	�Š�V¤U¨6£]Ó	3×	8Ñ	8¸Ó	;€BØ�[˜R�W×/Ñ/°Ó6Õ6¸)ÕD×JÑJÓL×RÑRÐSTÐV_Ó`€DÜ˜T $�ZÓ(¨AÕ-Ô0@ÀÀdÅÓ0LÐPQÕ0QÕQÔTdÐeiÐjnÕeoÓTpÕp€EÜ—:’:˜eÓ$�L€EØÕ€DÜ�KŠKœ˜Q �U A›¨Ð*´%·+±+ÀfÔM€EÜ�[Š[˜˜Q� �¬%¯+©+¸fÔE€FÜ�[Š[œ˜Q �U A›¬e¯k©kÀ&ÔI€FÜ
�+Š+�q˜1•u˜q•y !�n¬E¯M©MÀ&Ô
I€CÜ€AÜ×Ñ‘a˜“h¡ $£¨¨A¨e«H±a¸³iÁÀ6ÃÉAÈcËFÜ—j‘j×/Ñ/°Ó7×CÑCôEä�4Ÿ™› e¨G¨a°!­e n°cÓ:Ð:rY   r:  c                óÄ   € V ^8„  d   QhR\         R\        P                  R\        R\        \        P                  \        P                  \        P                  3,          /# )r’   Úbvhrö  Úmax_distr”   )rT  r•   r–   r§   r¶   )r—   s   "rW   r˜   r˜   ²  sH   € ÷ ñ Ü	ðÜŸ,™,ðÜ27ðä
Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñrY   c                 óê  € \        R VR4       V P                  P                  VP                  8w  d0   \        RV P                  P                   RVP                   24      hVP                  ^ ,          pVP                  4       pV^ 8  d   \        P                  ! V3\        V4      \        P                  VP                  R7      VP                  4       \        P                  ! V3R\        P                  VP                  R7      3# \        P                  ! V\        P                  VP                  R7      p\        P                  ! V^3\        P                  VP                  R7      p\        P                  ! V\        P                  VP                  R7      p\        p\        P                  V! V4      W7! V P                  4      V! V P                   4      V P                  P                  ^ ,          V! V P"                  4      V! V P$                  4      \        V4      V! V4      V! V4      V! V4      \        P&                  P)                  VP                  4      P*                  4       WEV3# )rö  ú
bvh is on z but points are on r%  r3  r4  )r5  rV  r“   rÊ  r  r¬   r•   rB  r§   r¢   r€  Úint64rl  r�  r  r   rr  rW  rX  r¹   r–  r—  )rv  rö  rw  rì   ÚdistÚclosestÚfacer-  s   &&&     rW   r   r   ²  sµ  € ô �˜6 9Ô-Ø
‡}�}×Ñ˜vŸ}™}Ô,Ü˜: c§m¡m×&:Ñ&:Ð%;Ð;NÈvÏ}É}ÈoÐ^Ó_Ð_Ø�‰�Q�€AØ×ÑÓ €FØ�!„|Ü—
’
˜A˜4¤ x£¼¿¹ÈfÏmÉmÔ\Ð^d×^jÑ^jÓ^lÜ—
’
˜A˜4 ¬5¯;©;¸v¿}¹}ÔMðOð 	Oä�;Š;�q¤§¡°f·m±mÔD€DÜ�kŠk˜1˜a˜&¬¯©¸f¿m¹mÔL€GÜ�;Š;�q¤§¡°F·M±MÔB€DÜ€AÜ×Ñ™Q˜v›Y¨¨1¨S¯]©]Ó+;¹Q¸s¿y¹y»\È3Ï=É=×K^ÑK^Ð_`ÕKaÑcdÐeh×enÑenÓcoÙ˜sŸw™w›Z¬¨x«¹!¸D»'Á1ÀWÃ:ÉqÐQUËwÜ"ŸZ™Z×6Ñ6°v·}±}ÓE×QÑQôSð ˜$ÐÐrY   c                óÞ   € V ^8„  d   QhR\         R\        P                  R\        P                  R\        R\        R\        R\
        \        P                  \        P                  3,          /# )r’   rv  ÚoriginsÚ
directionsÚtminÚtmaxÚcull_backfacer”   )rT  r•   r–   r§   rÏ   r¶   )r—   s   "rW   r˜   r˜   Ç  s`   € ÷ ñ Ü	ðÜ Ÿ<™<ðÜ5:·\±\ðÜINðÜ\aðäðô Œ5�<‰<œŸ™Ð%Õ&ñrY   c                 óF  € \        R VR4       \        RW!P                  ^ ,          ^34       V P                  P                  VP                  8w  g   VP                  VP                  8w  d=   \	        RV P                  P                   RVP                   RVP                   24      hVP                  ^ ,          p\
        P                  ! V\
        P                  VP                  R7      p\
        P                  ! V\
        P                  VP                  R7      p\        p	\        P                  V	! VP                  4       4      V	! VP                  4       4      Wi! V P                  4      V	! V P                  4      V P                  P                  ^ ,          V	! V P                  4      V	! V P                  4      \!        V4      \!        V4      \#        V4      V	! V4      V	! V4      \
        P$                  P'                  VP                  4      P(                  4       Wx3# )r  r€  ry  z, origins on z and directions on r%  r3  )r5  r  rV  r“   rÊ  r•   rl  r¢   rz  r�  r  r   r¬   rr  rW  rX  r§   rÏ   r¹   r–  r—  )
rv  r  r€  r�  r‚  rƒ  rì   rw  r}  r-  s
   &&&&&&    rW   r   r   Ç  s”  € ô �˜G YÔ/Ü�˜z¯M©M¸!Õ,<¸aÐ+@ÔAØ
‡}�}×Ñ˜wŸ~™~Ô-°×1BÑ1BÀgÇnÁnÔ1TÜ˜: c§m¡m×&:Ñ&:Ð%;¸=ÈÏÉÐHXÐXkØ&×-Ñ-Ð.ð0ó 1ð 	1à�‰�aÕ€AÜ�Š�AœUŸ]™]°7·>±>ÔB€AÜ�;Š;�q¤§¡°G·N±NÔC€DÜ€AÜ×Ñ™1˜W×/Ñ/Ó1Ó2±A°j×6KÑ6KÓ6MÓ4NÐPQÐSTÐUX×UbÑUbÓScÑefÐgj×gpÑgpÓeqØŸ=™=×.Ñ.¨qÕ1±1°S·Y±Y³<ÁÀ3Ç7Á7ÃÌUÐSWË[ÔZ_Ð`dÓZeÜ Ó.±°!³±a¸³g¼u¿z¹z×?XÑ?XÐY`×YgÑYgÓ?h×?tÑ?tôvð ˆ7€NrY   c                ó0   € V ^8„  d   QhR\         R\        /# )r’   r�  r”   )r;  rµ   )r—   s   "rW   r˜   r˜   Ú  s   € ÷ ñ ”sð ¤cñ rY   c                óŽ   € \        V\        4      '       g   \        V\        4      '       d   V^ 8  d   \        V  RV: 24      hV# )z6A size the kernels index with: a non-negative integer.z% must be a non-negative integer, got )r¡  rÏ   rµ   rÊ  )r�  r  s   &&rW   Ú_check_countr‡  Ú  s=   € ä�%œ×Ò¤j°¼×&<Ò&<ÀÈÄ	Ü˜D˜6Ð!FÀuÁiÐPÓQÐQØ€LrY   c                óT   € V ^8„  d   QhR\         R\        P                  R\        RR/# )r’   r�  r¡   r  r”   N)r;  r•   r–   r¶   )r—   s   "rW   r˜   r˜   á  s2   € ÷ Xñ X”sð X¤E§L¡Lð X¼ð XÀ4ñ XrY   c           	     óŠ  € VP                  4       \        V4      8w  gg   \        ;QJ d2    R \        VP                  VRR7       4       F  '       g   K   RM'	  RM#! R \        VP                  VRR7       4       4      '       d?   RP                  R V 4       4      p\        V  RV R\        VP                  4       24      hR	# )
z(`shape` entries of None accept any size.c              3   óF   "  € T F  w  rVR J;'       d    W8g  x € K  	  R # 5irN   rÛ   )rÜ   rw  rZ  s   &  rW   rß   Ú_check_shape.<locals>.<genexpr>ã  s%   é € Ð(tÑNsÁdÀa¨°$¨×)AÐ)A¸1¹6Ô)AÓNsùs   ‚!”!T)ÚstrictFú, c              3   óB   "  € T F  qf   RM
\        V4      x € K  	  R # 5i)NÚ*)r;  ©rÜ   rZ  s   & rW   rß   r‹  ä  s   é € ÐE¹u¸!¢	™¬s°1«vÔ5»uùs   ‚z must have shape (z), got N)r'  r¼  rã   Úzipr  ÚjoinrÊ  r¶   )r�  r¡   r  Úwants   &&& rW   r5  r5  á  s™   € à‡z�zƒ|”s˜5“zÔ!§S£SÑ(tÌcÐRX×R^ÑR^Ð`eÐnrÕNsÓ(t§S§S¢SÑ(tÌcÐRX×R^ÑR^Ð`eÐnrÕNsÓ(t×%tÒ%tØ�y‰yÑE¹uÓEÓEˆÜ˜D˜6Ð!3°D°6¸ÄÀvÇ|Á|ÓATÐ@UÐVÓWÐWñ &urY   c          
      ó`   € V ^8„  d   QhR\         R\        P                  R\        R\        RR/# )r’   r�  rr  rì   Úlowestr”   N)r;  r•   r–   rµ   )r—   s   "rW   r˜   r˜   è  s:   € ÷ 	\ñ 	\œð 	\¤U§\¡\ð 	\´cð 	\Ä3ð 	\Ètñ 	\rY   c                ó<  € VP                  4       ^ 8X  g%   \        P                  P                  4       '       d   R# \        P                  ! \        P
                  ! V4      4      P                  4       w  rEWC8  g   WR8¼  d   \        V  RV RV RV RV R2
4      hR# )a  The kernels index raw buffers with these values, so out-of-range ones would read or write past them.

Skipped while a CUDA graph is being captured: reading the bounds back would break the capture, and a replay
cannot check them anyway, so the caller who captures owns the indices.Nz must lie in [r�  z), got values in [Ú])r5  r•   r¹   Úis_current_stream_capturingr  Úaminmaxr  rÊ  )r�  rr  rì   r•  rC  rD  s   &&&&  rW   Ú_check_indicesrš  è  s†   € ð
 ‡{�{ƒ}˜ÔœUŸZ™Z×CÑC×EÒEÙÜ�[Š[œŸš uÓ-Ó.×5Ñ5Ó7�F€BØ	„{�b”gÜ˜D˜6 °¨x°r¸!¸Ð<NÈrÈdÐRTÐUWÐTXÐXYÐZÓ[Ð[ñ rY   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r’   ÚedgesÚ	num_nodesr”   r  )r—   s   "rW   r˜   r˜   ô  s)   € ÷ 
ñ 
¤§¡ð 
¼ð 
ÄÇÁñ 
rY   c                 óZ  € \        R V4      p\        RV R4       \        RW4       V P                  4       p \        P
                  ! V\        P                  V P                  R7      p\        P                  ! V\        P                  V P                  R7      p\        P                  \        V 4      V P                  ^ ,          V P                  \        P                  8H  V\        V4      \        V4      \        P                  P!                  V P                  4      P"                  4       V# )r�  rœ  r%  ©Nr’   )r‡  r5  rš  r¬   r•   r£   ró  r“   rl  rz  r  r   r�  r  r›   r¹   r–  r—  )rœ  r�  rs  Úlabelss   &&  rW   r   r   ô  sÏ   € Ü˜[¨)Ó4€IÜ�˜% Ô+Ü�7˜EÔ-Ø×ÑÓ€EÜ�\Š\˜)¬5¯;©;¸u¿|¹|ÔL€FÜ�[Š[˜¬%¯+©+¸e¿l¹lÔK€FÜ×ÑÔ,¨UÓ3°U·[±[Àµ^ÀUÇ[Á[ÔTY×T_ÑT_ÑE_ÐajÜ,¨VÓ4Ô6FÀvÓ6NÜ!ŸJ™J×5Ñ5°e·l±lÓC×OÑOôQð €MrY   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\        \         P                  \         P                  \         P                  3,          /# )r’   ÚverticesÚfacesrœ  Ú	positionsÚcos_thresholdr”   ©r•   r–   r§   r¶   )r—   s   "rW   r˜   r˜     sh   € ÷ "ñ "Ü�l‰lð"Ü#(§<¡<ð"Ü8=¿¹ð"ÜQV×Q]ÑQ]ð"äð"ô Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñ"rY   c                 óØ  € V P                   pV P                  ^ ,          VP                  ^ ,          rv\        RV R	4       \        RVR	4       \        RVR
4       \        RW7^34       \        RW4       \        RW&4       VP	                  \
        P                  4      P                  4       pVP                  R4      p\
        P                  ! V^,           \
        P                  VR7      p	\
        P                  ! \
        P                  ! W†R7      ^ V	R,          R7       \
        P                  ! V\
        P                  VR7      p
\
        P                  ! VP                  4       \
        P                  VR7      p\
        P                  ! V\
        P                  VR7      p\
        P                  ! V\
        P                  VR7      p\
        P                  ! V\
        P                  VR7      p\         P#                  \%        V P                  4       4      \%        V4      \%        V	4      \%        V
4      \%        V4      \%        VP	                  \
        P                  4      P                  4       4      \%        VP                  4       4      VP                  4       Wt\%        V4      \%        V4      \%        V4      \
        P&                  P)                  V4      P*                  4       WÍV3# )r   r¢  r£  rœ  r¤  r%  )Ú	minlength:r  NN)r'  rB  r3  rŸ  r4  )r“   r  r5  rš  rª   r•   ró  r¬   r6  r)  r  Úbincountrl  r5  r¢   rÏ   r  r   r�  r¹   r–  r—  )r¢  r£  rœ  r¤  r¥  r“   Ún_vertsÚn_edgesrI  ÚoffsetsÚcursorÚfanÚflipsÚskinnyÚlink_oks   &&&&&          rW   r   r     sû  € ð �_‰_€FØ—~‘~ aÕ(¨%¯+©+°a­.ˆWÜ�˜X yÔ1Ü�˜% Ô+Ü�˜% Ô+Ü�˜i°1¨Ô6Ü�7˜EÔ+Ü�7˜EÔ+Ø�H‰H”U—[‘[Ó!×,Ñ,Ó.€EØ�m‰m˜BÓ€Gä�kŠk˜' A�+¬U¯[©[ÀÔH€GÜ	‡L‚L”—’ Ô;ÀÈÐPRÍÕTÜ�[Š[˜¬¯©¸FÔC€FÜ
�+Š+�g—m‘m“o¬U¯[©[ÀÔ
H€CÜ�KŠK˜¤u§{¡{¸6ÔB€EÜ�[Š[˜¬¯©¸fÔE€FÜ�kŠk˜'¬¯©¸FÔC€GÜ×ÑÔ,¨X×-@Ñ-@Ó-BÓCÔEUÐV[ÓE\Ô^nÐovÓ^wÜ,¨VÓ4Ô6FÀsÓ6KÜ,¨U¯X©X´e·k±kÓ-B×-MÑ-MÓ-OÓPÔRbÐcl×cwÑcwÓcyÓRzØ#ŸM™M›O¨WÜ,¨UÓ3Ô5EÀfÓ5MÔO_Ð`gÓOhÜ!ŸJ™J×5Ñ5°fÓ=×IÑIôKð ˜'Ð!Ð!rY   c                óÄ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  /# )r’   ÚnbrrŽ  Ús_capÚt_capÚrelabel_everyr”   r  )r—   s   "rW   r˜   r˜      sL   € ÷ .ñ .”—‘ð .¤E§L¡Lð .¼¿¹ð .ÌeÏlÉlð .Üð.Ü(-¯©ñ.rY   c                 ó¼
  € V P                   pV P                  w  rgV^€8”  d   \        RV 24      h\        RWV34       \        RW&34       \        RW634       \	        RWR
R7       V^ 8X  d(   \
        P                  ! ^ \
        P                  VR7      # V P                  \
        P                  4      P                  4       p VP                  4       p\
        P                  ! V4      p\
        P                  ! Wg3\
        P                  VR7      p	VP                  4       p
VP                  4       p\
        P                  ! V\
        P                  VR7      p\        ^4       Uu. uF*  p\
        P                  ! V\
        P                  VR7      NK,  	  pp\        ^4       Uu. uF*  p\
        P                  ! ^\
        P                  VR7      NK,  	  pp\
        P                  ! ^\
        P                  VR7      p\
        P                  ! V\
        P                  VR7      p\
        P                  ! Wg3\
        P                   VR7      p\
        P                  ! V\
        P                  VR7      p\
        P                  ! ^V3\
        P                  VR7      p\
        P                  ! ^\
        P                  VR7      pVP#                  4       '       d!   R\%        VP'                  4       4      ,          MR	pV^,           p\
        P(                  P+                  V4      P,                  p\.        p\0        P3                  WgV! V 4      V! V4      V! V4      V! V	4      V4       ^ ^ pp VV,          ^ 8X  d¹   \0        P5                  WgV! V 4      V! V	4      V! V4      V! V4      VVV! V4      V! V4      V! V4      V4       \0        P7                  VV! V
4      V! V4      VVV! VV,          4      V! VV,          4      V! V4      V4	       \9        VV,          P;                  4       4      ^ 8X  d   MØ\0        P=                  WgV! V 4      V! V	4      V! V4      V! V
4      V! V4      V! V4      V! V4      V! V4      VVV! VV,          4      V! VV,          4      V! V^V,
          ,          4      V! V^V,
          ,          4      V! V4      V! V4      V4       ^V,
          pV^,          p\9        VP;                  4       4      ^ 8X  g   EKž   \0        P5                  WgV! V 4      V! V	4      V! V4      V! V4      VVV! V4      V! V4      V! V4      V4       VV8¬  # u upi u upi )rÏ  zFnbr can have at most 128 slots per node (reverse slots are int8), got rŽ  r´  rµ  r³  )r•  r%  r9  r£  r4  )r“   r  rÊ  r5  rš  r•   rl  rÏ   rª   ró  r¬   rÂ  r«   r€  rT  r)  r¢   r5  r§   r  r¹   r–  r—  r�  r  Úmin_cut_initÚmin_cut_global_relabelÚmin_cut_rebuild_activerµ   ÚitemÚmin_cut_iterate)r³  rŽ  r´  rµ  r¶  r“   rì   rí   ÚrÚrevÚeÚrtr,  r  ÚlistsÚcountsr  ÚqueuedÚincÚhnÚ	bfs_listsÚ
bfs_countsÚtolÚbigr‹  r-  ÚcurÚits   &&&&&                       rW   r   r      s  € à�Z‰Z€FØ�9‰9�D€AØˆ3„wÜÐaÐbcÐadÐeÓfÐfÜ�˜ ˜VÔ$Ü�˜% Ô&Ü�˜% Ô&Ü�5˜#¨Õ,ØˆA„vÜ�{Š{˜1¤E§J¡J°vÔ>Ð>Ø
�&‰&”—‘Ó
×
(Ñ
(Ó
*€CØ
�.‰.Ó
€CÜ×Ò˜Ó€AÜ
�+Š+�q�f¤E§J¡J°vÔ
>€CØ�‰‹€AØ	�‰‹€BÜ�Š�AœUŸ[™[°Ô8€AÜGLÈQÄxÓPÁxÀ!ŒU�[Š[˜¤%§+¡+°f×=Áx€EÐPÜHMÈaÌÓQÉÀ1Œe�kŠk˜!¤5§;¡;°v×>É€FÐQÜ�[Š[˜¤%§+¡+°fÔ=€FÜ�[Š[˜¤%§+¡+°fÔ=€FÜ
�+Š+�q�f¤E§M¡M¸&Ô
A€CÜ	�Š�QœeŸk™k°&Ô	9€BÜ—’˜Q ˜F¬%¯+©+¸fÔE€IÜ—’˜Q¤e§k¡k¸&ÔA€JØ&)§i¡i§k¢kˆ%”%˜Ÿ™›	Ó"Ö
"°s€CØ
ˆa�%€CÜ�Z‰Z×&Ñ& vÓ.×:Ñ:€FÜ€AÜ‡O�O�A™!˜C›&¡! C£&©!¨A«$±°#³¸Ô?Ø�ˆ€CØ
Ø�Õ Ô"Ü×%Ñ% a©A¨c«F±A°c³F¹A¸a»DÁ!ÀBÃ%ÈÈcÑSTÐUVÓSWÑYZÐ[dÓYeÑghÐisÓgtÐv|Ô}Ü×%Ñ% a©¨1«©q°«t°S¸#¹qÀÀsÅ»}ÉaÐPVÐWZÕP[ËnÑ^_Ð`fÓ^gÐioÔpÜ�6˜#•;×#Ñ#Ó%Ó&¨!Ô+ØÜ
×Ñ˜1¡ 3£©¨3«±°1³±q¸³t¹Q¸r»UÁAÀaÃDÉ!ÈCË&ÑRSÐTVÓRWÐY\Ð^aÑcdÐejÐknÕeoÓcpÙ˜V C�[›>©1¨U°1°sµ7­^Ó+<¹aÀÀqÈ3ÅwÅÓ>PÑRSÐTZÓR[Ñ]^Ð_eÓ]fÐhnô	pà�#�gˆØ
ˆa�ˆÜˆv�{‰{‹}Ó ×"ØÜ×Ñ˜a¡A c£F©A¨c«F±A°a³D¹!¸B»%ÀÀcÉ1ÈQË4ÑQRÐS\ÓQ]Ñ_`ÐakÓ_lÐntÔuØ�‰8€Oùò7 QùÚQs   Å#0UÆ"0Uc                ó„   € V ^8„  d   QhR\         R\        P                  R\        P                  R\        P                  /# )r’   rì   r“   r›   r”   )rµ   r•   r“   r›   r–   )r—   s   "rW   r˜   r˜   T  s7   € ÷ ñ ”Cð ¤§¡ð ´e·k±kð ÔV[×VbÑVbñ rY   c                ó¬   € WP                   VP                  V3p\        P                  V4      pVf"   \        P
                  ! WVR7      ;p\        V&   V# )zA cached length-n zero vector: the "no bias" / "no residual" operand of
the fused kernels, which read a vector unconditionally.r%  )rq  rr  Ú_ZERO_VECTORSr    r•   r)  )rì   r“   r›   rt  Úvecs   &&&  rW   Ú_zero_vectorrÐ  T  sL   € ð �k‰k˜6Ÿ<™<¨Ð
/€CÜ
×
Ñ
˜CÓ
 €CØ
‚{Ü#(§;¢;¨qÀfÔ#MÐMˆŒm˜CÑ Ø€JrY   c                ó8   € V ^8„  d   QhR\         P                  /# )r’   rw  ri  )r—   s   "rW   r˜   r˜   ^  s   € ÷ ñ ”e—l‘lñ rY   c                ó  € V P                   ^ ,          V P                   ^,          r!V P                  4       pV P                  4       ^8w  g   V^,          ^8w  g   V^,          V8w  d   R# V^,          V^,          V^,          V^8X  d   ^ MV^ ,          3p\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       g;   \        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d   R# V# )zš(w, h, d, n) element strides of an NDHWC-ordered view with a dense channel row, else None.
Strides must keep the kernel's 16-byte vector accesses aligned.Nc              3   ó2   "  € T F  q^,          x € K  	  R# 5i)r‰  NrÛ   r�  s   & rW   rß   Ú!_ndhwc_strides.<locals>.<genexpr>h  s   é € Ð
"™'�Qˆq�5Š5›'ùs   ‚TFc              3   ó*   "  € T F	  qR8„  x € K  	  R# 5i)r’   NéÿÿÿrÛ   r�  s   & rW   rß   rÔ  h  s   é € Ð)IÁ¸A¨i®-Ãùs   ‚)r  r˜  r'  rã   )rw  rì   ÚcÚstÚstridess   &    rW   Ú_ndhwc_stridesrÚ  ^  s»   € ð �7‰7�1�:�q—w‘w˜q•z€qØ	
�‰‹€BØ‡u�uƒw�!„|�r˜!•u ”z R¨¥U¨a¤ZÙð �!�u�b˜•e˜R �U¨¨a¬¡A°R¸µUÐ;€Gß
ƒsÑ
"™'Ó
"‡s‡s‚sÑ
"™'Ó
"×"Ò"§c£cÑ)IÁÓ)I§c§c¢cÑ)IÁÓ)I×&IÒ&IÙØ€NrY   c                óÔ
  € V P                   w  rxršpVP                   w  rÍrïpVw  ppp\        VVV4      ^8  d   R# Wž,
          V,          ^,           W¯,
          V,          ^,           VV,
          V,          ^,           pppV P                  \        P                  8H  ;'       Edc    VP                  \        P                  8H  ;'       Ed=    V P
                  ;'       Ed(    VP                  V P                  8H  ;'       Ed    VP                   ^,          V8H  ;'       dè    V^,          ^ 8H  ;'       g    V^8  ;'       dÇ    V^,          ^ 8H  ;'       d³    Wž8¬  ;'       d§    W¯8¬  ;'       d›    VV8¬  ;'       dŽ    VRJ ;'       g>    VP                  \        P                  8H  ;'       d]    VP                   V38H  ;'       dE    VRJ ;'       g9    VP                  \        P                  8H  ;'       d    VP                   W|VVV38H  pV'       g   R# V^8  du   \        P                  P                  P                  V ^ ^ ^ ^ ^ ^ ^ ^V,
          34      p \        P                  P                  P                  V^ ^ ^ ^ ^ ^ ^ ^V,
          34      p^p\        V 4      pVe'   V P                  \        P                  R7      '       d#   V P                  \        P                  R7      p RpVP                  \        P                  R7      pVe   VP                  4       M\        WÀP                  4      pVe!   VP                  \        P                  R7      M\        WÀP                  4      pVfH   \        P                  ! W|VVV3\        P                  V P                  \        P                  R7      pRpMÒVP                   W|VVV38w  g;   VP                  \        P                  8w  g   VP                  V P                  8w  d   \!        R4      hVP                  \        P                  R7      '       d   RpMO\        V4      pVe?   TTVV,          VV,          V,          V^8X  d   ^ MVV,          V,          V,          38w  d   R# \#        V 4      '       dE   \#        V4      '       d4   \#        V4      '       d#   \#        V4      '       d   \#        V4      '       g   R# \$        P&                  ! \)        V 4      \)        V4      \)        V4      \)        V4      \)        V4      WyW«WŒWïVVVV\        P*                  P-                  V P                  4      P.                  V.VOVO5!  pV'       d   V# R# )zèThe fused kernel, or None when it does not apply to this call.
config forces a tile config (benchmarking); -1 selects by shape. x and out may be
NDHWC-ordered views of larger tensors, so a tiled convolution needs no per-tile copies.N©r¬  ©r›   r“   r¬  zEfp16_conv3d: out must be an fp16 [N, K, Z, P, Q] tensor on x's device)r   r   r   r   )r  ré  r›   r•   r`  rõ   r“   r¢  r·  ÚpadrÚ  r  Úchannels_last_3dr¬   rÐ  rl  rÊ  rz  r  Úcutlass_fp16_conv3dr�  r¹   r–  r—  )r!  r  rÝ  r´  r˜  ÚconfigrB  rì   r×  rc  r,  r-  rí   r  rw  r½  rZ  ÚsdÚshÚswÚzr¡  r‚  rú   rÉ  rO   rc  s   &&&&&&&                    rW   Ú_cutlass_fp16_conv3dræ  m  sŸ  € ð —G‘G�M€Aˆ!�Ø—L‘L�M€Aˆ!�Ø�J€BˆˆBÜ
ˆ2ˆr�2ƒ˜ÔÙØ�u˜�m˜aÕ !¥%¨B¥°Õ!2°Q¸µU¸rµMÀAÕ4Eˆ!€q€Aà	�‰”5—=‘=Ñ ÷ 	jñ 	j V§\¡\´U·]±]Ñ%B÷ 	jñ 	jØ�I‰I÷	jñ 	jØ Ÿ-™-¨1¯8©8Ñ3÷	jñ 	jØ8>¿¹ÀQ½È1Ñ8L÷	jð 	jà��U�a‰Z× Ð ˜1˜q™5÷	jð 	jà&'¨!¥e¨q¡j÷	jð 	jà56±V÷	jð 	jà@AÁ÷	jð 	jàKLÐPQÉ6÷	jð 	jð �Tˆ\×QÐQ˜dŸj™j¬E¯M©MÑ9÷	jð Q¸d¿j¹jÈQÈDÑ>P÷	jð 	jð ˜Ð×hÐh (§.¡.´E·M±MÑ"A×"gÐ"gÀhÇnÁnÐYZÐ_`ÐbcÐefÐXgÑFgð ÷ ÙØˆ1„uä�H‰H×Ñ×#Ñ# A¨¨1¨a°°A°q¸!¸QÀ½UÐ'CÓDˆÜ—‘×$Ñ$×(Ñ(¨°!°Q¸¸1¸aÀÀAÀqÈ1ÅuÐ1MÓNˆØˆÜ	˜Ó	€BØ	‚z�Q—_‘_´5×3IÑ3I�_×JÒJØ�L‰L¤u×'=Ñ'=ˆLÓ>ˆØˆØ×Ñ¬U×-CÑ-CÐÓD€FØ $Ò 0ˆ4�?‰?Ô´lÀ1ÇhÁhÓ6O€DØMUÒMa�×#Ñ#´%×2HÑ2HÐ#ÔIÜ! !§X¡XÓ.ð à
‚{Ü�kŠk˜1  A q˜/´·±ÀqÇxÁxÔ_d×_uÑ_uÔvˆØ‰à�9‰9˜˜q ! Q˜Ô'¨3¯9©9¼¿¹Ô+EÈÏÉÐWX×W_ÑW_ÔI_ÜÐdÓeÐeØ×Ñ¬5×+AÑ+AÐ×BÒBØ‰Bô   Ó$ˆBØŠz˜R A q¨1¥u¨a°!­e°a­i¸aÀ1¼f¹È!ÈaÍ%ÐRSÍ)ÐVWÍ-Ð#XÔXÙÜ�q�MŠMœj¨×0Ò0´ZÀ×5EÒ5EÌ*ÐU]×J^ÒJ^ÔcmÐnq×crÒcrÙÜ	×	Ò	Ü˜ÓÔ-¨fÓ5Ô7GÈÓ7MÜ˜Ó"Ô$4°SÓ$9Ø	ˆa�A˜!  2 r¨2Ü�
‰
×!Ñ! !§(¡(Ó+×7Ñ7¸ð	
ð CEð	
ð HJó	
€B÷ ˆ3Ð˜$ÐrY   c                óö   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\        \        ,          R\         P                  /# )r’   r!  r  rÝ  Nr´  r˜  r”   ©r•   r–   r(  rµ   )r—   s   "rW   r˜   r˜   ¦  sf   € ÷ 7ñ 7Ü‡|�|ð7ä�L‰Lð7ô �,‰,˜Õ
ð7ô �l‰l˜TÕ!ð	7ô
 ”�Ið7ô ‡\�\ñ7rY   c                óÖ   € \        WW#V4      pVe   V# \        P                  P                  P	                  WW$R7      P                  \        P                  R7      pVf   V# WS,           # )z~fp16-accumulate conv3d with fused bias/residual, channels_last_3d in and out;
torch's conv when the kernel declines the shape.©r˜  rÜ  )ræ  r•   r¢  r·  Úconv3dr¬   rß  )r!  r  rÝ  r´  r˜  rB  s   &&&&& rW   r   r   ¦  sb   € ô ˜q¨$¸&Ó
A€CØ
‚Øˆ
Ü
�(‰(×
Ñ
×
$Ñ
$ Q°Ð
$Ó
D×
OÑ
OÔ^c×^tÑ^tÐ
OÓ
u€CØÒ"ˆ3Ð6¨­Ð6rY   c                óú   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\        \        ,          R\         P                  RR/# )	r’   r!  r  rÝ  Nr´  r˜  rB  r”   rè  )r—   s   "rW   r˜   r˜   ¶  sp   € ÷ ;ñ ;Ü‡|�|ð;ä�L‰Lð;ô �,‰,˜Õ
ð;ô �l‰l˜TÕ!ð	;ô
 ”�Ið;ô 
�‰ð;ð 
ñ;rY   c           	     óÆ   € \        WW#WER7      e   R# \        P                  P                  P	                  WW$R7      pTP                  Vf	   V4       R# Wc,           4       R# )zVfp16_conv3d into ``out``; torch's conv plus a copy when the kernel declines the shape.©rB  Nrê  )ræ  r•   r¢  r·  rë  r@  )r!  r  rÝ  r´  r˜  rB  Úress   &&&&&& rW   r   r   ¶  sM   € ô ˜A t°vÔGÒSÙÜ
�(‰(×
Ñ
×
$Ñ
$ Q°Ð
$Ó
D€CØ‡I�I�XÒ%ˆcÖ:¨3­>Ö:rY   c                ób  € V ^8„  d   QhR\         P                  R\         P                  R,          R\         P                  R,          R\        R\        R\        \        ,          R\
        R	\
        R
\         P                  R,          R\         P                  R,          R\         P                  /# )r’   r!  r  NrÝ  Ú
num_groupsrã  rÞ  ÚsiluÚzero_padrB  Ústatsr”   )r•   r–   rµ   r§   r(  rÏ   )r—   s   "rW   r˜   r˜   Å  s¦   € ÷ =ñ =Ü‡|�|ð=ä�L‰L˜4Õð=ô �,‰,˜Õ
ð=ô ð	=ô
 
ð=ô 
Œc�ð=ô ð=ô ð=ô 
�‰˜Õ	ð=ô �<‰<˜$Õð=ô ‡\�\ñ=rY   c
                ó°  € V P                   w  r«rÍpVw  ppppp\        V4      ^ 8  d   \        R4      hVew   VP                  V P                  4      P                  4       pVf!   \        W°P                  V P                  4      M(VP                  V P                  4      P                  4       pV	eq   V	P                   V	P                  V	P                  3W¬,          V^3\        P                  V P                  38w  d%   \        RW¬,           RV RV P                   24      hV^,          ;'       g†    VR8„  ;'       gy    VRJ;'       d    W³,          ;'       g]    VR8„  ;'       gP    V'       * ;'       d/    \        VV4      V8¬  ;'       g*    \        VV4      V8¬  ;'       g    W¬V,           ,          R8„  '       * pV'       d   \        V 4      MRpV'       d-   Vf)   Wë,          pV P                  \        P                  R	7      p T;'       d    \        V 4      pV'       g)   \        WW#WEWgV	4	      pVf   V# VP                  V4       V# Ve6   \!        V4      '       d%   \        V4      '       d   \#        W€4      '       g   TMRpVfc   \        P$                  ! W«VV,           VV,           V,           Wï,           V,           3V P                  V P                  \        P                  R
7      p\'        V P                  \        P                  4      pTpVf   T;rMiV	ff   WÞ,          ) R,          ) p\        P$                  ! ^V
,          V,          VV,          V,           ,          \        P                  V P                  R7      p\(        P+                  \-        V 4      \-        V4      \-        V4      \-        V4      \-        V4      \-        V	f   TMV	P                  4       4      W«WÍWãWOVVVVWgV\.        V P                  ,          \        P0                  P3                  V P                  4      P4                  4       Ve   VVJ d   V# VP                  V4       V# )a  Per-frame GroupNorm + SiLU + causal conv padding in one pass, channels_last_3d out.
``stats`` from group_norm_stats3d normalizes a row strip of a frame with that frame's
statistics. ``out`` is written in place where the kernel can index it, else copied into.z3group_norm_silu_pad3d: padding must be non-negativeNz.group_norm_silu_pad3d: stats must be float32 [r�  z, 2] on é   r  éÿÿ  rÜ  rÝ  r%  )r  ré  rÊ  rª   r›   r¬   rÐ  r“   r•   r¢   r  Ú_ndhwc_row_striderß  rz  Ú_eager_group_norm_silu_pad3dr@  Ú_writes_packed_ndhwcr   rl  ru  r  r   r�  re   r¹   r–  r—  )r!  r  rÝ  rñ  rã  rÞ  rò  ró  rB  rô  rš  r×  rw  r,  r-  ÚleftÚrightÚtopÚbottomÚfrontÚservedÚ
row_striderï  rÿ  Úplaceholderrm  Úchunkss   &&&&&&&&&&                 rW   r   r   Å  sž  € ð —G‘G�M€Aˆ!�Ø&)Ñ#€Dˆ%��f˜eÜ
ˆ3ƒx�!„|ÜÐNÓOÐOØÒØ—‘˜1Ÿ7™7Ó#×.Ñ.Ó0ˆØ59²\Œ|˜AŸx™x¨¯©Ô1ÀtÇwÁwÈqÏwÉwÓGW×GbÑGbÓGdˆØÒ˜eŸk™k¨5¯;©;¸¿¹ÐEÈ1Í5ÐR\Ð^_ÐJ`Ôbg×boÑboÐqr×qyÑqyÐIzÔzÜÐIÈ!Í%ÈÐPRÐS]ÐR^Ð^fÐgh×goÑgoÐfpÐqÓrÐrØ�a•%÷ -ð -˜1˜t™8÷ -ð -¨°dÐ(:×(dÐ(dÀÅ÷ -Ð@cÐR\Ð_cÑRc÷ -ð -Ø"”l×WÐW¬¨D°%Ó(8¸AÑ(=÷-Ø(VÄÀSÈ&ÓAQÐUVÑAV÷-ð -à˜e�)•_ uÑ,ô.€F÷ *0Ô" 1Ô%°T€Jß�*Ò$Ø•Uˆ
Ø�L‰L¤u×'=Ñ'=ˆLÓ>ˆØ×%Ð%œ
 1›€FßÜ*¨1°dÈÐRVÐbgÓhˆØŠ;ØˆJØ�	‰	�#ŒØˆ
ð ’Ô%9¸#×%>Ò%>Ä:ÈcÇ?Â?Ô[kÐlo×[sÒ[s‰#Ðy}€CØ
‚{Ü�kŠk˜1  U¥¨A°­G°fÕ,<¸a½hÈÕ>NÐOØ !§¡°·±Ì×H^ÑH^ô`ˆä$ Q§X¡X¬u¯}©}Ó=€KØ€IØ‚~Ø#Ð#ˆ�Ø	ŠØ•U�8˜tÕ#Ð$ˆÜ—K’K  A¥¨¥	¨V°a­Z¸*Õ-DÕ EÌUÏ]É]Ðcd×ckÑckÔlˆ	Ü×ÑÜ˜ÓÔ-¨fÓ5Ô7GÈÓ7MÜ˜ÓÔ/°	Ó:Ü¨ª™¸5×;KÑ;KÓ;MÓNØ	ˆa�A 3¨e°S¸&À%ÈÐYcÜ�a—g‘gÕ¤§
¡
× 9Ñ 9¸!¿(¹(Ó C× OÑ Oôð ‚{�c˜S“jØˆ
Ø‡I�Iˆc„NØ€JrY   c                ó8   € V ^8„  d   QhR\         P                  /# )r’   r!  ri  )r—   s   "rW   r˜   r˜     s   € ÷ ñ œŸ™ñ rY   c                ó„  € V P                   w  rr4pV^8X  d©   V P                  ^4      ^8X  d“   V P                  ^4      V8X  d}   V P                  ^4      WR,          8”  da   V P                  ^4      ^,          ^ 8X  dD   V^8X  g,   V P                  ^ 4      W@P                  ^4      ,          8X  d   V P                  ^4      # R# )z°Elements between image rows when x is a single frame with packed NHWC rows that sit farther
apart than W * C (a row-interleaved view the kernels read in place); None otherwise.N)r  r˜  )r!  rš  r×  rw  r,  r-  s   &     rW   rø  rø    s•   € ð —G‘G�M€Aˆ!�Ø	ˆQŒ�1—8‘8˜A“; !Ô#¨¯©°«°qÔ(8¸Q¿X¹XÀa»[È1Í5Ô=PÐUV×U]ÑU]Ð^_ÓU`ÐcdÕUdÐhiÔUiØ�a”˜1Ÿ8™8 A›;¨!¯h©h°q«k­/Ô9Ø�x‰x˜‹{ÐÙrY   c                óp   € V ^8„  d   QhR\         P                  R\        R\        R\         P                  /# )r’   r!  rñ  rã  r”   )r•   r–   rµ   r§   )r—   s   "rW   r˜   r˜     s0   € ÷ ñ œ%Ÿ,™,ð ´Cð ¼eð ÌÏÉñ rY   c                óÌ  € V P                   w  r4rVpV^,          ;'       g6    VR8„  ;'       g)    WA,          ;'       g    VR8„  ;'       g    W5,          R8„  '       * pV'       d   \        V 4      MRp	V'       d-   V	f)   Wt,          p	V P                  \        P                  R7      p T;'       d    \        V 4      pV'       g   \        WV4      # Wg,          ) R,          ) p
\        P                  ! ^V,          V,          V
,          V,          \        P                  V P                  R7      p\        P                  ! W5,          V^3\        P                  V P                  R7      p\        P                  \        V 4      \        V4      \        V4      W4WVWqW)\        V P                  ,          \        P                  P!                  V P                  4      P"                  4       V# )z•Per-frame GroupNorm (mean, rstd) of x [B, C, T, H, W] as float32 [B*T, num_groups, 2],
the same statistics group_norm_silu_pad3d computes for itself.rö  r  r÷  NrÜ  r%  )r  rø  r¬   r•   rß  rz  Ú_eager_group_norm_stats3drl  r¢   r“   r  r   r�  re   r›   r¹   r–  r—  )r!  rñ  rã  rš  r×  rw  r,  r-  r   r  r  rm  rô  s   &&&          rW   r   r     sh  € ð —G‘G�M€Aˆ!�Ø�a•%×[Ð[˜1˜t™8×[Ð[ q¥~×[Ð[¸ÀdÑ9J×[Ð[ÈaÍeÐV[ÉmÔ\€Fß)/Ô" 1Ô%°T€Jß�*Ò$Ø•Uˆ
Ø�L‰L¤u×'=Ñ'=ˆLÓ>ˆØ×%Ð%œ
 1›€FßÜ(¨¸Ó<Ð<Ø•ˆx˜4ÕÐ €FÜ—’˜A �E A�I¨Õ.°Õ2¼%¿-¹-ÐPQ×PXÑPXÔY€IÜ�KŠK˜� 
¨AÐ.´e·m±mÈAÏHÉHÔU€EÜ×ÑÜ˜ÓÔ-¨iÓ8Ô:JÈ5Ó:QØ	ˆa�A 3´MÀ!Ç'Á'Õ4JÌEÏJÉJ×LeÑLeÐfg×fnÑfnÓLo×L{ÑL{ôð €LrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# )r’   rB  r”   rò   )r—   s   "rW   r˜   r˜   %  s"   € ÷ Zñ ZœeŸl™lð Z¬tñ ZrY   c                ób  € V P                  \        P                  R7      '       d   R# V P                  w  rr4pV P	                  4       pV^8H  ;'       d\    V^,          ^8H  ;'       dH    V^,          V8H  ;'       d4    V^,          WR,          8H  ;'       d    V^,          WE,          V,          8H  # )zVPacked channels_last_3d, or for a batch of one a frame-offset view of a longer buffer.rÜ  T)r  r•   rß  r  r˜  )rB  rš  r×  r  r,  r-  rØ  s   &      rW   rú  rú  %  s�   € à
×Ñ¤u×'=Ñ'=Ð×>Ò>ÙØ—I‘I�M€Aˆ!�Ø	�‰‹€BØ�‰6×YÐY�b˜•e˜q‘j×YÐY R¨¥U¨a¡Z×YÐY°B°qµE¸Q½U±N×YÐYÀrÈ!ÅuÐPQÕPUÐXYÕPYÑGYÐYrY   c                ó   € V ^8„  d   QhRR/# )r’   r”   NrÛ   )r—   s   "rW   r˜   r˜   .  s   € ÷ ]ñ ]Ðimñ ]rY   c
                ó$   € \        WW#WEWgW‰4
       R# )zXgroup_norm_silu_pad3d into ``out``; a frame-offset view leaves room for a caller's halo.N)r   )
r!  r  rÝ  rñ  rã  rÞ  rò  ró  rB  rô  s
   &&&&&&&&&&rW   r   r   .  s   € ä˜! T°sÀÐQTÖ\rY   c          
      ó�   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\         P                  /# )r’   r!  Ú	freqs_cisÚ
split_halfÚinplacer”   rò   )r—   s   "rW   r˜   r˜   3  s@   € ÷ ñ Ü‡|�|ðÜ %§¡ðÜ=AðÜLPðä
‡\�\ñrY   c          	      ó>  € V'       g   \        W4      pV'       d   T M\        P                  ! V 4      p\        P                  P	                  V P
                  4      P                  p\        P                  \        V 4      \        V4      \        V4      R R VV4       V# rN   )
r   r•   rÂ  r¹   r–  r“   r—  r  r   r�  )r!  r  r  r  Úx_outr›  s   &&$$  rW   Ú_apply_rope1_cudar  3  sz   € ÷ Ü# AÓ1ˆ	ß‰Aœe×.Ò.¨qÓ1€EÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€Jä‡M�MÜ˜ÓÜ˜Ó#Ü˜ÓØØØØôð €LrY   c                óæ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\        \         P                  \         P                  3,          /# )r’   rÈ  Úxkr  r  r  r”   r•  )r—   s   "rW   r˜   r˜   G  s\   € ÷ ñ Ü�‰ðÜŸ,™,ðÜ38·<±<ðäðä#ðô Œ5�<‰<œŸ™Ð%Õ&ñrY   c          	      ó  € V P                   VP                   8w  d   \        WW4R 7      \        WW4R 7      3# V'       g   \        W4      pV'       d   T M\        P                  ! V 4      pV'       d   TM\        P                  ! V4      p\        P
                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V4      \        V4      \        V4      VV4       WV3# )©r  r  )r  r  r   r•   rÂ  r¹   r–  r“   r—  r  r   r�  )rÈ  r  r  r  r  Úxq_outÚxk_outr›  s   &&&$$   rW   Ú_apply_rope_cudar  G  sË   € ð 
‡x�x�2—8‘8Ôä˜b¸
ÔTÜ˜b¸
ÔTð
ð 	
÷ Ü# BÓ2ˆ	ß‰R¤× 0Ò 0°Ó 4€Fß‰R¤× 0Ò 0°Ó 4€FÜ—‘×*Ñ*¨2¯9©9Ó5×AÑA€Jä‡M�MÜ˜ÓÜ˜Ó#Ü˜Ó Ü˜ÓÜ˜Ó ØØôð ˆ>ÐrY   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# ©r’   r!  r  r”   ri  )r—   s   "rW   r˜   r˜   b  s2   € ÷ Lñ L”5—<‘<ð L¬E¯L©Lð L¼U¿\¹\ñ LrY   c                 ó   € \        WR R R7      # ©Fr  ©r  ©r!  r  s   &&rW   r   r   b  s   € Ü˜Q°eÀUÔKÐKrY   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# r  ri  )r—   s   "rW   r˜   r˜   f  s2   € ÷ Kñ K”E—L‘Lð K¬U¯\©\ð K¼e¿l¹lñ KrY   c                 ó8   € \        W3R 7       \        WRRR7      # ©©ÚreadonlyFTr  ©r   r  r   s   &&rW   r   r   f  s   € Ü�q <Õ0Ü˜Q°eÀTÔJÐJrY   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# ©r’   rÈ  r  r  r”   r÷  )r—   s   "rW   r˜   r˜   k  sP   € ÷ Pñ PÜ�‰ðPÜŸ,™,ðPÜ38·<±<ðPä
Œ5�<‰<œŸ™Ð%Õ&ñPrY   c                 ó    € \        WVR R R7      # r  ©r  ©rÈ  r  r  s   &&&rW   r   r   k  s   € ô ˜B I¸%ÈÔOÐOrY   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# r(  r÷  )r—   s   "rW   r˜   r˜   q  sP   € ÷ Oñ OÜ�‰ðOÜŸ,™,ðOÜ38·<±<ðOä
Œ5�<‰<œŸ™Ð%Õ&ñOrY   c                 ó<   € \        WV3R 7       \        WVRRR7      # r#  ©r   r  r+  s   &&&rW   r   r   q  s!   € ô �r¨¨Õ5Ü˜B I¸%ÈÔNÐNrY   c                óR   € V ^8„  d   QhR\         \        \        3,          R\        /# ©r’   Úkwargsr”   ©Údictr;  Úobjectr   )r—   s   "rW   r˜   r˜   x  s$   € ÷ !ñ !¤T¬#¬v¨+Õ%6ð !Ô;Kñ !rY   c                 ó   € V R ,          pV R,          p\        V\        P                  4      '       d!   \        V\        P                  4      '       g   Q h\        WRR7      f   \        P
                  ! R R4      # \        P                  ! 4       # )r!  r  r%   ©Úextension_opú6CUDA fused RMS-RoPE requires a supported native layout)r¡  r•   r–   Ú_native_rms_rope_layoutr   Úfailrc  )r1  r!  r  s   &  rW   Ú_validate_cuda_rms_rope1r;  x  sl   € Øˆs�€AØ�{Õ#€IÜ�aœŸ™×&Ò&¬:°iÄÇÁ×+NÒ+NÐNÐNÜ˜q¸+ÔFÒNÜ×$Ò$ SÐ*bÓcÐcÜ×ÒÓ Ð rY   c                óR   € V ^8„  d   QhR\         \        \        3,          R\        /# r0  r2  )r—   s   "rW   r˜   r˜   �  s$   € ÷ !ñ !¤D¬¬f¨Õ$5ð !Ô:Jñ !rY   c                 óò  € V R ,          V R,          r!V P                  R4      pVf   \        P                  ! RR4      # V P                  R4      pVf   Tp\        ;QJ d    R WW43 4       F  '       d   K   RM	  RM! R WW43 4       4      '       g   Q hV R,          p\	        V P                  R	4      ;'       g    ^ 4      pVP
                  VP
                  8X  d   R
MRp\        WWvR7      f   \        P                  ! R R4      # VP                  VP                  8w  d   \        P                  ! RR4      # VP
                  VP
                  8w  d(   \        W%RVR7      f   \        P                  ! RR4      # VP                  ^8w  g'   VP                  4       VP
                  R,          8w  d   \        P                  ! RR4      # VP                  ^8w  g'   VP                  4       VP
                  R,          8w  d   \        P                  ! RR4      # VP
                  VP
                  8X  d3   VP                  VP                  8w  d   \        P                  ! RR4      # \        P                  ! 4       # )r‚  rí   Úq_scalez#is required for fused CUDA RMS-RoPEÚk_scalec              3   óV   "  € T F  p\        V\        P                  4      x € K!  	  R # 5irN   )r¡  r•   r–   )rÜ   r  s   & rW   rß   Ú*_validate_cuda_rms_rope.<locals>.<genexpr>‰  s    é € ÐUÑ<T°5Œz˜%¤§¡×.Ð.Ó<Tùs   ‚')FTr  Úrot_dimr#   r%   ©r7  rB  r8  z5must have the same dtype as q for fused CUDA RMS-RoPEz'must be a 1D tensor matching q head_dimz'must be a 1D tensor matching k head_dimz0must match q_scale dtype for fused CUDA RMS-RoPEr4  )r    r   r:  Úallrµ   r  r9  r›   Úndimr5  rc  )r1  r‚  rí   r>  r?  r  rB  r7  s   &       rW   Ú_validate_cuda_rms_roperF  �  sâ  € Ø�#�;˜˜s�€qØ�j‰j˜Ó#€GØ‚Ü×$Ò$ YÐ0UÓVÐVØ�j‰j˜Ó#€GØ‚Øˆß‹3ÑU¸QÀ7Ñ<TÓU�3�3Š3ÑU¸QÀ7Ñ<TÓU×UÒUÐUÐUØ�{Õ#€IÜ�&—*‘*˜YÓ'×,Ð,¨1Ó-€GØ!"§¡¨A¯G©GÔ!3‘:¸€LÜ˜q¸,ÔXÒ`Ü×$Ò$ SÐ*bÓcÐcØ‡w�w�!—'‘'ÔÜ×$Ò$ SÐ*aÓbÐbØ‡w�w�!—'‘'ÔÔ5°aÐQ\ÐfmÔnÒvÜ×$Ò$ SÐ*bÓcÐcØ‡|�|�qÔ˜GŸM™M›O¨q¯w©w°r­{Ô:Ü×$Ò$ YÐ0YÓZÐZØ‡|�|�qÔ˜GŸM™M›O¨q¯w©w°r­{Ô:Ü×$Ò$ YÐ0YÓZÐZØ‡w�w�!—'‘'Ô˜gŸm™m¨w¯}©}Ô<Ü×$Ò$ YÐ0bÓcÐcÜ×ÒÓ Ð rY   rB  c                óŠ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        R,          /# )r’   r!  r  r7  rB  r”   Nr  )r—   s   "rW   r˜   r˜   œ  sF   € ÷ ñ Ü‡|�|ðä�|‰|ðô ð	ô
 ðô 
ˆD…[ñrY   c                ó4  € \        WVR 7      pVe…   \        e}   \        \        V4      '       dg   V P                  R,          ^ 8¼  dO   V P                  R,          ^ ,          ^ 8X  d0   V P                  \
        P                  \
        P                  39   g   R# V# )©rB  Nr4  )r   r  r  r  r›   r•   r`  ra  )r!  r  r7  rB  Úbnhds   &&$$ rW   r9  r9  œ  sm   € ô   °gÔ>€DØ‚|Ü
ŠÜ”B˜×%Ò%Ø�G‰G�B�K˜2ÔØ�G‰G�B�K˜"Õ Ô!Ø�G‰GœŸ™¤u§~¡~Ð6Ô6áØ€KrY   c                ó¼   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\        R\         P                  /# )r’   r!  r  r’  r¦  r  r  r”   )r•   r–   r§   rÏ   )r—   s   "rW   r˜   r˜   ¯  s^   € ÷ ñ Ü‡|�|ðä�|‰|ðô �<‰<ðô ð	ô ðô ðô ‡\�\ñrY   c          	      óL  € \        WR R7      f   Q hV'       d   T M\        P                  ! V 4      p\        P                  P	                  V P
                  4      P                  p\        P                  \        V 4      \        V4      \        V4      \        V4      VVV4       V# )r%   r6  )
r9  r•   rÂ  r¹   r–  r“   r—  r  r%   r�  )r!  r  r’  r¦  r  r  rB  r›  s   &&&&$$  rW   Ú_rms_rope1_cudarM  ¯  s…   € ô # 1¸kÔJÒVÐVÐVç‰!œE×,Ò,¨QÓ/€CÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ‡L�LÜ˜ÓÜ˜Ó#Ü˜ÓÜ˜ÓØØØôð €JrY   c                óL  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R	\        R
\        R\
        \         P                  \         P                  3,          /
# )r’   r‚  rí   r  r>  r?  Nr¦  r  r  rB  r”   )r•   r–   r§   rÏ   rµ   r¶   )r—   s   "rW   r˜   r˜   È  s—   € ÷ 1ñ 1Ü‡|�|ð1ä‡|�|ð1ô �|‰|ð1ô �\‰\ð	1ô
 �\‰\˜DÕ ð1ô ð1ô ð1ô ð1ô ð1ô Œ5�<‰<œŸ™Ð%Õ&ñ1rY   c                óÀ  € Vf   TpV P                   VP                   8w  dY   V'       d5   V'       d   \        P                  M\        P                  p	V	! WW#WEVR7      # \	        WW5WgR7      \	        WWEWgR7      3# \        WRVR7      f   Q hV'       d   T M\        P                  ! V 4      p
V'       d   TM\        P                  ! V4      p\        P                  P                  V P                  4      P                  p\        P                  \        V 4      \        V4      \        V4      \        V4      \        V4      \        V
4      \        V4      VVVV4       W«3# )NrI  r  r#   rC  )r  Ú_eager_roper(   r'   rM  r9  r•   rÂ  r¹   r–  r“   r—  r  r#   r�  )r‚  rí   r  r>  r?  r¦  r  r  rB  ÚfallbackÚq_outÚk_outr›  s   &&&&&&$$$    rW   Ú_rms_rope_cudarT  È  s0  € ð ‚Øˆà‡w�w�!—'‘'Ôßß<Cœ×8Ò8Ü(×<Ñ<ð á˜A )°gÐPWÔXÐXäØ˜gØ%ôô Ø˜gØ%ôð	
ð 		
ô # 1¸jÐRYÔZÒfÐfÐfç‰Aœe×.Ò.¨qÓ1€Eß‰Aœe×.Ò.¨qÓ1€EÜ—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ‡K�KÜ˜ÓÜ˜ÓÜ˜Ó#Ü˜Ó!Ü˜Ó!Ü˜ÓÜ˜ÓØØØØôð ˆ<ÐrY   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# ©r’   r!  r  r’  r¦  r”   rä  )r—   s   "rW   r˜   r˜   ü  sQ   € ÷ Zñ ZÜ‡|�|ðZä�|‰|ðZô �<‰<ðZô ð	Zô
 ‡\�\ñZrY   c           	      ó    € \        WW#R R R7      # r  ©rM  ©r!  r  r’  r¦  s   &&&&rW   r%   r%   ü  s   € ô ˜1¨ÀEÐSXÔYÐYrY   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rV  rä  )r—   s   "rW   r˜   r˜     sQ   € ÷ Yñ YÜ‡|�|ðYä�|‰|ðYô �<‰<ðYô ð	Yô
 ‡\�\ñYrY   c           	      ó<   € \        WV3R 7       \        WW#RRR7      # r#  ©r   rM  rY  s   &&&&rW   r&   r&     s#   € ô �q¨uÐ#5Õ6Ü˜1¨ÀEÐSWÔXÐXrY   c                ó(  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        \         P                  \         P                  3,          /# ©	r’   r‚  rí   r  r>  r?  Nr¦  r”   r¦  )r—   s   "rW   r˜   r˜     sy   € ÷ ñ Ü‡|�|ðä‡|�|ðô �|‰|ðô �\‰\ð	ô
 �\‰\˜DÕ ðô ðô Œ5�<‰<œŸ™Ð%Õ&ñrY   c                 ó"   € \        WW#WER R R7      # r  ©rT  ©r‚  rí   r  r>  r?  r¦  s   &&&&&&rW   r#   r#     s   € ô Ø	ˆi 'Ø %ôð rY   c                ó(  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        \         P                  \         P                  3,          /# r^  r¦  )r—   s   "rW   r˜   r˜     sy   € ÷ ñ Ü‡|�|ðä‡|�|ðô �|‰|ðô �\‰\ð	ô
 �\‰\˜DÕ ðô ðô Œ5�<‰<œŸ™Ð%Õ&ñrY   c                 óL   € Vf   Tp\        WW#V3R7       \        WW#WERRR7      # )Nr$  FTr  ©r   rT  ra  s   &&&&&&rW   r$   r$     s6   € ð ‚ØˆÜ�q y¸7Ð&CÕDÜØ	ˆi 'Ø $ôð rY   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rV  rä  )r—   s   "rW   r˜   r˜   .  sQ   € ÷ Yñ YÜ‡|�|ðYä�|‰|ðYô �<‰<ðYô ð	Yô
 ‡\�\ñYrY   c           	      ó    € \        WW#R RR7      # ©TFr  rX  rY  s   &&&&rW   r)   r)   .  s   € ô ˜1¨ÀDÐRWÔXÐXrY   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rV  rä  )r—   s   "rW   r˜   r˜   7  sQ   € ÷ Xñ XÜ‡|�|ðXä�|‰|ðXô �<‰<ðXô ð	Xô
 ‡\�\ñXrY   c           	      ó<   € \        WV3R 7       \        WW#RRR7      # ©r$  Tr  r\  rY  s   &&&&rW   r*   r*   7  s#   € ô �q¨uÐ#5Õ6Ü˜1¨ÀDÐRVÔWÐWrY   c                ó4  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R	\        \         P                  \         P                  3,          /# ©
r’   r‚  rí   r  r>  r?  Nr¦  rB  r”   ©r•   r–   r§   rµ   r¶   )r—   s   "rW   r˜   r˜   A  sƒ   € ÷ ñ Ü‡|�|ðä‡|�|ðô �|‰|ðô �\‰\ð	ô
 �\‰\˜DÕ ðô ðô ðô Œ5�<‰<œŸ™Ð%Õ&ñrY   c                 ó$   € \        WW#WER RVR7	      # )TF©r  r  rB  r`  ©r‚  rí   r  r>  r?  r¦  rB  s   &&&&&&&rW   r'   r'   A  s   € ô Ø	ˆi 'Ø °ôð rY   c                ó4  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R	\        \         P                  \         P                  3,          /# rl  rm  )r—   s   "rW   r˜   r˜   P  sƒ   € ÷ ñ Ü‡|�|ðä‡|�|ðô �|‰|ðô �\‰\ð	ô
 �\‰\˜DÕ ðô ðô ðô Œ5�<‰<œŸ™Ð%Õ&ñrY   c                 óN   € Vf   Tp\        WW#V3R7       \        WW#WERRVR7	      # )Nr$  Tro  rd  rp  s   &&&&&&&rW   r(   r(   P  s8   € ð ‚ØˆÜ�q y¸7Ð&CÕDÜØ	ˆi 'Ø ¨wôð rY   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# r  ri  )r—   s   "rW   r˜   r˜   c  s2   € ÷ Kñ KœeŸl™lð K´u·|±|ð KÌÏÉñ KrY   c                 ó   € \        WR RR7      # rg  r  r   s   &&rW   r!   r!   c  s   € Ü˜Q°dÀEÔJÐJrY   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# r  ri  )r—   s   "rW   r˜   r˜   g  s5   € ÷ Jñ JÜ‡|�|ðJÜ %§¡ðJä
‡\�\ñJrY   c                 ó8   € \        W3R 7       \        WRRR7      # rj  r&  r   s   &&rW   r"   r"   g  s   € ô �q <Õ0Ü˜Q°dÀDÔIÐIrY   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# r(  r÷  )r—   s   "rW   r˜   r˜   n  sT   € ÷ Oñ OÜ�‰ðOä�‰ðOô �|‰|ðOô Œ5�<‰<œŸ™Ð%Õ&ñ	OrY   c                 ó    € \        WVR RR7      # rg  r*  r+  s   &&&rW   r   r   n  s   € ô
 ˜B I¸$ÈÔNÐNrY   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# r(  r÷  )r—   s   "rW   r˜   r˜   v  sT   € ÷ Nñ NÜ�‰ðNä�‰ðNô �|‰|ðNô Œ5�<‰<œŸ™Ð%Õ&ñ	NrY   c                 ó<   € \        WV3R 7       \        WVRRR7      # rj  r.  r+  s   &&&rW   r    r    v  s!   € ô
 �r¨¨Õ5Ü˜B I¸$ÈÔMÐMrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# ©r’   Úwgtr”   rò   )r—   s   "rW   r˜   r˜   ˆ  s   € ÷ ñ ¬¯©ð ¼$ñ rY   c                 ó(   € V P                  4       ^8H  # )é   r&  ©r}  s   &rW   Ú_is_svdquant_tile_packed_weightr�  ˆ  s   € Ø�7‰7‹9˜‰>ÐrY   c                óD   € V ^8„  d   QhR\         P                  R\        /# r|  r  )r—   s   "rW   r˜   r˜   Œ  s   € ÷ ñ ¬E¯L©Lð ¼Sñ rY   c                 óª   € \        V 4      '       d(   \        V P                  ^ ,          4      \        ,          # \        V P                  ^ ,          4      # ©r   )r�  rµ   r  Ú_SVDQUANT_W4A4_BLOCK_Nr€  s   &rW   Ú"_svdquant_out_features_from_weightr†  Œ  s9   € Ü& s×+Ò+Ü�3—9‘9˜Q•<Ó Ô#9Õ9Ð9Üˆs�y‰y˜�|ÓÐrY   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r’   Úlora_upr”   ri  )r—   s   "rW   r˜   r˜   ’  s"   € ÷ ñ ¬u¯|©|ð ÄÇÁñ rY   c                ó  € V P                  4       ^8w  d   V # \        V RR4      pVey   VP                  V P                  8X  d^   VP                  V P                  8X  dC   VP                  V P                  ^ ,          \
        ,          V P                  ^,          38X  d   V# V P                  ^ ^^4      P                  V P                  ^ ,          \
        ,          V P                  ^,          4      P                  4       pW n	        V# )a  Return a natural (N, R) view/copy for tile-packed proj_up.

Converter layout is (N/128, R, 128). cuBLAS addmm_ wants a natural
row-major (N, R) tensor. Cache the one-time reorder on the source tensor so
the runtime path keeps cuBLAS speed without paying a per-forward permute.
Ú_ck_natural_lora_upN)
r'  rÉ   r“   r›   r  r…  r`  r6  r¬   rŠ  )rˆ  rÕ   Únaturals   &  rW   Ú!_natural_lora_up_from_tile_packedrŒ  ’  sÌ   € ð ‡{�{ƒ}˜ÔØˆÜ�WÐ3°TÓ:€FàÒØ�M‰M˜WŸ^™^Ô+Ø�L‰L˜GŸM™MÔ)Ø�L‰L˜WŸ]™]¨1Õ-Ô0FÕFÈÏÉÐVWÕHXÐYÔYàˆØ�o‰o˜a  AÓ&×.Ñ.Ø�‰�aÕÔ1Õ1°7·=±=ÀÕ3Cóç�jƒlð ð #*ÔØ€NrY   c                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\         P                  R,          R\        \         P                  \         P                  \         P                  3,          /# )	r’   r!  ÚsmoothÚ	lora_downÚpad_sizeÚact_unsignedÚlora_xNr”   )r•   r–   rµ   rÏ   r¶   )r—   s   "rW   r˜   r˜   ª  s„   € ÷ Q3ñ Q3Ü‡|�|ðQ3ä�L‰LðQ3ô �|‰|ðQ3ô ð	Q3ô
 ðQ3ô �L‰L˜4ÕðQ3ô Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñQ3rY   c                óÀ  € V P                  4       ^8X  d   V P                  4       '       g   Q R4       hV P                  w  rgVP                  ^,          p\        Wc4      p	\        p
Wz,          ^ 8X  g   Q RV RV
 24       h\
        P                  ! RR4      P                  4       R9   d   \        P                  MV P                  p\        P                  P                  V P                  4      P                  p\
        P                  ! RR4      P                  4       R9  pV'       Ed   V P                  P                  e   V P                  P                  M\        P                  P!                  4       pV\#        V4      V P                  W¹Wx3p\$        P'                  V4      pVf–   \        P(                  ! W—^,          \        P*                  V P                  R	7      \        P(                  ! Wz,          W�P                  V P                  R	7      \        P(                  ! W˜W°P                  R	7      3pV\$        V&   Vw  pppM�\        P(                  ! W—^,          \        P*                  V P                  R	7      p\        P(                  ! Wz,          W�P                  V P                  R	7      p\        P(                  ! W˜W°P                  R	7      p\,        P/                  \1        V 4      \1        V4      \1        V4      \1        V4      \1        V4      \1        V4      \3        V4      V4       Ve   TMT pV^ 8”  dk   VRV pVP                  VP                  8X  d1   VP                  4       '       d   \        P4                  ! VVVR
7       MVP7                  VV,          RR7       W–8”  d   VVR P9                  4        VP;                  \        P<                  4      VV3# )u#  SVDQuant W4A4 activation quantize + smooth + LoRA-down (CUDA).

Kitchen-native layouts:
  x         (M, K)       bf16/fp16  main-path input (pre-shifted if unsigned)
  smooth    (K,)         bf16/fp16
  lora_down (K, R)       bf16/fp16  natural row-major
  lora_x    (M, K)       bf16/fp16  pre-shift x for LoRA (defaults to x)
  q_x       (M_pad, K/2) int8       two int4 per byte
  ascales   (K/G, M_pad) bf16/fp16  per-row per-group
  lora_act  (M_pad, R)   bf16/fp16 by default (same dtype as x)

act_unsigned=True selects scale=max/15 + clamp [0,15] (u4 bit patterns for
downstream u4.s4 MMA). Caller must ensure x is non-negative â€” shift is a
model-topology concern kept out of the kernel API. Pass lora_x=raw_x when
x was pre-shifted (LoRA operates on pre-quantization, pre-shift activations).
zx must be 2D contiguousúK=z! must be divisible by group_size=Ú$COMFY_KITCHEN_SVDQUANT_LORA_ACT_FP32Ú r�   Ú&COMFY_KITCHEN_SVDQUANT_REUSE_WORKSPACENr%  rî  T)Únon_blocking>   r�   ÚonÚyesÚtrue>   rŒ   ÚnoÚoffÚfalse)r'  r  r  r…   Ú_SVDQUANT_W4A4_GROUP_SIZErO   ÚgetenvÚlowerr•   r¢   r›   r¹   r–  r“   r—  rr  rf  rµ   Ú_SVDQUANT_WORKSPACE_CACHEr    rl  r¤   r  Úsvdquant_quantize_w4a4r�  rÏ   Úmmr@  rB  r¥   r«   )r!  rŽ  r�  r�  r‘  r’  rë   rí   r½  Úm_padÚgÚlora_act_dtyper›  Úreuse_workspacer´   rt  rÕ   Úq_xÚascalesÚlora_actÚlora_srcÚlora_act_rowss   &&&&&&                rW   rI   rI   ª  sÛ  € ð0 �5‰5‹7�aŒ<˜AŸO™O×-Ò-ÐHÐ/HÓHÐ-Ø�7‰7�D€AØ�‰˜Õ€AÜ�AÓ €EÜ!€AØ�5�AŒ:ÐC˜˜A˜3Ð?À¸sÐCÓCˆ:ä&(§i¢iØ.°ó'ç�eƒgÐ+ô',”U—]’]à12·±ð ô —‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ—i’iØ0°#óç�eƒgÐ0ñ1€O÷ €Ø)*¯©¯©Ò)C�q—x‘x—~’~ÌÏÉ×IbÑIbÓIdˆØœS ›_¨a¯g©g°~ÈaÐSˆÜ*×.Ñ.¨sÓ3ˆØŠ>ä—’˜E¨¥6´·±ÀQÇXÁXÔNÜ—’˜A�F E·±ÀÇÁÔJÜ—’˜E¨NÇ8Á8ÔLðˆFð
 .4Ô% cÑ*Ø!'ÑˆˆW‘hä�kŠk˜% a¥¬u¯{©{À1Ç8Á8ÔLˆÜ—+’+˜a�f e·7±7À1Ç8Á8ÔLˆÜ—;’;˜u¨~ÇhÁhÔOˆä×ÑÜ˜ÓÜ˜Ó Ü˜Ó#Ü˜ÓÜ˜Ó!Ü˜Ó"Üˆ\ÓØô	ð"  Ò+‰v°€HØˆ1„uØ   !˜ˆØ×Ñ (§.¡.Ô0°]×5PÑ5P×5RÒ5RÜ�HŠH�X˜y¨mÖ<à×Ñ ¨9Õ 4À4ÐÔHØ„yØ��ˆ×ÑÔØ�8‰8”E—J‘JÓ ¨(Ð2Ð2rY   c                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\        R
\         P                  /	# )r’   Úactr}  rª  r9  Úlora_act_inrˆ  rÝ  Nr‘  r”   rò   )r—   s   "rW   r˜   r˜   þ  s’   € ÷ Oñ OÜ	�‰ðOä	�‰ðOô �\‰\ðOô �\‰\ð	Oô
 —‘ðOô �\‰\ðOô �,‰,˜Õ
ðOô ðOô ‡\�\ñOrY   c                óD  € V P                   ^ ,          p\        V4      p	\        P                  ! W‰VP                  V P
                  R7      p
\        P                  ! ^ VP                  V P
                  R7      p\        P                  ! RR4      P                  4       R9   p\        P                  ! R4      pVf   \        V4      pMVP                  4       R9   p\        V4      VP                  4       ^8H  8H  p\        P                  ! R4      pVP                  V
P                  8H  ;'       d*    T;'       d     VRJ ;'       g    VP                  4       R9   p\        P                  P                  V P
                  4      P                  p\        P                  \!        V P#                  \        P$                  4      4      \!        VP#                  \        P$                  4      4      \!        V4      \!        V4      \!        V4      \!        V4      \!        Ve   TMT4      \!        V
4      VVVVV4       V'       gd   VP                  V
P                  8X  d   TMVP'                  V
P                  4      p\)        V4      pV
P+                  VVP-                  4       4       V
# )aÄ  SVDQuant W4A4 int4 GEMM + LoRA-up + bias (CUDA).

int4 MMA + per-group dequant + bias run in one kernel. By default, when
lora_act_in already has the output dtype and proj_up's layout matches the
weight layout, LoRA-up is fused into the CUDA epilogue with bf16/fp16
tensor-core MMA. Set COMFY_KITCHEN_SVDQUANT_FUSE_LORA_UP=0 to force the
older Python/cuBLAS addmm_ epilogue for comparison.

Kitchen-native layouts:
  act         (M, K/2)   int8       two int4 per byte (signed or unsigned)
  wgt         (N, K/2)   int8       signed int4 weight, natural row-major
    or        (N/128, K/64, 32, 128) int8 kitchen_tile_packed_w4a4
  ascales     (K/G, M)   bf16/fp16  per-row per-group
  wscales     (K/G, N)   bf16/fp16  per-col per-group
    or        (N/128, K/G, 128) bf16/fp16 for tile-packed wgt
  lora_act_in (M, R)     bf16/fp16 or fp32
  lora_up     (N, R) or (N/128, R, 128) bf16/fp16
  bias        (N,) or (N/128, 128) bf16/fp16  (optional)
  out         (M, N)     bf16/fp16  (= lora_up.dtype)

act_unsigned: if True, A fragments go through u4.s4 MMA instead of s4.s4.
Set COMFY_KITCHEN_SVDQUANT_FAST_ACCUM=1 to use the experimental packed
half2/bfloat162 accumulator instead of the default fp32 accumulator.
r%  Ú!COMFY_KITCHEN_SVDQUANT_FAST_ACCUMr–  Ú#COMFY_KITCHEN_SVDQUANT_SHARED_SCALENÚ#COMFY_KITCHEN_SVDQUANT_FUSE_LORA_UP>   r�   r™  rš  r›  )r  r†  r•   rl  r›   r“   rO   r   r¡  r�  r'  r¹   r–  r—  r  Úsvdquant_scaled_mm_w4a4r�  r¥   r¤   rª   rŒ  r  rw  )r¯  r}  rª  r9  r°  rˆ  rÝ  r‘  rë   rì   rB  rl  Ú
fast_accumÚshared_scale_envÚshared_scaleÚlora_up_layout_matches_wgtÚfuse_lora_envÚ	fuse_lorar›  Ú	lora_bf16Ú
lora_up_mms   &&&&&&&&             rW   rK   rK   þ  s  € ðD 	�	‰	�!�€AÜ*¨3Ó/€AÜ
�+Š+�a '§-¡-¸¿
¹
Ô
C€CÜ�KŠK˜ §¡°s·z±zÔB€EÜ—’Ð>ÀÓC×IÑIÓKð Pñ €Jô —y’yÐ!FÓGÐØÒÜ6°sÓ;‰à'×-Ñ-Ó/Ð3MÑMˆä'¨Ó,°·±³À!Ñ1CÑDð ô —I’IÐCÓD€Mà×Ñ˜SŸY™YÑ&÷ 	[ð 	[Ø&÷	[ð 	[à˜dÐ"×YÐY m×&9Ñ&9Ó&;Ð?YÑ&Yð ô —‘×*Ñ*¨3¯:©:Ó6×BÑB€JÜ×ÑÜ˜Ÿ™¤%§+¡+Ó.Ó/Ü˜Ÿ™¤%§+¡+Ó.Ó/Ü˜Ó!Ü˜Ó!Ü˜Ó%Ü˜Ó!Ü Ò!1™°uÓ=Ü˜ÓØØØØØô÷  ð $/×#4Ñ#4¸¿	¹	Ô#A‘KÀ{Ç~Á~ÐVY×V_ÑV_ÓG`ˆ	Ü6°wÓ?ˆ
Ø�
‰
�9˜jŸl™l›nÔ-Ø€JrY   c                óÄ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  /# )r’   r!  r8  r9  ÚwzerosrG  r”   r  )r—   s   "rW   r˜   r˜   Z  sX   € ÷ ñ Ü‡|�|ðä�\‰\ðô �\‰\ðô �L‰Lð	ô
 ðô ‡\�\ñrY   c                óÊ  € VP                   w  rVV^,          pTpVP                  p	VP                  \        P                  4      p
V
^,          P                  \        P
                  4      pV
^,	          ^,          P                  \        P
                  4      p\        P                  ! W¼.RR7      P                  WW4      P                  V	4      pVP                  WWV,          V4      R,
          VP                  4       P                  R4      ,          VP                  4       P                  R4      ,           P                  WW4      pV P                  VP                  4       4      # )u!  Large-M fallback: dequantize qweight to bf16 then bf16 cuBLAS matmul.

qweight (N, K/2) int8 packed uint4 â†’ W (N, K) bf16 via group dequant, then
`out = x @ W.T`. Same algebra as eager.gemv_awq_w4a16 â€” used for M values
where the in-kitchen MMA kernel is slower than cuBLAS bf16 GEMM.
r&  g       @r4  )r  r›   rª   r•   ró  r«   r  r6  r¥   rw  rD  Úmatmul)r!  r8  r9  r¿  rG  rì   r  rí   r¦  Úcompute_dtypeÚx32rC  rD  Únibblesr-  s   &&&&&          rW   Ú_awq_w4a16_dequant_then_matmulrÅ  Z  sý   € ð —‘�I€AØ��
€AØ€AØ—M‘M€Mà
�*‰*”U—[‘[Ó
!€CØ
��)�‰œŸ
™
Ó	#€BØ�!�8�sÕ
×	Ñ	œuŸz™zÓ	*€BÜ�kŠk˜2˜(¨Ô+×3Ñ3°AÓ9×<Ñ<¸]ÓK€Gà	�‰�a˜a� Ó	# cÕ	)¨W¯Y©Y«[×-BÑ-BÀ2Ó-FÕFØ
�(‰(‹*×
Ñ
˜rÓ
"õ	#ç
�dˆ1ƒjð ð �8‰8�A—C‘C“E‹?ÐrY   c                óò   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\         P                  /# )	r’   r!  r8  r9  r¿  rÝ  NrG  r”   r  )r—   s   "rW   r˜   r˜   w  sk   € ÷ :.ñ :.Ü‡|�|ð:.ä�\‰\ð:.ô �\‰\ð:.ô �L‰Lð	:.ô
 �,‰,˜Õ
ð:.ô ð:.ô ‡\�\ñ:.rY   c           	     óþ  € V P                   pV P                  RVR,          4      pVP                   w  r‰VP                   ^ ,          p
W•,          ^ 8w  d   \        RV	 RV 24      hVP                   ^,          ^,          V	8w  d#   \        RVP                   ^,           RV	 24      hV\        8”  d6   \	        VP                  4       P                  VP                  4      WW54      pMì\        P                  ! WŠVP                  V P                  R7      p\        P                  P                  V P                  4      P                  p\        P                  \!        VP                  4       P                  VP                  4      4      \!        VP#                  \        P$                  4      4      \!        V4      \!        V4      \!        V4      VV4       Ve   VP'                  V4       VP                  ! . VRR OV
N5!  # )uÕ  AWQ W4A16 matmul: int4 weight @ fp activation (CUDA, kitchen-native).

Tiered routing:
  M â‰¤ 8                    naive 1-thread-per-output kernel (GEMV-style)
  8 < M â‰¤ 512              fused int4 x bf16/fp16 MMA kernel â€” dequant
                           into shmem, mma.m16n8k16.f32 along K, no
                           intermediate bf16 W workspace
  M > 512                  dequant + cuBLAS bf16 matmul fallback

bias is applied externally (`out.add_`), mirroring
scaled_mm_svdquant_w4a4's epilogue contract.

Layouts (match comfy_kitchen.backends.eager.awq):
  x        (M, K)   bf16/fp16  row-major activation. Leading dims are
                               flattened.
  qweight  (N, K/2) int8       two unsigned int4 per byte
  wscales  (K/G, N) bf16/fp16  per-group, per-output-col scale
  wzeros   (K/G, N) bf16/fp16  per-group, per-output-col zero
  bias     (N,)     bf16/fp16  optional (= wscales.dtype)
  out      (..., N) bf16/fp16  same dtype as wscales
r”  z not divisible by group_size=zqweight K//2=z inconsistent with x K=r%  Nr4  )r  r6  rÊ  Ú_AWQ_W4A16_MMA_M_LIMITrÅ  r¬   rª   r›   r•   rl  r“   r¹   r–  r—  r  Ú	awq_w4a16r�  r¥   r¤   Úadd_)r!  r8  r9  r¿  rÝ  rG  rÄ  r?  rë   rí   rì   Úout2dr›  s   &&&&&&       rW   rE   rE   w  s�  € ð: —‘€JØ
�)‰)�B˜
 2�Ó
'€CØ�9‰9�D€AØ�‰�aÕ€AØ…~˜ÔÜ˜2˜a˜SÐ =¸j¸\ÐJÓKÐKØ‡}�}�QÕ˜!Õ˜qÔ Ü˜=¨¯©°qÕ)9Ð(:Ð:QÐRSÐQTÐUÓVÐVàÔ!Ô!Ü.Ø�N‰NÓ×Ñ §¡Ó.°À&ó
‰ô —’˜A¨¯©¸a¿h¹hÔGˆÜ—Z‘Z×.Ñ.¨q¯x©xÓ8×DÑDˆ
Ü
�‰Ü˜SŸ^™^Ó-×0Ñ0°·±Ó?Ó@Ü˜WŸ\™\¬%¯+©+Ó6Ó7Ü˜WÓ%Ü˜VÓ$Ü˜UÓ#ØØô	
ð ÒØ�
‰
�4Ôà�=Š=Ð-˜* S b˜/Ð-¨1Ó-Ð-rY   c                ó$   € V ^8„  d   QhR\         /# rd  )r3  )r—   s   "rW   r˜   r˜   ´  s   € ÷ qñ qœDñ qrY   c            ,      óR  a€ R  oV3R lp V3R lp\        R04      p/ R\        R\        \        \        P                  \        P
                  04      \        ^4      3R7      R\        \        \        P                  \        P
                  04      \        ^4      3R7      R\        \        \        P                  \        P
                  04      \        ^4      3R7      /VRŒ\        3R	7      bR
\        R\        \        \        P
                  \        P                  04      \        ^4      3R7      R\        \        \        P
                  \        P                  04      \        ^4      3R7      R\        \        \        P
                  \        P                  04      \        ^4      3R7      /VRŒV 3R	7      bR\        R\        \        \        P
                  \        P                  04      \        ^4      3R7      R\        \        \        P
                  \        P                  04      \        ^4      3R7      R\        \        \        P
                  \        P                  04      \        ^4      3R7      /VRŒV3R	7      bR\        R\        \        \        P                  \        P                  04      \        ^4      3R7      /VR�R7      bR\        R\        \        \        P                  04      \        ^4      3R7      /VR�R7      bR\        R\        \        \        P                  04      \        ^4      3R7      /VR�R7      bR\        R\        \        \        P                  04      \        ^4      3R7      R\        \        \        P                  04      \        ^4      3R7      /VR�R7      bR\        R\        \        \        P                  \        P                  04      \        ^4      3R7      /VR�R7      bR\        R\        \        \        P                  04      \        ^4      3R7      R\        \        \        P                  \        P                  04      \        ^4      3R7      R\        \        \        P                  \        P                  04      \        ^4      3R7      R\        \        \        P                  04      \        ^4      3R7      /VR�R7      bR\        R\        \        \        P                  \        P                  04      \        ^4      3R7      R\        \        \        P                  04      \        ^4      3R7      R\        \        \        P                  04      \        ^4      3R7      R\        \        \        P                  04      \        ^4      3R7      /VR�R7      bR \        R!\        \        \        P                  \        P
                  \        P                  04      R"7      R#\        \        \        P                  \        P
                  \        P                  04      R"7      R$\        \        \        P                  \        P
                  \        P                  04      R"7      /VR%7      bR&\        R!\        \        \        P                  \        P
                  \        P                  04      R"7      R#\        \        \        P                  \        P
                  \        P                  04      R"7      R$\        \        \        P                  \        P
                  \        P                  04      R"7      /VR%7      bR'\        R!\        \        \        P
                  04      \        ^4      3R7      R(\        \        \        P
                  04      \        ^4      3R7      R)\        \        \        P
                  \        R*4      04      R"7      R+\        \        \        P
                  \        R*4      04      R"7      /VR%7      bR,\        R!\        \        \        P
                  \        P                  04      \        ^4      3R7      /VR%7      bR-\        R!\        \        \        P
                  \        P                  04      \        ^4      3R7      R(\        \        \        P                  \        P
                  \        P                  \        R*4      04      R"7      R)\        \        \        P                  \        P
                  \        P                  \        R*4      04      R"7      /VR%7      bR.\        R!\        \        \        P                  \        P
                  \        P                  04      R"7      R#\        \        \        P                  04      R"7      R/\        \        \        P                  \        P                  04      R"7      /VR%7      bR0\        R!\        \        \        P                  \        P                  04      R"7      R#\        \        \        P                  04      R"7      R/\        \        \        P                  \        P
                  \        P                  04      R"7      /VR%7      b/ R1\        R!\        \        \        P                  \        P
                  \        P                  04      R"7      R2\        \        \        P                  04      R"7      R/\        \        \        P                  \        P                  04      R"7      /VR%7      bR3\        R!\        \        \        P
                  \        P                  04      \        ^4      3R7      R4\        \        \        P                  04      R"7      /VR%7      bR5\        R!\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /VR%7      bR6\        R7\        \        \        P                  04      \        ^4      \!        ^ ^R87      3R7      R4\        \        \        P                  04      R"7      R9\        \        \        P                  04      R"7      R/\        \        \        P                  \        P
                  \        P                  04      R"7      /VR%7      bR:\        R!\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /VR%7      bR<\        R=\        \        \        P
                  \        P                  04      \        ^4      3R7      R>\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /VR%7      bR?\        R\        \        \        P
                  \        P                  04      \        ^4      3R7      R\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      R@\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      RA\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /V\"        3RB7      bRC\        R!\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      R#\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /V\$        3RB7      bRD\        R\        \        \        P
                  \        P                  04      \        ^4      3R7      R\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      R@\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      RA\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /V\"        3RB7      bRE\        R!\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      R#\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /V\$        3RB7      bRF\        R!\        \        \        P                  \        P
                  \        P                  04      R"7      R#\        \        \        P                  \        P
                  \        P                  \&        \(        04      R"7      RG\        \        \*        04      R"7      /VR%7      bRH\        R!\        \        \        P                  \        P
                  \        P                  04      R"7      RG\        \        \*        04      R"7      /VR%7      bRI\        R!\        \        \        P                  \        P
                  \        P                  04      R"7      RJ\        \        \        P                  \        P
                  \        P                  04      R"7      RK\        \        \*        04      R"7      RG\        \        \*        04      R"7      /VR%7      bRL\        R(\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      RK\        \        \*        04      R"7      RG\        \        \*        04      R"7      /VR%7      bRM\        R(\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      RK\        \        \*        04      R"7      /VR%7      bRN\        R(\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      RK\        \        \*        04      R"7      RO\        \        \*        04      R"7      RP\        \        \,        04      R"7      RQ\        \        \        P                  \        P                  04      R"7      RR\        \        \,        04      R"7      /VRŒR7      bRS\        RT\        \        \        P.                  04      \        ^4      3R7      RU\        \        \        P                  \        P                  04      \        ^4      3R7      RV\        \        \        P                  04      \        ^4      3R7      RR\        \        \        P                  04      \        ^4      3R7      RW\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      RK\        \        \*        04      R"7      RO\        \        \*        04      R"7      RX\        \        \        P                  \        P
                  \        P                  04      R"7      /VRŒR7      bCRY\        R!\        \        \        P                  \        P
                  \        P                  04      R"7      RT\        \        \        P.                  04      \        ^4      3R7      RU\        \        \        P                  \        P                  04      \        ^4      3R7      RV\        \        \        P                  04      \        ^4      3R7      RR\        \        \        P                  04      \        ^4      3R7      RW\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      R)\        \        \        P                  \        P
                  \        P                  04      R"7      RK\        \        \*        04      R"7      RO\        \        \*        04      R"7      RZ\        \        \        P                  \        P
                  \        P                  04      R"7      /
VRŒR7      R[\        R\        \        \        P.                  04      \        ^4      3R7      R#\        \        \        P                  \        P
                  \        P                  04      R"7      RK\        \        \*        04      R"7      /VR%7      R\\        R\        \        \        P.                  04      \        ^4      3R7      R#\        \        \        P                  \        P
                  \        P                  04      R"7      RK\        \        \*        04      R"7      R]\        \        \*        04      R"7      /VR%7      R^\        R\        \        \        P.                  04      R"7      R#\        \        \        P                  \        P
                  \        P                  04      R"7      /VR%7      R_\        R\        \        \        P.                  04      R"7      R#\        \        \        P                  \        P
                  \        P                  04      R"7      R]\        \        \*        04      R"7      /VR%7      R`\        R!\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /VR%7      Ra\        R=\        \        \        P
                  \        P                  04      \        ^4      3R7      R>\        \        \        P
                  \        P                  04      \        ^4      3R7      R;\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      /VR%7      Rb\        R!\        \        \        P
                  \        P                  04      \        ^4      \!        ^^@R87      3R7      Rc\        \        \        P
                  \        P                  04      R"7      Rd\        \        \        P
                  \        P                  04      \        ^4      3R7      /VRŒR7      Re\        Rf\        \        \        P.                  04      \        ^4      3R7      Rg\        \        \        P.                  04      R"7      Rh\        \        \        P
                  \        P                  04      R"7      Ri\        \        \        P
                  \        P                  04      R"7      Rj\        \        \        P                  \        P
                  \        P                  04      R"7      Rk\        \        \        P
                  \        P                  04      R"7      /VRŒR7      Rl\        R(\        \        \        P                  \        P
                  \        P                  04      \        ^4      \!        ^^@R87      3R7      RO\        \        \*        04      R"7      Rm\        \        \*        04      R"7      RG\        \        \*        04      R"7      /VR�R7      Rn\        RT\        \        \        P.                  04      \        ^4      \!        ^^ R87      3R7      Ro\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      RO\        \        \*        04      R"7      Rm\        \        \*        04      R"7      RX\        \        \        P                  \        P
                  \        P                  04      R"7      /VR�R7      Rp\        R!\        \        \        P                  \        P
                  \        P                  04      \1        ^4      3R7      Rq\        \        \        P.                  04      \        ^4      \!        ^^ R87      3R7      Ri\        \        \        P                  \        P
                  \        P                  04      \        ^4      3R7      R)\        \        \        P                  \        P
                  \        P                  04      R"7      RO\        \        \*        04      R"7      Rm\        \        \*        04      R"7      Rr\        \        \(        04      R"7      /VR�R7      Rs\        R(\        \        \        P.                  04      \        ^4      \!        ^^ R87      3R7      /VR%7      Rt\        R!\        \        \        P
                  \        P                  04      R"7      Rq\        \        \        P.                  04      \        ^4      3R7      Ri\        \        \        P
                  \        P                  04      \        ^4      3R7      Ru\        \        \        P
                  \        P                  04      \        ^4      3R7      /VRŒR7      /Cp\        R!\        \        \        P
                  04      \1        ^4      3R7      R(\        \        \        P
                  04      \        ^4      3R7      R)\        \        \        P
                  04      R"7      R+\        \        \        P
                  04      R"7      Rv\        \        \        P
                  04      R"7      /VRŒR7      VRw&   \2        '       Ed´   \        R!\        \        \        P                  \        P
                  \        P                  04      \1        ^4      3R7      R(\        \        \        P.                  04      \        ^4      3R7      Rx\        \        \        P                  04      R"7      R)\        \        \        P                  \        P
                  \        P                  04      R"7      RZ\        \        \        P                  \        P
                  \        P                  04      R"7      Ry\        \        \,        04      R"7      RO\        \        \*        04      R"7      /VR�R7      VRz&   \        R{\        \        \        P                  04      \        ^4      \!        ^^R87      3R7      R|\        \        \        P                  04      \        ^4      \!        ^^R87      3R7      R}\        \        \        P                  04      R"7      R~\        \        \        P                  04      R"7      R\        \        \        P                  04      R"7      R€\        \        \        P                  04      R"7      RZ\        \        \        P
                  \        P                  04      R"7      /VRŽR7      VR�&   R‚R<RƒR:R„RaR…R`R†R?R‡RCRˆRDR‰RE/P5                  4        F  w  rEW5,          W4&   K  	  R� F%  pWc9   g   K  \7        W6,          4      W6RŠ,           &   K'  	  \9        VRY,          4      VR‹&   V# )�c                 ó–  € V e²   V P                   R,          \        9  d   \        P                  ! RR\         24      # V P                   ^,          V P                   ^,          ,          R8”  g1   V P                   ^ ,          V P                   ^,          ,          R8”  d   \        P                  ! RR4      # \        P                  ! 4       # )Nr‚  zhead_dim must be one of r÷  zgrid dims exceed CUDA limitsr4  )r  Ú_NA3D_HEAD_DIMSr   r:  rc  ©r‚  s   &rW   Ú_na3d_launch_ruleÚ-_build_constraints.<locals>._na3d_launch_ruleµ  s�   € àŠ=Ø�w‰w�r�{¤/Ô1Ü'×,Ò,¨SÐ4LÌ_ÐL]Ð2^Ó_Ð_Ø�w‰w�q�z˜AŸG™G A�JÕ&¨Ô.°!·'±'¸!µ*¸q¿w¹wÀq½zÕ2IÈEÔ2QÜ'×,Ò,¨SÐ2PÓQÐQÜ×"Ò"Ó$Ð$rY   c                 óp   <€ \        V 4      pVP                  '       g   V# S! V P                  R 4      4      # rÐ  )r€   rŠ  r    )r1  ÚcommonrÑ  s   & €rW   Ú_na3d_call_ruleÚ+_build_constraints.<locals>._na3d_call_rule¾  s.   ø€ Ü& vÓ.ˆØ�~�~ˆ~ØˆMÙ  §¡¨C£Ó1Ð1rY   c                 ó:  <€ \        V 4      pVP                  '       g   V# S! V P                  R 4      4      pVP                  '       g   V# V P                  R4      pVe(   \        V4      ^8”  d   \        P
                  ! RR4      # \        P                  ! 4       # )r‚  r1  z$at most 8 distinct per-plane windows)r‚   rŠ  r    r¼  r   r:  rc  )r1  rÔ  Úlaunchr1  rÑ  s   &   €rW   Ú_window_attention_call_ruleÚ7_build_constraints.<locals>._window_attention_call_ruleÄ  s{   ø€ Ü2°6Ó:ˆØ�~�~ˆ~ØˆMÙ" 6§:¡:¨c£?Ó3ˆØ�~�~ˆ~ØˆMØ�Z‰Z˜ÓˆØŠ>œc "›g¨œkÜ#×(Ò(¨Ð/UÓVÐVÜ×"Ò"Ó$Ð$rY   r¹   r   r‚  )ÚdtypesÚshape_rulesrí   r|  )ÚparamsÚdefault_devicesÚmin_compute_capabilityÚ
call_rulesr	   r
   r   rö  )rÝ  rÞ  rß  r   rV  r   r   r  r€  r   rœ  r   r¢  r£  r¤  r   r³  rŽ  r´  rµ  r   r!  )rÛ  r’  râ  )rÝ  rÞ  r   r   r  rÝ  Nr´  r   r   rH   r“  r,   rL   r   rG   r¥  rF   r+   r²  )r'  Úfactorr¸  r   r  r   rÈ  r  r#   r>  r?  )rÝ  rÞ  rà  r%   r'   r)   r:   r¿  r;   rD   ÚHrG  rA   rB   r@   r$  rf  rg  r®   r2   rÓ  rN  rO  rõ  r/  r5   rÞ  r/   r0   r™  r-   r.   r!   r   rI   rŽ  r�  rK   r¯  r}  rª  r9  r°  rˆ  r?   r%  r1   r¯   r8   r8  r:  r9   rE   r¿  rµ  r3   rƒ  rÓ  r4   r™  rš  r›  rœ  r�  rž  rJ   r   r   r    r"   r$   r&   r(   r*   Ú_outr6   rˆ  rÑ   )é
   r   )r   r   )Ú	frozensetr|   r~   r•   ra  r`  r{   r�   r¢   r6  ró  rz  rq  r¦   Úfloat8_e5m2r¤   rz   rF  r;  r§   r;  rµ   rÏ   r«   r}   Ú_CUBLASLT_AVAILABLEÚitemsrƒ   r„   )rÕ  rÙ  Úcuda_devicesÚconstraintsÚinplace_nameÚfunctional_namer�  rÑ  s          @rW   Ú_build_constraintsrí  ´  s%  ø€ ò%õ2õ
%ô ˜f˜XÓ&€Lðx
ØÔ'à”_Ü$¤e§n¡n´e·m±mÐ%DÓEÜ!*¨1£ ôð ”_Ü$¤e§n¡n´e·m±mÐ%DÓEÜ!*¨1£ ôð ”_Ü$¤e§n¡n´e·m±mÐ%DÓEÜ!*¨1£ ôðð )Ø#)Ü1Ð3ô#
ðx
ð( 	Ô#à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôðð )Ø#)Ø'Ð)ô#
ð)x
ðN 	Ô/à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôðð )Ø#)Ø3Ð5ô#
ðOx
ðt 	Ô)àœ/´¼E¿M¹MÌ5Ï=É=Ð;YÓ1ZÔirÐstÓiuÐhwÔxðð )Ø#)ô
ðux
ðB 	Ô'Ø¤¼	Ä5Ç=Á=À/Ó8RÔajÐklÓamÐ`oÔ!pÐqØ(Ø#)ô
ðCx
ðL 	 Ô!4Øœo´YÄÇÁ¸Ó5OÔ^gÐhiÓ^jÐ]lÔmÐnØ(Ø#)ô"
ðMx
ðV 	Ô2àœ?´)¼U¿]¹]¸OÓ2LÔ[dÐefÓ[gÐZiÔjØœo´YÄÇÁ¸Ó5OÔ^gÐhiÓ^jÐ]lÔmðð )Ø#)ô 
ðWx
ðf 	Ô 3Øœ_´I¼u¿{¹{ÌEÏKÉKÐ>XÓ4YÔhqÐrsÓhtÐgvÔwÐxØ(Ø#)ô!
ðgx
ðp 	Ô 3àœO´9¼e¿m¹m¸_Ó3MÔ\eÐfgÓ\hÐ[jÔkØœ´	¼5¿;¹;ÌÏÉÐ:TÓ0UÔdmÐnoÓdpÐcrÔsØœ´	¼5¿;¹;ÌÏÉÐ:TÓ0UÔdmÐnoÓdpÐcrÔsØœ_´I¼u¿}¹}¸oÓ4NÔ]fÐghÓ]iÐ\kÔlð	ð )Ø#)ô	!
ðqx
ðD 	Ô&à”¬i¼¿¹ÄeÇkÁkÐ8RÓ.SÔbkÐlmÓbnÐapÔqØ”¬i¼¿¹¸Ó.HÔW`ÐabÓWcÐVeÔfØœ´	¼5¿=¹=¸/Ó0JÔYbÐcdÓYeÐXgÔhØœ´	¼5¿=¹=¸/Ó0JÔYbÐcdÓYeÐXgÔhð	ð )Ø#)ô	
ðEx
ðX 	Ô$à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð
ð )ô
ðYx
ðr Ô2à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð
ð )ô 
ðsx
ðN 	Ô*à”_¬I´u·}±}°oÓ,FÔU^Ð_`ÓUaÐTcÔdØœ/´¼E¿M¹M¸?Ó1KÔZcÐdeÓZfÐYhÔiØœ¬y¼%¿-¹-ÌÈdËÐ9TÓ/UÔVØœO´9¼e¿m¹mÌTÐRVËZÐ=XÓ3YÔZð	ð )ô
ðOx
ð` 	Ô1à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôðð )ô
ðax
ðr 	 Ô!4à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ/Ü$¤e§m¡m´U·]±]ÄEÇNÁNÔTXÐY]ÓT^Ð%_Ó`ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÔTXÐY]ÓT^Ð%_Ó`ôðð )ô"
ðsx
ðP 	"Ô#6à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œÜ$¤e§m¡m _Ó5ôð œÜ$¤e×&9Ñ&9¼5×;LÑ;LÐ%MÓNô ð
ð )ô$
ðQx
ðl 	$Ô%8à”_Ü$¤e×&9Ñ&9¼5×;LÑ;LÐ%MÓNôð œÜ$¤e§m¡m _Ó5ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTô ð
ð )ô&
ñmx
ðH 	"Ô#6à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð ”¬i¼¿¹¸Ó.FÔGØœÜ$¤e×&9Ñ&9¼5×;LÑ;LÐ%MÓNô ðð )ô$
ðIx
ð` 	Ô-à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð #¤OÜ$¤e§m¡m _Ó5ô%ðð )ô
ðax
ðx 	Ô-à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôðð )ô
ðyx
ðJ 	Ô/à”oÜ$¤e§k¡k ]Ó3Ü!*¨1£¬{¸qÈÔ/LÐ Môð #¤OÜ$¤e§m¡m _Ó5ô%ð ¤Ü$¤e×&9Ñ&9Ð%:Ó;ô!ð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTô ðð )ô!
ðKx
ðn 	Ô*à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð	ð )ô
ðox
ðH 	Ô)à”oÜ$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”oÜ$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôðð )ô
ðIx
ðj 	Ô'à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œ?Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œ?Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð#ð, )Ü/Ð1ô1
ðkx
ð^	 	Ô(à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôðð )Ü0Ð2ô!
ð_	x
ðB
 	Ô2à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œ?Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œ?Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð#ð, )Ü/Ð1ô1 
ðC
x
ðv
 	Ô 3à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôðð )Ü0Ð2ô!!
ðw
x
ðZ 	#Ô$7à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÔTYÔ[^Ð%_Ó`ôð &¤¼iÌÈÓ>NÔ'Oðð )ô%
ð[x
ðr 	 Ô!4à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð &¤¼iÌÈÓ>NÔ'Oð	ð )ô"
ðsx
ðD 	&Ô':à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð ”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œo´YÄ¸uÓ5EÔFØ%¤¼iÌÈÓ>NÔ'Oð	ð )ô(
ðEx
ð^ 	'Ô(;àœ/Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œo´YÄ¸uÓ5EÔFØ%¤¼iÌÈÓ>NÔ'Oðð )ô
)
ð_x
ðt 	%Ô&9àœ/Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œo´YÄ¸uÓ5EÔFðð )ô	'
ðux
ðH 	$Ô%8àœ/Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œo´YÄ¸uÓ5EÔFØ#¤_¼IÄsÀeÓ<LÔ%MØœ_´I¼t¸fÓ4EÔFØœ´iÄ×ATÑATÔV[×VcÑVcÐ@dÓ6eÔfØœO´9¼d¸VÓ3DÔEð
ð )Ø#)ô&
ðIx
ðf 	&Ô':àœÜ$¤e§j¡j \Ó2Ü!*¨1£ ôð œÜ$¤e×&9Ñ&9¼5¿=¹=Ð%IÓJÜ!*¨1£ ôð œ_Ü$¤e§m¡m _Ó5Ü!*¨1£ ôð œOÜ$¤e§m¡m _Ó5Ü!*¨1£ ôð œoÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œo´YÄ¸uÓ5EÔFØ#¤_¼IÄsÀeÓ<LÔ%MØ¤Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTô!ð/ð6 )Ø#)ô;(
ñgx
ðd 	Ô/à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œÜ$¤e§j¡j \Ó2Ü!*¨1£ ôð œÜ$¤e×&9Ñ&9¼5¿=¹=Ð%IÓJÜ!*¨1£ ôð œ_Ü$¤e§m¡m _Ó5Ü!*¨1£ ôð œOÜ$¤e§m¡m _Ó5Ü!*¨1£ ôð œoÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œo´YÄ¸uÓ5EÔFØ#¤_¼IÄsÀeÓ<LÔ%MØœ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð; ðB )Ø#)ôG$
ðJ 	)Ô*=à”_Ü$¤e§j¡j \Ó2Ü!*¨1£ ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œo´YÄ¸uÓ5EÔFð	ð )ô+
ð 	/Ô0Cà”_Ü$¤e§j¡j \Ó2Ü!*¨1£ ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œo´YÄ¸uÓ5EÔFØ#¤_¼IÄsÀeÓ<LÔ%Mð
ð )ô1
ð 	!Ô"5à”_Ü$¤e§j¡j \Ó2ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð	ð )ô
#
ð 	'Ô(;à”_Ü$¤e§j¡j \Ó2ôð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð $¤_¼IÄsÀeÓ<LÔ%Mðð )ô)
ð 	!Ô"5à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð	ð )ô#
ð 	 Ô!4à”oÜ$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð ”oÜ$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôðð )ô"
ð" 	!Ô"5à”_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£¬{¸qÈÔ/LÐ Môð œ/Ü$¤e§m¡m´U·^±^Ð%DÓEôð œ_Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôðð )Ø#)ô#
ð" 	"Ô#6à”¬i¼¿¹¸Ó.EÔT]Ð^_ÓT`ÐSbÔcØ”¬i¼¿¹¸Ó.EÔFØœ?´)¼U¿]¹]ÌEÏNÉNÐ<[Ó2\Ô]Øœ?´)¼U¿]¹]ÌEÏNÉNÐ<[Ó2\Ô]ØœÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTô ð œ?´)¼U¿]¹]ÌEÏNÉNÐ<[Ó2\Ô]ð	ð )Ø#)ô$
ð 	'Ô(;àœ/Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£¬{¸qÈÔ/LÐ Môð $¤_¼IÄsÀeÓ<LÔ%MØ"¤O¼9ÄcÀUÓ;KÔ$LØ%¤¼iÌÈÓ>NÔ'Oðð )Ø#)ô)
ð 	)Ô*=àœÜ$¤e§j¡j \Ó2Ü!*¨1£¬{¸qÈÔ/LÐ Môð œ/Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð $¤_¼IÄsÀeÓ<LÔ%MØ"¤O¼9ÄcÀUÓ;KÔ$LØ¤´yÄ%Ç-Á-ÔQV×Q^ÑQ^Ô`e×`nÑ`nÐAoÓ7pÔ qðð )Ø#)ô+
ð" 	Ô2à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!(¨£ ôð œ?Ü$¤e§j¡j \Ó2Ü!*¨1£¬{¸qÈÔ/LÐ Môð œ?Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!*¨1£ ôð œ¬y¼%¿-¹-ÌÏÉÔX]×XfÑXfÐ9gÓ/hÔiØ#¤_¼IÄsÀeÓ<LÔ%MØ"¤O¼9ÄcÀUÓ;KÔ$LØ¤´yÄ#ÀÓ7GÔ Hð!ð$ )Ø#)ô) 
ð, 	.Ô/Bàœ/Ü$¤e§j¡j \Ó2Ü!*¨1£¬{¸qÈÔ/LÐ Môðð )ô0
ð 	Ô-à”_Ü$¤e§m¡m´U·^±^Ð%DÓEôð œ?Ü$¤e§j¡j \Ó2Ü!*¨1£ ôð œ?Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôð œ/Ü$¤e§m¡m´U·^±^Ð%DÓEÜ!*¨1£ ôðð" )Ø#)ô'
ñGx
€Kôt "5à”Ü ¤%§-¡- Ó1Ü$ Q›Z˜Môð ”oÜ ¤%§-¡- Ó1Ü& q›\˜Oôð ”O¬9´e·m±m°_Ó+EÔFØœ¬y¼%¿-¹-¸Ó/IÔJØœo´YÄÇÁ¸Ó5OÔPð
ð %Ø%ô"€K�Ñ÷$ ÔÜ%8à”_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTÜ!(¨£ ôð œ/Ü$¤e§j¡j \Ó2Ü!*¨1£ ôð ¤Ü$¤e§m¡m _Ó5ô!ð œÜ$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œ_Ü$¤e§m¡m´U·]±]ÄEÇNÁNÐ%SÓTôð œ?´)¼T¸FÓ2CÔDØ#¤_¼IÄsÀeÓ<LÔ%Mð'ð* )Ø#)ô/&
ˆ�MÑ"ô2 *=à”_Ü$¤e§k¡k ]Ó3Ü!*¨1£¬{¸qÈÔ/LÐ Môð ”_Ü$¤e§k¡k ]Ó3Ü!*¨1£¬{¸qÈÔ/LÐ Môð !¤/Ü$¤e§m¡m _Ó5ô#ð !¤/Ü$¤e§m¡m _Ó5ô#ð  ¤Ü$¤e×&9Ñ&9Ð%:Ó;ô"ð  ¤Ü$¤e×&9Ñ&9Ð%:Ó;ô"ð œ_Ü$¤e§m¡m´U·^±^Ð%DÓEôð+ð2 )Ø#*ô7*
ˆÐ%Ñ&ð> 	�|Ø˜Ø Ð"9Ø!Ð#;Ø�ZØ�kØÐ 5ØÐ!7ð	*÷ �eƒgò	*Ñ%ˆð %0Õ$@ˆÓ!ñ	*ó 9ˆØÖÜ)7¸Õ8IÓ)JˆK˜v�Ó&ñ 9ô )AÀÐM_ÕA`Ó(a€KÐ$Ñ%ØÐrY   c                 ó:  € \         '       g   \        P                  ! R\        4       R# \        P
                  P                  4       '       g   \        P                  ! RR4       R# \        P                  ! R\        \        \        R7      \        4       R7       R# )z/Register CUDA backend with the global registry.r¹   Nz!CUDA not available on this system)Úfromlist)r�  ÚmoduleÚcapabilities)Ú_EXT_AVAILABLEr†   Úmark_unavailableÚ
_EXT_ERRORr•   r¹   Úis_availableÚregisterÚ
__import__r[  Ú__all__rí  rÛ   rY   rW   Ú	_registerrù  è  sb   € ç‹>Ü×!Ò! &¬*Ô5Ùä�:‰:×"Ñ"×$Ò$Ü×!Ò! &Ð*MÔNÙä×ÒØÜœ(¬WÔ5Ü'Ó)÷rY   c                óæ  € V ^8„  d   Qh/ ^ \         9   d%   \        \        \        P                  3,          ;R&   ^\         9   dL   \        \
        \        \        R,          \        P                  3,          \        P                  3,          ;R&   ^\         9   d   \        \        \        3,          ;R&   ^\         9   d   \        \        \        3,          ;R&   ^\         9   d   \        \        \        3,          ;R&   ^\         9   d,   \        \        \
        \        \        3,          3,          ;R&   ^\         9   d   \        \        \        3,          ;R&   ^\         9   d   \        \        \        3,          ;R	&   ^\         9   d/   \        \        P                  \        P                  3,          ;R
&   ^	\         9   d   \        \        \        3,          ;R&   ^
\         9   d%   \        \
        \        P                  3,          ;R&   ^\         9   d\   \        \
        \        R3,          \
        \        P                  \        P                  \        P                  3,          3,          ;R&   # )r’   rk  Nrs  rÔ   rá   r÷   r¸   rÀ   rÈ   rŸ   r<  rÎ  .r¢  )Ú__conditional_annotations__r3  rµ   r•   r–   r¶   r;  r›   rÏ   r“   r4  )r—   s   "rW   r˜   r˜      s¾  € × Ñ ÷T 1Ò 0”DœœeŸl™lÐ*Õ+Ñ 0ñU ÷V RÒ Q”Tœ%¤¤S¨4¥Z´·±Ð <Õ=¼u¿|¹|ÐKÕLÑ QñW ÷X +Ò *”dœ3¤˜9•oÑ *ñY ÷Z 5Ò 4¤¤S¬$ Y¥Ñ 4ñ[ ÷\ 1Ò 0œD¤¤d �OÑ 0ñ] ÷^ :Ò 9œ$œs¤E¬#¬s¨(¥OÐ3Õ4Ñ 9ñ_ ÷` 8Ò 7¤D¬¬c¨¥NÑ 7ña ÷b ,Ò +œœS¤#˜X�Ñ +ñc ÷p ?Ò >œœeŸl™l¬E¯L©LÐ8Õ9Ñ >ñq ÷R .Ò -œ$œs¤C˜x�.Ñ -ñS ÷Dr .Ò -Œt”Eœ5Ÿ<™<Ð'Õ(Ñ -ñEr ÷lC jÒ iœ4¤¤f¨c kÕ 2´E¼%¿,¹,ÌÏÉÔV[×VbÑVbÐ:bÕ4cÐ cÕdÑ iòmC rY   i   )Ú1660Ú1650Ú1630ÚT500ÚT550ÚT600ÚMX550ÚMX450zCMP 30HXÚT2000ÚT1000ÚT1200)r£  FTr„  rN   i ð )r   NNr£  )Nr   rô   )NNN)	NNFrœ   NNr£  NN)NNr£  rÖ  )rJ  r  é0   rK  rÏ  i   r\  )
ç      ð?NNNNr£  TNNr   )NNr  r£  NNNç�íµ ÷Æ°>TNNr   )r	  rÖ  i    )r£  )rK  )r4  N)FNN)Nr	  )Nr	  r   )rœ   FN)NF)NrK  (M  rû  Ú
contextlibÚctypesÚimportlib.utilÚ	importlibrO   Úsysr|  Útypingr   r•   Úcomfy_kitchen._rope_utilsr   r   r   r   Úcomfy_kitchen.allocationr   rø  Ú_dll_handleÚnvidia.cu13ÚnvidiaÚcu13Ú__path__Únvidia_cu13_pathÚ	ExceptionrX   ÚplatformÚlib_dirÚadd_dll_directoryÚlistdirÚfilenameÚsuppressÚCDLLÚpathr’  ÚRTLD_GLOBALr  ÚdirnameÚ__file__Ú_module_pathÚexistsÚextÚ	directoryÚ
startswithÚendswithÚutilÚspec_from_file_locationÚ_specÚloaderÚmodule_from_specÚmodulesÚexec_modulerò  rô  ÚImportErrorr¿  r;  Ú#comfy_kitchen.backends._activationsr^   rá  r_   r¹  r`   r{  ra   ry  Ú"comfy_kitchen.backends._modulationrb   Úcomfy_kitchen.backends.eagerrc   rP  Ú-comfy_kitchen.backends.eager.group_norm_pad3dr   rù  r   r  Ú)comfy_kitchen.backends.eager.quantizationrd   re   r-   r†  r:   r€  Ú%comfy_kitchen.backends.eager.sol_attnrf   rg   rh   ri   rj   rk   rl   rm   Ú%comfy_kitchen.backends.eager.svdquantrn   ro   Ú&comfy_kitchen.backends.eager.w4a8_int8rp   rq   rr   rs   rt   ru   rv   Ú comfy_kitchen.tensor.w4a8_packedrw   rx   ry   r5   r  Úcomfy_kitchen.constraintsrz   r{   r|   r}   r~   r   r€   r�   r‚   rƒ   r„   Úcomfy_kitchen.float_utilsr…   Úcomfy_kitchen.registryr†   Úcomfy_kitchen.tensor.int8_utilsr‡   rˆ   r‰   rÉ   rç  rk  rs  rÔ   rá   r÷   r¸   rÀ   rÈ   rî  rï  r    rþ   r  r  rµ   rê  r  rŸ   r±   rä   r¼   rÄ   rÌ   r×   ræ   ré   rï   rû   rÿ   r  r  r  r  r,  r0  r7  r<  r?  rE  rM  rT  rY  r_  rb  rg  rn  ru  rz  r�  r…  rˆ  r�  r¦   rH   ra  r,   rL   rG   r+   r;   r<   r=   r>   r×  rå  rë  rð  rø  r  rþ  r  r  r7   r9   r?   r¢   r1   r8   r>  rB   r  rl  rm  rQ  r\  r_  rd  r@   rq  rC   r]  rÿ  r.   rD   rA   r/   r0   rü  rF   rJ   r3   rÃ  rÏ  rz  r4   ró  r2   rý  r  r6   r  r  r  r	   rÏ  r
   rÇ  rÀ  rC  r^  r[  rg  rl  rp  rv  rz  r�  rƒ  r�  r   r   rÒ  rÊ  r›  rË  rî  r   r   r  r
  rû  r   rT  re  r   r§   r   r   r‡  r5  rš  r   r   r   rÎ  r`  rÐ  rÚ  ræ  r   r   rø  rú  r   r  r  r   r   r   r   r;  rF  r9  rM  rT  r%   r&   r#   r$   r)   r*   r'   r(   r!   r"   r   r    rŸ  r…  r¢  r�  r†  rŒ  rI   rK   rÈ  rÅ  rE   rí  rù  r˜   )rû  s   @rW   Ú<module>r?     s†  øð÷ Ò Û Û Û 	Û 
Û Ý ã ÷ó õ 8ðEØ
ðEàðEð ðEð ð	Eð
 ðEð ðEð ðEð ðEð ðEð ðEð ðEð ðEð ðEð ðEð ðEð   ð!Eð" ð#Eð$ ð%Eð& ð'Eð( ð)Eð* ð+Eð, ð-Eð. ð/Eð0 ð1Eð2 ð3Eð4 ð5Eð6 ð7Eð8 ð9Eð: ð;Eð< ð=Eð> ð?Eð@ ðAEðB ðCEðD ðEEðF ðGEðH  ðIEðJ ðKEðL #ðMEðN %ðOEðP +ðQEðR %ðSEðT "ðUEðV ðWEðX ðYEðZ ð[Eð\ ð]Eð^ ð_Eð` ðaEðb *ðcEðd ðeEðf ðgEðh ðiEðj &ðkEðl .ðmEðn #ðoEðp  ðqEðr #ðsEðt !ðuEðv &ðwEðx "ðyEðz ð{Eð| ð}Eð~ ðEð@ ðAEðB ðCEðD ðEEðF ðGEðH ðIE€ðP €ð	ð-ÛØ!Ÿ;™;×/Ñ/°Õ2Ðòð ‡|�|�wÔÙÐ/°Ó>ˆßØ×.Ò.¨wÓ7ˆKøáÐ/Ð1AÓBˆßØŸJšJ wÖ/�Ø Ö)¨e°xÖ.?Ø#×,Ò,¨Y×7Õ7ØŸš B§G¡G§L¡L°¸(Ó$CÈ&×J\ÑJ\Õ]÷ 8Ñ7ñ 0ð"Ø	€BØ—7‘7—<‘< §¡§¡°Ó 9ÈCÏLÉLÐ\cÔLc¹=ÐiuÓv€Là�7‰7�>‰>˜,×'Ò'ØŸ™¨Ô/‰f°UˆØ—G‘G—O‘O HÓ-ˆ	ØŸ
š
 9Ö-ˆHØ×"Ñ" 5×)Ô)¨h×.?Ñ.?À×.DÔ.DØ!Ÿw™wŸ|™|¨I°xÓ@’ñ .ð 
‡w�w‡~�~�l×#Ò#Ø—‘×6Ñ6Ø,¨ló
ˆ÷ �U—\—\�\Ø—‘×0Ñ0°Ó7ˆBØ<>ˆC�K‰KÐ8Ñ9Ø�L‰L×$Ñ$ RÔ(Ø!ˆNØ‰Jà"ˆNØ<¸\¸NÐK‰JàˆØ1°,°Ð@‰
õõõõõ EÝ <÷÷õõ÷	÷ 	ó 	÷÷÷ ñ ÷ ZÑ Yõ÷÷ ÷ ñ õ .Ý +÷ñ ð %×KÐK©°°^ÀUÓ)KÐ Ø.0Ð Ó 0ØOQÐ Ó QØ(*Ð Ó *Ø24Ð Ó 4Ø.0Ð Ó 0Ø79Ð Ó 9Ø57Ð "Ó 7Ø)+Ð Ó +á€ØŸJšJŸNšNÐ+SÐUXÓYÐ]`Ñ`Ð Ø"#Ð Ù ¡3 r§z¢z§~¢~Ð6^Ð`fÓ'gÓ#hÓiÐ Ø—
’
—’Ð;¸SÓAÀSÑH€Ø<>Ð Ó >õñ(Ð õ õõõõ	õõ"õ

õõõ"õWõ
>õ7õ7õCð ,.Ð Ó -õõ0õ$Gõ,FõFõõYõõõõ"õ-õXõõ(ð& FK×EXÒEX÷*ð6 FKÇ^Â^÷ð8  %×2Ò2÷!÷02ð 2ðr  %Ÿ~š~Ø÷ ÷FLð L÷.gð g÷.gð g÷6Mð Mõ6õ%õP-õ.õ/õdnõb÷Beð eõ,ðL !%Ø"Ÿ^š^÷Kõ\-ð !Ø,Ø&'÷	%ðD !Ø,Ø %§¢÷PðL !%Ø Ø,Ø÷TK÷nð õ6
ñ ˜RÐ!8Ó9Ð ñ +Ð õõ,.ñ4 ˜BÐ =Ó>Ð òGðD Ø ØØ$×2Ò2ØØ+/Ø ØØ÷3÷lð ÷4#ð #ðV Ð ð Ÿ
š
ŸšÐ'FÈÓLÐPSÑSÐ ÷cð cõPõ
"÷JQð Q÷%tò %t÷Pd÷
_÷6÷6!ò !÷H|ò |÷~>Mò >M÷B	ó$S÷N-÷]ò ]÷@	 ðN %)Ø&*ØØ Ø %§¢÷gñ g÷>÷"ð %)Ø&*Ø $ØØ Ø"Ÿ^š^Ø Ø,0ØØ$(Ø*.÷O)ñ O)ðn %)Ø&*Ø $ØØ Ø"Ÿ^š^Ø Ø,0ØØ$(Ø*.÷])ñ ])÷@÷yöI÷Xò ñD (€÷ ò  ðF €ÙÐ á Ð ÙÐ ó1ö>7ó8"óôYð �ôô,GôT÷Yó Y÷xg-ó g-ôTCôBôYô
P÷(ð (÷B7ó 7÷;ó ;ñ �
ØÑ ÙÑ ÷yOð yOöxˆjõ ÷-ð -÷;ð ;ò4 ;@Á»,÷ò ð, RUÒdiÑjoÓdpØ÷ò ÷&ð ÷Xð X÷	\ó 	\÷
ð 
÷"ó "÷>.ó .ðb ,.�Ó -ð EJÇMÂM÷ ò ÷ð ÷6÷r7ð 7÷ ;ð ;÷=ò =÷@ð ÷÷,Zð Z÷]ó ]÷
ð ÷(ð ÷6Lð L÷Kð K÷
Pð P÷Oð O÷!ð !÷!ð !ñ6ð
 ÷ò ÷&ð ñ21ð ÷1ò 1÷hZó Z÷Yó Y÷ó ÷ó ÷"Yó Y÷Xó X÷ó ÷ó ÷&Kð K÷Jð J÷Oð O÷Nð Nð Ñ ØÑ ØgiÑ Ó i÷ð ÷ð ÷ð ÷0Q3ó Q3÷hOó Oðr Ñ ÷ð ÷::.ó :.÷zqð qôhò" 
†øðWd ô -Ø Ÿ>™>¨!Õ,Óð-ú÷& 8×7Ñ7ûàô 	Úð	ûð@ ô Ø€NÙ�Q“€JØ	†BûØô Ø€NØ-¨a¨SÐ1€JØ	†BûðúsÄ   Ã!l: Ã,$m0 Äm0 Ä$m0 Ä5m0 Åm0 Åm0 Å69mÆ/m0 Æ<A*m> È'Am> Êm> ÊA m> Ë#m> Ìm> ÌA	m> Ím> Í'm> ì:mím0 ímím0 ím-í&
m0 í0m;í:m;í>n5înîn5î n5î!	n0î0n5