#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) #pragma once #ifndef USE_ROCM #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace at::cuda::tunable { namespace { template int64_t CublasltBatchCount(const GemmParams* /*params*/) { return 0; } template int64_t CublasltBatchCount(const GemmAndBiasParams* /*params*/) { return 0; } template int64_t CublasltBatchCount( const GemmStridedBatchedParams* params) { return params->batch; } template int64_t CublasltStrideA(const GemmParams* /*params*/) { return 0; } template int64_t CublasltStrideA(const GemmAndBiasParams* /*params*/) { return 0; } template int64_t CublasltStrideA(const GemmStridedBatchedParams* params) { return params->stride_a; } template int64_t CublasltStrideB(const GemmParams* /*params*/) { return 0; } template int64_t CublasltStrideB(const GemmAndBiasParams* /*params*/) { return 0; } template int64_t CublasltStrideB(const GemmStridedBatchedParams* params) { return params->stride_b; } template int64_t CublasltStrideC(const GemmParams* /*params*/) { return 0; } template int64_t CublasltStrideC(const GemmAndBiasParams* /*params*/) { return 0; } template int64_t CublasltStrideC(const GemmStridedBatchedParams* params) { return params->stride_c; } template at::opmath_type CublasltBeta(const GemmParams* params) { return params->beta; } template at::opmath_type CublasltBeta( const GemmStridedBatchedParams* params) { return params->beta; } template at::opmath_type CublasltBeta(const GemmAndBiasParams* params) { return params->bias ? 0 : 1; } template const void* CublasltBias(const ParamsT* /*params*/) { return nullptr; } template const void* CublasltBias(const GemmAndBiasParams* params) { return params->bias; } template at::cuda::blas::GEMMAndBiasActivationEpilogue CublasltActivation( const ParamsT* /*params*/) { return at::cuda::blas::GEMMAndBiasActivationEpilogue::None; } template at::cuda::blas::GEMMAndBiasActivationEpilogue CublasltActivation( const GemmAndBiasParams* params) { return params->activation; } // Shared cuBLASLt problem surface consumed by the templated Callable/TunableOp // helpers via static (duck-typed) dispatch. Holds the descriptors, preference, // workspace, and resolved data types common to every problem variant. Accessors // are non-virtual: helpers always operate on the concrete derived type, so // derived classes hide (rather than override) the defaults below where the two // variants diverge (D layout aliasing and heuristic descriptor selection). class CublasltGemmProblemBase { public: CublasltGemmProblemBase( cublasComputeType_t compute_type, cudaDataType_t scale_type, cudaDataType_t a_type, cudaDataType_t b_type, cudaDataType_t c_type, cudaDataType_t d_type) : compute_desc_(compute_type, scale_type), compute_type_(compute_type), scale_type_(scale_type), a_type_(a_type), b_type_(b_type), c_type_(c_type), d_type_(d_type) {} cublasComputeType_t compute_type() const { return compute_type_; } cudaDataType_t scale_type() const { return scale_type_; } cudaDataType_t a_type() const { return a_type_; } cudaDataType_t b_type() const { return b_type_; } cudaDataType_t c_type() const { return c_type_; } cudaDataType_t d_type() const { return d_type_; } cublasLtMatmulDesc_t compute_desc() const { return compute_desc_.descriptor(); } cublasLtMatrixLayout_t adesc() const { return adesc_->descriptor(); } cublasLtMatrixLayout_t bdesc() const { return bdesc_->descriptor(); } cublasLtMatrixLayout_t cdesc() const { return cdesc_->descriptor(); } // Default: D aliases C. Variants with a distinct D layout hide this. cublasLtMatrixLayout_t ddesc() const { return cdesc(); } // Defaults: heuristic descriptors are the real ones. Variants that feed // cuBLASLt a different shape for heuristic queries hide these. cublasLtMatrixLayout_t heuristic_bdesc() const { return bdesc(); } cublasLtMatrixLayout_t heuristic_cdesc() const { return cdesc(); } cublasLtMatmulPreference_t preference() const { return preference_.descriptor(); } const void* alpha_ptr() const { return alpha_ptr_; } const void* beta_ptr() const { return beta_ptr_; } void* workspace() const { return workspace_.ptr; } size_t workspace_size() const { return workspace_.size; } protected: at::cuda::blas::detail::CuBlasLtMatmulDescriptor compute_desc_; at::cuda::blas::detail::CuBlasLtMatmulPreference preference_; std::unique_ptr adesc_; std::unique_ptr bdesc_; std::unique_ptr cdesc_; at::cuda::blas::detail::CublasLtWorkspace workspace_; cublasComputeType_t compute_type_; cudaDataType_t scale_type_; cudaDataType_t a_type_; cudaDataType_t b_type_; cudaDataType_t c_type_; cudaDataType_t d_type_; const void* alpha_ptr_ = nullptr; const void* beta_ptr_ = nullptr; }; template class CublasltStandardGemmProblem : public CublasltGemmProblemBase { public: CublasltStandardGemmProblem( at::cuda::blas::detail::CublasLtTypeInfo type_info, char transa, char transb, int64_t m, int64_t n, int64_t k, at::opmath_type alpha, const T* a, int64_t lda, int64_t stride_a, const T* b, int64_t ldb, int64_t stride_b, at::opmath_type beta, C_Dtype* c, int64_t ldc, int64_t stride_c, int64_t batch_count, const void* bias, at::cuda::blas::GEMMAndBiasActivationEpilogue activation, bool set_epilogue_attribute) : CublasltGemmProblemBase( type_info.compute_type, type_info.scale_type, type_info.ab_type, type_info.ab_type, type_info.c_type, type_info.c_type), m_(m), n_(n), k_(k), lda_(lda), ldb_(ldb), ldc_(ldc), stride_a_(stride_a), stride_b_(stride_b), stride_c_(stride_c), batch_count_(batch_count), a_(a), b_(b), c_(c), alpha_(alpha), beta_(beta), opa_(at::cuda::blas::detail::cublasOpFromChar(transa)), opb_(at::cuda::blas::detail::cublasOpFromChar(transb)), bias_(bias), activation_(activation), set_epilogue_attribute_(set_epilogue_attribute) { at::cuda::blas::detail::cublasAdjustLdLevel3( transa, transb, m_, n_, k_, &lda_, &ldb_, &ldc_); alpha_ptr_ = &alpha_; beta_ptr_ = &beta_; if constexpr (std::is_same_v) { if (compute_type_ == CUBLAS_COMPUTE_16F) { halpha_ = alpha_; hbeta_ = beta_; alpha_ptr_ = &halpha_; beta_ptr_ = &hbeta_; } } compute_desc_.setAttribute(CUBLASLT_MATMUL_DESC_TRANSA, opa_); compute_desc_.setAttribute(CUBLASLT_MATMUL_DESC_TRANSB, opb_); if (set_epilogue_attribute_ || bias_ != nullptr || activation_ != at::cuda::blas::GEMMAndBiasActivationEpilogue::None) { epilogue_ = at::cuda::blas::detail::cublasLtEpilogue(activation_, bias_); compute_desc_.setAttribute(CUBLASLT_MATMUL_DESC_EPILOGUE, epilogue_); } if (bias_ != nullptr) { compute_desc_.setAttribute(CUBLASLT_MATMUL_DESC_BIAS_POINTER, bias_); } if (at::globalContext()._SMCarveout_EXPERIMENTAL().has_value()) { compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_SM_COUNT_TARGET, at::cuda::getCurrentDeviceProperties()->multiProcessorCount - at::globalContext()._SMCarveout_EXPERIMENTAL().value()); } if constexpr (std::is_same_v) { auto fp16_reduction = at::globalContext().allowFP16ReductionCuBLAS(); if (fp16_reduction != at::CuBLASReductionOption::AllowReducedPrecisionWithSplitK) { reduction_mask_ = fp16_reduction == at::CuBLASReductionOption:: DisallowReducedPrecisionAllowSplitK ? (CUBLASLT_REDUCTION_SCHEME_COMPUTE_TYPE | CUBLASLT_REDUCTION_SCHEME_NONE) : CUBLASLT_REDUCTION_SCHEME_NONE; preference_.setAttribute( CUBLASLT_MATMUL_PREF_REDUCTION_SCHEME_MASK, reduction_mask_); } } else if constexpr (std::is_same_v) { auto bf16_reduction = at::globalContext().allowBF16ReductionCuBLAS(); if (bf16_reduction != at::CuBLASReductionOption::AllowReducedPrecisionWithSplitK) { reduction_mask_ = bf16_reduction == at::CuBLASReductionOption:: DisallowReducedPrecisionAllowSplitK ? (CUBLASLT_REDUCTION_SCHEME_COMPUTE_TYPE | CUBLASLT_REDUCTION_SCHEME_NONE) : CUBLASLT_REDUCTION_SCHEME_NONE; preference_.setAttribute( CUBLASLT_MATMUL_PREF_REDUCTION_SCHEME_MASK, reduction_mask_); } } adesc_ = std::make_unique( a_type_, m_, k_, lda_, opa_ != CUBLAS_OP_N); bdesc_ = std::make_unique( b_type_, k_, n_, ldb_, opb_ != CUBLAS_OP_N); cdesc_ = std::make_unique( c_type_, m_, n_, ldc_); if (batch_count_ > 1) { int batch_as_int = static_cast(batch_count_); adesc_->setAttribute(CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, batch_as_int); bdesc_->setAttribute(CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, batch_as_int); cdesc_->setAttribute(CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, batch_as_int); adesc_->setAttribute( CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, stride_a_); bdesc_->setAttribute( CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, stride_b_); cdesc_->setAttribute( CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, stride_c_); } preference_.setAttribute( CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_A_BYTES, at::cuda::blas::detail::getAlignment(reinterpret_cast(a_))); preference_.setAttribute( CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_B_BYTES, at::cuda::blas::detail::getAlignment(reinterpret_cast(b_))); preference_.setAttribute( CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_C_BYTES, at::cuda::blas::detail::getAlignment(reinterpret_cast(c_))); if (bias_ != nullptr) { preference_.setAttribute( CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_D_BYTES, at::cuda::blas::detail::getAlignment( reinterpret_cast(bias_))); } preference_.setAttribute( CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, workspace_.size); lie_to_cublaslt_ = reduction_mask_ == CUBLASLT_REDUCTION_SCHEME_NONE && n_ == 1 && at::cuda::getCurrentDeviceProperties()->major >= 10; if (lie_to_cublaslt_) { const auto fake_ldb = ldb_ == 1 ? 2 : ldb_; const auto fake_ldc = ldc_ == 1 ? 2 : ldc_; fake_bdesc_ = std::make_unique( b_type_, k_, 2, fake_ldb, opb_ == CUBLAS_OP_T); fake_cdesc_ = std::make_unique( c_type_, m_, 2, fake_ldc); if (batch_count_ > 1) { int batch_as_int = static_cast(batch_count_); fake_bdesc_->setAttribute( CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, batch_as_int); fake_cdesc_->setAttribute( CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, batch_as_int); fake_bdesc_->setAttribute( CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, stride_b_); fake_cdesc_->setAttribute( CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, stride_c_); } } } cublasLtMatrixLayout_t heuristic_bdesc() const { return lie_to_cublaslt_ ? fake_bdesc_->descriptor() : bdesc(); } cublasLtMatrixLayout_t heuristic_cdesc() const { return lie_to_cublaslt_ ? fake_cdesc_->descriptor() : cdesc(); } cublasLtMatrixLayout_t heuristic_ddesc() const { return heuristic_cdesc(); } const T* a() const { return a_; } const T* b() const { return b_; } C_Dtype* c() const { return c_; } C_Dtype* d() const { return c_; } private: int64_t m_; int64_t n_; int64_t k_; int64_t lda_; int64_t ldb_; int64_t ldc_; int64_t stride_a_; int64_t stride_b_; int64_t stride_c_; int64_t batch_count_; const T* a_; const T* b_; C_Dtype* c_; at::opmath_type alpha_; at::opmath_type beta_; at::Half halpha_; at::Half hbeta_; uint32_t reduction_mask_ = std::numeric_limits::max(); cublasOperation_t opa_; cublasOperation_t opb_; const void* bias_ = nullptr; at::cuda::blas::GEMMAndBiasActivationEpilogue activation_; bool set_epilogue_attribute_; cublasLtEpilogue_t epilogue_ = CUBLASLT_EPILOGUE_DEFAULT; std::unique_ptr fake_bdesc_; std::unique_ptr fake_cdesc_; bool lie_to_cublaslt_ = false; }; template auto MakeCublasltStandardGemmProblem(const ParamsT* params) { return CublasltStandardGemmProblem( at::cuda::blas::detail::getCublasLtTypeInfo(), params->transa, params->transb, params->m, params->n, params->k, params->alpha, params->a, params->lda, CublasltStrideA(params), params->b, params->ldb, CublasltStrideB(params), CublasltBeta(params), params->c, params->ldc, CublasltStrideC(params), CublasltBatchCount(params), CublasltBias(params), CublasltActivation(params), true /* set_epilogue_attribute */); } template auto MakeCublasltStandardGemmProblem( const GemmStridedBatchedParams* params) { return CublasltStandardGemmProblem( at::cuda::blas::detail::getCublasLtTypeInfo(), params->transa, params->transb, params->m, params->n, params->k, params->alpha, params->a, params->lda, params->stride_a, params->b, params->ldb, params->stride_b, params->beta, params->c, params->ldc, params->stride_c, params->batch, nullptr, at::cuda::blas::GEMMAndBiasActivationEpilogue::None, // Keep batched GEMM descriptor behavior aligned with the legacy path; // setting EPILOGUE_DEFAULT changes some cuBLASLt heuristic results. false /* set_epilogue_attribute */); } template class CublasltScaledGemmProblem : public CublasltGemmProblemBase { public: explicit CublasltScaledGemmProblem(const ScaledGemmParams* params) : CublasltGemmProblemBase( CUBLAS_COMPUTE_32F, CUDA_R_32F, ScalarTypeToCudaDataType(params->a_dtype), ScalarTypeToCudaDataType(params->b_dtype), ScalarTypeToCudaDataType(params->bias_dtype), ScalarTypeToCudaDataType(params->c_dtype)), params_(params) { compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_TRANSA, at::cuda::blas::detail::cublasOpFromChar(params_->transa)); compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_TRANSB, at::cuda::blas::detail::cublasOpFromChar(params_->transb)); compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, params_->a_scale_ptr); compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, params_->b_scale_ptr); if (params_->c_scale_ptr != nullptr) { compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_D_SCALE_POINTER, params_->c_scale_ptr); } const int8_t fast_accum = params_->use_fast_accum ? 1 : 0; compute_desc_.setAttribute(CUBLASLT_MATMUL_DESC_FAST_ACCUM, fast_accum); if (at::globalContext()._SMCarveout_EXPERIMENTAL().has_value()) { compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_SM_COUNT_TARGET, at::cuda::getCurrentDeviceProperties()->multiProcessorCount - at::globalContext()._SMCarveout_EXPERIMENTAL().value()); } if (params_->bias_ptr != nullptr) { compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_BIAS_POINTER, params_->bias_ptr); compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_EPILOGUE, CUBLASLT_EPILOGUE_BIAS); compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_BIAS_DATA_TYPE, c_type_); } alpha_ptr_ = &alpha_val_; beta_ptr_ = &beta_val_; if (params_->alpha.has_value()) { const auto& alpha = params_->alpha.value(); if (alpha.is_cuda()) { auto pointer_mode = CUBLASLT_POINTER_MODE_DEVICE; compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_POINTER_MODE, pointer_mode); alpha_ptr_ = alpha.template const_data_ptr(); beta_ptr_ = at::cuda::detail::get_cublas_device_zero(); } else { alpha_val_ = alpha.template item(); } } #if CUDA_VERSION >= 12080 int a_scale_mode = at::cuda::blas::detail::cublasLtMatmulScaleMode( params_->a_scaling_type, params_->a_swizzle_type, params_->a_scale_dtype, params_->use_fast_accum); int b_scale_mode = at::cuda::blas::detail::cublasLtMatmulScaleMode( params_->b_scaling_type, params_->b_swizzle_type, params_->b_scale_dtype, params_->use_fast_accum); compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_A_SCALE_MODE, a_scale_mode); compute_desc_.setAttribute( CUBLASLT_MATMUL_DESC_B_SCALE_MODE, b_scale_mode); #endif adesc_ = std::make_unique( a_type_, params_->m, params_->k, params_->lda, params_->transa == 't'); bdesc_ = std::make_unique( b_type_, params_->k, params_->n, params_->ldb, params_->transb == 't'); cdesc_ = std::make_unique( c_type_, params_->m, params_->n, params_->ldc); ddesc_ = std::make_unique( d_type_, params_->m, params_->n, params_->ldc); preference_.setAttribute( CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, workspace_.size); } // Scaled GEMM has a distinct D layout, so it does not alias C. cublasLtMatrixLayout_t ddesc() const { return ddesc_->descriptor(); } cublasLtMatrixLayout_t heuristic_ddesc() const { return ddesc(); } const void* a() const { return params_->a; } const void* b() const { return params_->b; } // cuBLASLt requires a non-null C pointer even when beta is zero. Match the // non-tunable scaled_gemm path by passing an input pointer; beta = 0 means // C is never read. const void* c() const { return params_->a; } void* d() const { return params_->c; } private: const ScaledGemmParams* params_; std::unique_ptr ddesc_; float alpha_val_ = 1.0f; float beta_val_ = 0.0f; }; struct CublasltAlgoConfig { int32_t id = 0; uint32_t tile = 0; uint32_t stages = 0; int32_t splitk = 1; uint32_t reduction = 0; uint32_t swizzle = 0; uint32_t custom = 0; #if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 12000 uint16_t inner_shape = 0; uint16_t cluster_shape = 0; #endif }; template bool CublasltGetAlgoConfigAttribute( const cublasLtMatmulAlgo_t* algo, cublasLtMatmulAlgoConfigAttributes_t attr, T* value) { size_t written = 0; return cublasLtMatmulAlgoConfigGetAttribute( algo, attr, value, sizeof(T), &written) == CUBLAS_STATUS_SUCCESS; } inline std::optional CublasltAlgoConfigFromAlgo( const cublasLtMatmulAlgo_t& algo) { CublasltAlgoConfig config; bool ok = CublasltGetAlgoConfigAttribute(&algo, CUBLASLT_ALGO_CONFIG_ID, &config.id) && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_TILE_ID, &config.tile) && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_STAGES_ID, &config.stages) && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_SPLITK_NUM, &config.splitk) && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_REDUCTION_SCHEME, &config.reduction) && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_CTA_SWIZZLING, &config.swizzle) && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_CUSTOM_OPTION, &config.custom); #if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 12000 ok = ok && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_INNER_SHAPE_ID, &config.inner_shape) && CublasltGetAlgoConfigAttribute( &algo, CUBLASLT_ALGO_CONFIG_CLUSTER_SHAPE_ID, &config.cluster_shape); #endif if (!ok) { return std::nullopt; } return config; } inline std::string CublasltAlgoConfigName(const CublasltAlgoConfig& config) { std::string name = fmt::sprintf( "Gemm_Cublaslt_id_%d_tile_%u_stages_%u_splitk_%d_red_%u_swizzle_%u_custom_%u", config.id, config.tile, config.stages, config.splitk, config.reduction, config.swizzle, config.custom); #if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 12000 name += fmt::sprintf( "_inner_%hu_cluster_%hu", config.inner_shape, config.cluster_shape); #endif return name; } inline std::optional CublasltAlgoConfigFromName( const std::string& name) { CublasltAlgoConfig config; #if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 12000 int matched = std::sscanf( name.c_str(), "Gemm_Cublaslt_id_%d_tile_%u_stages_%u_splitk_%d_red_%u_swizzle_%u_custom_%u" "_inner_%hu_cluster_%hu", &config.id, &config.tile, &config.stages, &config.splitk, &config.reduction, &config.swizzle, &config.custom, &config.inner_shape, &config.cluster_shape); if (matched != 9) { return std::nullopt; } #else int matched = std::sscanf( name.c_str(), "Gemm_Cublaslt_id_%d_tile_%u_stages_%u_splitk_%d_red_%u_swizzle_%u_custom_%u", &config.id, &config.tile, &config.stages, &config.splitk, &config.reduction, &config.swizzle, &config.custom); if (matched != 7) { return std::nullopt; } #endif return config; } template bool CublasltSetAlgoConfigAttribute( cublasLtMatmulAlgo_t* algo, cublasLtMatmulAlgoConfigAttributes_t attr, const T& value) { return cublasLtMatmulAlgoConfigSetAttribute( algo, attr, &value, sizeof(value)) == CUBLAS_STATUS_SUCCESS; } template bool CublasltInitializeAlgo( const ProblemT& problem, const CublasltAlgoConfig& config, cublasLtMatmulAlgo_t* algo) { auto status = cublasLtMatmulAlgoInit( at::cuda::getCurrentCUDABlasLtHandle(), problem.compute_type(), problem.scale_type(), problem.a_type(), problem.b_type(), problem.c_type(), problem.d_type(), config.id, algo); if (status != CUBLAS_STATUS_SUCCESS) { return false; } bool ok = CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_TILE_ID, config.tile) && CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_STAGES_ID, config.stages) && CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_SPLITK_NUM, config.splitk) && CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_REDUCTION_SCHEME, config.reduction) && CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_CTA_SWIZZLING, config.swizzle) && CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_CUSTOM_OPTION, config.custom); #if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 12000 ok = ok && CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_INNER_SHAPE_ID, config.inner_shape) && CublasltSetAlgoConfigAttribute( algo, CUBLASLT_ALGO_CONFIG_CLUSTER_SHAPE_ID, config.cluster_shape); #endif return ok; } template bool CublasltValidateAlgo( const ProblemT& problem, cublasLtMatmulAlgo_t* algo) { cublasLtMatmulHeuristicResult_t check_result = {}; auto status = cublasLtMatmulAlgoCheck( at::cuda::getCurrentCUDABlasLtHandle(), problem.compute_desc(), problem.adesc(), problem.bdesc(), problem.cdesc(), problem.ddesc(), algo, &check_result); return status == CUBLAS_STATUS_SUCCESS && check_result.workspaceSize <= problem.workspace_size(); } template struct CublasltStandardGemmProblemFactory { template auto operator()(const ParamsT* params) const { return MakeCublasltStandardGemmProblem(params); } }; template struct CublasltScaledGemmProblemFactory { auto operator()(const ScaledGemmParams* params) const { return CublasltScaledGemmProblem(params); } }; template class CublasltMatmulOp : public Callable { public: explicit CublasltMatmulOp(std::string name) : name_(std::move(name)), config_(CublasltAlgoConfigFromName(name_)) {} CublasltMatmulOp(std::string name, CublasltAlgoConfig config) : name_(std::move(name)), config_(config) {} TuningStatus Call(const ParamsT* params) override { auto problem = problem_factory_(params); if (problem.workspace() == nullptr && problem.workspace_size() != 0) { return FAIL; } cublasLtMatmulAlgo_t algo = {}; if (!InitializeAlgo(problem, &algo)) { return FAIL; } auto status = cublasLtMatmul( at::cuda::getCurrentCUDABlasLtHandle(), problem.compute_desc(), problem.alpha_ptr(), problem.a(), problem.adesc(), problem.b(), problem.bdesc(), problem.beta_ptr(), problem.c(), problem.cdesc(), problem.d(), problem.ddesc(), &algo, problem.workspace(), problem.workspace_size(), at::cuda::getCurrentCUDAStream()); return status == CUBLAS_STATUS_SUCCESS ? OK : FAIL; } private: template bool InitializeAlgo(const ProblemT& problem, cublasLtMatmulAlgo_t* algo) { if (!config_.has_value() || !CublasltInitializeAlgo(problem, config_.value(), algo) || !CublasltValidateAlgo(problem, algo)) { return false; } return true; } ProblemFactory problem_factory_; std::string name_; std::optional config_; }; template std::vector> GetCublasltHeuristicCandidates(const ProblemT& problem) { std::vector> ret; if (problem.workspace() == nullptr && problem.workspace_size() != 0) { return ret; } std::vector heuristic_results( getTuningContext()->GetCublasLtRequestedAlgoCount()); int returned_result = 0; auto status = cublasLtMatmulAlgoGetHeuristic( at::cuda::getCurrentCUDABlasLtHandle(), problem.compute_desc(), problem.adesc(), problem.heuristic_bdesc(), problem.heuristic_cdesc(), problem.heuristic_ddesc(), problem.preference(), static_cast(heuristic_results.size()), heuristic_results.data(), &returned_result); if (status != CUBLAS_STATUS_SUCCESS || returned_result == 0) { return ret; } for (int i = 0; i < returned_result; ++i) { if (heuristic_results[i].state != CUBLAS_STATUS_SUCCESS) { continue; } auto config = CublasltAlgoConfigFromAlgo(heuristic_results[i].algo); if (!config.has_value()) { continue; } ret.emplace_back(CublasltAlgoConfigName(config.value()), config.value()); } return ret; } } // anonymous namespace template class CublasltMatmulTunableOp : public TunableOp { public: CublasltMatmulTunableOp() = default; protected: void RegisterOpCandidates(const ParamsT* params) override { const auto params_sig = params->Signature(); std::scoped_lock l{candidate_lock_}; if (candidate_names_by_params_.find(params_sig) != candidate_names_by_params_.end()) { return; } std::vector candidate_names{"Default"}; auto problem = problem_factory_(params); for (auto&& [name, config] : GetCublasltHeuristicCandidates(problem)) { if (!this->HasOp(name)) { this->RegisterOp( name, std::make_unique>( name, config)); } candidate_names.emplace_back(std::move(name)); } candidate_names_by_params_.emplace(params_sig, std::move(candidate_names)); } std::vector CandidateNames(const ParamsT* params) const override { const auto params_sig = params->Signature(); std::scoped_lock l{candidate_lock_}; auto it = candidate_names_by_params_.find(params_sig); if (it == candidate_names_by_params_.end()) { return {"Default"}; } return it->second; } bool RegisterOpForResult( const ResultEntry& result, const ParamsT* /*params*/) override { const auto name = result.GetKey(); if (this->HasOp(name)) { return true; } if (!CublasltAlgoConfigFromName(name).has_value()) { return false; } this->RegisterOp( name, std::make_unique>(name)); return true; } private: ProblemFactory problem_factory_; mutable std::mutex candidate_lock_; std::unordered_map> candidate_names_by_params_; }; template class CublasltGemmTunableOp : public CublasltMatmulTunableOp< ParamsT, CublasltStandardGemmProblemFactory> {}; template class CublasltScaledGemmTunableOp : public CublasltMatmulTunableOp< ParamsT, CublasltScaledGemmProblemFactory> {}; } // namespace at::cuda::tunable #endif // USE_ROCM #else #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)