#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) #pragma once // Light-weight version of CUDAContext.h with fewer transitive includes #include #include #include #include #include #include #include #ifdef CUDART_VERSION #include #endif #if defined(USE_CUDSS) #include #endif #if defined(USE_ROCM) #include #endif #include #include namespace c10 { struct Allocator; } namespace at::cuda { /* A common CUDA interface for ATen. This interface is distinct from CUDAHooks, which defines an interface that links to both CPU-only and CUDA builds. That interface is intended for runtime dispatch and should be used from files that are included in both CPU-only and CUDA builds. CUDAContext, on the other hand, should be preferred by files only included in CUDA builds. It is intended to expose CUDA functionality in a consistent manner. This means there is some overlap between the CUDAContext and CUDAHooks, but the choice of which to use is simple: use CUDAContext when in a CUDA-only file, use CUDAHooks otherwise. Note that CUDAContext simply defines an interface with no associated class. It is expected that the modules whose functions compose this interface will manage their own state. There is only a single CUDA context/state. */ /** * DEPRECATED: use device_count() instead */ inline int64_t getNumGPUs() { return c10::cuda::device_count(); } /** * CUDA is available if we compiled with CUDA, and there are one or more * devices. If we compiled with CUDA but there is a driver problem, etc., * this function will report CUDA is not available (rather than raise an error.) */ inline bool is_available() { return c10::cuda::device_count() > 0; } TORCH_CUDA_CPP_API cudaDeviceProp* getCurrentDeviceProperties(); TORCH_CUDA_CPP_API int warp_size(); TORCH_CUDA_CPP_API cudaDeviceProp* getDeviceProperties(c10::DeviceIndex device); TORCH_CUDA_CPP_API bool canDeviceAccessPeer( c10::DeviceIndex device, c10::DeviceIndex peer_device); TORCH_CUDA_CPP_API c10::Allocator* getCUDADeviceAllocator(); /* Handles */ TORCH_CUDA_CPP_API cusparseHandle_t getCurrentCUDASparseHandle(); // On CUDA, the public handle uses the BLAS library's default workspace unless // ATen workspace caching is explicitly enabled. On ROCm, ATen binds its // per-operation workspaces to separate handles, and each stream gets its own // public handle, bound at creation to a buffer from a caching-allocator pool // reserved for these buffers, at least as large as the arena rocBLAS would have // allocated. TORCH_CUDA_CPP_API cublasHandle_t getCurrentCUDABlasHandle(bool setup = true); // Internal scoped handle for ATen operations. When caching is disabled, it owns // an eager workspace and restores the default before releasing it; otherwise it // wraps the cache-backed handle. External users should continue to use // getCurrentCUDABlasHandle(). Eager scopes for the same underlying handle must // not overlap because restoring an inner scope replaces the outer workspace. class TORCH_CUDA_CPP_API CUDABlasHandleWithWorkspace { public: CUDABlasHandleWithWorkspace( cublasHandle_t handle, cudaStream_t stream, at::DataPtr workspace, bool restore_default_workspace); ~CUDABlasHandleWithWorkspace(); CUDABlasHandleWithWorkspace(CUDABlasHandleWithWorkspace&& other) noexcept; CUDABlasHandleWithWorkspace(const CUDABlasHandleWithWorkspace&) = delete; CUDABlasHandleWithWorkspace& operator=( const CUDABlasHandleWithWorkspace&) = delete; CUDABlasHandleWithWorkspace& operator=( CUDABlasHandleWithWorkspace&&) = delete; operator cublasHandle_t() const noexcept { return handle_; } private: cublasHandle_t handle_{nullptr}; cudaStream_t stream_{nullptr}; at::DataPtr workspace_; bool restore_default_workspace_{false}; }; TORCH_CUDA_CPP_API CUDABlasHandleWithWorkspace getCurrentCUDABlasHandleWithWorkspace(); TORCH_CUDA_CPP_API cublasLtHandle_t getCurrentCUDABlasLtHandle(); #ifdef USE_ROCM // hipblaslt handles are pooled per (device, stream) and cannot be created // while stream capture is active. Pre-stock the shared free list so other // threads (e.g. autograd workers) can reserve one mid-capture. TORCH_CUDA_CPP_API void ensureCublasLtHandlesAvailable(size_t n); // Under capture, getCurrentCUDABlasHandle() returns a capture handle: one per // (thread, device), with no rocBLAS arena, shared by every stream the thread // uses in the capture. Each request points it at the current stream and binds // that stream's workspace from the capture's memory pool. Once the public // handle has been requested on the current device, reserve this thread's // capture handle and pre-stock one spare for another thread, because creating // a handle is illegal under capture. TORCH_CUDA_CPP_API void prepareCaptureCublasHandles(); // Unbinds the capture handles used by this capture and frees their workspaces, // and the hipBLASLt workspaces getCUDABlasLtWorkspace handed out during it, // back to the capture's memory pool. TORCH_CUDA_CPP_API void releaseCaptureCublasWorkspaces( c10::CaptureId_t capture_id); #endif TORCH_CUDA_CPP_API void clearCublasWorkspaces(); TORCH_CUDA_CPP_API void clearCublasWorkspacesForStream(cudaStream_t stream); struct WorkspaceMapWithMutex { std::map, std::pair> map; std::shared_mutex mutex; }; TORCH_CUDA_CPP_API WorkspaceMapWithMutex& cublas_handle_stream_to_workspace(); TORCH_CUDA_CPP_API WorkspaceMapWithMutex& cublaslt_handle_stream_to_workspace(); TORCH_CUDA_CPP_API size_t getChosenWorkspaceSize(); TORCH_CUDA_CPP_API size_t getCUDABlasLtWorkspaceSize(); TORCH_CUDA_CPP_API bool isCUDABlasWorkspaceCachingEnabled(); // These return cache-owned allocations even when ATen workspace caching is // disabled. Use allocateCUDABlasWorkspace for operation-scoped storage. // On ROCm without workspace caching, under stream capture they return a buffer // from the capture's memory pool instead, one per (capture, stream), which the // graph owns once the capture ends. A pointer fetched before the capture must // not be used in it. TORCH_CUDA_CPP_API void* getCUDABlasLtWorkspace(); TORCH_CUDA_CPP_API void* getCUDABlasLtWorkspace(size_t workspace_size); TORCH_CUDA_CPP_API at::DataPtr allocateCUDABlasWorkspace(size_t size); TORCH_CUDA_CPP_API void setChosenWorkspaceSize(size_t size); TORCH_CUDA_CPP_API void setCUDABlasLtWorkspaceSize(size_t size); TORCH_CUDA_CPP_API void resetChosenWorkspaceSize(); TORCH_CUDA_CPP_API void resetCUDABlasLtWorkspaceSize(); TORCH_CUDA_CPP_API cusolverDnHandle_t getCurrentCUDASolverDnHandle(); #if defined(USE_CUDSS) TORCH_CUDA_CPP_API cudssHandle_t getCurrentCudssHandle(); #endif } // namespace at::cuda #else #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)