#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) #include #include #include namespace at::native { struct ScatterAddOp; struct ScatterMinOp; struct ScatterMaxOp; template inline bool fast_gather_kernel_eligible(const TensorIterator& iter, char * const out_ptr, char * const in_ptr, const size_t index_stride_bytes, const size_t element_size) { using at::native::memory::get_alignment; const auto index_element_size = iter.element_size(2); //TensorIterator strides and sizes are ordered fastest moving to slowest moving, //in contrast to regular sizes // we need contiguous source and dst slices and aligned pointers and strides and slice size to do vectorized loads // also we need idx to be expanded in the last dimension so we can copy entire slices // and we need the src tensor to keep 0 stride from restriding // (it could have been deleted by dimension collapse, in this case iterator would still be 2d // but we cannot use fast path) // the vectorized kernel launches at least a warp per index, so slices holding fewer // than a warp's worth of vectors leave most threads idle and run slower than the // elementwise kernel; require at least 32 vectors (512 bytes at 16-byte alignment) const auto slice_size_bytes = static_cast(iter.shape()[0]) * element_size; return iter.ndim() == 2 && iter.strides(2)[0]==0 && iter.strides(2)[1]==index_element_size && static_cast(iter.strides(0)[0])==element_size && static_cast(iter.strides(1)[0])==element_size && iter.strides(1)[1] == 0 && get_alignment(out_ptr) == alignment && get_alignment(in_ptr) == alignment && slice_size_bytes >= 32 * static_cast(alignment) && get_alignment(slice_size_bytes) == alignment && get_alignment(static_cast(index_stride_bytes)) == alignment && get_alignment(static_cast(iter.strides(0)[1])) == alignment; } // Minimum slice size for the vectorized/TMA scatter fast paths. They launch // at least a warp per entry, so below 256-byte slices the per-entry overhead // dominates and the elementwise kernel is faster (pre-#182675 routing). inline constexpr size_t kFastScatterMinSliceBytes = 256; // For scatter on dim=0: self is restrided (stride[dim]=0), src and index keep original strides. // In TensorIterator ordering (fast to slow): // dim 0 (slice): self stride = elem_size, src stride = elem_size, index stride = 0 // dim 1 (index): self stride = 0 (restrided), src stride = D*elem_size, index stride = idx_elem_size template inline bool fast_scatter_kernel_eligible(const TensorIterator& iter, char * const self_ptr, char * const src_ptr, const size_t self_stride_bytes, const size_t element_size) { using at::native::memory::get_alignment; const auto index_element_size = iter.element_size(2); const auto slice_size_bytes = static_cast(iter.shape()[0]) * element_size; return iter.ndim() == 2 && iter.strides(2)[0] == 0 && iter.strides(2)[1] == index_element_size && static_cast(iter.strides(0)[0]) == element_size && static_cast(iter.strides(1)[0]) == element_size && iter.strides(0)[1] == 0 && get_alignment(self_ptr) == alignment && get_alignment(src_ptr) == alignment && slice_size_bytes >= kFastScatterMinSliceBytes && get_alignment(slice_size_bytes) == alignment && get_alignment(self_stride_bytes) == alignment && get_alignment(static_cast(iter.strides(1)[1])) == alignment; } template void vectorized_gather_kernel_launch(char * out, char * inp, index_t * idx, int num_ind, int64_t slice_size_in_bytes, int64_t ind_dim_size, int64_t inp_stride_bytes, int64_t out_stride_bytes, bool allow_neg_indices=false); template void vectorized_scatter_kernel_launch( scalar_t* self_data, const scalar_t* src_data, index_t* idx, int num_ind, int64_t slice_size_in_bytes, int64_t self_dim_size, int64_t self_stride_bytes, int64_t src_stride_bytes); template void tma_scatter_kernel_launch( scalar_t* self_data, const scalar_t* src_data, index_t* idx, int num_ind, int D, int64_t self_dim_size, int64_t self_stride_bytes, int64_t src_stride_bytes); } #else #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)