#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) // Metal indexing primitives #pragma once #include #include #include namespace c10 { namespace metal { // Given coordinates and strides, calculates offset from the start of the // tensors template inline T offset_from_coord( thread T idx[max_ndim], constant long* strides, uint ndim) { T rc = 0; for (uint i = 0; i < ndim; ++i) { rc += idx[i] * T(strides[i]); } return rc; } // Given thread index calculates position in the ndim tensor template inline void pos_from_thread_index( T idx, thread T pos[max_ndim], constant long* sizes, uint ndim) { for (uint i = 0; i < ndim; ++i) { pos[i] = idx % T(sizes[i]); idx /= T(sizes[i]); } } // 2D dispatch overload: thread_pos.x maps directly to pos[0] (innermost dim), // thread_pos.y is decomposed across the remaining outer dims. Skips one // div/mod per element vs the 1D form. template inline void pos_from_thread_index( uint2 thread_pos, thread T pos[max_ndim], constant long* sizes, uint ndim) { pos[0] = static_cast(thread_pos.x); auto idx = static_cast(thread_pos.y); for (uint i = 1; i < ndim; ++i) { pos[i] = idx % T(sizes[i]); idx /= T(sizes[i]); } } inline long offset_from_thread_index( long idx, constant long* sizes, constant long* strides, uint ndim) { long pos[max_ndim]; pos_from_thread_index(idx, pos, sizes, ndim); return offset_from_coord(pos, strides, ndim); } template inline T val_at_offs(constant void* ptr, long offs) { return *reinterpret_cast( static_cast(ptr) + offs); } template inline T val_at_offs(device const void* ptr, long offs) { // A non-const `device` pointer coerces to this overload; the separate // `constant` overload above is required because address spaces don't convert. return *reinterpret_cast( static_cast(ptr) + offs); } template inline device T& ref_at_offs(device void* ptr, long offs) { return *reinterpret_cast(static_cast(ptr) + offs); } // One thread per element. Used for small dense tensors where the ILP variant's // per-thread overhead isn't amortized (see ILP_DISPATCH_THRESHOLD on the host). template kernel void unary_dense_scalar( device result_of* output [[buffer(0)]], constant T* input [[buffer(1)]], uint index [[thread_position_in_grid]]) { F f; output[index] = f(input[index]); } // Each thread loads ILP_PER_THREAD elements into thread-local memory, applies // the functor, then writes them back. Increases memory-level parallelism and // lets the compiler issue wide loads/stores when alignment permits. template kernel void unary_dense( device result_of* output [[buffer(0)]], constant T* input [[buffer(1)]], constant uint& numel [[buffer(2)]], uint index [[thread_position_in_grid]]) { F f; uint base = index * ILP_PER_THREAD; if (base + ILP_PER_THREAD <= numel) { array tmp_in; array, ILP_PER_THREAD> tmp_out; #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_in[j] = input[base + j]; } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_out[j] = f(tmp_in[j]); } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { output[base + j] = tmp_out[j]; } } else { for (uint i = base; i < numel; ++i) { output[i] = f(input[i]); } } } // 2D-grid variant: grid.x = thread_pos.x runs along the fastest-varying // (innermost) dim, grid.y enumerates the product of the remaining outer dims. // Saves one div+mod per thread vs the 1D form, and lets the SIMD-group walk // consecutive elements on the dense side (better memory coalescing). template kernel void unary_strided( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant long* sizes [[buffer(2)]], constant long* input_strides [[buffer(3)]], constant long* output_strides [[buffer(4)]], constant uint& ndim [[buffer(5)]], uint2 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; int pos[max_ndim]; pos_from_thread_index(thread_pos, pos, sizes, ndim); const auto input_offs = offset_from_coord(pos, input_strides, ndim); const auto output_offs = offset_from_coord(pos, output_strides, ndim); ref_at_offs(output, output_offs) = f(val_at_offs(input, input_offs)); } // Innermost dim is unit-stride for both operands (a slice/narrow view): compute // each operand's outer base offset once, then run unary_dense's ILP tile over // the contiguous inner run. Sits between unary_dense (fully contiguous) and // unary_strided (per-element offset calc). grid.x = inner tile, grid.y = linear // outer index; outer_* describe dims [1..ndim), inner is dim 0. template kernel void unary_inner_contiguous( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant long* outer_sizes [[buffer(2)]], constant long* input_outer_strides [[buffer(3)]], constant long* output_outer_strides [[buffer(4)]], constant uint2& ndim_outer_inner [[buffer(5)]], uint2 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const uint ndim_outer = ndim_outer_inner.x; const uint inner = ndim_outer_inner.y; int pos[max_ndim]; pos_from_thread_index(int(thread_pos.y), pos, outer_sizes, ndim_outer); const auto in_base = offset_from_coord(pos, input_outer_strides, ndim_outer); const auto out_base = offset_from_coord(pos, output_outer_strides, ndim_outer); constant T* in = reinterpret_cast( static_cast(input) + in_base); device res_t* out = reinterpret_cast( static_cast(output) + out_base); uint base = thread_pos.x * ILP_PER_THREAD; if (base + ILP_PER_THREAD <= inner) { array tmp_in; array tmp_out; #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_in[j] = in[base + j]; } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_out[j] = f(tmp_in[j]); } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { out[base + j] = tmp_out[j]; } } else { for (uint j = base; j < inner; ++j) { out[j] = f(in[j]); } } } // Forward declarations for the castout templates (defined after store_at_offs, // which they call). REGISTER_UNARY_OP expands to explicit template // instantiations of these, so they must be declared before the macro is used. template kernel void unary_dense_castout( device void* output, constant Tin* input, constant uint2& size_outtype, uint index); // ILP-wide castout. size_outtype_numel.x=elem_size, .y=out_type, .z=numel. template kernel void unary_dense_castout_ilp( device void* output, constant Tin* input, constant uint3& size_outtype_numel, uint index); template kernel void unary_strided_castout( device void* output, constant void* input, constant long* sizes, constant long* input_strides, constant long* output_strides, constant uint2& ndim_outtype, uint2 thread_pos); template kernel void unary_inner_contiguous_castout( device void* output, constant void* input, constant long* outer_sizes, constant long* input_outer_strides, constant long* output_outer_strides, constant uint4& ndimO_inner_elem_outtype, uint2 thread_pos); // Registers the direct per-(out,in) unary kernels and the castout variants // keyed on the input dtype. Castout kernels compute the functor in DTYPE0 // precision and cast the result to the user-supplied output dtype on store via // store_at_offs (runtime ScalarType switch), matching CPU semantics for // cross-dtype unary ops. Each (NAME, DTYPE0) pair must be registered at most // once across the library, since the castout host_names are keyed only on // DTYPE0. #define REGISTER_UNARY_OP(NAME, DTYPE0, DTYPE1) \ static_assert( \ ::metal:: \ is_same_v>, \ "Output dtype mismatch for unary op " #NAME " and input " #DTYPE0); \ template [[host_name(#NAME "_dense_" #DTYPE1 "_" #DTYPE0)]] kernel void :: \ c10::metal::unary_dense( \ device ::c10::metal::result_of * output, \ constant DTYPE0 * input, \ constant uint & numel, \ uint index); \ template \ [[host_name(#NAME "_dense_scalar_" #DTYPE1 "_" #DTYPE0)]] kernel void :: \ c10::metal::unary_dense_scalar( \ device ::c10::metal::result_of * output, \ constant DTYPE0 * input, \ uint index); \ template [[host_name(#NAME "_strided_" #DTYPE1 "_" #DTYPE0)]] kernel void :: \ c10::metal::unary_strided( \ device void* output, \ constant void* input, \ constant long* sizes, \ constant long* input_strides, \ constant long* output_strides, \ constant uint& ndim, \ uint2 thread_pos); \ template [[host_name(#NAME "_inner_contiguous_" #DTYPE1 "_" #DTYPE0)]] \ kernel void ::c10::metal::unary_inner_contiguous( \ device void* output, \ constant void* input, \ constant long* outer_sizes, \ constant long* input_outer_strides, \ constant long* output_outer_strides, \ constant uint2& ndim_outer_inner, \ uint2 thread_pos); \ template [[host_name(#NAME "_dense_castout_" #DTYPE0)]] kernel void ::c10:: \ metal::unary_dense_castout( \ device void* output, \ constant DTYPE0* input, \ constant uint2& size_outtype, \ uint index); \ template [[host_name(#NAME "_dense_castout_ilp_" #DTYPE0)]] kernel void :: \ c10::metal::unary_dense_castout_ilp( \ device void* output, \ constant DTYPE0* input, \ constant uint3& size_outtype_numel, \ uint index); \ template [[host_name(#NAME "_strided_castout_" #DTYPE0)]] kernel void :: \ c10::metal::unary_strided_castout( \ device void* output, \ constant void* input, \ constant long* sizes, \ constant long* input_strides, \ constant long* output_strides, \ constant uint2& ndim_outtype, \ uint2 thread_pos); \ template [[host_name(#NAME "_inner_contiguous_castout_" #DTYPE0)]] \ kernel void ::c10::metal:: \ unary_inner_contiguous_castout( \ device void* output, \ constant void* input, \ constant long* outer_sizes, \ constant long* input_outer_strides, \ constant long* output_outer_strides, \ constant uint4& ndimO_inner_elem_outtype, \ uint2 thread_pos) #define DEFINE_UNARY_FLOATING_FUNCTOR(NAME) \ struct NAME##_functor { \ template \ inline ::metal::enable_if_t<::metal::is_floating_point_v, T> operator()( \ const T x) { \ return T(NAME(x)); \ } \ template \ inline ::metal::enable_if_t<::metal::is_integral_v, float> operator()( \ const T x) { \ return NAME(static_cast(x)); \ } \ } #define DEFINE_BINARY_COMPARISON_FUNCTOR(NAME, OP) \ struct NAME##_functor { \ template \ inline bool operator()(const T a, const T b) { \ return a OP b; \ } \ } template kernel void unary_alpha_dense( device result_of* output [[buffer(0)]], constant T* input [[buffer(1)]], constant T2& alpha [[buffer(2)]], uint index [[thread_position_in_grid]]) { F f; output[index] = f(input[index], alpha); } // ILP variant of unary_alpha_dense; mirrors unary_dense. Selected by the host // only when the caller opts in via ilp_threshold (see // exec_unary_kernel_with_params). template kernel void unary_alpha_dense_ilp( device result_of* output [[buffer(0)]], constant T* input [[buffer(1)]], constant T2& alpha [[buffer(2)]], constant uint& numel [[buffer(3)]], uint index [[thread_position_in_grid]]) { F f; uint base = index * ILP_PER_THREAD; if (base + ILP_PER_THREAD <= numel) { array tmp_in; array, ILP_PER_THREAD> tmp_out; #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_in[j] = input[base + j]; } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_out[j] = f(tmp_in[j], alpha); } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { output[base + j] = tmp_out[j]; } } else { for (uint i = base; i < numel; ++i) { output[i] = f(input[i], alpha); } } } template kernel void unary_alpha_strided( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant long* sizes [[buffer(2)]], constant long* input_strides [[buffer(3)]], constant long* output_strides [[buffer(4)]], constant uint& ndim [[buffer(5)]], constant T2& alpha [[buffer(6)]], uint2 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; int pos[max_ndim]; pos_from_thread_index(thread_pos, pos, sizes, ndim); const auto input_offs = offset_from_coord(pos, input_strides, ndim); const auto output_offs = offset_from_coord(pos, output_strides, ndim); ref_at_offs(output, output_offs) = f(val_at_offs(input, input_offs), alpha); } // unary_inner_contiguous with a scalar argument (relu/clamp_min, ...): one // outer-offset calc per inner run, then an ILP tile over the unit-stride run. template kernel void unary_alpha_inner_contiguous( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant long* outer_sizes [[buffer(2)]], constant long* input_outer_strides [[buffer(3)]], constant long* output_outer_strides [[buffer(4)]], constant uint2& ndim_outer_inner [[buffer(5)]], constant T2& alpha [[buffer(6)]], uint2 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const uint ndim_outer = ndim_outer_inner.x; const uint inner = ndim_outer_inner.y; int pos[max_ndim]; pos_from_thread_index(int(thread_pos.y), pos, outer_sizes, ndim_outer); const auto in_base = offset_from_coord(pos, input_outer_strides, ndim_outer); const auto out_base = offset_from_coord(pos, output_outer_strides, ndim_outer); constant T* in = reinterpret_cast( static_cast(input) + in_base); device res_t* out = reinterpret_cast( static_cast(output) + out_base); uint base = thread_pos.x * ILP_PER_THREAD; if (base + ILP_PER_THREAD <= inner) { array tmp_in; array tmp_out; #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_in[j] = in[base + j]; } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_out[j] = f(tmp_in[j], alpha); } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { out[base + j] = tmp_out[j]; } } else { for (uint j = base; j < inner; ++j) { out[j] = f(in[j], alpha); } } } #define REGISTER_UNARY_ALPHA_OP(NAME, DTYPEI, DTYPEA, DTYPEO) \ static_assert( \ ::metal::is_same_v< \ DTYPEO, \ ::c10::metal::result_of>, \ "Output dtype mismatch for unary op " #NAME " and input " #DTYPEI); \ template [[host_name(#NAME "_dense_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ unary_alpha_dense( \ device ::c10::metal::result_of * \ output, \ constant DTYPEI * input, \ constant DTYPEA & alpha, \ uint index); \ template [[host_name(#NAME "_dense_ilp_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ unary_alpha_dense_ilp( \ device ::c10::metal::result_of * \ output, \ constant DTYPEI * input, \ constant DTYPEA & alpha, \ constant uint & numel, \ uint index); \ template [[host_name(#NAME "_strided_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ unary_alpha_strided( \ device void* output, \ constant void* input, \ constant long* sizes, \ constant long* input_strides, \ constant long* output_strides, \ constant uint& ndim, \ constant DTYPEA& alpha, \ uint2 thread_pos); \ template [[host_name(#NAME "_inner_contiguous_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ unary_alpha_inner_contiguous( \ device void* output, \ constant void* input, \ constant long* outer_sizes, \ constant long* input_outer_strides, \ constant long* output_outer_strides, \ constant uint2& ndim_outer_inner, \ constant DTYPEA& alpha, \ uint2 thread_pos) // Value at offset with dynamic cast from provided type template inline T val_at_offs(P ptr, long offs, ScalarType type) { switch (type) { #define _CASE_(EnumName, EnumValue, MetalType) \ case ScalarType::EnumName: \ return cast_to(val_at_offs(ptr, offs)); C10_METAL_ALL_TYPES_FUNCTOR(_CASE_) #undef _CASE_ } } // Store with dynamic cast to provided type. Mirrors val_at_offs's runtime // switch and is the store-side counterpart used by binary_*_castout kernels // (output dtype not known at compile time). template inline void store_at_offs( device void* ptr, long offs, ScalarType type, T value) { switch (type) { #define _CASE_(EnumName, EnumValue, MetalType) \ case ScalarType::EnumName: \ ref_at_offs(ptr, offs) = cast_to(value); \ return; C10_METAL_ALL_TYPES_FUNCTOR(_CASE_) #undef _CASE_ } } // Castout variants of the unary kernels: input is loaded at compile-time Tin // precision (no cast), the functor runs in Tin precision, and the result // (result_of) is cast to the user-supplied output dtype on store via // store_at_offs (runtime ScalarType switch). Matches CPU semantics for // cross-dtype unary ops (compute in input precision, cast on store). Mirrors // binary_*_castout but is keyed per input dtype, since unary has no separate // common-compute dtype. template kernel void unary_dense_castout( device void* output [[buffer(0)]], constant Tin* input [[buffer(1)]], constant uint2& size_outtype [[buffer(2)]], uint index [[thread_position_in_grid]]) { F f; using res_t = result_of; store_at_offs( output, long(index) * size_outtype.x, static_cast(size_outtype.y), f(input[index])); } // ILP-wide castout: mirrors unary_dense's per-thread tile but stores via the // runtime ScalarType switch. The out_type is uniform across all threads so the // compiler can hoist the switch out of the unrolled inner loop. template kernel void unary_dense_castout_ilp( device void* output [[buffer(0)]], constant Tin* input [[buffer(1)]], constant uint3& size_outtype_numel [[buffer(2)]], uint index [[thread_position_in_grid]]) { F f; using res_t = result_of; const uint elem_size = size_outtype_numel.x; const auto out_type = static_cast(size_outtype_numel.y); const uint numel = size_outtype_numel.z; uint base = index * ILP_PER_THREAD; if (base + ILP_PER_THREAD <= numel) { array tmp_in; array tmp_out; #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_in[j] = input[base + j]; } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_out[j] = f(tmp_in[j]); } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { store_at_offs( output, long(base + j) * elem_size, out_type, tmp_out[j]); } } else { for (uint i = base; i < numel; ++i) { store_at_offs(output, long(i) * elem_size, out_type, f(input[i])); } } } template kernel void unary_strided_castout( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant long* sizes [[buffer(2)]], constant long* input_strides [[buffer(3)]], constant long* output_strides [[buffer(4)]], constant uint2& ndim_outtype [[buffer(5)]], uint2 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; int pos[max_ndim]; pos_from_thread_index(thread_pos, pos, sizes, ndim_outtype.x); const auto input_offs = offset_from_coord(pos, input_strides, ndim_outtype.x); const auto output_offs = offset_from_coord(pos, output_strides, ndim_outtype.x); store_at_offs( output, output_offs, static_cast(ndim_outtype.y), f(val_at_offs(input, input_offs))); } // Castout variant of unary_inner_contiguous: output dtype is runtime (stored // via store_at_offs). ndimO_inner_elem_outtype = {outer ndim, inner extent, // output elem size (bytes), output ScalarType}. template kernel void unary_inner_contiguous_castout( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant long* outer_sizes [[buffer(2)]], constant long* input_outer_strides [[buffer(3)]], constant long* output_outer_strides [[buffer(4)]], constant uint4& ndimO_inner_elem_outtype [[buffer(5)]], uint2 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const uint ndim_outer = ndimO_inner_elem_outtype.x; const uint inner = ndimO_inner_elem_outtype.y; const uint elem = ndimO_inner_elem_outtype.z; const auto out_type = static_cast(ndimO_inner_elem_outtype.w); int pos[max_ndim]; pos_from_thread_index(int(thread_pos.y), pos, outer_sizes, ndim_outer); const auto in_base = offset_from_coord(pos, input_outer_strides, ndim_outer); const auto out_base = offset_from_coord(pos, output_outer_strides, ndim_outer); constant Tin* in = reinterpret_cast( static_cast(input) + in_base); uint base = thread_pos.x * ILP_PER_THREAD; if (base + ILP_PER_THREAD <= inner) { array tmp_in; array tmp_out; #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_in[j] = in[base + j]; } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { tmp_out[j] = f(tmp_in[j]); } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { store_at_offs( output, out_base + long(base + j) * elem, out_type, tmp_out[j]); } } else { for (uint j = base; j < inner; ++j) { store_at_offs( output, out_base + long(j) * elem, out_type, f(in[j])); } } } // Binary elementwise ops kernels // Right now the following flavors are available: // - binary_dense where both input, other and output are dense and share the // same type // - binary_strided when all inputs are of the same types, but some elements are // strided // - binary_inner_contiguous - strided, but the innermost dim is unit-stride for // every operand (a slice/narrow view) // - binary_dense_cast - inputs are dense, but of different dtypes // - binary_strided_cast - inputs or output are strided and of different dtypes // - binary_dense_broadcast - one input is dense, another one is broadcastable // Note about accuracy (for more info see // https://github.com/pytorch/pytorch/issues/152736) Sometimes when kernel is // invoked to produce `half` output, but one of the arguments is float arguments // should be upcast to float, rather than downcast to half At the moment this is // expressed with `om_t` optional argument (which stands for opmath_type) which // is identical to output type but could be something else template kernel void binary_strided( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant long* sizes [[buffer(3)]], constant long* output_strides [[buffer(4)]], constant long* input_strides [[buffer(5)]], constant long* other_strides [[buffer(6)]], constant uint3& ndim [[buffer(7)]], uint index [[thread_position_in_grid]]) { F f; using res_t = result_of; int pos[max_ndim]; pos_from_thread_index(int(index), pos, sizes, ndim.x); const auto input_offs = offset_from_coord(pos, input_strides, ndim.x); const auto other_offs = offset_from_coord(pos, other_strides, ndim.x); const auto output_offs = offset_from_coord(pos, output_strides, ndim.x); const auto a = val_at_offs(input, input_offs); const auto b = val_at_offs(other, other_offs); ref_at_offs(output, output_offs) = static_cast(f(om_t(a), om_t(b))); } template kernel void binary_alpha_strided( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant T2& alpha [[buffer(3)]], constant long* sizes [[buffer(4)]], constant long* output_strides [[buffer(5)]], constant long* input_strides [[buffer(6)]], constant long* other_strides [[buffer(7)]], constant uint3& ndim [[buffer(8)]], uint index [[thread_position_in_grid]]) { F f; int pos[max_ndim]; pos_from_thread_index(int(index), pos, sizes, ndim.x); const auto input_offs = offset_from_coord(pos, input_strides, ndim.x); const auto other_offs = offset_from_coord(pos, other_strides, ndim.x); const auto output_offs = offset_from_coord(pos, output_strides, ndim.x); const auto a = val_at_offs(input, input_offs); const auto b = val_at_offs(other, other_offs); ref_at_offs>(output, output_offs) = f(a, b, alpha); } template > kernel void binary_strided_cast( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant long* sizes [[buffer(3)]], constant long* output_strides [[buffer(4)]], constant long* input_strides [[buffer(5)]], constant long* other_strides [[buffer(6)]], constant uint4& ndim_types [[buffer(7)]], uint index [[thread_position_in_grid]]) { F f; using res_t = result_of; int pos[max_ndim]; pos_from_thread_index(int(index), pos, sizes, ndim_types.x); const auto input_offs = offset_from_coord(pos, input_strides, ndim_types.x); const auto other_offs = offset_from_coord(pos, other_strides, ndim_types.x); const auto output_offs = offset_from_coord(pos, output_strides, ndim_types.x); const auto a = val_at_offs( input, input_offs, static_cast(ndim_types.y)); const auto b = val_at_offs( other, other_offs, static_cast(ndim_types.z)); ref_at_offs(output, output_offs) = static_cast(f(a, b)); } // Variant of binary_strided_cast that also runtime-casts on store: the output // buffer's dtype is passed via ndim_types.w. Selected when the user-facing // output dtype diverges from the kernel's natural output (the iterator's // `cast_common_dtype_to_outputs(true)` path). The kernel's natural output // `res_t` is computed in compile-time precision, then runtime-cast to the // buffer's dtype on store. template > kernel void binary_strided_castout( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant long* sizes [[buffer(3)]], constant long* output_strides [[buffer(4)]], constant long* input_strides [[buffer(5)]], constant long* other_strides [[buffer(6)]], constant uint4& ndim_types [[buffer(7)]], uint index [[thread_position_in_grid]]) { F f; using res_t = result_of; int pos[max_ndim]; pos_from_thread_index(int(index), pos, sizes, ndim_types.x); const auto input_offs = offset_from_coord(pos, input_strides, ndim_types.x); const auto other_offs = offset_from_coord(pos, other_strides, ndim_types.x); const auto output_offs = offset_from_coord(pos, output_strides, ndim_types.x); const auto a = val_at_offs( input, input_offs, static_cast(ndim_types.y)); const auto b = val_at_offs( other, other_offs, static_cast(ndim_types.z)); const res_t result = static_cast(f(a, b)); store_at_offs( output, output_offs, static_cast(ndim_types.w), result); } template kernel void binary_alpha_strided_cast( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant T2& alpha [[buffer(3)]], constant long* sizes [[buffer(4)]], constant long* output_strides [[buffer(5)]], constant long* input_strides [[buffer(6)]], constant long* other_strides [[buffer(7)]], constant uint4& ndim_types [[buffer(8)]], uint index [[thread_position_in_grid]]) { F f; int pos[max_ndim]; pos_from_thread_index(int(index), pos, sizes, ndim_types.x); const auto input_offs = offset_from_coord(pos, input_strides, ndim_types.x); const auto other_offs = offset_from_coord(pos, other_strides, ndim_types.x); const auto output_offs = offset_from_coord(pos, output_strides, ndim_types.x); const auto a = val_at_offs(input, input_offs, static_cast(ndim_types.y)); const auto b = val_at_offs(other, other_offs, static_cast(ndim_types.z)); ref_at_offs>(output, output_offs) = f(a, b, alpha); } template > kernel void binary_dense( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], constant T* other [[buffer(2)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; out[tid] = static_cast(f(om_t(input[tid]), om_t(other[tid]))); } static_assert( ILP_PER_THREAD == 4, "C10_METAL_ILP_PER_THREAD_STR must match ILP_PER_THREAD"); // ILP variant of binary_dense: each thread processes ILP elements. // Mirrors unary_dense; selected on the host when both inputs and the output // are contiguous, share dtype (no cast), and the iterator has neither a scalar // nor a broadcast operand. The unroll width defaults to ILP_PER_THREAD; the // host-side kernel name encodes the value (e.g. `..._dense_ilp4_...`) so // future variants (ilp8, etc.) can coexist. See exec_binary_kernel for the // dispatch threshold. template < typename T, typename F, typename om_t = opmath_t, unsigned ILP = ILP_PER_THREAD> kernel void binary_dense_ilp( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], constant T* other [[buffer(2)]], constant uint& numel [[buffer(3)]], uint index [[thread_position_in_grid]]) { F f; using res_t = result_of; uint base = index * ILP; if (base + ILP <= numel) { array tmp_a; array tmp_b; array tmp_out; #pragma unroll for (uint j = 0; j < ILP; ++j) { tmp_a[j] = input[base + j]; tmp_b[j] = other[base + j]; } #pragma unroll for (uint j = 0; j < ILP; ++j) { tmp_out[j] = static_cast(f(om_t(tmp_a[j]), om_t(tmp_b[j]))); } #pragma unroll for (uint j = 0; j < ILP; ++j) { out[base + j] = tmp_out[j]; } } else { for (uint i = base; i < numel; ++i) { out[i] = static_cast(f(om_t(input[i]), om_t(other[i]))); } } } // Inner-contiguous binary: like unary_inner_contiguous, but runs // binary_dense_ilp's tile over the contiguous inner run (all three operands // unit-stride in dim 0). template > kernel void binary_inner_contiguous( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant long* outer_sizes [[buffer(3)]], constant long* output_outer_strides [[buffer(4)]], constant long* input_outer_strides [[buffer(5)]], constant long* other_outer_strides [[buffer(6)]], constant uint2& ndim_outer_inner [[buffer(7)]], uint2 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const uint ndim_outer = ndim_outer_inner.x; const uint inner = ndim_outer_inner.y; int pos[max_ndim]; pos_from_thread_index(int(thread_pos.y), pos, outer_sizes, ndim_outer); const auto out_base = offset_from_coord(pos, output_outer_strides, ndim_outer); const auto in_base = offset_from_coord(pos, input_outer_strides, ndim_outer); const auto oth_base = offset_from_coord(pos, other_outer_strides, ndim_outer); device res_t* out = reinterpret_cast( static_cast(output) + out_base); constant T* a = reinterpret_cast( static_cast(input) + in_base); constant T* b = reinterpret_cast( static_cast(other) + oth_base); uint base = thread_pos.x * ILP_PER_THREAD; if (base + ILP_PER_THREAD <= inner) { array ta; array tb; array to; #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { ta[j] = a[base + j]; tb[j] = b[base + j]; } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { to[j] = static_cast(f(om_t(ta[j]), om_t(tb[j]))); } #pragma unroll for (uint j = 0; j < ILP_PER_THREAD; ++j) { out[base + j] = to[j]; } } else { for (uint j = base; j < inner; ++j) { out[j] = static_cast(f(om_t(a[j]), om_t(b[j]))); } } } template kernel void binary_alpha_dense( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], constant T* other [[buffer(2)]], constant T2& alpha [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; out[tid] = f(input[tid], other[tid], alpha); } template kernel void binary_dense_cast( device result_of* out [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant uint4& sizes_types [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto a = val_at_offs( input, tid * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs( other, tid * sizes_types.y, static_cast(sizes_types.w)); out[tid] = static_cast(f(a, b)); } template kernel void binary_alpha_dense_cast( device result_of* out [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other [[buffer(2)]], constant T2& alpha [[buffer(3)]], constant uint4& sizes_types [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; const auto a = val_at_offs( input, tid * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs( other, tid * sizes_types.y, static_cast(sizes_types.w)); out[tid] = f(a, b, alpha); } template > kernel void binary_dense_broadcast( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], constant T* broadcast [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; out[tid] = static_cast( f(om_t(input[tid]), om_t(broadcast[tid % broadcast_numel]))); } template > kernel void binary_dense_broadcast_rhs( device result_of* out [[buffer(0)]], constant T* broadcast [[buffer(1)]], constant T* input [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; out[tid] = static_cast( f(om_t(broadcast[tid % broadcast_numel]), om_t(input[tid]))); } template kernel void binary_alpha_dense_broadcast( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], constant T* broadcast [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], constant T2& alpha [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; out[tid] = f(input[tid], broadcast[tid % broadcast_numel], alpha); } template kernel void binary_alpha_dense_broadcast_rhs( device result_of* out [[buffer(0)]], constant T* broadcast [[buffer(1)]], constant T* input [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], constant T2& alpha [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; out[tid] = f(broadcast[tid % broadcast_numel], input[tid], alpha); } template kernel void binary_dense_broadcast_cast( device result_of* out [[buffer(0)]], constant void* input [[buffer(1)]], constant void* broadcast [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], constant uint4& sizes_types [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto a = val_at_offs( input, tid * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs( broadcast, (tid % broadcast_numel) * sizes_types.y, static_cast(sizes_types.w)); out[tid] = static_cast(f(a, b)); } template kernel void binary_dense_broadcast_rhs_cast( device result_of* out [[buffer(0)]], constant void* broadcast [[buffer(1)]], constant void* input [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], constant uint4& sizes_types [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto a = val_at_offs( broadcast, (tid % broadcast_numel) * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs( input, tid * sizes_types.y, static_cast(sizes_types.w)); out[tid] = static_cast(f(a, b)); } template kernel void binary_alpha_dense_broadcast_cast( device result_of* out [[buffer(0)]], constant void* input [[buffer(1)]], constant void* broadcast [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], constant T2& alpha [[buffer(4)]], constant uint4& sizes_types [[buffer(5)]], uint tid [[thread_position_in_grid]]) { F f; const auto a = val_at_offs( input, tid * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs( broadcast, (tid % broadcast_numel) * sizes_types.y, static_cast(sizes_types.w)); out[tid] = f(a, b, alpha); } template kernel void binary_alpha_dense_broadcast_rhs_cast( device result_of* out [[buffer(0)]], constant void* broadcast [[buffer(1)]], constant void* input [[buffer(2)]], constant long& broadcast_numel [[buffer(3)]], constant T2& alpha [[buffer(4)]], constant uint4& sizes_types [[buffer(5)]], uint tid [[thread_position_in_grid]]) { F f; const auto a = val_at_offs( broadcast, (tid % broadcast_numel) * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs( input, tid * sizes_types.y, static_cast(sizes_types.w)); out[tid] = f(a, b, alpha); } template > kernel void binary_dense_scalar( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], device const T* scalar [[buffer(2)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; out[tid] = static_cast(f(om_t(input[tid]), om_t(scalar[0]))); } template > kernel void binary_dense_scalar_lhs( device result_of* out [[buffer(0)]], device const T* scalar [[buffer(1)]], constant T* input [[buffer(2)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; out[tid] = static_cast(f(om_t(scalar[0]), om_t(input[tid]))); } template kernel void binary_dense_scalar_cast( device result_of* out [[buffer(0)]], constant void* input [[buffer(1)]], device const uchar* scalar [[buffer(2)]], constant uint4& sizes_types [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto a = val_at_offs( input, tid * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs(scalar, 0, static_cast(sizes_types.w)); out[tid] = static_cast(f(a, b)); } template kernel void binary_dense_scalar_lhs_cast( device result_of* out [[buffer(0)]], device const uchar* scalar [[buffer(1)]], constant void* input [[buffer(2)]], constant uint4& sizes_types [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto a = val_at_offs(scalar, 0, static_cast(sizes_types.z)); const auto b = val_at_offs( input, tid * sizes_types.y, static_cast(sizes_types.w)); out[tid] = static_cast(f(a, b)); } template kernel void binary_alpha_dense_scalar( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], device const T* scalar [[buffer(2)]], constant T2& alpha [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; out[tid] = f(input[tid], scalar[0], alpha); } template kernel void binary_alpha_dense_scalar_lhs( device result_of* out [[buffer(0)]], device const T* scalar [[buffer(1)]], constant T* input [[buffer(2)]], constant T2& alpha [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; out[tid] = f(scalar[0], input[tid], alpha); } template kernel void binary_alpha_dense_scalar_cast( device result_of* out [[buffer(0)]], constant void* input [[buffer(1)]], device const uchar* scalar [[buffer(2)]], constant T2& alpha [[buffer(3)]], constant uint4& sizes_types [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; const auto a = val_at_offs( input, tid * sizes_types.x, static_cast(sizes_types.z)); const auto b = val_at_offs(scalar, 0, static_cast(sizes_types.w)); out[tid] = f(a, b, alpha); } template kernel void binary_alpha_dense_scalar_lhs_cast( device result_of* out [[buffer(0)]], device const uchar* scalar [[buffer(1)]], constant void* input [[buffer(2)]], constant T2& alpha [[buffer(3)]], constant uint4& sizes_types [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; const auto a = val_at_offs(scalar, 0, static_cast(sizes_types.z)); const auto b = val_at_offs( input, tid * sizes_types.y, static_cast(sizes_types.w)); out[tid] = f(a, b, alpha); } #define REGISTER_BINARY_OP_(NAME, DTYPEI, DTYPEO, OMT) \ static_assert( \ ::metal::is_same_v< \ DTYPEO, \ ::c10::metal::result_of>, \ "Output dtype mismatch for binary op " #NAME " and input " #DTYPEI); \ template [[host_name(#NAME "_strided_" #DTYPEO "_" #DTYPEI)]] kernel void :: \ c10::metal::binary_strided( \ device void* out, \ constant void* input, \ constant void* other, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other_strides, \ constant uint3& ndim, \ uint tid); \ template [[host_name(#NAME "_inner_contiguous_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal:: \ binary_inner_contiguous( \ device void* out, \ constant void* input, \ constant void* other, \ constant long* outer_sizes, \ constant long* output_outer_strides, \ constant long* input_outer_strides, \ constant long* other_outer_strides, \ constant uint2& ndim_outer_inner, \ uint2 thread_pos); \ template \ [[host_name(#NAME "_strided_cast_" #DTYPEO "_" #DTYPEI)]] kernel void :: \ c10::metal::binary_strided_cast( \ device void* out, \ constant void* input, \ constant void* other, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other_strides, \ constant uint4& ndim_types, \ uint tid); \ template [[host_name(#NAME "_dense_" #DTYPEO "_" #DTYPEI)]] kernel void :: \ c10::metal::binary_dense( \ device ::c10::metal::result_of * \ out_, \ constant DTYPEI * input_, \ constant DTYPEI * other_, \ uint tid); \ template [[host_name(#NAME "_dense_ilp" C10_METAL_ILP_PER_THREAD_STR \ "_" #DTYPEO "_" #DTYPEI)]] kernel void ::c10:: \ metal::binary_dense_ilp( \ device ::c10::metal::result_of * \ out_, \ constant DTYPEI * input_, \ constant DTYPEI * other_, \ constant uint & numel, \ uint tid); \ template \ [[host_name(#NAME "_dense_cast_" #DTYPEO "_" #DTYPEI)]] kernel void :: \ c10::metal::binary_dense_cast( \ device ::c10::metal::result_of * \ out_, \ constant void* input, \ constant void* other, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal:: \ binary_dense_broadcast( \ device ::c10::metal::result_of * \ out_, \ constant DTYPEI * input_, \ constant DTYPEI * broadcast_, \ constant long& broadcast_numel, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_rhs_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal:: \ binary_dense_broadcast_rhs( \ device ::c10::metal::result_of * \ out_, \ constant DTYPEI * broadcast_, \ constant DTYPEI * input_, \ constant long& broadcast_numel, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_cast_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal:: \ binary_dense_broadcast_cast( \ device ::c10::metal::result_of * \ out_, \ constant void* input_, \ constant void* broadcast_, \ constant long& broadcast_numel, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_rhs_cast_" #DTYPEO \ "_" #DTYPEI)]] kernel void ::c10::metal:: \ binary_dense_broadcast_rhs_cast( \ device ::c10::metal::result_of * \ out_, \ constant void* broadcast_, \ constant void* input_, \ constant long& broadcast_numel, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal::binary_dense_scalar( \ device ::c10::metal::result_of * out_, \ constant DTYPEI * input_, \ device const DTYPEI* scalar_, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_lhs_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal:: \ binary_dense_scalar_lhs( \ device ::c10::metal::result_of * \ out_, \ device const DTYPEI* scalar_, \ constant DTYPEI* input_, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_cast_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal:: \ binary_dense_scalar_cast( \ device ::c10::metal::result_of * \ out_, \ constant void* input_, \ device const uchar* scalar_, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_lhs_cast_" #DTYPEO \ "_" #DTYPEI)]] kernel void ::c10::metal:: \ binary_dense_scalar_lhs_cast( \ device ::c10::metal::result_of * \ out_, \ device const uchar* scalar_, \ constant void* input_, \ constant uint4& sizes_types, \ uint tid) // OpMath Binary Op promotes inputs to higher precision type before Functor call #define REGISTER_OPMATH_BINARY_OP(NAME, DTYPEI, DTYPEO) \ REGISTER_BINARY_OP_(NAME, DTYPEI, DTYPEO, ::c10::metal::opmath_t) #define REGISTER_BINARY_OP(NAME, DTYPEI, DTYPEO) \ REGISTER_BINARY_OP_(NAME, DTYPEI, DTYPEO, DTYPEI) // Opt-in castout variant: enables `_strided_castout__` // for ops whose kernel-natural output dtype is fixed (comparison: bool) and // whose user-facing `out=` may differ. Arithmetic ops that produce // `iter.dtype(0)` directly should NOT register this -- the existing // `__` cast kernel matrix covers them. #define REGISTER_BINARY_CASTOUT_OP(NAME, DTYPEI, DTYPEO) \ template [[host_name(#NAME "_strided_castout_" #DTYPEO "_" #DTYPEI)]] \ kernel void ::c10::metal:: \ binary_strided_castout( \ device void* out, \ constant void* input, \ constant void* other, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other_strides, \ constant uint4& ndim_types, \ uint tid) #define REGISTER_BINARY_ALPHA_OP(NAME, DTYPEI, DTYPEA, DTYPEO) \ static_assert( \ ::metal::is_same_v< \ DTYPEO, \ ::c10::metal::result_of>, \ "Output dtype mismatch for binary op " #NAME " and input " #DTYPEI); \ template [[host_name(#NAME "_strided_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_strided( \ device void* out, \ constant void* input, \ constant void* other, \ constant DTYPEA& alpha, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other_strides, \ constant uint3& ndim, \ uint tid); \ template [[host_name(#NAME "_strided_cast_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_strided_cast( \ device void* out, \ constant void* input, \ constant void* other, \ constant DTYPEA& alpha, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other_strides, \ constant uint4& ndim_types, \ uint tid); \ template [[host_name(#NAME "_dense_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense( \ device ::c10::metal:: \ result_of * \ out_, \ constant DTYPEI * input_, \ constant DTYPEI * other_, \ constant DTYPEA & alpha, \ uint tid); \ template [[host_name(#NAME "_dense_cast_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_cast( \ device ::c10::metal:: \ result_of * \ out_, \ constant void* input, \ constant void* other, \ constant DTYPEA& alpha, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_broadcast( \ device ::c10::metal:: \ result_of * \ out_, \ constant DTYPEI * input_, \ constant DTYPEI * broadcast_, \ constant long& broadcast_numel, \ constant DTYPEA& alpha, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_rhs_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_broadcast_rhs( \ device ::c10::metal:: \ result_of * \ out_, \ constant DTYPEI * broadcast_, \ constant DTYPEI * input_, \ constant long& broadcast_numel, \ constant DTYPEA& alpha, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_cast_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_broadcast_cast( \ device ::c10::metal:: \ result_of * \ out_, \ constant void* input_, \ constant void* broadcast_, \ constant long& broadcast_numel, \ constant DTYPEA& alpha, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_broadcast_rhs_cast_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_broadcast_rhs_cast( \ device ::c10::metal:: \ result_of * \ out_, \ constant void* broadcast_, \ constant void* input_, \ constant long& broadcast_numel, \ constant DTYPEA& alpha, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_scalar( \ device ::c10::metal:: \ result_of * \ out_, \ constant DTYPEI * input_, \ device const DTYPEI* scalar_, \ constant DTYPEA& alpha, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_lhs_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_scalar_lhs( \ device ::c10::metal:: \ result_of * \ out_, \ device const DTYPEI* scalar_, \ constant DTYPEI* input_, \ constant DTYPEA& alpha, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_cast_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_scalar_cast( \ device ::c10::metal:: \ result_of * \ out_, \ constant void* input_, \ device const uchar* scalar_, \ constant DTYPEA& alpha, \ constant uint4& sizes_types, \ uint tid); \ template [[host_name(#NAME "_dense_scalar_lhs_cast_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ binary_alpha_dense_scalar_lhs_cast( \ device ::c10::metal:: \ result_of * \ out_, \ device const uchar* scalar_, \ constant void* input_, \ constant DTYPEA& alpha, \ constant uint4& sizes_types, \ uint tid) // Decodes the dispatch coordinates and accumulates all four operand offsets in // one pass. The obvious alternative -- materialize `pos[max_ndim]`, then call // offset_from_coord per operand -- costs ~3x on index-bound shapes such as // broadcast operands: with a runtime trip count the array lands in scratch // memory instead of registers. inline long4 ternary_offsets( uint3 thread_pos, constant long* sizes, uint ndim, constant long* strides0, constant long* strides1, constant long* strides2, constant long* strides3) { // 32-bit offset math: the host splits any iterator that fails // can_use_32bit_indexing(), which bounds every operand's byte offset to // int32. 64-bit accumulation here costs ~1.8x on index-bound 3D shapes. const int p0 = int(thread_pos.x); int4 offs = int4( p0 * int(strides0[0]), p0 * int(strides1[0]), p0 * int(strides2[0]), p0 * int(strides3[0])); if (ndim == 1) { return long4(offs); } const int p1 = int(thread_pos.y); offs += int4( p1 * int(strides0[1]), p1 * int(strides1[1]), p1 * int(strides2[1]), p1 * int(strides3[1])); if (ndim == 2) { return long4(offs); } // `ndim` lives in a constant buffer, so these branches are uniform across the // threadgroup. Up to 3 dims the coordinates come straight from the dispatch // grid and no division is needed at all; only dims past the third pay for it. if (ndim == 3) { const int p2 = int(thread_pos.z); return long4( offs + int4( p2 * int(strides0[2]), p2 * int(strides1[2]), p2 * int(strides2[2]), p2 * int(strides3[2]))); } int idx = int(thread_pos.z); for (uint i = 2; i < ndim; ++i) { const int sz = int(sizes[i]); const int p = idx % sz; idx /= sz; offs += int4( p * int(strides0[i]), p * int(strides1[i]), p * int(strides2[i]), p * int(strides3[i])); } return long4(offs); } // Ternary elementwise ops kernels // Right now the following flavors are available: // - ternary_dense where both input, other1, other2, and output are dense and // share the same type // - ternary_strided when all inputs are of the same types, but some elements // are strided // - ternary_dense_cast - inputs are dense, but of different dtypes // - ternary_strided_cast - inputs or output are strided and of different dtypes // The cast flavors are instantiated for the iterator's common dtype and load // and store through each buffer's own runtime dtype, i.e. they compute at the // same precision CPU does and cast the result into `out` on store // Each of those has an `alpha` counterpart (see ternary_alpha_strided below) // for ops that take a scalar alongside the three tensors, such as addcmul // Note about accuracy (for more info see // https://github.com/pytorch/pytorch/issues/152736) Sometimes when kernel is // invoked to produce `half` output, but one of the arguments is float arguments // should be upcast to float, rather than downcast to half At the moment this is // expressed with `om_t` optional argument (which stands for opmath_type) which // is identical to output type but could be something else template kernel void ternary_strided( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other1 [[buffer(2)]], constant void* other2 [[buffer(3)]], constant long* sizes [[buffer(4)]], constant long* output_strides [[buffer(5)]], constant long* input_strides [[buffer(6)]], constant long* other1_strides [[buffer(7)]], constant long* other2_strides [[buffer(8)]], constant uint& ndim [[buffer(9)]], uint3 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto offs = ternary_offsets( thread_pos, sizes, ndim, output_strides, input_strides, other1_strides, other2_strides); const auto a = val_at_offs(input, offs.y); const auto b = val_at_offs(other1, offs.z); const auto c = val_at_offs(other2, offs.w); ref_at_offs(output, offs.x) = static_cast(f(om_t(a), om_t(b), om_t(c))); } template > kernel void ternary_strided_cast( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other1 [[buffer(2)]], constant void* other2 [[buffer(3)]], constant long* sizes [[buffer(4)]], constant long* output_strides [[buffer(5)]], constant long* input_strides [[buffer(6)]], constant long* other1_strides [[buffer(7)]], constant long* other2_strides [[buffer(8)]], constant uint& ndim [[buffer(9)]], constant uint4& types [[buffer(10)]], uint3 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto offs = ternary_offsets( thread_pos, sizes, ndim, output_strides, input_strides, other1_strides, other2_strides); const auto a = val_at_offs(input, offs.y, static_cast(types.x)); const auto b = val_at_offs(other1, offs.z, static_cast(types.y)); const auto c = val_at_offs(other2, offs.w, static_cast(types.z)); store_at_offs( output, offs.x, static_cast(types.w), static_cast(f(a, b, c))); } template , typename T0 = T> kernel void ternary_dense( device result_of* out [[buffer(0)]], constant T0* input [[buffer(1)]], constant T* other1 [[buffer(2)]], constant T* other2 [[buffer(3)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; out[tid] = static_cast( f(om_t(input[tid]), om_t(other1[tid]), om_t(other2[tid]))); } template kernel void ternary_dense_cast( device void* out [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other1 [[buffer(2)]], constant void* other2 [[buffer(3)]], constant uint4& sizes [[buffer(4)]], constant uint4& types [[buffer(5)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto a = val_at_offs(input, tid * sizes.x, static_cast(types.x)); const auto b = val_at_offs( other1, tid * sizes.y, static_cast(types.y)); const auto c = val_at_offs( other2, tid * sizes.z, static_cast(types.z)); store_at_offs( out, tid * sizes.w, static_cast(types.w), static_cast(f(a, b, c))); } // Alpha flavors of the ternary kernels, for ops that carry a scalar next to // the three tensors (addcmul/addcdiv's `value`). The scalar is bound right // after the tensors, at buffer 4, which shifts shapes and strides by one // compared to the plain flavors above. template kernel void ternary_alpha_strided( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other1 [[buffer(2)]], constant void* other2 [[buffer(3)]], constant A& alpha [[buffer(4)]], constant long* sizes [[buffer(5)]], constant long* output_strides [[buffer(6)]], constant long* input_strides [[buffer(7)]], constant long* other1_strides [[buffer(8)]], constant long* other2_strides [[buffer(9)]], constant uint& ndim [[buffer(10)]], uint3 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto offs = ternary_offsets( thread_pos, sizes, ndim, output_strides, input_strides, other1_strides, other2_strides); const auto a = val_at_offs(input, offs.y); const auto b = val_at_offs(other1, offs.z); const auto c = val_at_offs(other2, offs.w); ref_at_offs(output, offs.x) = static_cast(f(om_t(a), om_t(b), om_t(c), alpha)); } template > kernel void ternary_alpha_strided_cast( device void* output [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other1 [[buffer(2)]], constant void* other2 [[buffer(3)]], constant A& alpha [[buffer(4)]], constant long* sizes [[buffer(5)]], constant long* output_strides [[buffer(6)]], constant long* input_strides [[buffer(7)]], constant long* other1_strides [[buffer(8)]], constant long* other2_strides [[buffer(9)]], constant uint& ndim [[buffer(10)]], constant uint4& types [[buffer(11)]], uint3 thread_pos [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto offs = ternary_offsets( thread_pos, sizes, ndim, output_strides, input_strides, other1_strides, other2_strides); const auto a = val_at_offs(input, offs.y, static_cast(types.x)); const auto b = val_at_offs(other1, offs.z, static_cast(types.y)); const auto c = val_at_offs(other2, offs.w, static_cast(types.z)); store_at_offs( output, offs.x, static_cast(types.w), static_cast(f(a, b, c, alpha))); } template > kernel void ternary_alpha_dense( device result_of* out [[buffer(0)]], constant T* input [[buffer(1)]], constant T* other1 [[buffer(2)]], constant T* other2 [[buffer(3)]], constant A& alpha [[buffer(4)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; out[tid] = static_cast( f(om_t(input[tid]), om_t(other1[tid]), om_t(other2[tid]), alpha)); } template kernel void ternary_alpha_dense_cast( device void* out [[buffer(0)]], constant void* input [[buffer(1)]], constant void* other1 [[buffer(2)]], constant void* other2 [[buffer(3)]], constant A& alpha [[buffer(4)]], constant uint4& sizes [[buffer(5)]], constant uint4& types [[buffer(6)]], uint tid [[thread_position_in_grid]]) { F f; using res_t = result_of; const auto a = val_at_offs(input, tid * sizes.x, static_cast(types.x)); const auto b = val_at_offs( other1, tid * sizes.y, static_cast(types.y)); const auto c = val_at_offs( other2, tid * sizes.z, static_cast(types.z)); store_at_offs( out, tid * sizes.w, static_cast(types.w), static_cast(f(a, b, c, alpha))); } #define REGISTER_TERNARY_OP_(NAME, DTYPEI, DTYPEO, OMT) \ static_assert( \ ::metal::is_same_v< \ DTYPEO, \ ::c10::metal::result_of>, \ "Output dtype mismatch for ternary op " #NAME " and input " #DTYPEI); \ template [[host_name(#NAME "_strided_" #DTYPEO "_" #DTYPEI)]] kernel void :: \ c10::metal::ternary_strided( \ device void* out, \ constant void* input, \ constant void* other1, \ constant void* other2, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other1_strides, \ constant long* other2_strides, \ constant uint& ndim, \ uint3 tid); \ template [[host_name(#NAME "_strided_cast_" #DTYPEI)]] kernel void ::c10:: \ metal::ternary_strided_cast( \ device void* out, \ constant void* input, \ constant void* other1, \ constant void* other2, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other1_strides, \ constant long* other2_strides, \ constant uint& ndim, \ constant uint4& types, \ uint3 tid); \ template [[host_name(#NAME "_dense_" #DTYPEO "_" #DTYPEI)]] kernel void :: \ c10::metal::ternary_dense( \ device ::c10::metal:: \ result_of * \ out_, \ constant DTYPEI * input_, \ constant DTYPEI * other1_, \ constant DTYPEI * other2_, \ uint tid); \ template [[host_name(#NAME "_dense_cast_" #DTYPEI)]] kernel void ::c10:: \ metal::ternary_dense_cast( \ device void* out_, \ constant void* input, \ constant void* other1, \ constant void* other2, \ constant uint4& sizes, \ constant uint4& types, \ uint tid) // OpMath ternary Op promotes inputs to higher precision type before Functor // call #define REGISTER_OPMATH_TERNARY_OP(NAME, DTYPEI, DTYPEO) \ REGISTER_TERNARY_OP_(NAME, DTYPEI, DTYPEO, ::c10::metal::opmath_t) #define REGISTER_TERNARY_OP(NAME, DTYPEI, DTYPEO) \ REGISTER_TERNARY_OP_(NAME, DTYPEI, DTYPEO, DTYPEI) #define REGISTER_TERNARY_ALPHA_OP_(NAME, DTYPEI, DTYPEA, DTYPEO, OMT) \ static_assert( \ ::metal::is_same_v< \ DTYPEO, \ ::c10::metal:: \ result_of>, \ "Output dtype mismatch for ternary op " #NAME " and input " #DTYPEI); \ template [[host_name(#NAME "_strided_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ ternary_alpha_strided( \ device void* out, \ constant void* input, \ constant void* other1, \ constant void* other2, \ constant DTYPEA& alpha, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other1_strides, \ constant long* other2_strides, \ constant uint& ndim, \ uint3 tid); \ template [[host_name(#NAME "_strided_cast_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ ternary_alpha_strided_cast( \ device void* out, \ constant void* input, \ constant void* other1, \ constant void* other2, \ constant DTYPEA& alpha, \ constant long* sizes, \ constant long* output_strides, \ constant long* input_strides, \ constant long* other1_strides, \ constant long* other2_strides, \ constant uint& ndim, \ constant uint4& types, \ uint3 tid); \ template [[host_name(#NAME "_dense_" #DTYPEO "_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ ternary_alpha_dense( \ device ::c10::metal:: \ result_of * \ out_, \ constant DTYPEI * input_, \ constant DTYPEI * other1_, \ constant DTYPEI * other2_, \ constant DTYPEA & alpha, \ uint tid); \ template [[host_name(#NAME "_dense_cast_" #DTYPEI \ "_" #DTYPEA)]] kernel void ::c10::metal:: \ ternary_alpha_dense_cast( \ device void* out_, \ constant void* input, \ constant void* other1, \ constant void* other2, \ constant DTYPEA& alpha, \ constant uint4& sizes, \ constant uint4& types, \ uint tid) // OpMath ternary alpha op promotes inputs to higher precision type before // Functor call #define REGISTER_OPMATH_TERNARY_ALPHA_OP(NAME, DTYPEI, DTYPEA, DTYPEO) \ REGISTER_TERNARY_ALPHA_OP_( \ NAME, DTYPEI, DTYPEA, DTYPEO, ::c10::metal::opmath_t) #define REGISTER_TERNARY_ALPHA_OP(NAME, DTYPEI, DTYPEA, DTYPEO) \ REGISTER_TERNARY_ALPHA_OP_(NAME, DTYPEI, DTYPEA, DTYPEO, DTYPEI) } // namespace metal } // namespace c10 #else #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)