/* //@HEADER // ************************************************************************ // // Kokkos v. 3.0 // Copyright (2020) National Technology & Engineering // Solutions of Sandia, LLC (NTESS). // // Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // 3. Neither the name of the Corporation nor the names of the // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) // // ************************************************************************ //@HEADER */ /// \file Kokkos_DynRankView.hpp /// \brief Declaration and definition of Kokkos::DynRankView. /// /// This header file declares and defines Kokkos::DynRankView and its /// related nonmember functions. #ifndef KOKKOS_DYNRANKVIEW_HPP #define KOKKOS_DYNRANKVIEW_HPP #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE #define KOKKOS_IMPL_PUBLIC_INCLUDE #define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DYNRANKVIEW #endif #include #include #include namespace Kokkos { template class DynRankView; // forward declare namespace Impl { template struct DynRankDimTraits { enum : size_t { unspecified = KOKKOS_INVALID_INDEX }; // Compute the rank of the view from the nonzero dimension arguments. KOKKOS_INLINE_FUNCTION static size_t computeRank(const size_t N0, const size_t N1, const size_t N2, const size_t N3, const size_t N4, const size_t N5, const size_t N6, const size_t /* N7 */) { return ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified && N1 == unspecified && N0 == unspecified) ? 0 : ((N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified && N1 == unspecified) ? 1 : ((N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified) ? 2 : ((N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified) ? 3 : ((N6 == unspecified && N5 == unspecified && N4 == unspecified) ? 4 : ((N6 == unspecified && N5 == unspecified) ? 5 : ((N6 == unspecified) ? 6 : 7))))))); } // Compute the rank of the view from the nonzero layout arguments. template KOKKOS_INLINE_FUNCTION static size_t computeRank(const Layout& layout) { return computeRank(layout.dimension[0], layout.dimension[1], layout.dimension[2], layout.dimension[3], layout.dimension[4], layout.dimension[5], layout.dimension[6], layout.dimension[7]); } // Extra overload to match that for specialize types v2 template KOKKOS_INLINE_FUNCTION static size_t computeRank( const Kokkos::Impl::ViewCtorProp& /* prop */, const Layout& layout) { return computeRank(layout); } // Create the layout for the rank-7 view. // Non-strided Layout template KOKKOS_INLINE_FUNCTION static std::enable_if_t< (std::is_same::value || std::is_same::value), Layout> createLayout(const Layout& layout) { return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, layout.dimension[1] != unspecified ? layout.dimension[1] : 1, layout.dimension[2] != unspecified ? layout.dimension[2] : 1, layout.dimension[3] != unspecified ? layout.dimension[3] : 1, layout.dimension[4] != unspecified ? layout.dimension[4] : 1, layout.dimension[5] != unspecified ? layout.dimension[5] : 1, layout.dimension[6] != unspecified ? layout.dimension[6] : 1, layout.dimension[7] != unspecified ? layout.dimension[7] : 1); } // LayoutStride template KOKKOS_INLINE_FUNCTION static std::enable_if_t< (std::is_same::value), Layout> createLayout(const Layout& layout) { return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, layout.stride[0], layout.dimension[1] != unspecified ? layout.dimension[1] : 1, layout.stride[1], layout.dimension[2] != unspecified ? layout.dimension[2] : 1, layout.stride[2], layout.dimension[3] != unspecified ? layout.dimension[3] : 1, layout.stride[3], layout.dimension[4] != unspecified ? layout.dimension[4] : 1, layout.stride[4], layout.dimension[5] != unspecified ? layout.dimension[5] : 1, layout.stride[5], layout.dimension[6] != unspecified ? layout.dimension[6] : 1, layout.stride[6], layout.dimension[7] != unspecified ? layout.dimension[7] : 1, layout.stride[7]); } // Extra overload to match that for specialize types template KOKKOS_INLINE_FUNCTION static std::enable_if_t< (std::is_same::value || std::is_same::value || std::is_same::value), typename Traits::array_layout> createLayout(const Kokkos::Impl::ViewCtorProp& /* prop */, const typename Traits::array_layout& layout) { return createLayout(layout); } // Create a view from the given dimension arguments. // This is only necessary because the shmem constructor doesn't take a layout. // NDE shmem View's are not compatible with the added view_alloc value_type // / fad_dim deduction functionality template static ViewType createView(const ViewArg& arg, const size_t N0, const size_t N1, const size_t N2, const size_t N3, const size_t N4, const size_t N5, const size_t N6, const size_t N7) { return ViewType(arg, N0 != unspecified ? N0 : 1, N1 != unspecified ? N1 : 1, N2 != unspecified ? N2 : 1, N3 != unspecified ? N3 : 1, N4 != unspecified ? N4 : 1, N5 != unspecified ? N5 : 1, N6 != unspecified ? N6 : 1, N7 != unspecified ? N7 : 1); } }; // Non-strided Layout template KOKKOS_INLINE_FUNCTION static std::enable_if_t< (std::is_same::value || std::is_same::value) && std::is_integral::value, Layout> reconstructLayout(const Layout& layout, iType dynrank) { return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX); } // LayoutStride template KOKKOS_INLINE_FUNCTION static std::enable_if_t< (std::is_same::value) && std::is_integral::value, Layout> reconstructLayout(const Layout& layout, iType dynrank) { return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, dynrank > 0 ? layout.stride[0] : (0), dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, dynrank > 1 ? layout.stride[1] : (0), dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, dynrank > 2 ? layout.stride[2] : (0), dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, dynrank > 3 ? layout.stride[3] : (0), dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, dynrank > 4 ? layout.stride[4] : (0), dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, dynrank > 5 ? layout.stride[5] : (0), dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, dynrank > 6 ? layout.stride[6] : (0), dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX, dynrank > 7 ? layout.stride[7] : (0)); } /** \brief Debug bounds-checking routines */ // Enhanced debug checking - most infrastructure matches that of functions in // Kokkos_ViewMapping; additional checks for extra arguments beyond rank are 0 template KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( const iType0&, const MapType&) { return true; } template KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( const iType0& rank, const MapType& map, const iType1& i, Args... args) { if (static_cast(R) < rank) { return (size_t(i) < map.extent(R)) && dyn_rank_view_verify_operator_bounds(rank, map, args...); } else if (i != 0) { KOKKOS_IMPL_DO_NOT_USE_PRINTF( "DynRankView Debug Bounds Checking Error: at rank %u\n Extra " "arguments beyond the rank must be zero \n", R); return (false) && dyn_rank_view_verify_operator_bounds(rank, map, args...); } else { return (true) && dyn_rank_view_verify_operator_bounds(rank, map, args...); } } template inline void dyn_rank_view_error_operator_bounds(char*, int, const MapType&) {} template inline void dyn_rank_view_error_operator_bounds(char* buf, int len, const MapType& map, const iType& i, Args... args) { const int n = snprintf( buf, len, " %ld < %ld %c", static_cast(i), static_cast(map.extent(R)), (sizeof...(Args) ? ',' : ')')); dyn_rank_view_error_operator_bounds(buf + n, len - n, map, args...); } // op_rank = rank of the operator version that was called template KOKKOS_INLINE_FUNCTION void dyn_rank_view_verify_operator_bounds( const iType0& op_rank, const iType1& rank, const Kokkos::Impl::SharedAllocationTracker& tracker, const MapType& map, Args... args) { if (static_cast(rank) > op_rank) { Kokkos::abort( "DynRankView Bounds Checking Error: Need at least rank arguments to " "the operator()"); } if (!dyn_rank_view_verify_operator_bounds<0>(rank, map, args...)) { KOKKOS_IF_ON_HOST( (enum {LEN = 1024}; char buffer[LEN]; const std::string label = tracker.template get_label(); int n = snprintf(buffer, LEN, "DynRankView bounds error of view %s (", label.c_str()); dyn_rank_view_error_operator_bounds<0>(buffer + n, LEN - n, map, args...); Kokkos::Impl::throw_runtime_exception(std::string(buffer));)) KOKKOS_IF_ON_DEVICE( ((void)tracker; Kokkos::abort("DynRankView bounds error");)) } } /** \brief Assign compatible default mappings */ struct ViewToDynRankViewTag {}; } // namespace Impl namespace Impl { template class ViewMapping< DstTraits, SrcTraits, std::enable_if_t<(std::is_same::value && std::is_void::value && std::is_void::value && (std::is_same::value || ((std::is_same::value || std::is_same::value || std::is_same::value) && (std::is_same::value || std::is_same::value || std::is_same::value)))), Kokkos::Impl::ViewToDynRankViewTag>> { private: enum { is_assignable_value_type = std::is_same::value || std::is_same::value }; enum { is_assignable_layout = std::is_same::value || std::is_same::value }; public: enum { is_assignable = is_assignable_value_type && is_assignable_layout }; using DstType = ViewMapping; using SrcType = ViewMapping; template KOKKOS_INLINE_FUNCTION static void assign( Kokkos::DynRankView& dst, const Kokkos::View& src) { static_assert( is_assignable_value_type, "View assignment must have same value type or const = non-const"); static_assert( is_assignable_layout, "View assignment must have compatible layout or have rank <= 1"); // Removed dimension checks... using dst_offset_type = typename DstType::offset_type; dst.m_map.m_impl_offset = dst_offset_type( std::integral_constant(), src.layout()); // Check this for integer input1 for padding, etc dst.m_map.m_impl_handle = Kokkos::Impl::ViewDataHandle::assign( src.m_map.m_impl_handle, src.m_track.m_tracker); dst.m_track.assign(src.m_track.m_tracker, DstTraits::is_managed); dst.m_rank = src.Rank; } }; } // namespace Impl /* \class DynRankView * \brief Container that creates a Kokkos view with rank determined at runtime. * Essentially this is a rank 7 view * * Changes from View * 1. The rank of the DynRankView is returned by the method rank() * 2. Max rank of a DynRankView is 7 * 3. subview called with 'subview(...)' or 'subdynrankview(...)' (backward * compatibility) * 4. Every subview is returned with LayoutStride * 5. Copy and Copy-Assign View to DynRankView * 6. deep_copy between Views and DynRankViews * 7. rank( view ); returns the rank of View or DynRankView * */ template struct is_dyn_rank_view : public std::false_type {}; template struct is_dyn_rank_view> : public std::true_type { }; template class DynRankView : public ViewTraits { static_assert(!std::is_array::value && !std::is_pointer::value, "Cannot template DynRankView with array or pointer datatype - " "must be pod"); private: template friend class DynRankView; template friend class Kokkos::Impl::ViewMapping; public: using drvtraits = ViewTraits; using view_type = View; using traits = ViewTraits; private: using map_type = Kokkos::Impl::ViewMapping; using track_type = Kokkos::Impl::SharedAllocationTracker; track_type m_track; map_type m_map; unsigned m_rank; public: KOKKOS_INLINE_FUNCTION view_type& DownCast() const { return (view_type&)(*this); } KOKKOS_INLINE_FUNCTION const view_type& ConstDownCast() const { return (const view_type&)(*this); } // Types below - at least the HostMirror requires the value_type, NOT the rank // 7 data_type of the traits /** \brief Compatible view of array of scalar types */ using array_type = DynRankView< typename drvtraits::scalar_array_type, typename drvtraits::array_layout, typename drvtraits::device_type, typename drvtraits::memory_traits>; /** \brief Compatible view of const data type */ using const_type = DynRankView< typename drvtraits::const_data_type, typename drvtraits::array_layout, typename drvtraits::device_type, typename drvtraits::memory_traits>; /** \brief Compatible view of non-const data type */ using non_const_type = DynRankView< typename drvtraits::non_const_data_type, typename drvtraits::array_layout, typename drvtraits::device_type, typename drvtraits::memory_traits>; /** \brief Compatible HostMirror view */ using HostMirror = DynRankView; //---------------------------------------- // Domain rank and extents // enum { Rank = map_type::Rank }; //Will be dyn rank of 7 always, keep the // enum? template KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< std::is_integral::value, size_t> extent(const iType& r) const { return m_map.extent(r); } template KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< std::is_integral::value, int> extent_int(const iType& r) const { return static_cast(m_map.extent(r)); } KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() const; //---------------------------------------- /* Deprecate all 'dimension' functions in favor of * ISO/C++ vocabulary 'extent'. */ KOKKOS_INLINE_FUNCTION constexpr size_t size() const { return m_map.extent(0) * m_map.extent(1) * m_map.extent(2) * m_map.extent(3) * m_map.extent(4) * m_map.extent(5) * m_map.extent(6) * m_map.extent(7); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { return m_map.stride_0(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { return m_map.stride_1(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { return m_map.stride_2(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { return m_map.stride_3(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { return m_map.stride_4(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { return m_map.stride_5(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { return m_map.stride_6(); } KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { return m_map.stride_7(); } template KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { m_map.stride(s); } //---------------------------------------- // Range span is the span which contains all members. using reference_type = typename map_type::reference_type; using pointer_type = typename map_type::pointer_type; enum { reference_type_is_lvalue_reference = std::is_lvalue_reference::value }; KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { return m_map.span_is_contiguous(); } KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { return m_map.data(); } KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const { return (m_map.data() != nullptr); } //---------------------------------------- // Allow specializations to query their specialized map KOKKOS_INLINE_FUNCTION const Kokkos::Impl::ViewMapping& impl_map() const { return m_map; } //---------------------------------------- private: enum { is_layout_left = std::is_same::value, is_layout_right = std::is_same::value, is_layout_stride = std::is_same::value, is_default_map = std::is_void::value && (is_layout_left || is_layout_right || is_layout_stride) }; // Bounds checking macros #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) // rank of the calling operator - included as first argument in ARG #define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ Kokkos::Impl::runtime_check_memory_access_violation< \ typename traits::memory_space>( \ "Kokkos::DynRankView ERROR: attempt to access inaccessible memory " \ "space"); \ Kokkos::Impl::dyn_rank_view_verify_operator_bounds< \ typename traits::memory_space> \ ARG; #else #define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ Kokkos::Impl::runtime_check_memory_access_violation< \ typename traits::memory_space>( \ "Kokkos::DynRankView ERROR: attempt to access inaccessible memory " \ "space"); #endif public: KOKKOS_INLINE_FUNCTION constexpr unsigned rank() const { return m_rank; } // operators () // Rank 0 KOKKOS_INLINE_FUNCTION reference_type operator()() const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) return impl_map().reference(); // return m_map.reference(0,0,0,0,0,0,0); } // Rank 1 // This assumes a contiguous underlying memory (i.e. no padding, no // striding...) template KOKKOS_INLINE_FUNCTION std::enable_if_t< std::is_same::value && std::is_integral::value, reference_type> operator[](const iType& i0) const { // Phalanx is violating this, since they use the operator to access ALL // elements in the allocation KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , // this->rank(), m_track, m_map) ) return data()[i0]; } // This assumes a contiguous underlying memory (i.e. no padding, no // striding... AND a Trilinos/Sacado scalar type ) template KOKKOS_INLINE_FUNCTION std::enable_if_t< !std::is_same::value && std::is_integral::value, reference_type> operator[](const iType& i0) const { // auto map = impl_map(); const size_t dim_scalar = m_map.dimension_scalar(); const size_t bytes = this->span() / dim_scalar; using tmp_view_type = Kokkos::View< DataType*, typename traits::array_layout, typename traits::device_type, Kokkos::MemoryTraits>; tmp_view_type rankone_view(this->data(), bytes, dim_scalar); return rankone_view(i0); } // Rank 1 parenthesis template KOKKOS_INLINE_FUNCTION std::enable_if_t<(std::is_void::value && std::is_integral::value), reference_type> operator()(const iType& i0) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) return m_map.reference(i0); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> operator()(const iType& i0) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) return m_map.reference(i0, 0, 0, 0, 0, 0, 0); } // Rank 2 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) return m_map.reference(i0, i1); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) return m_map.reference(i0, i1, 0, 0, 0, 0, 0); } // Rank 3 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3, this->rank(), m_track, m_map, i0, i1, i2)) return m_map.reference(i0, i1, i2); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3, this->rank(), m_track, m_map, i0, i1, i2)) return m_map.reference(i0, i1, i2, 0, 0, 0, 0); } // Rank 4 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) return m_map.reference(i0, i1, i2, i3); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) return m_map.reference(i0, i1, i2, i3, 0, 0, 0); } // Rank 5 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) return m_map.reference(i0, i1, i2, i3, i4); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) return m_map.reference(i0, i1, i2, i3, i4, 0, 0); } // Rank 6 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4, const iType5& i5) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) return m_map.reference(i0, i1, i2, i3, i4, i5); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4, const iType5& i5) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) return m_map.reference(i0, i1, i2, i3, i4, i5, 0); } // Rank 7 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> operator()(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4, const iType5& i5, const iType6& i6) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) return m_map.reference(i0, i1, i2, i3, i4, i5, i6); } // Rank 0 KOKKOS_INLINE_FUNCTION reference_type access() const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) return impl_map().reference(); // return m_map.reference(0,0,0,0,0,0,0); } // Rank 1 // Rank 1 parenthesis template KOKKOS_INLINE_FUNCTION std::enable_if_t<(std::is_void::value && std::is_integral::value), reference_type> access(const iType& i0) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) return m_map.reference(i0); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> access(const iType& i0) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) return m_map.reference(i0, 0, 0, 0, 0, 0, 0); } // Rank 2 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) return m_map.reference(i0, i1); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) return m_map.reference(i0, i1, 0, 0, 0, 0, 0); } // Rank 3 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3, this->rank(), m_track, m_map, i0, i1, i2)) return m_map.reference(i0, i1, i2); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3, this->rank(), m_track, m_map, i0, i1, i2)) return m_map.reference(i0, i1, i2, 0, 0, 0, 0); } // Rank 4 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) return m_map.reference(i0, i1, i2, i3); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) return m_map.reference(i0, i1, i2, i3, 0, 0, 0); } // Rank 5 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) return m_map.reference(i0, i1, i2, i3, i4); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) return m_map.reference(i0, i1, i2, i3, i4, 0, 0); } // Rank 6 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_void::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4, const iType5& i5) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) return m_map.reference(i0, i1, i2, i3, i4, i5); } template KOKKOS_INLINE_FUNCTION std::enable_if_t::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4, const iType5& i5) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) return m_map.reference(i0, i1, i2, i3, i4, i5, 0); } // Rank 7 template KOKKOS_INLINE_FUNCTION std::enable_if_t< (std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type> access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, const iType4& i4, const iType5& i5, const iType6& i6) const { KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) return m_map.reference(i0, i1, i2, i3, i4, i5, i6); } #undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY //---------------------------------------- // Standard constructor, destructor, and assignment operators... KOKKOS_DEFAULTED_FUNCTION ~DynRankView() = default; KOKKOS_INLINE_FUNCTION DynRankView() : m_track(), m_map(), m_rank() {} // Default ctor KOKKOS_INLINE_FUNCTION DynRankView(const DynRankView& rhs) : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} KOKKOS_INLINE_FUNCTION DynRankView(DynRankView&& rhs) : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} KOKKOS_INLINE_FUNCTION DynRankView& operator=(const DynRankView& rhs) { m_track = rhs.m_track; m_map = rhs.m_map; m_rank = rhs.m_rank; return *this; } KOKKOS_INLINE_FUNCTION DynRankView& operator=(DynRankView&& rhs) { m_track = rhs.m_track; m_map = rhs.m_map; m_rank = rhs.m_rank; return *this; } //---------------------------------------- // Compatible view copy constructor and assignment // may assign unmanaged from managed. template KOKKOS_INLINE_FUNCTION DynRankView(const DynRankView& rhs) : m_track(rhs.m_track, traits::is_managed), m_map(), m_rank(rhs.m_rank) { using SrcTraits = typename DynRankView::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible DynRankView copy construction"); Mapping::assign(m_map, rhs.m_map, rhs.m_track); } template KOKKOS_INLINE_FUNCTION DynRankView& operator=( const DynRankView& rhs) { using SrcTraits = typename DynRankView::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible DynRankView copy construction"); Mapping::assign(m_map, rhs.m_map, rhs.m_track); m_track.assign(rhs.m_track, traits::is_managed); m_rank = rhs.rank(); return *this; } // Copy/Assign View to DynRankView template KOKKOS_INLINE_FUNCTION DynRankView(const View& rhs) : m_track(), m_map(), m_rank(rhs.Rank) { using SrcTraits = typename View::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible View to DynRankView copy construction"); Mapping::assign(*this, rhs); } template KOKKOS_INLINE_FUNCTION DynRankView& operator=(const View& rhs) { using SrcTraits = typename View::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, "Incompatible View to DynRankView copy assignment"); Mapping::assign(*this, rhs); return *this; } //---------------------------------------- // Allocation tracking properties KOKKOS_INLINE_FUNCTION int use_count() const { return m_track.use_count(); } inline const std::string label() const { return m_track.template get_label(); } //---------------------------------------- // Allocation according to allocation properties and array layout // unused arg_layout dimensions must be set to KOKKOS_INVALID_INDEX so that // rank deduction can properly take place template explicit inline DynRankView( const Kokkos::Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout) : m_track(), m_map(), m_rank(Impl::DynRankDimTraits:: template computeRank( arg_prop, arg_layout)) { // Append layout and spaces if not input using alloc_prop_input = Kokkos::Impl::ViewCtorProp; // use 'std::integral_constant' for non-types // to avoid duplicate class error. using alloc_prop = Kokkos::Impl::ViewCtorProp< P..., std::conditional_t, std::string>, std::conditional_t, typename traits::device_type::memory_space>, std::conditional_t, typename traits::device_type::execution_space>>; static_assert(traits::is_managed, "View allocation constructor requires managed memory"); if (alloc_prop::initialize && !alloc_prop::execution_space::impl_is_initialized()) { // If initializing view data then // the execution space must be initialized. Kokkos::Impl::throw_runtime_exception( "Constructing DynRankView and initializing data with uninitialized " "execution space"); } // Copy the input allocation properties with possibly defaulted properties alloc_prop prop_copy(arg_prop); //------------------------------------------------------------ #if defined(KOKKOS_ENABLE_CUDA) // If allocating in CudaUVMSpace must fence before and after // the allocation to protect against possible concurrent access // on the CPU and the GPU. // Fence using the trait's executon space (which will be Kokkos::Cuda) // to avoid incomplete type errors from usng Kokkos::Cuda directly. if (std::is_same::value) { typename traits::device_type::memory_space::execution_space().fence( "Kokkos::DynRankView<>::DynRankView: fence before UVM allocation"); } #endif //------------------------------------------------------------ Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( prop_copy, Impl::DynRankDimTraits:: template createLayout(arg_prop, arg_layout), Impl::ViewCtorProp::has_execution_space); //------------------------------------------------------------ #if defined(KOKKOS_ENABLE_CUDA) if (std::is_same::value) { typename traits::device_type::memory_space::execution_space().fence( "Kokkos::DynRankView<>::DynRankView: fence after UVM allocation"); } #endif //------------------------------------------------------------ // Setup and initialization complete, start tracking m_track.assign_allocated_record_to_uninitialized(record); } // Wrappers template explicit KOKKOS_INLINE_FUNCTION DynRankView( const Kokkos::Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout) : m_track() // No memory tracking , m_map(arg_prop, Impl::DynRankDimTraits:: template createLayout(arg_prop, arg_layout)), m_rank(Impl::DynRankDimTraits:: template computeRank( arg_prop, arg_layout)) { static_assert( std::is_same::pointer_type>::value, "Constructing DynRankView to wrap user memory must supply matching " "pointer type"); } //---------------------------------------- // Constructor(s) // Simple dimension-only layout template explicit inline DynRankView( const Kokkos::Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, size_t> const arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, const size_t arg_N4 = KOKKOS_INVALID_INDEX, const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) : DynRankView(arg_prop, typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) {} template explicit KOKKOS_INLINE_FUNCTION DynRankView( const Kokkos::Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, size_t> const arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, const size_t arg_N4 = KOKKOS_INVALID_INDEX, const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) : DynRankView(arg_prop, typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) {} // Allocate with label and layout template explicit inline DynRankView( const Label& arg_label, std::enable_if_t::value, typename traits::array_layout> const& arg_layout) : DynRankView(Kokkos::Impl::ViewCtorProp(arg_label), arg_layout) {} // Allocate label and layout, must disambiguate from subview constructor template explicit inline DynRankView( const Label& arg_label, std::enable_if_t::value, const size_t> arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, const size_t arg_N4 = KOKKOS_INVALID_INDEX, const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) : DynRankView( Kokkos::Impl::ViewCtorProp(arg_label), typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) {} //---------------------------------------- // Memory span required to wrap these dimensions. static constexpr size_t required_allocation_size( const size_t arg_N0 = 0, const size_t arg_N1 = 0, const size_t arg_N2 = 0, const size_t arg_N3 = 0, const size_t arg_N4 = 0, const size_t arg_N5 = 0, const size_t arg_N6 = 0, const size_t arg_N7 = 0) { return map_type::memory_span(typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); } explicit KOKKOS_INLINE_FUNCTION DynRankView( pointer_type arg_ptr, const size_t arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, const size_t arg_N4 = KOKKOS_INVALID_INDEX, const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) : DynRankView(Kokkos::Impl::ViewCtorProp(arg_ptr), arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7) {} explicit KOKKOS_INLINE_FUNCTION DynRankView( pointer_type arg_ptr, typename traits::array_layout& arg_layout) : DynRankView(Kokkos::Impl::ViewCtorProp(arg_ptr), arg_layout) {} //---------------------------------------- // Shared scratch memory constructor static inline size_t shmem_size(const size_t arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, const size_t arg_N4 = KOKKOS_INVALID_INDEX, const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) { const size_t num_passed_args = (arg_N0 != KOKKOS_INVALID_INDEX) + (arg_N1 != KOKKOS_INVALID_INDEX) + (arg_N2 != KOKKOS_INVALID_INDEX) + (arg_N3 != KOKKOS_INVALID_INDEX) + (arg_N4 != KOKKOS_INVALID_INDEX) + (arg_N5 != KOKKOS_INVALID_INDEX) + (arg_N6 != KOKKOS_INVALID_INDEX) + (arg_N7 != KOKKOS_INVALID_INDEX); if (std::is_void::value && num_passed_args != traits::rank_dynamic) { Kokkos::abort( "Kokkos::View::shmem_size() rank_dynamic != number of arguments.\n"); } {} return map_type::memory_span(typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); } explicit KOKKOS_INLINE_FUNCTION DynRankView( const typename traits::execution_space::scratch_memory_space& arg_space, const typename traits::array_layout& arg_layout) : DynRankView( Kokkos::Impl::ViewCtorProp( reinterpret_cast( arg_space.get_shmem(map_type::memory_span( Impl::DynRankDimTraits:: createLayout(arg_layout) // is this correct? )))), arg_layout) {} explicit KOKKOS_INLINE_FUNCTION DynRankView( const typename traits::execution_space::scratch_memory_space& arg_space, const size_t arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, const size_t arg_N4 = KOKKOS_INVALID_INDEX, const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) : DynRankView( Kokkos::Impl::ViewCtorProp( reinterpret_cast( arg_space.get_shmem(map_type::memory_span( Impl::DynRankDimTraits:: createLayout(typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)))))), typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) {} }; template KOKKOS_INLINE_FUNCTION constexpr unsigned rank( const DynRankView& DRV) { return DRV.rank(); } // needed for transition to common constexpr method in view and dynrankview // to return rank //---------------------------------------------------------------------------- // Subview mapping. // Deduce destination view type from source view traits and subview arguments namespace Impl { struct DynRankSubviewTag {}; } // namespace Impl namespace Impl { template class ViewMapping< std::enable_if_t<(std::is_void::value && (std::is_same::value || std::is_same::value || std::is_same::value)), Kokkos::Impl::DynRankSubviewTag>, SrcTraits, Args...> { private: enum { RZ = false, R0 = bool(is_integral_extent<0, Args...>::value), R1 = bool(is_integral_extent<1, Args...>::value), R2 = bool(is_integral_extent<2, Args...>::value), R3 = bool(is_integral_extent<3, Args...>::value), R4 = bool(is_integral_extent<4, Args...>::value), R5 = bool(is_integral_extent<5, Args...>::value), R6 = bool(is_integral_extent<6, Args...>::value) }; enum { rank = unsigned(R0) + unsigned(R1) + unsigned(R2) + unsigned(R3) + unsigned(R4) + unsigned(R5) + unsigned(R6) }; using array_layout = Kokkos::LayoutStride; using value_type = typename SrcTraits::value_type; using data_type = value_type*******; public: using traits_type = Kokkos::ViewTraits; using type = Kokkos::View; template struct apply { static_assert(Kokkos::is_memory_traits::value, ""); using traits_type = Kokkos::ViewTraits; using type = Kokkos::View; }; using dimension = typename SrcTraits::dimension; template struct ExtentGenerator { KOKKOS_INLINE_FUNCTION static SubviewExtents<7, rank> generator( const dimension& dim, Arg0 arg0 = Arg0(), Arg1 arg1 = Arg1(), Arg2 arg2 = Arg2(), Arg3 arg3 = Arg3(), Arg4 arg4 = Arg4(), Arg5 arg5 = Arg5(), Arg6 arg6 = Arg6()) { return SubviewExtents<7, rank>(dim, arg0, arg1, arg2, arg3, arg4, arg5, arg6); } }; using ret_type = Kokkos::DynRankView; template KOKKOS_INLINE_FUNCTION static ret_type subview( const unsigned src_rank, Kokkos::DynRankView const& src, Args... args) { using DstType = ViewMapping; using DstDimType = std::conditional_t< (rank == 0), ViewDimension<>, std::conditional_t< (rank == 1), ViewDimension<0>, std::conditional_t< (rank == 2), ViewDimension<0, 0>, std::conditional_t< (rank == 3), ViewDimension<0, 0, 0>, std::conditional_t< (rank == 4), ViewDimension<0, 0, 0, 0>, std::conditional_t< (rank == 5), ViewDimension<0, 0, 0, 0, 0>, std::conditional_t< (rank == 6), ViewDimension<0, 0, 0, 0, 0, 0>, ViewDimension<0, 0, 0, 0, 0, 0, 0>>>>>>>>; using dst_offset_type = ViewOffset; using dst_handle_type = typename DstType::handle_type; ret_type dst; const SubviewExtents<7, rank> extents = ExtentGenerator::generator( src.m_map.m_impl_offset.m_dim, args...); dst_offset_type tempdst(src.m_map.m_impl_offset, extents); dst.m_track = src.m_track; dst.m_map.m_impl_offset.m_dim.N0 = tempdst.m_dim.N0; dst.m_map.m_impl_offset.m_dim.N1 = tempdst.m_dim.N1; dst.m_map.m_impl_offset.m_dim.N2 = tempdst.m_dim.N2; dst.m_map.m_impl_offset.m_dim.N3 = tempdst.m_dim.N3; dst.m_map.m_impl_offset.m_dim.N4 = tempdst.m_dim.N4; dst.m_map.m_impl_offset.m_dim.N5 = tempdst.m_dim.N5; dst.m_map.m_impl_offset.m_dim.N6 = tempdst.m_dim.N6; dst.m_map.m_impl_offset.m_stride.S0 = tempdst.m_stride.S0; dst.m_map.m_impl_offset.m_stride.S1 = tempdst.m_stride.S1; dst.m_map.m_impl_offset.m_stride.S2 = tempdst.m_stride.S2; dst.m_map.m_impl_offset.m_stride.S3 = tempdst.m_stride.S3; dst.m_map.m_impl_offset.m_stride.S4 = tempdst.m_stride.S4; dst.m_map.m_impl_offset.m_stride.S5 = tempdst.m_stride.S5; dst.m_map.m_impl_offset.m_stride.S6 = tempdst.m_stride.S6; dst.m_map.m_impl_handle = dst_handle_type(src.m_map.m_impl_handle + src.m_map.m_impl_offset( extents.domain_offset(0), extents.domain_offset(1), extents.domain_offset(2), extents.domain_offset(3), extents.domain_offset(4), extents.domain_offset(5), extents.domain_offset(6))); dst.m_rank = (src_rank > 0 ? unsigned(R0) : 0) + (src_rank > 1 ? unsigned(R1) : 0) + (src_rank > 2 ? unsigned(R2) : 0) + (src_rank > 3 ? unsigned(R3) : 0) + (src_rank > 4 ? unsigned(R4) : 0) + (src_rank > 5 ? unsigned(R5) : 0) + (src_rank > 6 ? unsigned(R6) : 0); return dst; } }; } // namespace Impl template using Subdynrankview = typename Kokkos::Impl::ViewMapping::ret_type; template KOKKOS_INLINE_FUNCTION Subdynrankview, Args...> subdynrankview(const Kokkos::DynRankView& src, Args... args) { if (src.rank() > sizeof...(Args)) // allow sizeof...(Args) >= src.rank(), // ignore the remaining args { Kokkos::abort( "subdynrankview: num of args must be >= rank of the source " "DynRankView"); } using metafcn = Kokkos::Impl::ViewMapping, Args...>; return metafcn::subview(src.rank(), src, args...); } // Wrapper to allow subview function name template KOKKOS_INLINE_FUNCTION Subdynrankview, Args...> subview(const Kokkos::DynRankView& src, Args... args) { return subdynrankview(src, args...); } } // namespace Kokkos namespace Kokkos { // overload == and != template KOKKOS_INLINE_FUNCTION bool operator==(const DynRankView& lhs, const DynRankView& rhs) { // Same data, layout, dimensions using lhs_traits = ViewTraits; using rhs_traits = ViewTraits; return std::is_same::value && std::is_same::value && std::is_same::value && lhs.rank() == rhs.rank() && lhs.data() == rhs.data() && lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7); } template KOKKOS_INLINE_FUNCTION bool operator!=(const DynRankView& lhs, const DynRankView& rhs) { return !(operator==(lhs, rhs)); } } // namespace Kokkos //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { namespace Impl { template struct DynRankViewFill { using const_value_type = typename OutputView::traits::const_value_type; const OutputView output; const_value_type input; KOKKOS_INLINE_FUNCTION void operator()(const size_t i0) const { const size_t n1 = output.extent(1); const size_t n2 = output.extent(2); const size_t n3 = output.extent(3); const size_t n4 = output.extent(4); const size_t n5 = output.extent(5); const size_t n6 = output.extent(6); for (size_t i1 = 0; i1 < n1; ++i1) { for (size_t i2 = 0; i2 < n2; ++i2) { for (size_t i3 = 0; i3 < n3; ++i3) { for (size_t i4 = 0; i4 < n4; ++i4) { for (size_t i5 = 0; i5 < n5; ++i5) { for (size_t i6 = 0; i6 < n6; ++i6) { output.access(i0, i1, i2, i3, i4, i5, i6) = input; } } } } } } } DynRankViewFill(const OutputView& arg_out, const_value_type& arg_in) : output(arg_out), input(arg_in) { using execution_space = typename OutputView::execution_space; using Policy = Kokkos::RangePolicy; Kokkos::parallel_for("Kokkos::DynRankViewFill", Policy(0, output.extent(0)), *this); } }; template struct DynRankViewFill> { DynRankViewFill(const OutputView& dst, const typename OutputView::const_value_type& src) { Kokkos::Impl::DeepCopy( dst.data(), &src, sizeof(typename OutputView::const_value_type)); } }; template struct DynRankViewRemap { const OutputView output; const InputView input; const size_t n0; const size_t n1; const size_t n2; const size_t n3; const size_t n4; const size_t n5; const size_t n6; const size_t n7; DynRankViewRemap(const ExecSpace& exec_space, const OutputView& arg_out, const InputView& arg_in) : output(arg_out), input(arg_in), n0(std::min((size_t)arg_out.extent(0), (size_t)arg_in.extent(0))), n1(std::min((size_t)arg_out.extent(1), (size_t)arg_in.extent(1))), n2(std::min((size_t)arg_out.extent(2), (size_t)arg_in.extent(2))), n3(std::min((size_t)arg_out.extent(3), (size_t)arg_in.extent(3))), n4(std::min((size_t)arg_out.extent(4), (size_t)arg_in.extent(4))), n5(std::min((size_t)arg_out.extent(5), (size_t)arg_in.extent(5))), n6(std::min((size_t)arg_out.extent(6), (size_t)arg_in.extent(6))), n7(std::min((size_t)arg_out.extent(7), (size_t)arg_in.extent(7))) { using Policy = Kokkos::RangePolicy; Kokkos::parallel_for("Kokkos::DynRankViewRemap", Policy(exec_space, 0, n0), *this); } DynRankViewRemap(const OutputView& arg_out, const InputView& arg_in) : output(arg_out), input(arg_in), n0(std::min((size_t)arg_out.extent(0), (size_t)arg_in.extent(0))), n1(std::min((size_t)arg_out.extent(1), (size_t)arg_in.extent(1))), n2(std::min((size_t)arg_out.extent(2), (size_t)arg_in.extent(2))), n3(std::min((size_t)arg_out.extent(3), (size_t)arg_in.extent(3))), n4(std::min((size_t)arg_out.extent(4), (size_t)arg_in.extent(4))), n5(std::min((size_t)arg_out.extent(5), (size_t)arg_in.extent(5))), n6(std::min((size_t)arg_out.extent(6), (size_t)arg_in.extent(6))), n7(std::min((size_t)arg_out.extent(7), (size_t)arg_in.extent(7))) { using Policy = Kokkos::RangePolicy; Kokkos::parallel_for("Kokkos::DynRankViewRemap", Policy(0, n0), *this); } KOKKOS_INLINE_FUNCTION void operator()(const size_t i0) const { for (size_t i1 = 0; i1 < n1; ++i1) { for (size_t i2 = 0; i2 < n2; ++i2) { for (size_t i3 = 0; i3 < n3; ++i3) { for (size_t i4 = 0; i4 < n4; ++i4) { for (size_t i5 = 0; i5 < n5; ++i5) { for (size_t i6 = 0; i6 < n6; ++i6) { output.access(i0, i1, i2, i3, i4, i5, i6) = input.access(i0, i1, i2, i3, i4, i5, i6); } } } } } } } }; } /* namespace Impl */ } /* namespace Kokkos */ namespace Kokkos { namespace Impl { /* \brief Returns a View of the requested rank, aliasing the underlying memory, to facilitate implementation of deep_copy() and other routines that are defined on View */ template KOKKOS_FUNCTION auto as_view_of_rank_n(DynRankView v) { if (v.rank() != N) { KOKKOS_IF_ON_HOST( const std::string message = "Converting DynRankView of rank " + std::to_string(v.rank()) + " to a View of mis-matched rank " + std::to_string(N) + "!"; Kokkos::abort(message.c_str());) KOKKOS_IF_ON_DEVICE( Kokkos::abort("Converting DynRankView to a View of mis-matched rank!");) } return View::type, Args...>( v.data(), v.impl_map().layout()); } template void apply_to_view_of_static_rank(Function&& f, DynRankView a) { switch (rank(a)) { case 0: f(as_view_of_rank_n<0>(a)); break; case 1: f(as_view_of_rank_n<1>(a)); break; case 2: f(as_view_of_rank_n<2>(a)); break; case 3: f(as_view_of_rank_n<3>(a)); break; case 4: f(as_view_of_rank_n<4>(a)); break; case 5: f(as_view_of_rank_n<5>(a)); break; case 6: f(as_view_of_rank_n<6>(a)); break; case 7: f(as_view_of_rank_n<7>(a)); break; default: KOKKOS_IF_ON_HOST( Kokkos::abort( std::string( "Trying to apply a function to a view of unexpected rank " + std::to_string(rank(a))) .c_str());) KOKKOS_IF_ON_DEVICE( Kokkos::abort( "Trying to apply a function to a view of unexpected rank");) } } } // namespace Impl template KOKKOS_INLINE_FUNCTION constexpr auto DynRankView::layout() const -> typename traits::array_layout { switch (rank()) { case 0: return Impl::as_view_of_rank_n<0>(*this).layout(); case 1: return Impl::as_view_of_rank_n<1>(*this).layout(); case 2: return Impl::as_view_of_rank_n<2>(*this).layout(); case 3: return Impl::as_view_of_rank_n<3>(*this).layout(); case 4: return Impl::as_view_of_rank_n<4>(*this).layout(); case 5: return Impl::as_view_of_rank_n<5>(*this).layout(); case 6: return Impl::as_view_of_rank_n<6>(*this).layout(); case 7: return Impl::as_view_of_rank_n<7>(*this).layout(); default: KOKKOS_IF_ON_HOST( Kokkos::abort( std::string( "Calling DynRankView::layout on DRV of unexpected rank " + std::to_string(rank())) .c_str());) KOKKOS_IF_ON_DEVICE( Kokkos::abort( "Calling DynRankView::layout on DRV of unexpected rank");) } // control flow should never reach here return m_map.layout(); } /** \brief Deep copy a value from Host memory into a view. */ template inline void deep_copy( const ExecSpace& e, const DynRankView& dst, typename ViewTraits::const_value_type& value, std::enable_if_t::specialize, void>::value>* = nullptr) { static_assert( std::is_same::non_const_value_type, typename ViewTraits::value_type>::value, "deep_copy requires non-const type"); Impl::apply_to_view_of_static_rank( [=](auto view) { deep_copy(e, view, value); }, dst); } template inline void deep_copy( const DynRankView& dst, typename ViewTraits::const_value_type& value, std::enable_if_t::specialize, void>::value>* = nullptr) { Impl::apply_to_view_of_static_rank([=](auto view) { deep_copy(view, value); }, dst); } /** \brief Deep copy into a value in Host memory from a view. */ template inline void deep_copy( const ExecSpace& e, typename ViewTraits::non_const_value_type& dst, const DynRankView& src, std::enable_if_t::specialize, void>::value>* = 0) { deep_copy(e, dst, Impl::as_view_of_rank_n<0>(src)); } template inline void deep_copy( typename ViewTraits::non_const_value_type& dst, const DynRankView& src, std::enable_if_t::specialize, void>::value>* = 0) { deep_copy(dst, Impl::as_view_of_rank_n<0>(src)); } //---------------------------------------------------------------------------- /** \brief A deep copy between views of the default specialization, compatible * type, same rank, same contiguous layout. * * A rank mismatch will error out in the attempt to convert to a View */ template inline void deep_copy( const ExecSpace& exec_space, const DstType& dst, const SrcType& src, std::enable_if_t< (std::is_void::value && std::is_void::value && (Kokkos::is_dyn_rank_view::value || Kokkos::is_dyn_rank_view::value))>* = nullptr) { static_assert( std::is_same::value, "deep_copy requires non-const destination type"); switch (rank(dst)) { case 0: deep_copy(exec_space, Impl::as_view_of_rank_n<0>(dst), Impl::as_view_of_rank_n<0>(src)); break; case 1: deep_copy(exec_space, Impl::as_view_of_rank_n<1>(dst), Impl::as_view_of_rank_n<1>(src)); break; case 2: deep_copy(exec_space, Impl::as_view_of_rank_n<2>(dst), Impl::as_view_of_rank_n<2>(src)); break; case 3: deep_copy(exec_space, Impl::as_view_of_rank_n<3>(dst), Impl::as_view_of_rank_n<3>(src)); break; case 4: deep_copy(exec_space, Impl::as_view_of_rank_n<4>(dst), Impl::as_view_of_rank_n<4>(src)); break; case 5: deep_copy(exec_space, Impl::as_view_of_rank_n<5>(dst), Impl::as_view_of_rank_n<5>(src)); break; case 6: deep_copy(exec_space, Impl::as_view_of_rank_n<6>(dst), Impl::as_view_of_rank_n<6>(src)); break; case 7: deep_copy(exec_space, Impl::as_view_of_rank_n<7>(dst), Impl::as_view_of_rank_n<7>(src)); break; default: Kokkos::Impl::throw_runtime_exception( "Calling DynRankView deep_copy with a view of unexpected rank " + std::to_string(rank(dst))); } } template inline void deep_copy( const DstType& dst, const SrcType& src, std::enable_if_t< (std::is_void::value && std::is_void::value && (Kokkos::is_dyn_rank_view::value || Kokkos::is_dyn_rank_view::value))>* = nullptr) { static_assert( std::is_same::value, "deep_copy requires non-const destination type"); switch (rank(dst)) { case 0: deep_copy(Impl::as_view_of_rank_n<0>(dst), Impl::as_view_of_rank_n<0>(src)); break; case 1: deep_copy(Impl::as_view_of_rank_n<1>(dst), Impl::as_view_of_rank_n<1>(src)); break; case 2: deep_copy(Impl::as_view_of_rank_n<2>(dst), Impl::as_view_of_rank_n<2>(src)); break; case 3: deep_copy(Impl::as_view_of_rank_n<3>(dst), Impl::as_view_of_rank_n<3>(src)); break; case 4: deep_copy(Impl::as_view_of_rank_n<4>(dst), Impl::as_view_of_rank_n<4>(src)); break; case 5: deep_copy(Impl::as_view_of_rank_n<5>(dst), Impl::as_view_of_rank_n<5>(src)); break; case 6: deep_copy(Impl::as_view_of_rank_n<6>(dst), Impl::as_view_of_rank_n<6>(src)); break; case 7: deep_copy(Impl::as_view_of_rank_n<7>(dst), Impl::as_view_of_rank_n<7>(src)); break; default: Kokkos::Impl::throw_runtime_exception( "Calling DynRankView deep_copy with a view of unexpected rank " + std::to_string(rank(dst))); } } } // namespace Kokkos //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- namespace Kokkos { namespace Impl { // Deduce Mirror Types template struct MirrorDRViewType { // The incoming view_type using src_view_type = typename Kokkos::DynRankView; // The memory space for the mirror view using memory_space = typename Space::memory_space; // Check whether it is the same memory space enum { is_same_memspace = std::is_same::value }; // The array_layout using array_layout = typename src_view_type::array_layout; // The data type (we probably want it non-const since otherwise we can't even // deep_copy to it. using data_type = typename src_view_type::non_const_data_type; // The destination view type if it is not the same memory space using dest_view_type = Kokkos::DynRankView; // If it is the same memory_space return the existsing view_type // This will also keep the unmanaged trait if necessary using view_type = std::conditional_t; }; template struct MirrorDRVType { // The incoming view_type using src_view_type = typename Kokkos::DynRankView; // The memory space for the mirror view using memory_space = typename Space::memory_space; // Check whether it is the same memory space enum { is_same_memspace = std::is_same::value }; // The array_layout using array_layout = typename src_view_type::array_layout; // The data type (we probably want it non-const since otherwise we can't even // deep_copy to it. using data_type = typename src_view_type::non_const_data_type; // The destination view type if it is not the same memory space using view_type = Kokkos::DynRankView; }; } // namespace Impl namespace Impl { template inline typename DynRankView::HostMirror create_mirror( const DynRankView& src, const Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_memory_space>* = nullptr) { using src_type = DynRankView; using dst_type = typename src_type::HostMirror; using alloc_prop_input = Impl::ViewCtorProp; static_assert( !alloc_prop_input::has_label, "The view constructor arguments passed to Kokkos::create_mirror " "must not include a label!"); static_assert( !alloc_prop_input::has_pointer, "The view constructor arguments passed to Kokkos::create_mirror must " "not include a pointer!"); static_assert( !alloc_prop_input::allow_padding, "The view constructor arguments passed to Kokkos::create_mirror must " "not explicitly allow padding!"); using alloc_prop = Impl::ViewCtorProp; alloc_prop prop_copy(arg_prop); static_cast&>(prop_copy).value = std::string(src.label()).append("_mirror"); return dst_type(prop_copy, Impl::reconstructLayout(src.layout(), src.rank())); } template inline auto create_mirror( const DynRankView& src, const Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_memory_space>* = nullptr) { using dst_type = typename Impl::MirrorDRVType< typename Impl::ViewCtorProp::memory_space, T, P...>::view_type; using alloc_prop_input = Impl::ViewCtorProp; static_assert( !alloc_prop_input::has_label, "The view constructor arguments passed to Kokkos::create_mirror " "must not include a label!"); static_assert( !alloc_prop_input::has_pointer, "The view constructor arguments passed to Kokkos::create_mirror must " "not include a pointer!"); static_assert( !alloc_prop_input::allow_padding, "The view constructor arguments passed to Kokkos::create_mirror must " "not explicitly allow padding!"); using alloc_prop = Impl::ViewCtorProp; alloc_prop prop_copy(arg_prop); static_cast&>(prop_copy).value = std::string(src.label()).append("_mirror"); return dst_type(prop_copy, Impl::reconstructLayout(src.layout(), src.rank())); } } // namespace Impl // Create a mirror in host space template inline typename DynRankView::HostMirror create_mirror( const DynRankView& src, std::enable_if_t::specialize, void>::value>* = nullptr) { return Impl::create_mirror(src, Kokkos::Impl::ViewCtorProp<>{}); } template inline typename DynRankView::HostMirror create_mirror( Kokkos::Impl::WithoutInitializing_t wi, const DynRankView& src, std::enable_if_t::specialize, void>::value>* = nullptr) { return Impl::create_mirror(src, Kokkos::view_alloc(wi)); } template inline typename DynRankView::HostMirror create_mirror( const Impl::ViewCtorProp& arg_prop, const DynRankView& src, std::enable_if_t< std::is_void::specialize>::value && !Impl::ViewCtorProp::has_memory_space>* = nullptr) { return Impl::create_mirror(src, arg_prop); } // Create a mirror in a new space template ::value && std::is_void::specialize>::value>> typename Impl::MirrorDRVType::view_type create_mirror( const Space&, const Kokkos::DynRankView& src) { return Impl::create_mirror( src, Kokkos::view_alloc(typename Space::memory_space{})); } template typename Impl::MirrorDRVType::view_type create_mirror( Kokkos::Impl::WithoutInitializing_t wi, const Space&, const Kokkos::DynRankView& src, std::enable_if_t::specialize, void>::value>* = nullptr) { return Impl::create_mirror( src, Kokkos::view_alloc(wi, typename Space::memory_space{})); } template inline auto create_mirror( const Impl::ViewCtorProp& arg_prop, const DynRankView& src, std::enable_if_t< std::is_void::specialize>::value && Impl::ViewCtorProp::has_memory_space>* = nullptr) { using ReturnType = typename Impl::MirrorDRVType< typename Impl::ViewCtorProp::memory_space, T, P...>::view_type; return ReturnType{Impl::create_mirror(src, arg_prop)}; } namespace Impl { template inline std::enable_if_t< std::is_same< typename DynRankView::memory_space, typename DynRankView