Kokkos_CudaSpace.hpp

/*
//@HEADER
// ************************************************************************
//
//   Kokkos: Manycore Performance-Portable Multidimensional Arrays
//              Copyright (2012) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact  H. Carter Edwards (hcedwar@sandia.gov)
//
// ************************************************************************
//@HEADER
*/

#ifndef KOKKOS_CUDASPACE_HPP
#define KOKKOS_CUDASPACE_HPP

#if defined( KOKKOS_HAVE_CUDA )

#include <iosfwd>
#include <typeinfo>
#include <string>

#include <Kokkos_Core_fwd.hpp>
#include <Kokkos_HostSpace.hpp>
#include <Cuda/Kokkos_Cuda_abort.hpp>

/*--------------------------------------------------------------------------*/

namespace Kokkos {

class CudaSpace {
public:

  typedef Impl::MemorySpaceTag  kokkos_tag ;
  typedef CudaSpace             memory_space ;
  typedef unsigned int          size_type ;
  typedef Kokkos::Cuda          execution_space ;

  static void * allocate( const std::string    & label ,
                          const std::type_info & scalar_type ,
                          const size_t           scalar_size ,
                          const size_t           scalar_count );

  static void increment( const void * );

  static void decrement( const void * );

  static int count( const void * );

  static void print_memory_view( std::ostream & );

  static std::string query_label( const void * );

  /*--------------------------------*/
#if defined( __CUDACC__ )
  static void texture_object_attach( const void            * const arg_ptr
                                   , ::cudaChannelFormatDesc const & arg_desc
                                   , ::cudaTextureObject_t * const arg_tex_obj
                                   , void const           ** const arg_alloc_ptr
                                   , int                   * const arg_offset
                                   );
#endif

  /*--------------------------------*/
  static void access_error();
  static void access_error( const void * const );
};

} // namespace Kokkos

/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/

namespace Kokkos {

class CudaUVMSpace {
public:

  typedef Impl::MemorySpaceTag  kokkos_tag ;
  typedef CudaUVMSpace          memory_space ;
  typedef unsigned int          size_type ;
  typedef Cuda                  execution_space ;

  static void * allocate( const std::string    & label ,
                          const std::type_info & scalar_type ,
                          const size_t           scalar_size ,
                          const size_t           scalar_count );

  static void increment( const void * );

  static void decrement( const void * );

  static int count( const void * );

  static void print_memory_view( std::ostream & );

  static std::string query_label( const void * );

#if defined( __CUDACC__ )
  static void texture_object_attach( const void            * const arg_ptr
                                   , ::cudaChannelFormatDesc const & arg_desc
                                   , ::cudaTextureObject_t * const arg_tex_obj
                                   , void const           ** const arg_alloc_ptr
                                   , int                   * const arg_offset
                                   );
#endif
};

} // namespace Kokkos

/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/

namespace Kokkos {

class CudaHostPinnedSpace {
public:

  typedef Impl::MemorySpaceTag        kokkos_tag ;
  typedef CudaHostPinnedSpace         memory_space ;
  typedef unsigned int                size_type ;

  typedef HostSpace::execution_space  execution_space ;

  static void * allocate( const std::string    & label ,
                          const std::type_info & scalar_type ,
                          const size_t           scalar_size ,
                          const size_t           scalar_count );

  static void increment( const void * );

  static int count( const void * );

  static void decrement( const void * );

  static void print_memory_view( std::ostream & );

  static std::string query_label( const void * );
};

} // namespace Kokkos

/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/

namespace Kokkos {
namespace Impl {

template<> struct DeepCopy< CudaSpace , CudaSpace > { DeepCopy( void * dst , const void * src , size_t ); };
template<> struct DeepCopy< CudaSpace , HostSpace > { DeepCopy( void * dst , const void * src , size_t ); };
template<> struct DeepCopy< HostSpace , CudaSpace > { DeepCopy( void * dst , const void * src , size_t ); };

template<> struct DeepCopy< CudaSpace , CudaUVMSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< CudaSpace , CudaSpace >( dst , src , n ); }
};

template<> struct DeepCopy< CudaSpace , CudaHostPinnedSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< CudaSpace , HostSpace >( dst , src , n ); }
};


template<> struct DeepCopy< CudaUVMSpace , CudaSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< CudaSpace , CudaSpace >( dst , src , n ); }
};

template<> struct DeepCopy< CudaUVMSpace , CudaUVMSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< CudaSpace , CudaSpace >( dst , src , n ); }
};

template<> struct DeepCopy< CudaUVMSpace , CudaHostPinnedSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< CudaSpace , HostSpace >( dst , src , n ); }
};

template<> struct DeepCopy< CudaUVMSpace , HostSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< CudaSpace , HostSpace >( dst , src , n ); }
};


template<> struct DeepCopy< CudaHostPinnedSpace , CudaSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< HostSpace , CudaSpace >( dst , src , n ); }
};

template<> struct DeepCopy< CudaHostPinnedSpace , CudaUVMSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< HostSpace , CudaSpace >( dst , src , n ); }
};

template<> struct DeepCopy< CudaHostPinnedSpace , CudaHostPinnedSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< HostSpace , HostSpace >( dst , src , n ); }
};

template<> struct DeepCopy< CudaHostPinnedSpace , HostSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< HostSpace , HostSpace >( dst , src , n ); }
};


template<> struct DeepCopy< HostSpace , CudaUVMSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< HostSpace , CudaSpace >( dst , src , n ); }
};

template<> struct DeepCopy< HostSpace , CudaHostPinnedSpace >
{
  inline
  DeepCopy( void * dst , const void * src , size_t n )
  { (void) DeepCopy< HostSpace , HostSpace >( dst , src , n ); }
};

} // namespace Impl
} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {
namespace Impl {

template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::CudaSpace , Kokkos::HostSpace >
{
  enum {value = 0};
  KOKKOS_INLINE_FUNCTION static void verify( void )
    { Kokkos::cuda_abort("Cuda code attempted to access HostSpace memory"); }

  KOKKOS_INLINE_FUNCTION static void verify( const void * )
    { Kokkos::cuda_abort("Cuda code attempted to access HostSpace memory"); }
};

template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::CudaSpace , Kokkos::CudaUVMSpace >
{
  enum {value = 1};
  inline static void verify( void ) { }
  inline static void verify( const void * ) { }
};

template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::CudaSpace , Kokkos::CudaHostPinnedSpace >
{
  enum {value = 1};
  inline static void verify( void ) { }
  inline static void verify( const void * ) { }
};

template< class OtherSpace >
struct VerifyExecutionCanAccessMemorySpace<
  typename enable_if< ! is_same<Kokkos::CudaSpace,OtherSpace>::value , Kokkos::CudaSpace >::type ,
  OtherSpace >
{
  enum {value = 0};
  KOKKOS_INLINE_FUNCTION static void verify( void )
    { Kokkos::cuda_abort("Cuda code attempted to access unknown Space memory"); }

  KOKKOS_INLINE_FUNCTION static void verify( const void * )
    { Kokkos::cuda_abort("Cuda code attempted to access unknown Space memory"); }
};

//----------------------------------------------------------------------------
template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::CudaSpace >
{
#if defined( KOKKOS_USE_CUDA_UVM )
  enum {value = 1};
  inline static void verify( void ) { }
  inline static void verify( const void * ) { }
#else
  enum {value = 0};
  inline static void verify( void ) { CudaSpace::access_error(); }
  inline static void verify( const void * p ) { CudaSpace::access_error(p); }
#endif
};

template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::CudaUVMSpace >
{
  enum {value = 1};
  inline static void verify( void ) { }
  inline static void verify( const void * ) { }
};

template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::CudaHostPinnedSpace >
{
  enum {value = 1};
  KOKKOS_INLINE_FUNCTION static void verify( void ) {}
  KOKKOS_INLINE_FUNCTION static void verify( const void * ) {}
};

//----------------------------------------------------------------------------

} // namespace Impl
} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

#endif /* #if defined( KOKKOS_HAVE_CUDA ) */
#endif /* #define KOKKOS_CUDASPACE_HPP */