Kokkos_Parallel.hpp

Go to the documentation of this file.

/*
//@HEADER
// ************************************************************************
//
//   Kokkos: Manycore Performance-Portable Multidimensional Arrays
//              Copyright (2012) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact  H. Carter Edwards (hcedwar@sandia.gov)
//
// ************************************************************************
//@HEADER
*/


#ifndef KOKKOS_PARALLEL_HPP
#define KOKKOS_PARALLEL_HPP

#include <cstddef>
#include <Kokkos_Core_fwd.hpp>
#include <Kokkos_View.hpp>
#include <Kokkos_ExecPolicy.hpp>
#include <impl/Kokkos_Traits.hpp>
#include <impl/Kokkos_Tags.hpp>

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {
namespace Impl {

//----------------------------------------------------------------------------
template< class Functor
        , class Policy
        , class EnableFunctor = void
        , class EnablePolicy  = void
        >
struct FunctorPolicyExecutionSpace {
  typedef Kokkos::DefaultExecutionSpace execution_space ;
};

template< class Functor , class Policy >
struct FunctorPolicyExecutionSpace
  < Functor , Policy
  , typename enable_if_type< typename Functor::device_type     >::type
  , typename enable_if_type< typename Policy ::execution_space >::type
  >
{
  typedef typename Policy ::execution_space execution_space ;
};

template< class Functor , class Policy , class EnableFunctor >
struct FunctorPolicyExecutionSpace
  < Functor , Policy
  , EnableFunctor
  , typename enable_if_type< typename Policy::execution_space >::type
  >
{
  typedef typename Policy ::execution_space execution_space ;
};

template< class Functor , class Policy , class EnablePolicy >
struct FunctorPolicyExecutionSpace
  < Functor , Policy
  , typename enable_if_type< typename Functor::device_type >::type
  , EnablePolicy
  >
{
  typedef typename Functor::device_type execution_space ;
};

//----------------------------------------------------------------------------

template< class FunctorType , class Enable = void >
struct ReduceAdapterValueType ;

template< class FunctorType >
struct ReduceAdapterValueType< FunctorType , typename enable_if_type< typename FunctorType::value_type >::type >
{
  typedef typename FunctorType::value_type type ;
};

template< class FunctorType ,
          class ValueType = typename ReduceAdapterValueType< FunctorType >::type >
struct ReduceAdapter ;

//----------------------------------------------------------------------------
template< class FunctorType , class ExecPolicy > class ParallelFor ;

template< class FunctorType , class ExecPolicy > class ParallelReduce ;

template< class FunctorType , class ExecPolicy > class ParallelScan ;

} // namespace Impl
} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {

template< class ExecPolicy , class FunctorType >
inline
void parallel_for( const ExecPolicy  & policy
                 , const FunctorType & functor
                 , typename Impl::enable_if< ! Impl::is_integral< ExecPolicy >::value >::type * = 0
                 )
{
  (void) Impl::ParallelFor< FunctorType , ExecPolicy >( functor , policy );
}

template< class FunctorType >
inline
void parallel_for( const size_t        work_count ,
                   const FunctorType & functor )
{
  typedef typename
    Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space
      execution_space ;
  typedef RangePolicy< execution_space > policy ;
  (void) Impl::ParallelFor< FunctorType , policy >( functor , policy(0,work_count) );
}

} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {

template< class ExecPolicy , class FunctorType >
inline
void parallel_reduce( const ExecPolicy  & policy 
                    , const FunctorType & functor
                    , typename Impl::enable_if< ! Impl::is_integral< ExecPolicy >::value >::type * = 0
                    )
{
  (void) Impl::ParallelReduce< FunctorType , ExecPolicy >( functor , policy );
}

// integral range policy
template< class FunctorType >
inline
void parallel_reduce( const size_t        work_count
                    , const FunctorType & functor
                    )
{
  typedef typename
    Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space
      execution_space ;

  typedef RangePolicy< execution_space > policy ;

  typedef Kokkos::Impl::ReduceAdapter< FunctorType >  Reduce ;

  typedef typename Kokkos::Impl::if_c< Reduce::StaticValueSize
                                     , typename Reduce::scalar_type
                                     , typename Reduce::pointer_type
                                     >::type value_type ;

  Kokkos::View< value_type
              , typename execution_space::host_mirror_device_type
              , Kokkos::MemoryUnmanaged
              >
    result_view ;

  (void) Impl::ParallelReduce< FunctorType , policy >( functor , policy(0,work_count) , result_view );
}

// general policy and view ouput
template< class ExecPolicy , class FunctorType , class ViewType >
inline
void parallel_reduce( const ExecPolicy  & policy 
                    , const FunctorType & functor 
                    , const ViewType    & result_view
                    , typename Impl::enable_if<
                      ( Impl::is_view<ViewType>::value && ! Impl::is_integral< ExecPolicy >::value
                      )>::type * = 0 )
{
  (void) Impl::ParallelReduce< FunctorType, ExecPolicy >( functor , policy , result_view );
}

// general policy and pod or array of pod output
template< class ExecPolicy , class FunctorType >
inline
void parallel_reduce( const ExecPolicy  & policy 
                    , const FunctorType & functor 
                    , typename Impl::enable_if<
                      ( ! Impl::is_integral< ExecPolicy >::value )
                      , typename Kokkos::Impl::ReduceAdapter< FunctorType >::reference_type
                      >::type result_ref )
{
  typedef typename
    Kokkos::Impl::FunctorPolicyExecutionSpace< FunctorType , ExecPolicy >::execution_space
      execution_space ;

  typedef Kokkos::Impl::ReduceAdapter< FunctorType >  Reduce ;

  // Wrap the result output request in a view to inform the implementation
  // of the type and memory space.

  typedef typename Kokkos::Impl::if_c< Reduce::StaticValueSize
                                     , typename Reduce::scalar_type
                                     , typename Reduce::pointer_type
                                     >::type value_type ;

  Kokkos::View< value_type
              , typename execution_space::host_mirror_device_type
              , Kokkos::MemoryUnmanaged
              >
    result_view( Reduce::pointer( result_ref )
               , Reduce::value_count( functor )
               );

  (void) Impl::ParallelReduce< FunctorType, ExecPolicy >( functor , policy , result_view );
}

// integral range policy and view ouput
template< class FunctorType , class ViewType >
inline
void parallel_reduce( const size_t        work_count
                    , const FunctorType & functor 
                    , const ViewType    & result_view
                    , typename Impl::enable_if<( Impl::is_view<ViewType>::value )>::type * = 0 )
{
  typedef typename
    Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space
      execution_space ;

  typedef RangePolicy< execution_space > ExecPolicy ;

  (void) Impl::ParallelReduce< FunctorType, ExecPolicy >( functor , ExecPolicy(0,work_count) , result_view );
}

// integral range policy and pod or array of pod output
template< class FunctorType >
inline
void parallel_reduce( const size_t        work_count ,
                      const FunctorType & functor ,
                      typename Kokkos::Impl::ReduceAdapter< FunctorType >::reference_type result )
{
  typedef typename
    Kokkos::Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space
      execution_space ;

  typedef Kokkos::RangePolicy< execution_space > policy ;

  typedef Kokkos::Impl::ReduceAdapter< FunctorType >  Reduce ;

  // Wrap the result output request in a view to inform the implementation
  // of the type and memory space.

  typedef typename Kokkos::Impl::if_c< Reduce::StaticValueSize
                                     , typename Reduce::scalar_type
                                     , typename Reduce::pointer_type
                                     >::type value_type ;

  Kokkos::View< value_type
              , typename execution_space::host_mirror_device_type
              , Kokkos::MemoryUnmanaged
              >
    result_view( Reduce::pointer( result )
               , Reduce::value_count( functor )
               );
  
  (void) Impl::ParallelReduce< FunctorType , policy >( functor , policy(0,work_count) , result_view );
}

} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {

template< class ExecutionPolicy , class FunctorType >
inline
void parallel_scan( const ExecutionPolicy & policy 
                  , const FunctorType     & functor 
                  , typename Impl::enable_if< ! Impl::is_integral< ExecutionPolicy >::value >::type * = 0
                  )
{
  Impl::ParallelScan< FunctorType , ExecutionPolicy > scan( functor , policy );
}

template< class FunctorType >
inline
void parallel_scan( const size_t        work_count ,
                    const FunctorType & functor )
{
  typedef typename
    Kokkos::Impl::FunctorPolicyExecutionSpace< FunctorType , void >::execution_space
      execution_space ;

  typedef Kokkos::RangePolicy< execution_space > policy ;

  (void) Impl::ParallelScan< FunctorType , policy >( functor , policy(0,work_count) );
}

} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {
namespace Impl {

template< class FunctorType , class Enable = void >
struct FunctorHasInit : public false_type {};

template< class FunctorType >
struct FunctorHasInit< FunctorType , typename enable_if< 0 < sizeof( & FunctorType::init ) >::type >
  : public true_type {};

template< class FunctorType , class Enable = void >
struct FunctorHasJoin : public false_type {};

template< class FunctorType >
struct FunctorHasJoin< FunctorType , typename enable_if< 0 < sizeof( & FunctorType::join ) >::type >
  : public true_type {};

template< class FunctorType , class Enable = void >
struct FunctorHasFinal : public false_type {};

template< class FunctorType >
struct FunctorHasFinal< FunctorType , typename enable_if< 0 < sizeof( & FunctorType::final ) >::type >
  : public true_type {};

template< class FunctorType , class Enable = void >
struct FunctorTeamShmemSize
{
  static inline size_t value( const FunctorType & , int ) { return 0 ; }
};

template< class FunctorType >
struct FunctorTeamShmemSize< FunctorType , typename enable_if< sizeof( & FunctorType::team_shmem_size ) >::type >
{
  static inline size_t value( const FunctorType & f , int team_size ) { return f.team_shmem_size( team_size ) ; }
};

template< class FunctorType >
struct FunctorTeamShmemSize< FunctorType , typename enable_if< sizeof( & FunctorType::shmem_size ) >::type >
{
  static inline size_t value( const FunctorType & f , int team_size ) { return f.shmem_size( team_size ) ; }
};

} // namespace Impl
} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {
namespace Impl {

#if defined( KOKKOS_HAVE_CXX11 )

template< class FunctionPtr >
struct ReduceAdapterFunctorOperatorArgType ;

template< class Functor , class Arg0 , class Arg1 >
struct ReduceAdapterFunctorOperatorArgType< void ( Functor::*)( Arg0 , Arg1 & ) const > {
  typedef Arg1 type ;
};

// Functor does not have a 'typedef ... value_type' and C++11 is enabled.
// Deduce the value type from the functor's argument list.
template< class FunctorType , class Enable >
struct ReduceAdapterValueType {
private:
  typedef decltype( & FunctorType::operator() ) function_pointer_type ;
public:
  typedef typename ReduceAdapterFunctorOperatorArgType< function_pointer_type >::type type ;
};

#endif /* #if defined( KOKKOS_HAVE_CXX11 ) */

template< class FunctorType , class ScalarType >
struct ReduceAdapter
{
  enum { StaticValueSize = sizeof(ScalarType) };

  typedef ScalarType & reference_type  ;
  typedef ScalarType * pointer_type  ;
  typedef ScalarType   scalar_type  ;

  KOKKOS_INLINE_FUNCTION static
  reference_type reference( void * p ) { return *((ScalarType*) p); }

  KOKKOS_INLINE_FUNCTION static
  reference_type reference( void * p , unsigned i ) { return ((ScalarType*) p)[i]; }

  KOKKOS_INLINE_FUNCTION static
  pointer_type pointer( reference_type p ) { return & p ; }

  KOKKOS_INLINE_FUNCTION static
  unsigned value_count( const FunctorType & ) { return 1 ; }

  KOKKOS_INLINE_FUNCTION static
  unsigned value_size( const FunctorType & ) { return sizeof(ScalarType); }

  KOKKOS_INLINE_FUNCTION static
  void copy( const FunctorType & , void * const dst , const void * const src )
    { *((scalar_type*)dst) = *((const scalar_type*)src); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void join( const F & f
           , volatile void * update
           , typename enable_if< is_same<F,FunctorType>::value &&
                                 FunctorHasJoin<F>::value
                               , volatile const void *
                               >::type input )
    { f.join( *((volatile ScalarType*)update) , *((volatile const ScalarType*)input) ); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void join( const F & f
           , volatile void * update
           , typename enable_if< is_same<F,FunctorType>::value &&
                                 ! FunctorHasJoin<F>::value
                               , volatile const void *
                               >::type input )
    { *((volatile ScalarType*)update) += *((volatile const ScalarType*)input); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  reference_type
  init( const F & f ,
        typename enable_if< ( is_same<F,FunctorType>::value &&
                              FunctorHasInit<F>::value )
                          >::type * p )
    { f.init( *((ScalarType *) p ) );  return *((ScalarType *) p ); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  reference_type
  init( const F & ,
        typename enable_if< ( is_same<F,FunctorType>::value &&
                              ! FunctorHasInit<F>::value )
                          >::type * p )
    { return *( new(p) ScalarType() ); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void final( const F & f ,
              typename enable_if< ( is_same<F,FunctorType>::value &&
                                    FunctorHasFinal<F>::value )
                                >::type * p )
    { f.final( *((ScalarType *) p ) ); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void final( const F & ,
              typename enable_if< ( is_same<F,FunctorType>::value &&
                                    ! FunctorHasFinal<F>::value )
                                >::type * )
    {}
};

template< class FunctorType , class ScalarType >
struct ReduceAdapter< FunctorType , ScalarType[] >
{
  enum { StaticValueSize = 0 };

  typedef ScalarType * reference_type  ;
  typedef ScalarType * pointer_type  ;
  typedef ScalarType   scalar_type  ;

  KOKKOS_INLINE_FUNCTION static
  ScalarType * reference( void * p ) { return (ScalarType*) p ; }

  KOKKOS_INLINE_FUNCTION static
  reference_type reference( void * p , unsigned i ) { return ((ScalarType*) p)+i; }

  KOKKOS_INLINE_FUNCTION static
  pointer_type pointer( reference_type p ) { return p ; }

  KOKKOS_INLINE_FUNCTION static
  unsigned value_count( const FunctorType & f ) { return f.value_count ; }

  KOKKOS_INLINE_FUNCTION static
  unsigned value_size( const FunctorType & f ) { return f.value_count * sizeof(ScalarType); }

  KOKKOS_INLINE_FUNCTION static
  void copy( const FunctorType & f , void * const dst , const void * const src )
    {
      for ( int i = 0 ; i < int(f.value_count) ; ++i ) {
        ((scalar_type*)dst)[i] = ((const scalar_type*)src)[i];
      }
    }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void join( const F & f
           , volatile void * const update
           , typename enable_if< is_same<F,FunctorType>::value &&
                                 FunctorHasJoin<F>::value
                               , volatile const void * const
                               >::type input )
    { f.join( ((volatile ScalarType*)update) , ((volatile const ScalarType*)input) ); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void join( const F & f
           , volatile void * const update
           , typename enable_if< is_same<F,FunctorType>::value &&
                                 ! FunctorHasJoin<F>::value
                               , volatile const void * const
                               >::type input )
    {
      for ( int i = 0 ; i < int(f.value_count) ; ++i ) {
        ((volatile ScalarType*)update)[i] += ((volatile const ScalarType*)input)[i] ;
      }
    }


  template< class F >
  KOKKOS_INLINE_FUNCTION static
  reference_type
  init( const F & f ,
        typename enable_if< ( is_same<F,FunctorType>::value &&
                              FunctorHasInit<F>::value )
                          >::type * p )
    { f.init( ((ScalarType *) p ) ); return (ScalarType*) p ; }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  reference_type
  init( const F & f ,
        typename enable_if< ( is_same<F,FunctorType>::value &&
                              ! FunctorHasInit<F>::value )
                          >::type * p )
    {
      for ( int i = 0 ; i < int(f.value_count) ; ++i ) {
        new(((ScalarType*)p)+i) ScalarType();
      }
      return (ScalarType*)p ;
    }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void final( const F & f ,
              typename enable_if< ( is_same<F,FunctorType>::value &&
                                    FunctorHasFinal<F>::value )
                                >::type * p )
    { f.final( ((ScalarType *) p ) ); }

  template< class F >
  KOKKOS_INLINE_FUNCTION static
  void final( const F & ,
              typename enable_if< ( is_same<F,FunctorType>::value &&
                                    ! FunctorHasFinal<F>::value )
                                >::type * )
    {}
};

} // namespace Impl
} // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

#endif /* KOKKOS_PARALLEL_HPP */