Tpetra_DistObjectKA_decl.hpp

// @HEADER
// ***********************************************************************
//
//          Tpetra: Templated Linear Algebra Services Package
//                 Copyright (2008) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ************************************************************************
// @HEADER

#ifndef TPETRA_DISTOBJECT_KA_DECL_HPP
#define TPETRA_DISTOBJECT_KA_DECL_HPP

#include "Tpetra_ConfigDefs.hpp"
#include "Tpetra_Map.hpp"
#include "Tpetra_Import.hpp"
#include "Tpetra_Export.hpp"
#include "Tpetra_SrcDistObject.hpp"

#if TPETRA_USE_KOKKOS_DISTOBJECT

#include "KokkosCompat_View.hpp"
#include "Kokkos_Core.hpp"

// #ifndef HAVE_TPETRA_TRANSFER_TIMERS
// #  define HAVE_TPETRA_TRANSFER_TIMERS 1
// #endif // HAVE_TPETRA_TRANSFER_TIMERS

#ifdef HAVE_TPETRA_TRANSFER_TIMERS
#  undef HAVE_TPETRA_TRANSFER_TIMERS
#endif // HAVE_TPETRA_TRANSFER_TIMERS


namespace Tpetra {
  template <class Packet,
            class LocalOrdinal = Map<>::local_ordinal_type,
            class GlobalOrdinal = typename Map<LocalOrdinal>::global_ordinal_type,
            class Node = typename Map<LocalOrdinal, GlobalOrdinal>::node_type>
  class DistObjectKA :
    virtual public SrcDistObject,
    virtual public Teuchos::Describable {
  public:


    typedef Packet packet_type;
    typedef LocalOrdinal local_ordinal_type;
    typedef GlobalOrdinal global_ordinal_type;
    typedef Node node_type;

    typedef typename Kokkos::Compat::NodeDevice<node_type>::type device_type;
    typedef typename device_type::size_type view_size_type;




    explicit DistObjectKA (const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& map);

    DistObjectKA (const DistObjectKA<Packet,LocalOrdinal,GlobalOrdinal,Node>& rhs);

    virtual ~DistObjectKA ();




    void
    doImport (const SrcDistObject& source,
              const Import<LocalOrdinal,GlobalOrdinal,Node>& importer,
              CombineMode CM);

    void
    doExport (const SrcDistObject& source,
              const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter,
              CombineMode CM);

    void
    doImport (const SrcDistObject& source,
              const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter,
              CombineMode CM);

    void
    doExport (const SrcDistObject& source,
              const Import<LocalOrdinal,GlobalOrdinal,Node>& importer,
              CombineMode CM);




    bool isDistributed () const;

    virtual Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >
    getMap() const { return map_; }




    void print (std::ostream &os) const;




    virtual std::string description () const;


    virtual void
    describe (Teuchos::FancyOStream &out,
              const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const;



    virtual void
    removeEmptyProcessesInPlace (const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& newMap);

    // Forward declaration of nonmember function.
    template<class PT, class LO, class GO, class NT>
    friend void
    removeEmptyProcessesInPlace (Teuchos::RCP<Tpetra::DistObjectKA<PT, LO, GO, NT> >& input,
                                 const Teuchos::RCP<const Map<LO, GO, NT> >& newMap);
    // Forward declaration of nonmember function.
    template<class PT, class LO, class GO, class NT>
    friend void
    removeEmptyProcessesInPlace (Teuchos::RCP<Tpetra::DistObjectKA<PT, LO, GO, NT> >& input);

  protected:
    enum ReverseOption {
      DoForward, //*!< Perform the transfer in forward mode.
      DoReverse  //*!< Perform the transfer in reverse mode.
    };

    virtual size_t constantNumberOfPackets () const;

    virtual void
    doTransfer (const SrcDistObject& src,
                CombineMode CM,
                size_t numSameIDs,
                const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
                const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
                const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
                const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
                Distributor &distor,
                ReverseOption revOp);



    virtual bool
    checkSizes (const SrcDistObject& source) = 0;

    virtual void
    copyAndPermute (
      const SrcDistObject& source,
      size_t numSameIDs,
      const Kokkos::View<const LocalOrdinal*, device_type> &permuteToLIDs,
      const Kokkos::View<const LocalOrdinal*, device_type> &permuteFromLIDs) = 0;

    virtual void
    packAndPrepare (
      const SrcDistObject& source,
      const Kokkos::View<const LocalOrdinal*, device_type> &exportLIDs,
      Kokkos::View<Packet*, device_type> &exports,
      const Kokkos::View<size_t*, device_type> &numPacketsPerLID,
      size_t& constantNumPackets,
      Distributor &distor) = 0;

    virtual void
    unpackAndCombine (
      const Kokkos::View<const LocalOrdinal*, device_type> &importLIDs,
      const Kokkos::View<const Packet*, device_type> &imports,
      const Kokkos::View<size_t*, device_type> &numPacketsPerLID,
      size_t constantNumPackets,
      Distributor &distor,
      CombineMode CM) = 0;

    virtual void createViews () const;

    virtual void createViewsNonConst (KokkosClassic::ReadWriteOption rwo);

    virtual void releaseViews () const;

    Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;

  private:
    Kokkos::View<packet_type*,device_type> imports_;

    Kokkos::View<size_t*,device_type> numImportPacketsPerLID_;
    typename Kokkos::View<size_t*,device_type>::HostMirror host_numImportPacketsPerLID_;

    Kokkos::View<packet_type*,device_type> exports_;

    Kokkos::View<size_t*,device_type> numExportPacketsPerLID_;

#ifdef HAVE_TPETRA_TRANSFER_TIMERS
    Teuchos::RCP<Teuchos::Time> doXferTimer_;
    Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
    Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
    Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
    Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
#endif // HAVE_TPETRA_TRANSFER_TIMERS

  }; // class DistObject


  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
                               const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
                                                            typename DistObjectType::global_ordinal_type,
                                                            typename DistObjectType::node_type> >& newMap);

  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);

} // namespace Tpetra

#endif /* TPETRA_USE_KOKKOS_DISTOBJECT */

#endif /* TPETRA_DISTOBJECT_DECL_HPP */