Tpetra Matrix/Vector Services: Tpetra_KokkosRefactor_DistObject

00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 //
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #ifndef TPETRA_KOKKOS_REFACTOR_DISTOBJECT_DECL_HPP
00043 #define TPETRA_KOKKOS_REFACTOR_DISTOBJECT_DECL_HPP
00044 
00045 /* These are provided in base DistObject template
00046 #include "Tpetra_ConfigDefs.hpp"
00047 #include "Tpetra_Map.hpp"
00048 #include "Tpetra_Import.hpp"
00049 #include "Tpetra_Export.hpp"
00050 #include "Tpetra_SrcDistObject.hpp"
00051 */
00052 
00053 #include "KokkosCompat_ClassicNodeAPI_Wrapper.hpp"
00054 #include "KokkosCompat_View.hpp"
00055 #include "Kokkos_Core.hpp"
00056 
00057 /* These are provided in base DistObject template
00058 // #ifndef HAVE_TPETRA_TRANSFER_TIMERS
00059 // #  define HAVE_TPETRA_TRANSFER_TIMERS 1
00060 // #endif // HAVE_TPETRA_TRANSFER_TIMERS
00061 
00062 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00063 #  undef HAVE_TPETRA_TRANSFER_TIMERS
00064 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00065 */
00066 
00067 namespace Tpetra {
00071   template <class Packet,
00072             class LocalOrdinal,
00073             class GlobalOrdinal,
00074             class Device>
00075   class DistObject<Packet, LocalOrdinal, GlobalOrdinal,
00076                    Kokkos::Compat::KokkosDeviceWrapperNode<Device> > :
00077     virtual public SrcDistObject,
00078     virtual public Teuchos::Describable {
00079   public:
00080 
00081     typedef Kokkos::Compat::KokkosDeviceWrapperNode<Device>  Node;
00082 
00084 
00085 
00090     typedef Packet packet_type;
00092     typedef LocalOrdinal local_ordinal_type;
00094     typedef GlobalOrdinal global_ordinal_type;
00096     typedef Node node_type;
00097 
00099     typedef typename Node::device_type device_type;
00100     typedef typename Kokkos::View<Packet*, device_type>::size_type view_size_type;
00101 
00103 
00104 
00105 
00107     explicit DistObject (const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& map);
00108 
00110     DistObject (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& rhs);
00111 
00113     virtual ~DistObject ();
00114 
00116 
00117 
00118 
00136     void
00137     doImport (const SrcDistObject& source,
00138               const Import<LocalOrdinal,GlobalOrdinal,Node>& importer,
00139               CombineMode CM);
00140 
00158     void
00159     doExport (const SrcDistObject& source,
00160               const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter,
00161               CombineMode CM);
00162 
00181     void
00182     doImport (const SrcDistObject& source,
00183               const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter,
00184               CombineMode CM);
00185 
00204     void
00205     doExport (const SrcDistObject& source,
00206               const Import<LocalOrdinal,GlobalOrdinal,Node>& importer,
00207               CombineMode CM);
00208 
00210 
00211 
00212 
00218     bool isDistributed () const;
00219 
00226     virtual Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >
00227     getMap() const { return map_; }
00228 
00230 
00231 
00232 
00237     void print (std::ostream &os) const;
00238 
00240 
00241 
00242 
00247     virtual std::string description () const;
00248 
00249 
00254     virtual void
00255     describe (Teuchos::FancyOStream &out,
00256               const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const;
00258 
00259 
00260 
00306     virtual void
00307     removeEmptyProcessesInPlace (const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& newMap);
00308 
00309     // Forward declaration of nonmember function.
00310     template<class PT, class LO, class GO, class NT>
00311     friend void
00312     removeEmptyProcessesInPlace (Teuchos::RCP<Tpetra::DistObject<PT, LO, GO, NT> >& input,
00313                                  const Teuchos::RCP<const Map<LO, GO, NT> >& newMap);
00314     // Forward declaration of nonmember function.
00315     template<class PT, class LO, class GO, class NT>
00316     friend void
00317     removeEmptyProcessesInPlace (Teuchos::RCP<Tpetra::DistObject<PT, LO, GO, NT> >& input);
00319 
00320   protected:
00328     enum ReverseOption {
00329       DoForward, //*!< Perform the transfer in forward mode.
00330       DoReverse  //*!< Perform the transfer in reverse mode.
00331     };
00332 
00348     virtual size_t constantNumberOfPackets () const;
00349 
00372     virtual void
00373     doTransfer (const SrcDistObject& src,
00374                 CombineMode CM,
00375                 size_t numSameIDs,
00376                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00377                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00378                 const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00379                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00380                 Distributor &distor,
00381                 ReverseOption revOp);
00382 
00383     virtual void
00384     doTransferOld (const SrcDistObject& src,
00385                 CombineMode CM,
00386                 size_t numSameIDs,
00387                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00388                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00389                 const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00390                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00391                 Distributor &distor,
00392                 ReverseOption revOp);
00393     virtual void
00394     doTransferNew (const SrcDistObject& src,
00395                 CombineMode CM,
00396                 size_t numSameIDs,
00397                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00398                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00399                 const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00400                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00401                 Distributor &distor,
00402                 ReverseOption revOp);
00403 
00414 
00415 
00419     virtual bool
00420     checkSizes (const SrcDistObject& source) = 0;
00421 
00423     virtual bool useNewInterface () { return false; }
00424 
00442     virtual void
00443     copyAndPermute (const SrcDistObject& source,
00444                     size_t numSameIDs,
00445                     const Teuchos::ArrayView<const LocalOrdinal>& permuteToLIDs,
00446                     const Teuchos::ArrayView<const LocalOrdinal>& permuteFromLIDs) {}
00447     virtual void
00448     copyAndPermuteNew (
00449       const SrcDistObject& source,
00450       size_t numSameIDs,
00451       const Kokkos::View<const LocalOrdinal*, device_type> &permuteToLIDs,
00452       const Kokkos::View<const LocalOrdinal*, device_type> &permuteFromLIDs) {}
00453 
00476     virtual void
00477     packAndPrepare (const SrcDistObject& source,
00478                     const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
00479                     Teuchos::Array<Packet>& exports,
00480                     const Teuchos::ArrayView<size_t>& numPacketsPerLID,
00481                     size_t& constantNumPackets,
00482                     Distributor &distor) {}
00483     virtual void
00484     packAndPrepareNew (
00485       const SrcDistObject& source,
00486       const Kokkos::View<const LocalOrdinal*, device_type> &exportLIDs,
00487       Kokkos::View<Packet*, device_type> &exports,
00488       const Kokkos::View<size_t*, device_type> &numPacketsPerLID,
00489       size_t& constantNumPackets,
00490       Distributor &distor) {}
00491 
00513     virtual void
00514     unpackAndCombine (const Teuchos::ArrayView<const LocalOrdinal> &importLIDs,
00515                       const Teuchos::ArrayView<const Packet> &imports,
00516                       const Teuchos::ArrayView<size_t> &numPacketsPerLID,
00517                       size_t constantNumPackets,
00518                       Distributor &distor,
00519                       CombineMode CM) {}
00520     virtual void
00521     unpackAndCombineNew (
00522       const Kokkos::View<const LocalOrdinal*, device_type> &importLIDs,
00523       const Kokkos::View<const Packet*, device_type> &imports,
00524       const Kokkos::View<size_t*, device_type> &numPacketsPerLID,
00525       size_t constantNumPackets,
00526       Distributor &distor,
00527       CombineMode CM) {}
00529 
00536     virtual void createViews () const;
00537 
00553     virtual void createViewsNonConst (KokkosClassic::ReadWriteOption rwo);
00554 
00565     virtual void releaseViews () const;
00566 
00568     Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;
00569 
00570   private:
00572     Kokkos::View<packet_type*,device_type> imports_;
00573   protected:
00574     Teuchos::Array<Packet> imports_old_;
00575   private:
00576 
00587     Kokkos::View<size_t*,device_type> numImportPacketsPerLID_;
00588     typename Kokkos::View<size_t*,device_type>::HostMirror host_numImportPacketsPerLID_;
00589   protected:
00590     Teuchos::Array<size_t> numImportPacketsPerLID_old_;
00591   private:
00592 
00594     Kokkos::View<packet_type*,device_type> exports_;
00595   protected:
00596     Teuchos::Array<Packet> exports_old_;
00597   private:
00598 
00609     Kokkos::View<size_t*,device_type> numExportPacketsPerLID_;
00610   protected:
00611     Teuchos::Array<size_t> numExportPacketsPerLID_old_;
00612 
00613 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00614   private:
00615     Teuchos::RCP<Teuchos::Time> doXferTimer_;
00616     Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
00617     Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
00618     Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
00619     Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
00620 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00621 
00622   }; // class DistObject
00623 
00624   /* These are provided in base DistObject template
00707   template<class DistObjectType>
00708   void
00709   removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
00710                                const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
00711                                                             typename DistObjectType::global_ordinal_type,
00712                                                             typename DistObjectType::node_type> >& newMap);
00713 
00750   template<class DistObjectType>
00751   void
00752   removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);
00753   */
00754 
00755 } // namespace Tpetra
00756 
00757 #endif /* TPETRA_DISTOBJECT_DECL_HPP */