BulkData.hpp

/*------------------------------------------------------------------------*/
/*                 Copyright 2010 Sandia Corporation.                     */
/*  Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive   */
/*  license for use of this work by or on behalf of the U.S. Government.  */
/*  Export of this program may require a license from the                 */
/*  United States Government.                                             */
/*------------------------------------------------------------------------*/

#ifndef stk_mesh_BulkData_hpp
#define stk_mesh_BulkData_hpp

//----------------------------------------------------------------------

#include <stk_util/parallel/Parallel.hpp>
#include <stk_util/parallel/DistributedIndex.hpp>

#include <stk_mesh/base/Types.hpp>
#include <stk_mesh/base/Field.hpp>
#include <stk_mesh/base/MetaData.hpp>
#include <stk_mesh/base/Ghosting.hpp>
#include <stk_mesh/base/Selector.hpp>
#include <stk_mesh/base/Trace.hpp>
#include <stk_mesh/base/EntityComm.hpp>

#include <stk_mesh/baseImpl/EntityRepository.hpp>
#include <stk_mesh/baseImpl/BucketRepository.hpp>

#include <algorithm>
#include <map>

//----------------------------------------------------------------------

namespace stk_classic {
namespace mesh {

//----------------------------------------------------------------------
class BulkData {
public:

  void optimize_buckets_at_modification_end(bool b) { m_optimize_buckets = b; }

  inline static BulkData & get( const Bucket & bucket);
  inline static BulkData & get( const Entity & entity);
  inline static BulkData & get( const Ghosting & ghost);

  enum BulkDataSyncState { MODIFIABLE = 1 , SYNCHRONIZED = 2 };

  ~BulkData();

  BulkData( MetaData & mesh_meta_data ,
      ParallelMachine parallel ,
      unsigned bucket_max_size = 1000 ,
      bool use_memory_pool = true );

  //------------------------------------
  const MetaData & mesh_meta_data() const { return m_mesh_meta_data ; }

  ParallelMachine parallel() const { return m_parallel_machine ; }

  unsigned parallel_size()   const { return m_parallel_size ; }

  unsigned parallel_rank()   const { return m_parallel_rank ; }

  //------------------------------------
  BulkDataSyncState synchronized_state() const { return m_sync_state ; }

  size_t synchronized_count() const { return m_sync_count ; }

  bool modification_begin();

  bool modification_end();

  bool final_modification_end()
  {
    optimize_buckets_at_modification_end(true);
    const bool mod_flag =  modification_end();

    //call modification_begin and end one last time to free deleted entities
    modification_begin();
    modification_end();

    m_mesh_finalized = true;

    return mod_flag;
  }

  void change_entity_owner( const std::vector<EntityProc> & arg_change);

  void update_field_data_states() const { m_bucket_repository.update_field_data_states(); }

  void copy_entity_fields( const Entity & src, Entity & dest) {
    //TODO fix const correctness for src
    Entity & non_const_src = const_cast<Entity &>(src);
    m_bucket_repository.copy_fields(
        dest.bucket(),
        dest.bucket_ordinal(),
        non_const_src.bucket(),
        non_const_src.bucket_ordinal()
        );
  }

  //------------------------------------
  const std::vector<Bucket*> & buckets( EntityRank rank ) const
  { return m_bucket_repository.buckets(rank); }

#ifndef SWIG //SRK
  AllBucketsRange get_bucket_range() const
  {
    return m_bucket_repository.get_bucket_range();
  }

  AllBucketsRange get_bucket_range( EntityRank rank ) const
  {
    return m_bucket_repository.get_bucket_range(rank);
  }
#endif

  Entity * get_entity( EntityRank entity_rank , EntityId entity_id ) const {
    require_good_rank_and_id(entity_rank, entity_id);
    return m_entity_repo.get_entity( EntityKey(entity_rank, entity_id));
  }

  Entity * get_entity( const EntityKey key ) const  {
    return m_entity_repo.get_entity(key);
  }

  //------------------------------------
  Entity & declare_entity( EntityRank ent_rank ,
      EntityId ent_id , const PartVector& parts);

  void change_entity_id( EntityId id, Entity & entity);

  void change_entity_parts( Entity & entity,
      const PartVector & add_parts ,
      const PartVector & remove_parts = PartVector() )
  {
    change_entity_parts(entity,
                        add_parts.begin(), add_parts.end(),
                        remove_parts.begin(), remove_parts.end());
  }

//Optional parameter 'always_propagate_internal_changes' is always true except when this function
//is being called from the sierra-framework. The fmwk redundantly does its own propagation of the
//internal part changes (mostly induced-part stuff), so it's a performance optimization to avoid
//the propagation that stk-mesh does.
  template<typename AddIterator, typename RemoveIterator>
  void change_entity_parts( Entity & entity,
                            AddIterator begin_add_parts, AddIterator end_add_parts,
                            RemoveIterator begin_remove_parts, RemoveIterator end_remove_parts,
                            bool always_propagate_internal_changes=true );

  bool destroy_entity( Entity * & entity );

  //------------------------------------

  void generate_new_entities(const std::vector<size_t>& requests,
      std::vector<Entity *>& requested_entities);

  //------------------------------------
  void declare_relation( Entity & e_from ,
      Entity & e_to ,
      const RelationIdentifier local_id );

  void declare_relation( Entity & entity, const std::vector<Relation> & rel);

  bool destroy_relation( Entity & e_from ,
                         Entity & e_to,
                         const RelationIdentifier local_id );

  //------------------------------------
  //------------------------------------
  const std::vector<Entity*> & entity_comm() const
    { return m_entity_comm ; }

  //------------------------------------
  Ghosting & shared_aura() const { return * m_ghosting[1] ; }

  Ghosting & create_ghosting( const std::string & name );

  void change_ghosting( Ghosting & ghosts,
                        const std::vector<EntityProc> & add_send ,
                        const std::vector<Entity*> & remove_receive );

  void destroy_all_ghosting();

  const std::vector<Ghosting*> & ghostings() const { return m_ghosting ; }

  PairIterEntityComm entity_comm(const EntityKey & key) const { return m_entity_comm_map.comm(key); }
  PairIterEntityComm entity_comm_sharing(const EntityKey & key) const { return m_entity_comm_map.sharing(key); }
  PairIterEntityComm entity_comm(const EntityKey & key, const Ghosting & sub ) const { return m_entity_comm_map.comm(key,sub); }
  bool entity_comm_insert( const EntityKey & key, const EntityCommInfo & val) { return m_entity_comm_map.insert(key,val); }
  bool entity_comm_erase(  const EntityKey & key, const EntityCommInfo & val) { return m_entity_comm_map.erase(key,val); }
  bool entity_comm_erase(  const EntityKey & key, const Ghosting & ghost) { return m_entity_comm_map.erase(key,ghost); }
  void entity_comm_clear_ghosting(const EntityKey & key ) { m_entity_comm_map.comm_clear_ghosting(key); }
  void entity_comm_clear(const EntityKey & key) { m_entity_comm_map.comm_clear(key); }
  void entity_comm_swap(const EntityKey & key1, const EntityKey & key2) { m_entity_comm_map.comm_swap(key1, key2); }

private:

  MetaData & meta_data() const { return m_mesh_meta_data ; }

#ifndef DOXYGEN_COMPILE

  BulkData();
  BulkData( const BulkData & );
  BulkData & operator = ( const BulkData & );

  parallel::DistributedIndex          m_entities_index ;
  impl::EntityRepository              m_entity_repo ;
  impl::BucketRepository              m_bucket_repository ;
  std::vector<Entity*>                m_entity_comm ;
  std::vector<Ghosting*>              m_ghosting ; 
  // Other information:
  MetaData &         m_mesh_meta_data ;
  ParallelMachine    m_parallel_machine ;
  unsigned           m_parallel_size ;
  unsigned           m_parallel_rank ;
  size_t             m_sync_count ;
  BulkDataSyncState  m_sync_state ;
  bool               m_meta_data_verified ;
  bool               m_optimize_buckets;
  bool               m_mesh_finalized;
  EntityComm         m_entity_comm_map;

  unsigned determine_new_owner( Entity & ) const ;

  /*  Entity modification consequences:
   *  1) Change entity relation => update via part relation => change parts
   *  2) Change parts => update forward relations via part relation
   *                  => update via field relation
   */
  void internal_change_entity_parts( Entity & ,
                                     const PartVector & add_parts ,
                                     const PartVector & remove_parts );

//Optional parameter 'always_propagate_internal_changes' is always true except when this function
//is being called from the sierra-framework. The fmwk redundantly does its own propagation of the
//internal part changes (mostly induced-part stuff), so it's a performance optimization to avoid
//the propagation that stk-mesh does.
  void internal_change_entity_parts( Entity & ,
                                     const OrdinalVector & add_parts ,
                                     const OrdinalVector & remove_parts,
                                     bool always_propagate_internal_changes=true);

  void internal_propagate_part_changes( Entity & entity, const PartVector & removed );
  void internal_propagate_part_changes( Entity & entity, const OrdinalVector & removed );

  void internal_change_ghosting( Ghosting & ghosts,
                                 const std::vector<EntityProc> & add_send ,
                                 const std::vector<Entity*> & remove_receive );

  bool internal_modification_end( bool regenerate_aura );
    void internal_resolve_shared_modify_delete();
    void internal_resolve_shared_modify_delete_second_pass();
    void internal_resolve_ghosted_modify_delete();
    void internal_resolve_parallel_create();
    void internal_resolve_shared_membership();

  void internal_update_distributed_index( std::vector<Entity*> & shared_new );

  void internal_regenerate_shared_aura();

  void internal_basic_part_check(const Part* part,
                                 const unsigned entity_rank,
                                 const unsigned undef_rank,
                                 bool& intersection_ok,
                                 bool& rel_target_ok,
                                 bool& rank_ok) const;

  // Returns false if there is a problem. It is expected that
  // verify_change_parts will be called if quick_verify_change_part detects
  // a problem, therefore we leave the generation of an exception to
  // verify_change_parts. We want this function to be as fast as
  // possible.
  bool internal_quick_verify_change_part(const Part* part,
                                         const unsigned entity_rank,
                                         const unsigned undef_rank) const;

  void internal_verify_change_parts( const MetaData   & meta ,
                                     const Entity     & entity ,
                                     const PartVector & parts ) const;

  void internal_verify_change_parts( const MetaData   & meta ,
                                     const Entity     & entity ,
                                     const OrdinalVector & parts ) const;

  //------------------------------------

  void require_ok_to_modify() const ;

  void require_entity_owner( const Entity & entity, unsigned owner) const ;

  void require_metadata_committed() const;

  void require_good_rank_and_id(EntityRank ent_rank, EntityId ent_id) const;

  void require_valid_relation( const char action[] ,
                               const BulkData & mesh ,
                               const Entity   & e_from ,
                               const Entity   & e_to );

  //------------------------------------

  // FIXME: Remove this friend once unit-testing has been refactored
  friend class UnitTestModificationEndWrapper;
  friend class ::stk_classic::mesh::MetaData;
#endif /* DOXYGEN_COMPILE */
};

BulkData & BulkData::get( const Bucket & bucket) {
  return bucket.bulk_data();
}

BulkData & BulkData::get( const Entity & entity) {
  return BulkData::get(entity.bucket());
}

BulkData & BulkData::get( const Ghosting & ghost) {
  return ghost.bulk_data();
}

// TODO - Does this need to be in the public API? It's only used internally.
// In what context would a client ever call this?
void set_field_relations( Entity & e_from ,
                          Entity & e_to ,
                          const unsigned ident );

inline
void BulkData::internal_basic_part_check(const Part* part,
                                         const unsigned entity_rank,
                                         const unsigned undef_rank,
                                         bool& intersection_ok,
                                         bool& rel_target_ok,
                                         bool& rank_ok) const
{
  // const unsigned part_rank = part->primary_entity_rank();

  intersection_ok = part->intersection_of().empty();
  rel_target_ok   = ( part->relations().empty() ||
                      part != part->relations().begin()->m_target );
  // Do we allow arbitrary part changes to entities regardless of part rank? For the sake of the migration, we will for now.
#ifdef SIERRA_MIGRATION
  rank_ok = true;
#else
  const unsigned part_rank = part->primary_entity_rank();
  rank_ok         = ( entity_rank == part_rank ||
                      undef_rank  == part_rank );
#endif
}

inline bool BulkData::internal_quick_verify_change_part(const Part* part,
                                                        const unsigned entity_rank,
                                                        const unsigned undef_rank) const
{
  bool intersection_ok, rel_target_ok, rank_ok;
  internal_basic_part_check(part, entity_rank, undef_rank, intersection_ok, rel_target_ok, rank_ok);
  return intersection_ok && rel_target_ok && rank_ok;
}

//Optional parameter 'always_propagate_internal_changes' is always true except when this function
//is being called from the sierra-framework. The fmwk redundantly does its own propagation of the
//internal part changes (mostly induced-part stuff), so it's a performance optimization to avoid
//the propagation that stk-mesh does.
template<typename AddIterator, typename RemoveIterator>
void BulkData::change_entity_parts( Entity & entity,
                                    AddIterator begin_add_parts, AddIterator end_add_parts,
                                    RemoveIterator begin_remove_parts, RemoveIterator end_remove_parts,
                                    bool always_propagate_internal_changes)
{
  TraceIfWatching("stk_classic::mesh::BulkData::change_entity_parts", LOG_ENTITY, entity.key());
  DiagIfWatching(LOG_ENTITY, entity.key(), "entity state: " << entity);
  //DiagIfWatching(LOG_ENTITY, entity.key(), "add_parts: " << add_parts);
  //DiagIfWatching(LOG_ENTITY, entity.key(), "remove_parts: " << remove_parts);

  require_ok_to_modify();

  require_entity_owner( entity , m_parallel_rank );

  const EntityRank entity_rank = entity.entity_rank();
  const EntityRank undef_rank  = InvalidEntityRank;

  // Transitive addition and removal:
  // 1) Include supersets of add_parts
  // 2) Do not include a remove_part if it appears in the add_parts
  // 3) Include subsets of remove_parts

  // most parts will at least have universal and topology part as supersets
  const unsigned expected_min_num_supersets = 2;

  OrdinalVector a_parts;
  a_parts.reserve( std::distance(begin_add_parts, end_add_parts) * (expected_min_num_supersets + 1) );
  for(AddIterator add_iter=begin_add_parts; add_iter!=end_add_parts; ++add_iter) {
    a_parts.push_back((*add_iter)->mesh_meta_data_ordinal());
  }
  bool quick_verify_check = true;

  for ( AddIterator ia = begin_add_parts; ia != end_add_parts ; ++ia ) {
    quick_verify_check = quick_verify_check &&
      internal_quick_verify_change_part(*ia, entity_rank, undef_rank);
    const PartVector& supersets = (*ia)->supersets();
    for(PartVector::const_iterator s_iter=supersets.begin(), s_end=supersets.end();
        s_iter!=s_end; ++s_iter) {
      a_parts.push_back((*s_iter)->mesh_meta_data_ordinal());
    }
  }

  order(a_parts);

  OrdinalVector::const_iterator a_parts_begin = a_parts.begin(),
                                a_parts_end   = a_parts.end();
  OrdinalVector r_parts ;

  for ( RemoveIterator ir = begin_remove_parts; ir != end_remove_parts ; ++ir ) {

    // The following guards should be in the public interface to
    // changing parts.  However, internal mechanisms such as changing
    // ownership calls this function to add or remove an entity from
    // the three special parts.  Without refactoring, these guards
    // cannot be put in place.
    /*
    ThrowErrorMsgIf( m_mesh_meta_data.universal_part() == **ir,
                     "Cannot remove entity from universal part" );
    ThrowErrorMsgIf( m_mesh_meta_data.locally_owned_part() == **ir,
                     "Cannot remove entity from locally owned part" );
    ThrowErrorMsgIf( m_mesh_meta_data.globally_shared_part() == **ir,
                     "Cannot remove entity from globally shared part" );
    */

    quick_verify_check = quick_verify_check &&
      internal_quick_verify_change_part(*ir, entity_rank, undef_rank);

    if ( ! contains_ordinal( a_parts_begin, a_parts_end , (*ir)->mesh_meta_data_ordinal() ) ) {
      r_parts.push_back( (*ir)->mesh_meta_data_ordinal() );
      for ( PartVector::const_iterator  cur_part = (*ir)->subsets().begin() ;
            cur_part != (*ir)->subsets().end() ;
            ++cur_part )
        if ( entity.bucket().member ( **cur_part ) )
          r_parts.push_back ( (*cur_part)->mesh_meta_data_ordinal() );
    }
  }

  order(r_parts);

  // If it looks like we have a problem, run the full check and we should
  // expect to see an exception thrown; otherwise, only do the full check in
  // debug mode because it incurs significant overhead.
  if ( ! quick_verify_check ) {
    internal_verify_change_parts( m_mesh_meta_data , entity , a_parts );
    internal_verify_change_parts( m_mesh_meta_data , entity , r_parts );
    ThrowRequireMsg(false, "Expected throw from verify methods above.");
  }
  else {
#ifndef NDEBUG
    internal_verify_change_parts( m_mesh_meta_data , entity , a_parts );
    internal_verify_change_parts( m_mesh_meta_data , entity , r_parts );
#endif
  }

  internal_change_entity_parts( entity , a_parts , r_parts , always_propagate_internal_changes );

  return ;
}

} // namespace mesh
} // namespace stk_classic

//----------------------------------------------------------------------
//----------------------------------------------------------------------

#endif //  stk_mesh_BulkData_hpp