threads.c

Go to the documentation of this file.

/****************************/
/* THIS IS OPEN SOURCE CODE */
/****************************/

/* 
* File:    threads.c
* Author:  Philip Mucci
*          mucci@cs.utk.edu
* Mods:    Kevin London
*          london@cs.utk.edu
*/

/* This file contains thread allocation and bookkeeping functions */

#include "papi.h"
#include "papi_internal.h"
#include "papi_vector.h"
#include "papi_memory.h"
#include <string.h>
#include <unistd.h>

/*****************/
/* BEGIN GLOBALS */
/*****************/

/* The following globals get initialized and cleared by:
   extern int _papi_hwi_init_global_threads(void);
   extern int _papi_hwi_shutdown_thread(ThreadInfo_t *thread); */

/* list of threads, gets initialized to master process with TID of getpid() */

volatile ThreadInfo_t *_papi_hwi_thread_head;

/* If we have TLS, this variable ALWAYS points to our thread descriptor. It's like magic! */

#if defined(HAVE_THREAD_LOCAL_STORAGE)
THREAD_LOCAL_STORAGE_KEYWORD ThreadInfo_t *_papi_hwi_my_thread;
#endif

/* Function that returns and unsigned long thread identifier */

unsigned long ( *_papi_hwi_thread_id_fn ) ( void );

/* Function that sends a signal to other threads */

#ifdef ANY_THREAD_GETS_SIGNAL
int ( *_papi_hwi_thread_kill_fn ) ( int, int );
#endif

/*****************/
/*  END  GLOBALS */
/*****************/

static int
lookup_and_set_thread_symbols( void )
{
#if defined(ANY_THREAD_GETS_SIGNAL)
    int retval;
    char *error_ptc = NULL, *error_ptk = NULL;
    void *symbol_ptc = NULL, *symbol_ptk = NULL, *handle = NULL;

    handle = dlopen( NULL, RTLD_LAZY );
    if ( handle == NULL ) {
        PAPIERROR( "Error from dlopen(NULL, RTLD_LAZY): %d %s", errno,
                   dlerror(  ) );
        return ( PAPI_ESYS );
    }

    symbol_ptc = dlsym( handle, "pthread_self" );
    if ( symbol_ptc == NULL ) {
        error_ptc = dlerror(  );
        THRDBG( "dlsym(%p,pthread_self) returned NULL: %s\n",
                ( error_ptc ? error_ptc : "No error, NULL symbol!" ) );
    }

    symbol_ptk = dlsym( handle, "pthread_kill" );
    if ( symbol_ptk == NULL ) {
        error_ptk = dlerror(  );
        THRDBG( "dlsym(%p,pthread_kill) returned NULL: %s\n",
                ( error_ptk ? error_ptk : "No error, NULL symbol!" ) );
    }

    dlclose( handle );

    if ( !( ( _papi_hwi_thread_kill_fn && _papi_hwi_thread_id_fn ) ||
            ( !_papi_hwi_thread_kill_fn && !_papi_hwi_thread_id_fn ) ) )
        return ( PAPI_EMISC );

    _papi_hwi_thread_kill_fn = ( int ( * )( int, int ) ) symbol_ptk;
    _papi_hwi_thread_id_fn = ( unsigned long ( * )( void ) ) symbol_ptc;
#endif
    return ( PAPI_OK );
}

static ThreadInfo_t *
allocate_thread( int tid )
{
    ThreadInfo_t *thread;
    int i;

    /* The Thread EventSet is special. It is not in the EventSet list, but is pointed
       to by each EventSet of that particular thread. */

    thread = ( ThreadInfo_t * ) papi_malloc( sizeof ( ThreadInfo_t ) );
    if ( thread == NULL )
        return ( NULL );
    memset( thread, 0x00, sizeof ( ThreadInfo_t ) );

    thread->context =
        ( hwd_context_t ** ) papi_malloc( sizeof ( hwd_context_t * ) *
                                          ( size_t ) papi_num_components );
    if ( !thread->context ) {
        papi_free( thread );
        return ( NULL );
    }

    thread->running_eventset =
        ( EventSetInfo_t ** ) papi_malloc( sizeof ( EventSetInfo_t * ) *
                                           ( size_t ) papi_num_components );
    if ( !thread->running_eventset ) {
        papi_free( thread->context );
        papi_free( thread );
        return ( NULL );
    }

    for ( i = 0; i < papi_num_components; i++ ) {
        thread->context[i] =
            ( void * ) papi_malloc( ( size_t ) _papi_hwd[i]->size.context );
        thread->running_eventset[i] = NULL;
        if ( thread->context[i] == NULL ) {
            for ( i--; i >= 0; i-- )
                papi_free( thread->context[i] );
            papi_free( thread->context );
            papi_free( thread );
            return ( NULL );
        }
        memset( thread->context[i], 0x00,
                ( size_t ) _papi_hwd[i]->size.context );
    }

    if ( _papi_hwi_thread_id_fn ) {
           thread->tid = ( *_papi_hwi_thread_id_fn ) (  );
    }
    else {
       thread->tid = ( unsigned long ) getpid(  );
    }

    thread->allocator_tid=thread->tid;

    if (tid == 0 ) {
    }
    else {
      thread->tid=tid;
    }

    THRDBG( "Allocated thread %ld at %p, allocator: %ld\n", thread->tid, 
        thread,
        thread->allocator_tid );

    return thread;
}

static void
free_thread( ThreadInfo_t ** thread )
{
    int i;
    THRDBG( "Freeing thread %ld at %p\n", ( *thread )->tid, *thread );

    for ( i = 0; i < papi_num_components; i++ ) {
        if ( ( *thread )->context[i] )
            papi_free( ( *thread )->context[i] );
    }

    if ( ( *thread )->context )
        papi_free( ( *thread )->context );

    if ( ( *thread )->running_eventset )
        papi_free( ( *thread )->running_eventset );

    memset( *thread, 0x00, sizeof ( ThreadInfo_t ) );
    papi_free( *thread );
    *thread = NULL;
}

static void
insert_thread( ThreadInfo_t * entry, int tid )
{
    _papi_hwi_lock( THREADS_LOCK );

    if ( _papi_hwi_thread_head == NULL ) {  /* 0 elements */
        THRDBG( "_papi_hwi_thread_head is NULL\n" );
        entry->next = entry;
    } else if ( _papi_hwi_thread_head->next == _papi_hwi_thread_head ) {    /* 1 elements */
        THRDBG( "_papi_hwi_thread_head was thread %ld at %p\n",
                _papi_hwi_thread_head->tid, _papi_hwi_thread_head );
        _papi_hwi_thread_head->next = entry;
        entry->next = ( ThreadInfo_t * ) _papi_hwi_thread_head;
    } else {                 /* 2+ elements */

        THRDBG( "_papi_hwi_thread_head was thread %ld at %p\n",
                _papi_hwi_thread_head->tid, _papi_hwi_thread_head );
        entry->next = _papi_hwi_thread_head->next;
        _papi_hwi_thread_head->next = entry;
    }

    _papi_hwi_thread_head = entry;

    THRDBG( "_papi_hwi_thread_head now thread %ld at %p\n",
            _papi_hwi_thread_head->tid, _papi_hwi_thread_head );

    _papi_hwi_unlock( THREADS_LOCK );

#if defined(HAVE_THREAD_LOCAL_STORAGE)
    /* Don't set the current local thread if we are a fake attach thread */
        if (tid==0) {
       _papi_hwi_my_thread = entry;
       THRDBG( "TLS for thread %ld is now %p\n", entry->tid,
            _papi_hwi_my_thread );
    }
#endif
}

static int
remove_thread( ThreadInfo_t * entry )
{
    ThreadInfo_t *tmp = NULL, *prev = NULL;

    _papi_hwi_lock( THREADS_LOCK );

    THRDBG( "_papi_hwi_thread_head was thread %ld at %p\n",
            _papi_hwi_thread_head->tid, _papi_hwi_thread_head );

    /* Find the preceding element and the matched element,
       short circuit if we've seen the head twice */

    for ( tmp = ( ThreadInfo_t * ) _papi_hwi_thread_head;
          ( entry != tmp ) || ( prev == NULL ); tmp = tmp->next ) {
        prev = tmp;
    }

    if ( tmp != entry ) {
        THRDBG( "Thread %ld at %p was not found in the thread list!\n",
                entry->tid, entry );
        return ( PAPI_EBUG );
    }

    /* Only 1 element in list */

    if ( prev == tmp ) {
        _papi_hwi_thread_head = NULL;
        tmp->next = NULL;
        THRDBG( "_papi_hwi_thread_head now NULL\n" );
    } else {
        prev->next = tmp->next;
        /* If we're removing the head, better advance it! */
        if ( _papi_hwi_thread_head == tmp ) {
            _papi_hwi_thread_head = tmp->next;
            THRDBG( "_papi_hwi_thread_head now thread %ld at %p\n",
                    _papi_hwi_thread_head->tid, _papi_hwi_thread_head );
        }
        THRDBG( "Removed thread %p from list\n", tmp );
    }

    _papi_hwi_unlock( THREADS_LOCK );

#if defined(HAVE_THREAD_LOCAL_STORAGE)
    _papi_hwi_my_thread = NULL;
    THRDBG( "TLS for thread %ld is now %p\n", entry->tid,
            _papi_hwi_my_thread );
#endif

    return PAPI_OK;
}

int
_papi_hwi_initialize_thread( ThreadInfo_t ** dest, int tid )
{
    int retval;
    ThreadInfo_t *thread;
    int i;

    if ( ( thread = allocate_thread( tid  ) ) == NULL ) {
        *dest = NULL;
        return PAPI_ENOMEM;
    }

    /* Call the component to fill in anything special. */

    for ( i = 0; i < papi_num_components; i++ ) {
        if (_papi_hwd[i]->cmp_info.disabled) continue;
        retval = _papi_hwd[i]->init_thread( thread->context[i] );
        if ( retval ) {
           free_thread( &thread );
           *dest = NULL;
           return retval;
        }
    }

    insert_thread( thread, tid );

    *dest = thread;
    return PAPI_OK;
}

#if defined(ANY_THREAD_GETS_SIGNAL)

/* This is ONLY defined for systems that enable ANY_THREAD_GETS_SIGNAL
   since we must forward signals sent to non-PAPI threads.

   This is NOT compatible with thread local storage, since to broadcast
   the signal, we need a list of threads. */

int
_papi_hwi_broadcast_signal( unsigned int mytid )
{
    int i, retval, didsomething = 0;
    volatile ThreadInfo_t *foo = NULL;

    _papi_hwi_lock( THREADS_LOCK );

    for ( foo = _papi_hwi_thread_head; foo != NULL; foo = foo->next ) {
        /* xxxx Should this be hardcoded to index 0 or walk the list or what? */
        for ( i = 0; i < papi_num_components; i++ ) {
            if ( ( foo->tid != mytid ) && ( foo->running_eventset[i] ) &&
                 ( foo->running_eventset[i]->
                   state & ( PAPI_OVERFLOWING | PAPI_MULTIPLEXING ) ) ) {
                /* xxxx mpx_info inside _papi_mdi_t _papi_hwi_system_info is commented out.
                   See papi_internal.h for details. The multiplex_timer_sig value is now part of that structure */
              THRDBG("Thread %ld sending signal %d to thread %ld\n",mytid,foo->tid,
                  (foo->running_eventset[i]->state & PAPI_OVERFLOWING ? _papi_hwd[i]->cmp_info.hardware_intr_sig : _papi_os_info.itimer_sig));
              retval = (*_papi_hwi_thread_kill_fn)(foo->tid, 
                  (foo->running_eventset[i]->state & PAPI_OVERFLOWING ? _papi_hwd[i]->cmp_info.hardware_intr_sig : _papi_os_info.itimer_sig));
              if (retval != 0)
                return(PAPI_EMISC);
            }
        }
        if ( foo->next == _papi_hwi_thread_head )
          break;
    }
    _papi_hwi_unlock( THREADS_LOCK );

    return ( PAPI_OK );
}
#endif

/* This is undefined for systems that enable ANY_THREAD_GETS_SIGNAL
   since we always must enable threads for safety. */

int
_papi_hwi_set_thread_id_fn( unsigned long ( *id_fn ) ( void ) )
{
#if !defined(ANY_THREAD_GETS_SIGNAL)
    /* Check for multiple threads still in the list, if so, we can't change it */

    if ( _papi_hwi_thread_head->next != _papi_hwi_thread_head )
        return ( PAPI_EINVAL );

    /* We can't change the thread id function from one to another, 
       only NULL to non-NULL and vice versa. */

    if ( ( id_fn != NULL ) && ( _papi_hwi_thread_id_fn != NULL ) )
        return ( PAPI_EINVAL );

    _papi_hwi_thread_id_fn = id_fn;

    THRDBG( "Set new thread id function to %p\n", id_fn );

    if ( id_fn )
        _papi_hwi_thread_head->tid = ( *_papi_hwi_thread_id_fn ) (  );
    else
        _papi_hwi_thread_head->tid = ( unsigned long ) getpid(  );

    THRDBG( "New master tid is %ld\n", _papi_hwi_thread_head->tid );
#else
    THRDBG( "Skipping set of thread id function\n" );
#endif

    return PAPI_OK;
}


static int _papi_hwi_thread_free_eventsets(long tid) {

   EventSetInfo_t *ESI;
   ThreadInfo_t *master;
   DynamicArray_t *map = &_papi_hwi_system_info.global_eventset_map;
   int i;

   master = _papi_hwi_lookup_thread( tid );

   _papi_hwi_lock( INTERNAL_LOCK );

   for( i = 0; i < map->totalSlots; i++ ) {
      ESI = map->dataSlotArray[i];
      if ( ( ESI ) && (ESI->master!=NULL) ) {

     if ( ESI->master == master ) {
        THRDBG("Attempting to remove %d from tid %ld\n",ESI->EventSetIndex,tid);

        /* Code copied from _papi_hwi_remove_EventSet(ESI);      */
        _papi_hwi_free_EventSet( ESI );
        map->dataSlotArray[i] = NULL;
        map->availSlots++;
        map->fullSlots--;
     } 
      }
   }

   _papi_hwi_unlock( INTERNAL_LOCK );

   return PAPI_OK;
}




int
_papi_hwi_shutdown_thread( ThreadInfo_t * thread )
{
    int retval = PAPI_OK;
    unsigned long tid;
    int i, failure = 0;

    if ( _papi_hwi_thread_id_fn )
        tid = ( *_papi_hwi_thread_id_fn ) (  );
    else
        tid = ( unsigned long ) getpid(  );

        THRDBG("Want to shutdown thread %ld, alloc %ld, our_tid: %ld\n",
           thread->tid,
           thread->allocator_tid,
           tid);

    if ((thread->tid==tid) || ( thread->allocator_tid == tid )) {

                _papi_hwi_thread_free_eventsets(tid);

        remove_thread( thread );
        THRDBG( "Shutting down thread %ld at %p\n", thread->tid, thread );
        for( i = 0; i < papi_num_components; i++ ) {
           if (_papi_hwd[i]->cmp_info.disabled) continue;
           retval = _papi_hwd[i]->shutdown_thread( thread->context[i]);
           if ( retval != PAPI_OK ) failure = retval;
        }
        free_thread( &thread );
        return ( failure );
    }

    THRDBG( "Skipping shutdown thread %ld at %p, thread %ld not allocator!\n",
            thread->tid, thread, tid );
    return PAPI_EBUG;
}

/* THESE MUST BE CALLED WITH A GLOBAL LOCK */

int
_papi_hwi_shutdown_global_threads( void )
{
        int err,num_threads,i;
    ThreadInfo_t *tmp,*next;
    unsigned long our_tid;

    tmp = _papi_hwi_lookup_thread( 0 );

    if ( tmp == NULL ) {
       THRDBG( "Did not find my thread for shutdown!\n" );
       err = PAPI_EBUG;
    }
    else {
       our_tid=tmp->tid;

       THRDBG("Shutting down %ld\n",our_tid);

       err = _papi_hwi_shutdown_thread( tmp );


       /* count threads */
       tmp = ( ThreadInfo_t * ) _papi_hwi_thread_head;
       num_threads=0;
       while(tmp!=NULL) {
          num_threads++;
          if (tmp->next==_papi_hwi_thread_head) break;
          tmp=tmp->next;
       }

       /* Shut down all threads allocated by this thread */
       /* Urgh it's a circular list where we removed in the loop  */
       /* so the only sane way to do it is get a count in advance */
       tmp = ( ThreadInfo_t * ) _papi_hwi_thread_head;

       for(i=0;i<num_threads;i++) {

          next=tmp->next;

          THRDBG("looking at #%d %ld our_tid: %ld alloc_tid: %ld\n",
             i,tmp->tid,our_tid,tmp->allocator_tid);
        
          if (tmp->allocator_tid==our_tid) {
         THRDBG("Also removing thread %ld\n",tmp->tid);
             err = _papi_hwi_shutdown_thread( tmp );
          }
  
          tmp=next;

       }
    }


#ifdef DEBUG
    if ( ISLEVEL( DEBUG_THREADS ) ) {
        if ( _papi_hwi_thread_head ) {
            THRDBG( "Thread head %p still exists!\n", _papi_hwi_thread_head );
        }
    }
#endif

#if defined(HAVE_THREAD_LOCAL_STORAGE)
    _papi_hwi_my_thread = NULL;
#endif
    _papi_hwi_thread_head = NULL;
    _papi_hwi_thread_id_fn = NULL;
#if defined(ANY_THREAD_GETS_SIGNAL)
    _papi_hwi_thread_kill_fn = NULL;
#endif

    return err;
}

int
_papi_hwi_init_global_threads( void )
{
    int retval;
    ThreadInfo_t *tmp;

    _papi_hwi_lock( GLOBAL_LOCK );

#if defined(HAVE_THREAD_LOCAL_STORAGE)
    _papi_hwi_my_thread = NULL;
#endif
    _papi_hwi_thread_head = NULL;
    _papi_hwi_thread_id_fn = NULL;
#if defined(ANY_THREAD_GETS_SIGNAL)
    _papi_hwi_thread_kill_fn = NULL;
#endif

    retval = _papi_hwi_initialize_thread( &tmp , 0);
    if ( retval == PAPI_OK ) {
       retval = lookup_and_set_thread_symbols(  );
    }

    _papi_hwi_unlock( GLOBAL_LOCK );

    return ( retval );
}

int
_papi_hwi_gather_all_thrspec_data( int tag, PAPI_all_thr_spec_t * where )
{
    int didsomething = 0;
    ThreadInfo_t *foo = NULL;

    _papi_hwi_lock( THREADS_LOCK );

    for ( foo = ( ThreadInfo_t * ) _papi_hwi_thread_head; foo != NULL;
          foo = foo->next ) {
        /* If we want thread ID's */
        if ( where->id )
            memcpy( &where->id[didsomething], &foo->tid,
                    sizeof ( where->id[didsomething] ) );

        /* If we want data pointers */
        if ( where->data )
            where->data[didsomething] = foo->thread_storage[tag];

        didsomething++;

        if ( ( where->id ) || ( where->data ) ) {
            if ( didsomething >= where->num )
                break;
        }

        if ( foo->next == _papi_hwi_thread_head )
            break;
    }

    where->num = didsomething;
    _papi_hwi_unlock( THREADS_LOCK );

    return ( PAPI_OK );

}