linux-infiniband_umad.c File Reference

This file has the source code for a component that enables PAPI-C to access hardware monitoring counters for InfiniBand devices through the. More...

Include dependency graph for linux-infiniband_umad.c:

Go to the source code of this file.

Macros
#define	infiniband_native_table   subscriptions
#define	InitStruct(var, type)   type var; memset(&var, 0, sizeof(type))

Functions
static counter_info *	addCounter (const char *name, const char *desc, const char *unit)
static void	addIBPort (const char *ca_name, umad_port_t *port)
static int	init_ib_port (ib_port *portdata)
static int	read_ib_counter ()
void	host_read_values (long long *data)
static counter_info *	counterFromName (const char *cntr)
static uint64_t	host_subscribe (const char *cntr)
static string_list *	host_listCounter (int num_counters1)
static void	host_finalize ()
static void	host_deleteStringList (string_list *to_delete)
int	INFINIBAND_init_thread (hwd_context_t *ctx)
int	INFINIBAND_init_component (int cidx)
static int	linkInfinibandLibraries ()
int	INFINIBAND_init_control_state (hwd_control_state_t *ctrl)
int	INFINIBAND_start (hwd_context_t *ctx, hwd_control_state_t *ctrl)
int	INFINIBAND_stop (hwd_context_t *ctx, hwd_control_state_t *ctrl)
int	INFINIBAND_read (hwd_context_t *ctx, hwd_control_state_t *ctrl, long_long **events, int flags)
int	INFINIBAND_shutdown_thread (hwd_context_t *ctx)
int	INFINIBAND_shutdown_component (void)
int	INFINIBAND_ctl (hwd_context_t *ctx, int code, _papi_int_option_t *option)
int	INFINIBAND_update_control_state (hwd_control_state_t *ptr, NativeInfo_t *native, int count, hwd_context_t *ctx)
int	INFINIBAND_set_domain (hwd_control_state_t *cntrl, int domain)
int	INFINIBAND_reset (hwd_context_t *ctx, hwd_control_state_t *ctrl)
int	INFINIBAND_ntv_enum_events (unsigned int *EventCode, int modifier)
int	INFINIBAND_ntv_code_to_name (unsigned int EventCode, char *name, int len)
int	INFINIBAND_ntv_code_to_descr (unsigned int EventCode, char *name, int len)
int	INFINIBAND_ntv_code_to_bits (unsigned int EventCode, hwd_register_t *bits)

Variables
void(*	_dl_non_dynamic_init )(void)
papi_vector_t	_infiniband_umad_vector

Detailed Description

Author

Heike Jagode (in collaboration with Michael Kluge, TU Dresden) jagod.nosp@m.e@ee.nosp@m.cs.ut.nosp@m.k.ed.nosp@m.u

Tony Castaldo; minor changes; the infiniband_umad.h differs on the declaration of umad_get_ca(), and demands a const char* instead of a char*. Corrected the prototypes below.

InfiniBand component

Tested version of OFED: 1.4

OFED library. Since a new interface was introduced with OFED version 1.4 (released Dec 2008), the current InfiniBand component does not support OFED versions < 1.4.

Definition in file linux-infiniband_umad.c.

Macro Definition Documentation

infiniband_native_table

#define infiniband_native_table   subscriptions

InitStruct

#define InitStruct	(		var,
			type
	)		type var; memset(&var, 0, sizeof(type))

Function Documentation

addCounter()

static counter_info* addCounter ( const char *  name, const char *  desc, const char *  unit  )

static

add a counter to the list of available counters

Parameters

name	the short name of the counter
desc	a longer description
unit	the unit for this counter

Definition at line 157 of file linux-infiniband_umad.c.

{
    counter_info *cntr, *last;

    cntr = ( counter_info * ) malloc( sizeof ( counter_info ) );
    if ( cntr == NULL ) {
        fprintf( stderr, "can not allocate memory for new counter\n" );
        exit( 1 );
    }
    cntr->name = strdup( name );
    cntr->description = strdup( desc );
    cntr->unit = strdup( unit );
    cntr->value = 0;
    cntr->next = NULL;

    if ( root_counter == NULL ) {
        root_counter = cntr;
    } else {
        last = root_counter;
        while ( last->next != NULL )
            last = last->next;
        last->next = cntr;
    }

    return cntr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

addIBPort()

static void addIBPort ( const char *  ca_name, umad_port_t *  port  )

static

add one IB port to the list of available ports and add the counters related to this port to the global counter list

Definition at line 190 of file linux-infiniband_umad.c.

{
    ib_port *nwif, *last;
    char counter_name[512];

    nwif = ( ib_port * ) malloc( sizeof ( ib_port ) );

    if ( nwif == NULL ) {
        fprintf( stderr, "can not allocate memory for IB port description\n" );
        exit( 1 );
    }

    sprintf( counter_name, "%s_%d", ca_name, port->portnum );
    nwif->name = strdup( counter_name );

    sprintf( counter_name, "%s_%d_recv", ca_name, port->portnum );
    nwif->recv_cntr =
        addCounter( counter_name, "bytes received on this IB port", "bytes" );

    sprintf( counter_name, "%s_%d_send", ca_name, port->portnum );
    nwif->send_cntr =
        addCounter( counter_name, "bytes written to this IB port", "bytes" );

    nwif->port_rate = port->rate;
    nwif->is_initialized = 0;
    nwif->port_number = port->portnum;
    nwif->next = NULL;

    num_counters += 2;

    if ( root_ib_port == NULL ) {
        root_ib_port = nwif;
    } else {
        last = root_ib_port;
        while ( last->next != NULL )
            last = last->next;
        last->next = nwif;
    }
}

Here is the call graph for this function:

counterFromName()

static counter_info* counterFromName ( const char *  cntr )

static

find the pointer for a counter_info structure based on the counter name

Definition at line 357 of file linux-infiniband_umad.c.

{
    int loop = 0;
    char tmp[512];
    counter_info *local_cntr = root_counter;

    while ( local_cntr != NULL ) {
        if ( strcmp( cntr, local_cntr->name ) == 0 )
            return local_cntr;

        local_cntr = local_cntr->next;
        loop++;
    }

    gethostname( tmp, 512 );
    fprintf( stderr, "can not find host counter: %s on %s\n", cntr, tmp );
    fprintf( stderr, "we only have: " );
    local_cntr = root_counter;

    while ( local_cntr != NULL ) {
        fprintf( stderr, "'%s' ", local_cntr->name );
        local_cntr = local_cntr->next;
        loop++;
    }

    fprintf( stderr, "\n" );
    exit( 1 );
    /* never reached */
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

host_deleteStringList()

static void host_deleteStringList ( string_list *  to_delete )

static

delete a list of strings

Definition at line 525 of file linux-infiniband_umad.c.

{
    int loop;

    if ( to_delete->data != NULL ) {
        for ( loop = 0; loop < to_delete->count; loop++ )
            free( to_delete->data[loop] );

        free( to_delete->data );
    }

    free( to_delete );
}

Here is the call graph for this function:

Here is the caller graph for this function:

host_finalize()

static void host_finalize ( void  )

static

finalizes the library

Definition at line 485 of file linux-infiniband_umad.c.

{
    counter_info *cntr, *next;

    if ( is_finalized )
        return;

    cntr = root_counter;

    while ( cntr != NULL ) {
        next = cntr->next;
        free( cntr->name );
        free( cntr->description );
        free( cntr->unit );
        free( cntr );
        cntr = next;
    }

    root_counter = NULL;

    ib_port *nwif, *last;
   last = root_ib_port;
   while ( last != NULL ) {                                    // While we have ports; 
      nwif = last;                                             // Copy the pointer.
      last = last->next;                                       // update the loop pointer now.
      if (nwif->name) free(nwif->name);                        // Free any name malloc.

      free(nwif);                                              // free the chain link itself.
    }

   root_ib_port = NULL;                                     // All done with this.

    is_finalized = 1;
} // end host_finalize()

Here is the caller graph for this function:

host_listCounter()

static string_list* host_listCounter ( int  num_counters1 )

static

return a newly allocated list of strings containing all counter names

Definition at line 452 of file linux-infiniband_umad.c.

{
    string_list *list;
    counter_info *cntr = root_counter;

    list = malloc( sizeof ( string_list ) );
    if ( list == NULL ) {
        fprintf( stderr, "unable to allocate memory for new string_list" );
        exit( 1 );
    }
    list->count = 0;
    list->data = ( char ** ) malloc( num_counters1 * sizeof ( char * ) );

    if ( list->data == NULL ) {
        fprintf( stderr,
                 "unable to allocate memory for %d pointers in a new string_list\n",
                 num_counters1 );
        exit( 1 );
    }

    while ( cntr != NULL ) {
        list->data[list->count++] = strdup( cntr->name );
        cntr = cntr->next;
    }

    return list;
}

Here is the call graph for this function:

Here is the caller graph for this function:

host_read_values()

void host_read_values

(

long long *

data

)

Definition at line 338 of file linux-infiniband_umad.c.

{
    int loop;

    read_ib_counter(  );

    for ( loop = 0; loop < INFINIBAND_MAX_COUNTERS; loop++ ) {
        if ( subscriptions[loop] == NULL )
            break;

        data[loop] = subscriptions[loop]->value;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

host_subscribe()

static uint64_t host_subscribe ( const char *  cntr )

static

allow external code to subscribe to a counter based on the counter name

Definition at line 393 of file linux-infiniband_umad.c.

{
    int loop;
    int len;
    char tmp_name[512];
    ib_port *aktp;

    counter_info *counter = counterFromName( cntr );

    for ( loop = 0; loop < INFINIBAND_MAX_COUNTERS; loop++ ) {
        if ( subscriptions[loop] == NULL ) {
            subscriptions[loop] = counter;
            counter->idx = loop;

            /* we have an IB counter if the name ends with _send or _recv and
               the prefix before that is in the ib_port list */
            if ( ( len = strlen( cntr ) ) > 5 ) {
                if ( strcmp( &cntr[len - 5], "_recv" ) == 0 ||
                     strcmp( &cntr[len - 5], "_send" ) == 0 ) {
                    /* look through all IB_counters */
                    strncpy( tmp_name, cntr, len - 5 );
                    tmp_name[len - 5] = 0;
                    aktp = root_ib_port;
                    // printf("looking for IB port '%s'\n", tmp_name);
                    while ( aktp != NULL ) {
                        if ( strcmp( aktp->name, tmp_name ) == 0 ) {
                            if ( !aktp->is_initialized ) {
                                init_ib_port( aktp );
                                active_ib_port = aktp;
                            }
                            return loop + 1;
                        }
                        /* name does not match, if this counter is
                           initialized, we can't have two active IB ports */
                        if ( aktp->is_initialized ) {
#if 0   /* not necessary with OFED version >= 1.4 */
                            fprintf( stderr,
                                     "unable to activate IB port monitoring for more than one port\n" );
                            exit( 1 );
#endif
                        }
                        aktp = aktp->next;
                    }
                }
            }
            return loop + 1;
        }
    }
    fprintf( stderr, "please subscribe only once to each counter\n" );
    exit( 1 );
    /* never reached */
    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

INFINIBAND_ctl()

int INFINIBAND_ctl	(	hwd_context_t *	ctx,
		int	code,
		_papi_int_option_t *	option
	)

Definition at line 803 of file linux-infiniband_umad.c.

{
    ( void ) ctx;
    ( void ) code;
    ( void ) option;
    return ( PAPI_OK );
}

INFINIBAND_init_component()

int INFINIBAND_init_component

(

int

cidx

)

Definition at line 585 of file linux-infiniband_umad.c.

{
    SUBDBG ("Entry: cidx: %d\n", cidx);
    int i;

    /* link in all the infiniband libraries and resolve the symbols we need to use */
    if (linkInfinibandLibraries() != PAPI_OK) {
        SUBDBG ("Dynamic link of Infiniband libraries failed, component will be disabled.\n");
        SUBDBG ("See disable reason in papi_component_avail output for more details.\n");
        return (PAPI_ENOSUPP);
    }

    /* make sure that the infiniband library finds the kernel module loaded. */
    if ( (*umad_initPtr)(  ) < 0 ) {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Call to initialize umad library failed.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }

    for ( i = 0; i < INFINIBAND_MAX_COUNTERS; i++ ) {
        _papi_hwd_infiniband_register_start[i] = -1;
        _papi_hwd_infiniband_register[i] = -1;
    }

    /* Export the component id */
    _infiniband_umad_vector.cmp_info.CmpIdx = cidx;

    return ( PAPI_OK );
}

Here is the call graph for this function:

INFINIBAND_init_control_state()

int INFINIBAND_init_control_state

(

hwd_control_state_t *

ctrl

)

Definition at line 703 of file linux-infiniband_umad.c.

{
    ( void ) ctrl;
    return PAPI_OK;
}

INFINIBAND_init_thread()

int INFINIBAND_init_thread

(

hwd_context_t *

ctx

)

Definition at line 548 of file linux-infiniband_umad.c.

{
    string_list *counter_list = NULL;
    int i;
    int loop;

    /* initialize portid struct of type ib_portid_t to 0 */
    InitStruct( portid, ib_portid_t );

    if ( is_initialized )
        return PAPI_OK;

    is_initialized = 1;

    init_ib_counter(  );

    for ( loop = 0; loop < INFINIBAND_MAX_COUNTERS; loop++ )
        subscriptions[loop] = NULL;

    counter_list = host_listCounter( num_counters );

    for ( i = 0; i < counter_list->count; i++ )
        host_subscribe( counter_list->data[i] );

    ( ( INFINIBAND_context_t * ) ctx )->state.ncounter = counter_list->count;

    host_deleteStringList( counter_list );

    return PAPI_OK;
}

Here is the call graph for this function:

INFINIBAND_ntv_code_to_bits()

int INFINIBAND_ntv_code_to_bits	(	unsigned int	EventCode,
		hwd_register_t *	bits
	)

Definition at line 913 of file linux-infiniband_umad.c.

{
    memcpy( ( INFINIBAND_register_t * ) bits,
            infiniband_native_table[EventCode],
            sizeof ( INFINIBAND_register_t ) );

    return PAPI_OK;
}

INFINIBAND_ntv_code_to_descr()

int INFINIBAND_ntv_code_to_descr	(	unsigned int	EventCode,
		char *	name,
		int	len
	)

Definition at line 901 of file linux-infiniband_umad.c.

{
    strncpy( name, infiniband_native_table[EventCode]->description, len );

    return PAPI_OK;
}

INFINIBAND_ntv_code_to_name()

int INFINIBAND_ntv_code_to_name	(	unsigned int	EventCode,
		char *	name,
		int	len
	)

Definition at line 889 of file linux-infiniband_umad.c.

{
    strncpy( name, infiniband_native_table[EventCode]->name, len );

    return PAPI_OK;
}

INFINIBAND_ntv_enum_events()

int INFINIBAND_ntv_enum_events	(	unsigned int *	EventCode,
		int	modifier
	)

Definition at line 865 of file linux-infiniband_umad.c.

{
    if ( modifier == PAPI_ENUM_FIRST ) {
        *EventCode = 0;
        return PAPI_OK;
    }

    if ( modifier == PAPI_ENUM_EVENTS ) {
        int index = *EventCode;

        if ( infiniband_native_table[index + 1] ) {
            *EventCode = *EventCode + 1;
            return ( PAPI_OK );
        } else
            return ( PAPI_ENOEVNT );
    } else
        return ( PAPI_EINVAL );
}

INFINIBAND_read()

int INFINIBAND_read	(	hwd_context_t *	ctx,
		hwd_control_state_t *	ctrl,
		long_long **	events,
		int	flags
	)

Definition at line 753 of file linux-infiniband_umad.c.

{
    int i;
    ( void ) flags;

    host_read_values( _papi_hwd_infiniband_register );

    for ( i = 0; i < ( ( INFINIBAND_context_t * ) ctx )->state.ncounter; i++ ) {
        ( ( INFINIBAND_control_state_t * ) ctrl )->counts[i] =
            _papi_hwd_infiniband_register[i] -
            _papi_hwd_infiniband_register_start[i];
    }

    *events = ( ( INFINIBAND_control_state_t * ) ctrl )->counts;
    return ( PAPI_OK );
}

Here is the call graph for this function:

INFINIBAND_reset()

int INFINIBAND_reset	(	hwd_context_t *	ctx,
		hwd_control_state_t *	ctrl
	)

Definition at line 854 of file linux-infiniband_umad.c.

{
    INFINIBAND_start( ctx, ctrl );
    return ( PAPI_OK );
}

Here is the call graph for this function:

INFINIBAND_set_domain()

int INFINIBAND_set_domain	(	hwd_control_state_t *	cntrl,
		int	domain
	)

Definition at line 840 of file linux-infiniband_umad.c.

{
    (void) cntrl;
    if ( PAPI_DOM_ALL != domain )
        return ( PAPI_EINVAL );

    return ( PAPI_OK );
}

INFINIBAND_shutdown_component()

int INFINIBAND_shutdown_component

(

void

)

Definition at line 788 of file linux-infiniband_umad.c.

{
    // close the dynamic libraries needed by this component (opened in the init substrate call)
    dlclose(dl1);
    dlclose(dl2);

    return ( PAPI_OK );
} // end Shutdown component.

INFINIBAND_shutdown_thread()

int INFINIBAND_shutdown_thread

(

hwd_context_t *

ctx

)

Definition at line 776 of file linux-infiniband_umad.c.

{
    ( void ) ctx;
    host_finalize(  );
    return ( PAPI_OK );
}

Here is the call graph for this function:

INFINIBAND_start()

int INFINIBAND_start	(	hwd_context_t *	ctx,
		hwd_control_state_t *	ctrl
	)

Definition at line 714 of file linux-infiniband_umad.c.

{
    ( void ) ctx;
    ( void ) ctrl;

    host_read_values( _papi_hwd_infiniband_register_start );

    memcpy( _papi_hwd_infiniband_register, _papi_hwd_infiniband_register_start,
            INFINIBAND_MAX_COUNTERS * sizeof ( long long ) );

    return ( PAPI_OK );
}

Here is the call graph for this function:

Here is the caller graph for this function:

INFINIBAND_stop()

int INFINIBAND_stop	(	hwd_context_t *	ctx,
		hwd_control_state_t *	ctrl
	)

Definition at line 732 of file linux-infiniband_umad.c.

{
    int i;
    ( void ) ctx;

    host_read_values( _papi_hwd_infiniband_register );

    for ( i = 0; i < ( ( INFINIBAND_context_t * ) ctx )->state.ncounter; i++ ) {
        ( ( INFINIBAND_control_state_t * ) ctrl )->counts[i] =
            _papi_hwd_infiniband_register[i] -
            _papi_hwd_infiniband_register_start[i];
    }

    return ( PAPI_OK );
}

Here is the call graph for this function:

INFINIBAND_update_control_state()

int INFINIBAND_update_control_state	(	hwd_control_state_t *	ptr,
		NativeInfo_t *	native,
		int	count,
		hwd_context_t *	ctx
	)

Definition at line 819 of file linux-infiniband_umad.c.

{
    ( void ) ptr;
    ( void ) ctx;
    int i, index;

    for ( i = 0; i < count; i++ ) {
        index = native[i].ni_event;
        native[i].ni_position = index;
    }

    return ( PAPI_OK );
}

init_ib_port()

static int init_ib_port ( ib_port *  portdata )

static

initialize one IB port so that we are able to read values from it

Definition at line 235 of file linux-infiniband_umad.c.

{
    int mgmt_classes[4] = { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS, IB_SA_CLASS,
        IB_PERFORMANCE_CLASS
    };
    char *ca = 0;
    static uint8_t pc[1024];
    int mask = 0xFFFF;

    srcport = (*mad_rpc_open_portPtr)( ca, portdata->port_number, mgmt_classes, 4 );
    if ( !srcport ) {
        fprintf( stderr, "Failed to open '%s' port '%d'\n", ca,
                 portdata->port_number );
        exit( 1 );
    }

    if ( (*ib_resolve_self_viaPtr)( &portid, &ibportnum, 0, srcport ) < 0 ) {
        fprintf( stderr, "can't resolve self port\n" );
        exit( 1 );
    }

    /* PerfMgt ClassPortInfo is a required attribute */
    /* might be redundant, could be left out for fast implementation */
    if ( !(*pma_query_viaPtr) ( pc, &portid, ibportnum, ib_timeout, CLASS_PORT_INFO, srcport ) ) {
        fprintf( stderr, "classportinfo query\n" );
        exit( 1 );
    }

    if ( !(*performance_reset_viaPtr) ( pc, &portid, ibportnum, mask, ib_timeout, IB_GSI_PORT_COUNTERS, srcport ) ) {
        fprintf( stderr, "perf reset\n" );
        exit( 1 );
    }

    /* read the initial values */
    (*mad_decode_fieldPtr)( pc, IB_PC_XMT_BYTES_F, &portdata->last_send_val );
    portdata->sum_send_val = 0;
    (*mad_decode_fieldPtr)( pc, IB_PC_RCV_BYTES_F, &portdata->last_recv_val );
    portdata->sum_recv_val = 0;

    portdata->is_initialized = 1;

    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

linkInfinibandLibraries()

static int linkInfinibandLibraries ( )

static

Definition at line 622 of file linux-infiniband_umad.c.

{
    /* Attempt to guess if we were statically linked to libc, if so bail */
    if ( _dl_non_dynamic_init != NULL ) {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "The Infiniband component does not support statically linking of libc.", PAPI_MAX_STR_LEN);
        return PAPI_ENOSUPP;
    }

    /* Need to link in the Infiniband libraries, if not found disable the component */
    dl1 = dlopen("libibumad.so", RTLD_NOW | RTLD_GLOBAL);
    if (!dl1)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband library libibumad.so not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    umad_initPtr = dlsym(dl1, "umad_init");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function umad_init not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    umad_get_cas_namesPtr = dlsym(dl1, "umad_get_cas_names");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function umad_get_cas_names not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    umad_get_caPtr = dlsym(dl1, "umad_get_ca");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function umad_get_ca not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }

    /* Need to link in the Infiniband libraries, if not found disable the component */
    dl2 = dlopen("libibmad.so", RTLD_NOW | RTLD_GLOBAL);
    if (!dl2)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband library libibmad.so not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    mad_decode_fieldPtr = dlsym(dl2, "mad_decode_field");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function mad_decode_field not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    mad_rpc_open_portPtr = dlsym(dl2, "mad_rpc_open_port");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function mad_rpc_open_port not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    ib_resolve_self_viaPtr = dlsym(dl2, "ib_resolve_self_via");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function ib_resolve_self_via not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    performance_reset_viaPtr = dlsym(dl2, "performance_reset_via");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function performance_reset_via not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }
    pma_query_viaPtr = dlsym(dl2, "pma_query_via");
    if (dlerror() != NULL)
    {
        strncpy(_infiniband_umad_vector.cmp_info.disabled_reason, "Infiniband function pma_query_via not found.",PAPI_MAX_STR_LEN);
        return ( PAPI_ENOSUPP );
    }

    return ( PAPI_OK );
}

Here is the caller graph for this function:

read_ib_counter()

static int read_ib_counter ( )

static

read and reset IB counters (reset on demand)

Definition at line 284 of file linux-infiniband_umad.c.

{
    uint32_t send_val;
    uint32_t recv_val;
    uint8_t pc[1024];
    /* 32 bit counter FFFFFFFF */
    uint32_t max_val = 4294967295;
    /* if it is bigger than this -> reset */
    uint32_t reset_limit = max_val * 0.7;
    int mask = 0xFFFF;

    if ( active_ib_port == NULL )
        return 0;

    /* reading cost ~70 mirco secs */
    if ( !(*pma_query_viaPtr) ( pc, &portid, ibportnum, ib_timeout, IB_GSI_PORT_COUNTERS, srcport ) ) {
        fprintf( stderr, "perfquery\n" );
        exit( 1 );
    }

    (*mad_decode_fieldPtr)( pc, IB_PC_XMT_BYTES_F, &send_val );
    (*mad_decode_fieldPtr)( pc, IB_PC_RCV_BYTES_F, &recv_val );

    /* multiply the numbers read by 4 as the IB port counters are not
       counting bytes. they always count 32dwords. see man page of
       perfquery for details
       internally a uint64_t ia used to sum up the values */
    active_ib_port->sum_send_val +=
        ( send_val - active_ib_port->last_send_val ) * 4;
    active_ib_port->sum_recv_val +=
        ( recv_val - active_ib_port->last_recv_val ) * 4;

    active_ib_port->send_cntr->value = active_ib_port->sum_send_val;
    active_ib_port->recv_cntr->value = active_ib_port->sum_recv_val;

    if ( send_val > reset_limit || recv_val > reset_limit ) {
        /* reset cost ~70 mirco secs */
        if ( !(*performance_reset_viaPtr) ( pc, &portid, ibportnum, mask, ib_timeout, IB_GSI_PORT_COUNTERS, srcport ) ) {
            fprintf( stderr, "perf reset\n" );
            exit( 1 );
        }

        (*mad_decode_fieldPtr)( pc, IB_PC_XMT_BYTES_F, &active_ib_port->last_send_val );
        (*mad_decode_fieldPtr)( pc, IB_PC_RCV_BYTES_F, &active_ib_port->last_recv_val );
    } else {
        active_ib_port->last_send_val = send_val;
        active_ib_port->last_recv_val = recv_val;
    }

    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

_dl_non_dynamic_init

void(* _dl_non_dynamic_init) (void)

use libumad to discover IB ports

Definition at line 36 of file linux-infiniband_umad.c.

{
    char names[20][UMAD_CA_NAME_LEN];
    int n, i;
    char *ca_name;
    umad_ca_t ca;
    int r;
    int portnum;

//  if ( umad_init(  ) < 0 ) {
//      fprintf( stderr, "can't init UMAD library\n" );
//      exit( 1 );
//  }

    if ( ( n = (*umad_get_cas_namesPtr)( ( void * ) names, UMAD_CA_NAME_LEN ) ) < 0 ) {
        fprintf( stderr, "can't list IB device names\n" );
        exit( 1 );
    }

    for ( i = 0; i < n; i++ ) {
        ca_name = names[i];

        if ( ( r = (*umad_get_caPtr)( ca_name, &ca ) ) < 0 ) {
            fprintf( stderr, "can't read ca from IB device\n" );
            exit( 1 );
        }

        if ( !ca.node_type )
            continue;

        /* port numbers are '1' based in OFED */
        for ( portnum = 1; portnum <= ca.numports; portnum++ )
            addIBPort( ca.ca_name, ca.ports[portnum] );
    }
}

_infiniband_umad_vector

papi_vector_t _infiniband_umad_vector

Definition at line 926 of file linux-infiniband_umad.c.