linux-infiniband.c File Reference

This file has the source code for a component that enables PAPI-C to access the infiniband performance monitor through the Linux sysfs interface. This code will dynamically create a native events table for all the events that can be accesed through the sysfs interface. The counters exported by this component cannot be reset programatically. More...

Include dependency graph for linux-infiniband.c:

Go to the source code of this file.

Data Structures
struct	infiniband_register_t
struct	ib_device_t
struct	ib_counter_t
struct	infiniband_native_event_entry_t
struct	infiniband_control_state_t
struct	infiniband_context_t

Macros
#define	INFINIBAND_MAX_COUNTERS   128

Functions
static char *	make_ib_event_description (const char *input_str, int extended)
static ib_device_t *	add_ib_device (const char *name, int port)
static ib_counter_t *	add_ib_counter (const char *name, const char *file_name, int extended, ib_device_t *device)
static int	find_ib_device_events (ib_device_t *dev, int extended)
static int	find_ib_devices ()
static long long	read_ib_counter_value (int index)
static void	deallocate_infiniband_resources ()
static int	_infiniband_init_thread (hwd_context_t *ctx)
static int	_infiniband_init_component (int cidx)
static int	_infiniband_init_control_state (hwd_control_state_t *ctl)
static int	_infiniband_start (hwd_context_t *ctx, hwd_control_state_t *ctl)
static int	_infiniband_stop (hwd_context_t *ctx, hwd_control_state_t *ctl)
static int	_infiniband_read (hwd_context_t *ctx, hwd_control_state_t *ctl, long_long **events, int flags)
static int	_infiniband_shutdown_component (void)
static int	_infiniband_shutdown_thread (hwd_context_t *ctx)
static int	_infiniband_ctl (hwd_context_t *ctx, int code, _papi_int_option_t *option)
static int	_infiniband_update_control_state (hwd_control_state_t *ctl, NativeInfo_t *native, int count, hwd_context_t *ctx)
static int	_infiniband_set_domain (hwd_control_state_t *ctl, int domain)
static int	_infiniband_reset (hwd_context_t *ctx, hwd_control_state_t *ctl)
static int	_infiniband_ntv_enum_events (unsigned int *EventCode, int modifier)
static int	_infiniband_ntv_code_to_name (unsigned int EventCode, char *name, int len)
static int	_infiniband_ntv_code_to_descr (unsigned int EventCode, char *name, int len)
static int	_infiniband_ntv_code_to_info (unsigned int EventCode, PAPI_event_info_t *info)

Variables
static const char *	ib_dir_path = "/sys/class/infiniband"
static infiniband_native_event_entry_t *	infiniband_native_events = 0
static int	num_events = 0
papi_vector_t	_infiniband_vector
static ib_device_t *	root_device = 0
static ib_counter_t *	root_counter = 0

Detailed Description

Author

Heike Jagode jagod.nosp@m.e@ic.nosp@m.l.utk.nosp@m..edu

Gabriel Marin gmari.nosp@m.n@ee.nosp@m.cs.ut.nosp@m.k.ed.nosp@m.u

Infiniband component

Definition in file linux-infiniband.c.

Macro Definition Documentation

INFINIBAND_MAX_COUNTERS

#define INFINIBAND_MAX_COUNTERS   128

Definition at line 44 of file linux-infiniband.c.

Function Documentation

_infiniband_ctl()

static int _infiniband_ctl ( hwd_context_t *  ctx, int  code, _papi_int_option_t *  option  )

static

Definition at line 626 of file linux-infiniband.c.

{
   ( void ) ctx;
   ( void ) code;
   ( void ) option;
   return PAPI_OK;
}

_infiniband_init_component()

static int _infiniband_init_component ( int  cidx )

static

Definition at line 482 of file linux-infiniband.c.

{
   /* discover Infiniband devices and available events */
   int result = find_ib_devices();
   
   if (result != PAPI_OK)  // we couldn't initialize the component
   {
      // deallocate any eventually allocated memory
      deallocate_infiniband_resources();
   }
    
   _infiniband_vector.cmp_info.num_native_events = num_events;

   _infiniband_vector.cmp_info.num_cntrs = num_events;
   _infiniband_vector.cmp_info.num_mpx_cntrs = num_events;


   /* Export the component id */
   _infiniband_vector.cmp_info.CmpIdx = cidx;

   return (result);
}

Here is the call graph for this function:

_infiniband_init_control_state()

static int _infiniband_init_control_state ( hwd_control_state_t *  ctl )

static

Definition at line 511 of file linux-infiniband.c.

{
   infiniband_control_state_t* control = (infiniband_control_state_t*) ctl;
   int i;

   for (i=0 ; i<INFINIBAND_MAX_COUNTERS ; ++i) {
      control->being_measured[i] = 0;
   }

   return PAPI_OK;
}

_infiniband_init_thread()

static int _infiniband_init_thread ( hwd_context_t *  ctx )

static

Definition at line 470 of file linux-infiniband.c.

{
   (void) ctx;
   return PAPI_OK;
}

_infiniband_ntv_code_to_descr()

static int _infiniband_ntv_code_to_descr ( unsigned int  EventCode, char *  name, int  len  )

static

Definition at line 756 of file linux-infiniband.c.

{
   int index = EventCode;

   if (index>=0 && index<num_events) {
      strncpy(name, infiniband_native_events[index].description, len);
   }
   return PAPI_OK;
}

_infiniband_ntv_code_to_info()

static int _infiniband_ntv_code_to_info ( unsigned int  EventCode, PAPI_event_info_t *  info  )

static

Definition at line 767 of file linux-infiniband.c.

{
   int index = EventCode;

   if ( ( index < 0) || (index >= num_events )) return PAPI_ENOEVNT; 

   if (infiniband_native_events[index].name)
   {
      unsigned int len = strlen(infiniband_native_events[index].name);
      if (len > sizeof(info->symbol)-1) len = sizeof(info->symbol)-1;
      strncpy(info->symbol, infiniband_native_events[index].name, len);
      info->symbol[len] = '\0';
   }
   if (infiniband_native_events[index].description)
   {
      unsigned int len = strlen(infiniband_native_events[index].description);
      if (len > sizeof(info->long_descr)-1) len = sizeof(info->long_descr)-1;
      strncpy(info->long_descr, infiniband_native_events[index].description, len);
      info->long_descr[len] = '\0';
   }

   strncpy(info->units, "\0", 1);
       /* infiniband_native_events[index].units, sizeof(info->units)); */

/*   info->data_type = infiniband_native_events[index].return_type;
 */
   return PAPI_OK;
}

_infiniband_ntv_code_to_name()

static int _infiniband_ntv_code_to_name ( unsigned int  EventCode, char *  name, int  len  )

static

Definition at line 741 of file linux-infiniband.c.

{
   int index = EventCode;

   if (index>=0 && index<num_events) {
      strncpy( name, infiniband_native_events[index].name, len );
   }

   return PAPI_OK;
}

_infiniband_ntv_enum_events()

static int _infiniband_ntv_enum_events ( unsigned int *  EventCode, int  modifier  )

static

Definition at line 709 of file linux-infiniband.c.

{
   switch (modifier) {
      case PAPI_ENUM_FIRST:
         if (num_events == 0)
            return (PAPI_ENOEVNT);
            
         *EventCode = 0;
         return PAPI_OK;

      case PAPI_ENUM_EVENTS:
      {
         int index = *EventCode & PAPI_NATIVE_AND_MASK;

         if (index < num_events - 1) {
            *EventCode = *EventCode + 1;
            return PAPI_OK;
         } else
            return PAPI_ENOEVNT;

         break;
      }
      default:
         return PAPI_EINVAL;
   }
   return PAPI_EINVAL;
}

_infiniband_read()

static int _infiniband_read ( hwd_context_t *  ctx, hwd_control_state_t *  ctl, long_long **  events, int  flags  )

static

Definition at line 589 of file linux-infiniband.c.

{
   ( void ) flags;
    
   _infiniband_stop(ctx, ctl);  /* we cannot actually stop the counters */
   /* Pass back a pointer to our results */
   *events = ((infiniband_control_state_t*) ctl)->counts;
   
   return PAPI_OK;
}

Here is the call graph for this function:

_infiniband_reset()

static int _infiniband_reset ( hwd_context_t *  ctx, hwd_control_state_t *  ctl  )

static

Definition at line 697 of file linux-infiniband.c.

{
   (void) ctx;
   (void) ctl;
   return PAPI_OK;
}

_infiniband_set_domain()

static int _infiniband_set_domain ( hwd_control_state_t *  ctl, int  domain  )

static

Definition at line 672 of file linux-infiniband.c.

{
   int found = 0;
   (void) ctl;
    
   if (PAPI_DOM_USER & domain)
      found = 1;

   if (PAPI_DOM_KERNEL & domain)
      found = 1;

   if (PAPI_DOM_OTHER & domain)
      found = 1;

   if (!found)
      return (PAPI_EINVAL);

   return (PAPI_OK);
}

_infiniband_shutdown_component()

static int _infiniband_shutdown_component ( void  )

static

Definition at line 603 of file linux-infiniband.c.

{
   /* Cleanup resources used by this component before leaving */
   deallocate_infiniband_resources();
   
   return PAPI_OK;
}

Here is the call graph for this function:

_infiniband_shutdown_thread()

static int _infiniband_shutdown_thread ( hwd_context_t *  ctx )

static

Definition at line 612 of file linux-infiniband.c.

{
   ( void ) ctx;

   return PAPI_OK;
}

_infiniband_start()

static int _infiniband_start ( hwd_context_t *  ctx, hwd_control_state_t *  ctl  )

static

Definition at line 527 of file linux-infiniband.c.

{
   infiniband_context_t* context = (infiniband_context_t*) ctx;
   infiniband_control_state_t* control = (infiniband_control_state_t*) ctl;
   long long now = PAPI_get_real_usec();
   int i;

   for (i=0 ; i<INFINIBAND_MAX_COUNTERS ; ++i) {
      if (control->being_measured[i] && control->need_difference[i]) {
         context->start_value[i] = read_ib_counter_value(i);
      }
   }
   control->lastupdate = now;

   return PAPI_OK;
}

Here is the call graph for this function:

_infiniband_stop()

static int _infiniband_stop ( hwd_context_t *  ctx, hwd_control_state_t *  ctl  )

static

Definition at line 549 of file linux-infiniband.c.

{
   infiniband_context_t* context = (infiniband_context_t*) ctx;
   infiniband_control_state_t* control = (infiniband_control_state_t*) ctl;
   long long now = PAPI_get_real_usec();
   int i;
   long long temp;

   for (i=0 ; i<INFINIBAND_MAX_COUNTERS ; ++i) {
      if (control->being_measured[i])
      {
         temp = read_ib_counter_value(i);
         if (context->start_value[i] && control->need_difference[i]) {
            /* Must subtract values, but check for wraparound. 
             * We cannot even detect all wraparound cases. Using the short,
             * auto-resetting IB counters is error prone.
             */
            if (temp < context->start_value[i]) {
               SUBDBG("Wraparound!\nstart:\t%#016x\ttemp:\t%#016x",
                        (unsigned)context->start_value[i], (unsigned)temp);
               /* The counters auto-reset. I cannot even adjust them to 
                * account for a simple wraparound. 
                * Just use the current reading of the counter, which is useless.
                */
            } else
               temp -= context->start_value[i];
         }
         control->counts[i] = temp;
      }
   }
   control->lastupdate = now;

   return PAPI_OK;
}

Here is the call graph for this function:

Here is the caller graph for this function:

_infiniband_update_control_state()

static int _infiniband_update_control_state ( hwd_control_state_t *  ctl, NativeInfo_t *  native, int  count, hwd_context_t *  ctx  )

static

Definition at line 636 of file linux-infiniband.c.

{
   int i, index;
   ( void ) ctx;
    
   infiniband_control_state_t* control = (infiniband_control_state_t*) ctl;
   
   for (i=0 ; i<INFINIBAND_MAX_COUNTERS ; ++i) {
      control->being_measured[i] = 0;
   }
   
   for (i=0 ; i<count ; ++i) {
      index = native[i].ni_event & PAPI_NATIVE_AND_MASK;
      native[i].ni_position =
                  infiniband_native_events[index].resources.selector - 1;
      control->being_measured[index] = 1;
      control->need_difference[index] = 1;
   }
   return PAPI_OK;
}

add_ib_counter()

static ib_counter_t* add_ib_counter ( const char *  name, const char *  file_name, int  extended, ib_device_t *  device  )

static

Definition at line 176 of file linux-infiniband.c.

{
   ib_counter_t *new_cnt = (ib_counter_t*) papi_calloc(sizeof(ib_counter_t), 1);
   if (new_cnt == 0) {
      PAPIERROR("cannot allocate memory for new IB counter structure");
      return (0);
   }
   
   new_cnt->ev_name = strdup(name);
   new_cnt->ev_file_name = strdup(file_name);
   new_cnt->extended = extended;
   new_cnt->ev_device = device;
   if (new_cnt->ev_name==0 || new_cnt->ev_file_name==0)
   {
      PAPIERROR("cannot allocate memory for counter internal fields");
      papi_free(new_cnt);
      return (0);
   }

   // prepend the new counter to the counter list
   new_cnt->next = root_counter;
   root_counter = new_cnt;
   
   return (new_cnt);
}

Here is the call graph for this function:

Here is the caller graph for this function:

add_ib_device()

static ib_device_t* add_ib_device ( const char *  name, int  port  )

static

Definition at line 151 of file linux-infiniband.c.

{
   ib_device_t *new_dev = (ib_device_t*) papi_calloc(sizeof(ib_device_t), 1);
   if (new_dev == 0) {
      PAPIERROR("cannot allocate memory for new IB device structure");
      return (0);
   }
   
   new_dev->dev_name = strdup(name);
   new_dev->dev_port = port;
   if (new_dev->dev_name==0)
   {
      PAPIERROR("cannot allocate memory for device internal fields");
      papi_free(new_dev);
      return (0);
   }

   // prepend the new device to the device list
   new_dev->next = root_device;
   root_device = new_dev;
   
   return (new_dev);
}

Here is the call graph for this function:

Here is the caller graph for this function:

deallocate_infiniband_resources()

static void deallocate_infiniband_resources ( )

static

Definition at line 432 of file linux-infiniband.c.

{
   int i;
   
   if (infiniband_native_events)
   {
      for (i=0 ; i<num_events ; ++i) {
         if (infiniband_native_events[i].name)
            free(infiniband_native_events[i].name);
         if (infiniband_native_events[i].file_name)
            free(infiniband_native_events[i].file_name);
         if (infiniband_native_events[i].description)
            papi_free(infiniband_native_events[i].description);
      }
      papi_free(infiniband_native_events);
   }
   
   ib_device_t *iter = root_device;
   while (iter != 0) 
   {
      if (iter->dev_name)
         free(iter->dev_name);
   
      ib_device_t *tmp = iter;
      iter = iter->next;
      papi_free(tmp);
   }
   root_device = 0;
}

Here is the caller graph for this function:

find_ib_device_events()

static int find_ib_device_events ( ib_device_t *  dev, int  extended  )

static

Definition at line 204 of file linux-infiniband.c.

{
   int nevents = 0;
   DIR *cnt_dir = NULL;
   char counters_path[128];

   if ( extended ) {
      /* mofed driver version <4.0 */
      snprintf(counters_path, sizeof(counters_path), "%s/%s/ports/%d/counters%s", 
             ib_dir_path, dev->dev_name, dev->dev_port, (extended?"_ext":""));
   
      cnt_dir = opendir(counters_path);
      if (cnt_dir == NULL) {
         /* directory counters_ext in sysfs fs has changed to hw_counters */
         /* in 4.0 version of mofed driver */
         SUBDBG("cannot open counters directory `%s'\n", counters_path);

         snprintf(counters_path, sizeof(counters_path), "%s/%s/ports/%d/%scounters", 
                ib_dir_path, dev->dev_name, dev->dev_port, "hw_");
   
         cnt_dir = opendir(counters_path);
      }
   }
   else {
      snprintf(counters_path, sizeof(counters_path), "%s/%s/ports/%d/counters", 
             ib_dir_path, dev->dev_name, dev->dev_port);
      cnt_dir = opendir(counters_path);
   }
   
   if (cnt_dir == NULL) {
      SUBDBG("cannot open counters directory `%s'\n", counters_path);
      goto out;
   }
   
   struct dirent *ev_ent;
   /* iterate over all the events */
   while ((ev_ent = readdir(cnt_dir)) != NULL) {
      char *ev_name = ev_ent->d_name;
      long long value = -1;
      char event_path[160];
      char counter_name[80];

      if (ev_name[0] == '.')
         continue;

      /* Check that we can read an integer from the counter file */
      snprintf(event_path, sizeof(event_path), "%s/%s", counters_path, ev_name);
      if (pscanf(event_path, "%lld", &value) != 1) {
        SUBDBG("cannot read value for event '%s'\n", ev_name);
        continue;
      }

      /* Create new counter */
      snprintf(counter_name, sizeof(counter_name), "%s_%d%s:%s", 
            dev->dev_name, dev->dev_port, (extended?"_ext":""), ev_name);
      if (add_ib_counter(counter_name, ev_name, extended, dev))
      {
         SUBDBG("Added new counter `%s'\n", counter_name);
         nevents += 1;
      }
   }

 out:
  if (cnt_dir != NULL)
    closedir(cnt_dir);

  return (nevents);
}

Here is the call graph for this function:

Here is the caller graph for this function:

find_ib_devices()

static int find_ib_devices ( )

static

Definition at line 274 of file linux-infiniband.c.

{
  DIR *ib_dir = NULL;
  int result = PAPI_OK;
  num_events = 0;

  ib_dir = opendir(ib_dir_path);
  if (ib_dir == NULL) {
     SUBDBG("cannot open `%s'\n", ib_dir_path);
     strncpy(_infiniband_vector.cmp_info.disabled_reason,
                 "Infiniband sysfs interface not found", PAPI_MAX_STR_LEN);
     result = PAPI_ENOSUPP;
     goto out;
  }

  struct dirent *hca_ent;
  while ((hca_ent = readdir(ib_dir)) != NULL) {
     char *hca = hca_ent->d_name;
     char ports_path[80];
     DIR *ports_dir = NULL;

     if (hca[0] == '.')
        goto next_hca;

     snprintf(ports_path, sizeof(ports_path), "%s/%s/ports", ib_dir_path, hca);
     ports_dir = opendir(ports_path);
     if (ports_dir == NULL) {
        SUBDBG("cannot open `%s'\n", ports_path);
        goto next_hca;
     }

     struct dirent *port_ent;
     while ((port_ent = readdir(ports_dir)) != NULL) {
        int port = atoi(port_ent->d_name);
        if (port <= 0)
           continue;

        /* Check that port is active. .../HCA/ports/PORT/state should read "4: ACTIVE." */
        int state = -1;
        char state_path[80];
        snprintf(state_path, sizeof(state_path), "%s/%s/ports/%d/state", ib_dir_path, hca, port);
        if (pscanf(state_path, "%d", &state) != 1) {
           SUBDBG("cannot read state of IB HCA `%s' port %d\n", hca, port);
           continue;
        }

        if (state != 4) {
           SUBDBG("skipping inactive IB HCA `%s', port %d, state %d\n", hca, port, state);
           continue;
        }

        /* Create dev name (HCA/PORT) and get stats for dev. */
        SUBDBG("Found IB device `%s', port %d\n", hca, port);
        ib_device_t *dev = add_ib_device(hca, port);
        if (!dev)
           continue;
        // do we want to check for short counters only if no extended counters found?
        num_events += find_ib_device_events(dev, 1);  // check if we have extended (64bit) counters
        num_events += find_ib_device_events(dev, 0);  // check also for short counters
     }

   next_hca:
      if (ports_dir != NULL)
         closedir(ports_dir);
   }

   if (root_device == 0)  // no active devices found
   {
     strncpy(_infiniband_vector.cmp_info.disabled_reason,
                 "No active Infiniband ports found", PAPI_MAX_STR_LEN);
     result = PAPI_ENOIMPL;
   } else if (num_events == 0)
   {
     strncpy(_infiniband_vector.cmp_info.disabled_reason,
                 "No supported Infiniband events found", PAPI_MAX_STR_LEN);
     result = PAPI_ENOIMPL;
   } else
   {
      // Events are stored in a linked list, in reverse order than how I found them
      // Revert them again, so that they are in finding order, not that it matters.
      int i = num_events - 1;
      // now allocate memory to store the counters into the native table
      infiniband_native_events = (infiniband_native_event_entry_t*)
           papi_calloc(num_events, sizeof(infiniband_native_event_entry_t));
      ib_counter_t *iter = root_counter;
      while (iter != 0)
      {
         infiniband_native_events[i].name = iter->ev_name;
         infiniband_native_events[i].file_name = iter->ev_file_name;
         infiniband_native_events[i].device = iter->ev_device;
         infiniband_native_events[i].extended = iter->extended;
         infiniband_native_events[i].resources.selector = i + 1;
         infiniband_native_events[i].description = 
                  make_ib_event_description(iter->ev_file_name, iter->extended);
         
         ib_counter_t *tmp = iter;
         iter = iter->next;
         papi_free(tmp);
         -- i;
      }
      root_counter = 0;
   }
   
   out:
      if (ib_dir != NULL)
         closedir(ib_dir);
    
   return (result);
}

Here is the call graph for this function:

Here is the caller graph for this function:

make_ib_event_description()

static char* make_ib_event_description ( const char *  input_str, int  extended  )

static

Definition at line 125 of file linux-infiniband.c.

{
   int i, len;
   char *desc = 0;
   if (! input_str)
      return (0);
   
   desc = (char*) papi_calloc(PAPI_MAX_STR_LEN, 1);
   if (desc == 0) {
      PAPIERROR("cannot allocate memory for event description");
      return (0);
   }
   len = strlen(input_str);
   
   snprintf(desc, PAPI_MAX_STR_LEN, "%s (%s).",
           input_str, (extended ? "free-running 64bit counter" :
            "overflowing, auto-resetting counter"));
   desc[0] = toupper(desc[0]);
   for (i=0 ; i<len ; ++i)
      if (desc[i] == '_')
         desc[i] = ' ';
   
   return (desc);
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_ib_counter_value()

static long long read_ib_counter_value ( int  index )

static

Definition at line 385 of file linux-infiniband.c.

{
   char ev_file[128];
   char counters_path[128];
   DIR *cnt_dir = NULL;
   long long value = 0ll;
   infiniband_native_event_entry_t *iter = &infiniband_native_events[index];
   
   if ( iter->extended ) {
      /* mofed driver version <4.0 */
      snprintf(counters_path, sizeof(counters_path), "%s/%s/ports/%d/counters%s",
             ib_dir_path, iter->device->dev_name, iter->device->dev_port, "_ext");
   
      cnt_dir = opendir(counters_path);
      if (cnt_dir == NULL) {
         /* directory counters_ext in sysfs fs has changed to hw_counters */
         /* in 4.0 version of mofed driver */
         snprintf(counters_path, sizeof(counters_path), "%s/%s/ports/%d/%scounters",
                ib_dir_path, iter->device->dev_name, iter->device->dev_port, "hw_");
   
         cnt_dir = opendir(counters_path);
      }
   }
   else {
      snprintf(counters_path, sizeof(counters_path), "%s/%s/ports/%d/counters",
             ib_dir_path, iter->device->dev_name, iter->device->dev_port );
      cnt_dir = opendir(counters_path);
   }


   if (cnt_dir != NULL)
      closedir(cnt_dir);

 
   snprintf(ev_file, sizeof(ev_file), "%s/%s",
           counters_path, iter->file_name);

   if (pscanf(ev_file, "%lld", &value) != 1) {
      PAPIERROR("cannot read value for counter '%s'\n", iter->name);
   } else
   {
      SUBDBG("Counter '%s': %lld\n", iter->name, value);
   }
   return (value);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

_infiniband_vector

papi_vector_t _infiniband_vector

Definition at line 115 of file linux-infiniband.c.

ib_dir_path

const char* ib_dir_path = "/sys/class/infiniband"

static

Definition at line 75 of file linux-infiniband.c.

infiniband_native_events

infiniband_native_event_entry_t* infiniband_native_events = 0

static

Definition at line 110 of file linux-infiniband.c.

num_events

int num_events = 0

static

Definition at line 112 of file linux-infiniband.c.

root_counter

ib_counter_t* root_counter = 0

static

Definition at line 122 of file linux-infiniband.c.

root_device

ib_device_t* root_device = 0

static

Definition at line 121 of file linux-infiniband.c.