linux-mx.c

Go to the documentation of this file.

#include "papi.h"
#include "papi_internal.h"
#include "papi_vector.h"

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <ctype.h>

#define MX_MAX_COUNTERS 100
#define MX_MAX_COUNTER_TERMS  MX_MAX_COUNTERS

#define LINELEN 128

typedef struct MX_register
{
    /* indicate which counters this event can live on */
    unsigned int selector;
} MX_register_t;

typedef struct MX_native_event_entry
{
    /* description of the resources required by this native event */
    MX_register_t resources;
    /* If it exists, then this is the name of this event */
    char *name;
    /* If it exists, then this is the description of this event */
    char *description;
} MX_native_event_entry_t;

typedef struct MX_reg_alloc
{
    MX_register_t ra_bits;
} MX_reg_alloc_t;

typedef struct MX_control_state
{
  long long start_count[MX_MAX_COUNTERS];
  long long current_count[MX_MAX_COUNTERS];
  long long difference[MX_MAX_COUNTERS];
  int which_counter[MX_MAX_COUNTERS];
  int num_events;
} MX_control_state_t;

typedef struct MX_context
{
    MX_control_state_t state;
} MX_context_t;


static const MX_native_event_entry_t mx_native_table[] = {
    {{1,  }, "LANAI_UPTIME", "Lanai uptime (seconds)"},
    {{2,  }, "COUNTERS_UPTIME", "Counters uptime (seconds)"},
    {{3,  }, "BAD_CRC8", "Bad CRC8 (Port 0)"},
    {{4,  }, "BAD_CRC32", "Bad CRC32 (Port 0)"},
    {{5,  }, "UNSTRIPPED_ROUTE", "Unstripped route (Port 0)"},
    {{6,  }, "PKT_DESC_INVALID", "pkt_desc_invalid (Port 0)"},
    {{7,  }, "RECV_PKT_ERRORS", "recv_pkt_errors (Port 0)"},
    {{8,  }, "PKT_MISROUTED", "pkt_misrouted (Port 0)"},
    {{9,  }, "DATA_SRC_UNKNOWN", "data_src_unknown"},
    {{10,  }, "DATA_BAD_ENDPT", "data_bad_endpt"},
    {{11,  }, "DATA_ENDPT_CLOSED", "data_endpt_closed"},
    {{12,  }, "DATA_BAD_SESSION", "data_bad_session"},
    {{13,  }, "PUSH_BAD_WINDOW", "push_bad_window"},
    {{14,  }, "PUSH_DUPLICATE", "push_duplicate"},
    {{15,  }, "PUSH_OBSOLETE", "push_obsolete"},
    {{16,  }, "PUSH_RACE_DRIVER", "push_race_driver"},
    {{17,  }, "PUSH_BAD_SEND_HANDLE_MAGIC", "push_bad_send_handle_magic"},
    {{18,  }, "PUSH_BAD_SRC_MAGIC", "push_bad_src_magic"},
    {{19,  }, "PULL_OBSOLETE", "pull_obsolete"},
    {{20,  }, "PULL_NOTIFY_OBSOLETE", "pull_notify_obsolete"},
    {{21,  }, "PULL_RACE_DRIVER", "pull_race_driver"},
    {{22,  }, "ACK_BAD_TYPE", "ack_bad_type"},
    {{23,  }, "ACK_BAD_MAGIC", "ack_bad_magic"},
    {{24,  }, "ACK_RESEND_RACE", "ack_resend_race"},
    {{25,  }, "LATE_ACK", "Late ack"},
    {{26,  }, "ACK_NACK_FRAMES_IN_PIPE", "ack_nack_frames_in_pipe"},
    {{27,  }, "NACK_BAD_ENDPT", "nack_bad_endpt"},
    {{28,  }, "NACK_ENDPT_CLOSED", "nack_endpt_closed"},
    {{29,  }, "NACK_BAD_SESSION", "nack_bad_session"},
    {{30,  }, "NACK_BAD_RDMAWIN", "nack_bad_rdmawin"},
    {{31,  }, "NACK_EVENTQ_FULL", "nack_eventq_full"},
    {{32,  }, "SEND_BAD_RDMAWIN", "send_bad_rdmawin"},
    {{33,  }, "CONNECT_TIMEOUT", "connect_timeout"},
    {{34,  }, "CONNECT_SRC_UNKNOWN", "connect_src_unknown"},
    {{35,  }, "QUERY_BAD_MAGIC", "query_bad_magic"},
    {{36,  }, "QUERY_TIMED_OUT", "query_timed_out"},
    {{37,  }, "QUERY_SRC_UNKNOWN", "query_src_unknown"},
    {{38,  }, "RAW_SENDS", "Raw sends (Port 0)"},
    {{39,  }, "RAW_RECEIVES", "Raw receives (Port 0)"},
    {{40,  }, "RAW_OVERSIZED_PACKETS", "Raw oversized packets (Port 0)"},
    {{41,  }, "RAW_RECV_OVERRUN", "raw_recv_overrun"},
    {{42,  }, "RAW_DISABLED", "raw_disabled"},
    {{43,  }, "CONNECT_SEND", "connect_send"},
    {{44,  }, "CONNECT_RECV", "connect_recv"},
    {{45,  }, "ACK_SEND", "ack_send (Port 0)"},
    {{46,  }, "ACK_RECV", "ack_recv (Port 0)"},
    {{47,  }, "PUSH_SEND", "push_send (Port 0)"},
    {{48,  }, "PUSH_RECV", "push_recv (Port 0)"},
    {{49,  }, "QUERY_SEND", "query_send (Port 0)"},
    {{50,  }, "QUERY_RECV", "query_recv (Port 0)"},
    {{51,  }, "REPLY_SEND", "reply_send (Port 0)"},
    {{52,  }, "REPLY_RECV", "reply_recv (Port 0)"},
    {{53,  }, "QUERY_UNKNOWN", "query_unknown (Port 0)"},
/*   {{ 54,  }, "QUERY_UNKNOWN", "query_unknown (Port 0)"},*/
    {{55,  }, "DATA_SEND_NULL", "data_send_null (Port 0)"},
    {{56,  }, "DATA_SEND_SMALL", "data_send_small (Port 0)"},
    {{57,  }, "DATA_SEND_MEDIUM", "data_send_medium (Port 0)"},
    {{58,  }, "DATA_SEND_RNDV", "data_send_rndv (Port 0)"},
    {{59,  }, "DATA_SEND_PULL", "data_send_pull (Port 0)"},
    {{60,  }, "DATA_RECV_NULL", "data_recv_null (Port 0)"},
    {{61,  }, "DATA_RECV_SMALL_INLINE", "data_recv_small_inline (Port 0)"},
    {{62,  }, "DATA_RECV_SMALL_COPY", "data_recv_small_copy (Port 0)"},
    {{63,  }, "DATA_RECV_MEDIUM", "data_recv_medium (Port 0)"},
    {{64,  }, "DATA_RECV_RNDV", "data_recv_rndv (Port 0)"},
    {{65,  }, "DATA_RECV_PULL", "data_recv_pull (Port 0)"},
    {{66,  }, "ETHER_SEND_UNICAST_CNT", "ether_send_unicast_cnt (Port 0)"},
    {{67,  }, "ETHER_SEND_MULTICAST_CNT", "ether_send_multicast_cnt (Port 0)"},
    {{68,  }, "ETHER_RECV_SMALL_CNT", "ether_recv_small_cnt (Port 0)"},
    {{69,  }, "ETHER_RECV_BIG_CNT", "ether_recv_big_cnt (Port 0)"},
    {{70,  }, "ETHER_OVERRUN", "ether_overrun"},
    {{71,  }, "ETHER_OVERSIZED", "ether_oversized"},
    {{72,  }, "DATA_RECV_NO_CREDITS", "data_recv_no_credits"},
    {{73,  }, "PACKETS_RECENT", "Packets resent"},
    {{74,  }, "PACKETS_DROPPED", "Packets dropped (data send side)"},
    {{75,  }, "MAPPER_ROUTES_UPDATE", "Mapper routes update"},
    {{76,  }, "ROUTE_DISPERSION", "Route dispersion (Port 0)"},
    {{77,  }, "OUT_OF_SEND_HANDLES", "out_of_send_handles"},
    {{78,  }, "OUT_OF_PULL_HANDLES", "out_of_pull_handles"},
    {{79,  }, "OUT_OF_PUSH_HANDLES", "out_of_push_handles"},
    {{80,  }, "MEDIUM_CONT_RACE", "medium_cont_race"},
    {{81,  }, "CMD_TYPE_UNKNOWN", "cmd_type_unknown"},
    {{82,  }, "UREQ_TYPE_UNKNOWN", "ureq_type_unknown"},
    {{83,  }, "INTERRUPTS_OVERRUN", "Interrupts overrun"},
    {{84,  }, "WAITING_FOR_INTERRUPT_DMA", "Waiting for interrupt DMA"},
    {{85,  }, "WAITING_FOR_INTERRUPT_ACK", "Waiting for interrupt Ack"},
    {{86,  }, "WAITING_FOR_INTERRUPT_TIMER", "Waiting for interrupt Timer"},
    {{87,  }, "SLABS_RECYCLING", "Slabs recycling"},
    {{88,  }, "SLABS_PRESSURE", "Slabs pressure"},
    {{89,  }, "SLABS_STARVATION", "Slabs starvation"},
    {{90,  }, "OUT_OF_RDMA_HANDLES", "out_of_rdma handles"},
    {{91,  }, "EVENTQ_FULL", "eventq_full"},
    {{92,  }, "BUFFER_DROP", "buffer_drop (Port 0)"},
    {{93,  }, "MEMORY_DROP", "memory_drop (Port 0)"},
    {{94,  }, "HARDWARE_FLOW_CONTROL", "Hardware flow control (Port 0)"},
    {{95,  }, "SIMULATED_PACKETS_LOST", "(Devel) Simulated packets lost (Port 0)"},
    {{96,  }, "LOGGING_FRAMES_DUMPED", "(Logging) Logging frames dumped"},
    {{97,  }, "WAKE_INTERRUPTS", "Wake interrupts"},
    {{98,  }, "AVERTED_WAKEUP_RACE", "Averted wakeup race"},
    {{99,  }, "DMA_METADATA_RACE", "Dma metadata race"},
    {{0, }, "", ""}
};

static int num_events=0;
papi_vector_t _mx_vector;

static char mx_counters_exe[BUFSIZ];

static int
read_mx_counters( long long *counters )
{
    FILE *fp;
    char line[LINELEN];
    int i, linenum;

    /* Open a pipe to the mx_counters executable */

    fp = popen( mx_counters_exe, "r" );
    if ( !fp ) {
       perror( "popen" );
       return PAPI_ECMP;
    }


    /* A line of output looks something similar to:       */
        /* "    Lanai uptime (seconds):     766268 (0xbb13c)" */

    /* This code may fail if number of ports on card > 1 */

    linenum = 0;
    while ( fgets( line, LINELEN, fp ) ) {
      //       printf("%s",line);
       for(i=0; line[i]!= '\0' && i<LINELEN-1;i++) {

          /* skip to colon */
          if (line[i]==':') {

             /* read in value */
             if (line[i+1]!='\0') {
           //       printf("Line %d trying %s",linenum,&line[i+1]);
                sscanf(&line[i+1],"%lld",&counters[linenum]);
            linenum++;
            break;
         }
          }
       }
       if (linenum>=MX_MAX_COUNTERS) break;
    }

    pclose( fp );

    return PAPI_OK;
}



/*
 * Component setup and shutdown
 */

/* Initialize hardware counters, setup the function vector table
 * and get hardware information, this routine is called when the 
 * PAPI process is initialized (IE PAPI_library_init)
 */
static int
_mx_init_component( int cidx )
{

    FILE *fff;
    char test_string[BUFSIZ];

    /* detect if MX available */

    strncpy(mx_counters_exe,"mx_counters 2> /dev/null",BUFSIZ);
    fff=popen(mx_counters_exe,"r");
    /* popen only returns NULL if "sh" fails, not the actual command */
    if (fgets(test_string,BUFSIZ,fff)==NULL) {
       pclose(fff);
       strncpy(mx_counters_exe,"./components/mx/utils/fake_mx_counters 2> /dev/null",BUFSIZ);
       fff=popen(mx_counters_exe,"r");
       if (fgets(test_string,BUFSIZ,fff)==NULL) {
          pclose(fff);
          /* neither real nor fake found */
          strncpy(_mx_vector.cmp_info.disabled_reason,
              "No MX utilities found",PAPI_MAX_STR_LEN);
          return PAPI_ECMP;
       }
    }
    pclose(fff);

    num_events=MX_MAX_COUNTERS;
    _mx_vector.cmp_info.num_native_events=num_events;

    /* Export the component id */
    _mx_vector.cmp_info.CmpIdx = cidx;


    return PAPI_OK;
}


/*
 * This is called whenever a thread is initialized
 */
static int
_mx_init_thread( hwd_context_t * ctx )
{
    ( void ) ctx;            /*unused */
    return PAPI_OK;
}


static int
_mx_shutdown_component(void) 
{
  return PAPI_OK;
}

static int
_mx_shutdown_thread( hwd_context_t * ctx )
{
    ( void ) ctx;            /*unused */
    return PAPI_OK;
}



/*
 * Control of counters (Reading/Writing/Starting/Stopping/Setup)
 * functions
 */
static int
_mx_init_control_state( hwd_control_state_t *ctl )
{
    ( void ) ctl;            /*unused */

    return PAPI_OK;
}

static int
_mx_update_control_state( hwd_control_state_t *ctl, NativeInfo_t *native,
              int count, hwd_context_t *ctx )
{
    ( void ) ctx;            /*unused */
    int i, index;
    
    MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl;

    for(i=0; i<count; i++ ) {
        index = native[i].ni_event;
        mx_ctl->which_counter[i]=index;
        //      printf("Mapping event# %d to HW counter %d (count=%d)\n",
        //     i,index,count); 
        native[i].ni_position = i;
    }

    mx_ctl->num_events=count;

    return PAPI_OK;
}


static int
_mx_start( hwd_context_t *ctx, hwd_control_state_t *ctl )
{

        long long mx_counters[MX_MAX_COUNTERS];

    ( void ) ctx;            /*unused */

    MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl;
    int i;

    read_mx_counters( mx_counters );

    //  for(i=0;i<MX_MAX_COUNTERS;i++) printf("%d %lld\n",i,mx_counters[i]);

    for(i=0;i<mx_ctl->num_events;i++) {
           mx_ctl->current_count[i]=
             mx_counters[mx_ctl->which_counter[i]];
       mx_ctl->start_count[i]=mx_ctl->current_count[i];
    }

    return PAPI_OK;
}


static int
_mx_stop( hwd_context_t *ctx, hwd_control_state_t *ctl )
{
    ( void ) ctx;            /*unused */

        long long mx_counters[MX_MAX_COUNTERS];
    MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl;
    int i;

    read_mx_counters( mx_counters );

    for(i=0;i<mx_ctl->num_events;i++) {
           mx_ctl->current_count[i]=
             mx_counters[mx_ctl->which_counter[i]];
    }

    return PAPI_OK;
}

static int
_mx_read( hwd_context_t *ctx, hwd_control_state_t *ctl, long long **events,
         int flags )
{
    ( void ) ctx;            /*unused */
    ( void ) flags;          /*unused */
    int i;
        long long mx_counters[MX_MAX_COUNTERS];

    MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl;

    read_mx_counters( mx_counters );

    for ( i = 0; i < mx_ctl->num_events; i++ ) {
            mx_ctl->current_count[i]=
             mx_counters[mx_ctl->which_counter[i]];
        mx_ctl->difference[i] = mx_ctl->current_count[i]-
                                   mx_ctl->start_count[i];
    }
    *events = mx_ctl->difference;

    return PAPI_OK;
}


static int
_mx_reset( hwd_context_t * ctx, hwd_control_state_t * ctrl )
{
    _mx_start( ctx, ctrl );
    return PAPI_OK;
}

/* Unused write function */
/* static int */
/* _mx_write( hwd_context_t * ctx, hwd_control_state_t * ctrl, long long *from ) */
/* { */
/*  ( void ) ctx;            /\*unused *\/ */
/*  ( void ) ctrl;           /\*unused *\/ */
/*  ( void ) from;           /\*unused *\/ */

/*  return PAPI_OK; */
/* } */

/*
 * Functions for setting up various options
 */

/* This function sets various options in the component
 * The valid codes being passed in are PAPI_SET_DEFDOM,
 * PAPI_SET_DOMAIN, PAPI_SETDEFGRN, PAPI_SET_GRANUL * and PAPI_SET_INHERIT
 */
static int
_mx_ctl( hwd_context_t * ctx, int code, _papi_int_option_t * option )
{
    ( void ) ctx;            /*unused */
    ( void ) code;           /*unused */
    ( void ) option;         /*unused */

    return PAPI_OK;
}

/*
 * This function has to set the bits needed to count different domains
 * In particular: PAPI_DOM_USER, PAPI_DOM_KERNEL PAPI_DOM_OTHER
 * By default return PAPI_EINVAL if none of those are specified
 * and PAPI_OK with success
 * PAPI_DOM_USER is only user context is counted
 * PAPI_DOM_KERNEL is only the Kernel/OS context is counted
 * PAPI_DOM_OTHER  is Exception/transient mode (like user TLB misses)
 * PAPI_DOM_ALL   is all of the domains
 */
static int
_mx_set_domain( hwd_control_state_t * cntrl, int domain )
{
    ( void ) cntrl;          /*unused */
    if ( PAPI_DOM_ALL != domain ) {
        return PAPI_EINVAL;
    }

    return PAPI_OK;
}



static int
_mx_ntv_code_to_name( unsigned int EventCode, char *name, int len )
{

    int event=EventCode;

    if (event >=0 && event < num_events) {
      strncpy( name, mx_native_table[event].name, len );
      return PAPI_OK;
    }
    return PAPI_ENOEVNT;


}

static int
_mx_ntv_code_to_descr( unsigned int EventCode, char *name, int len )
{
    int event=EventCode;

    if (event >=0 && event < num_events) {
       strncpy( name, mx_native_table[event].description, len );
       return PAPI_OK;
    }
    return PAPI_ENOEVNT;
}



static int
_mx_ntv_enum_events( unsigned int *EventCode, int modifier )
{

    if ( modifier == PAPI_ENUM_FIRST ) {
       if (num_events==0) return PAPI_ENOEVNT;
       *EventCode = 0;
       return PAPI_OK;
    }

    if ( modifier == PAPI_ENUM_EVENTS ) {
        int index = *EventCode;

        if ( mx_native_table[index + 1].resources.selector ) {
            *EventCode = *EventCode + 1;
            return PAPI_OK;
        } else {
            return PAPI_ENOEVNT;
        }
    }
        
        return PAPI_EINVAL;
}


papi_vector_t _mx_vector = {
    .cmp_info = {
        .name = "mx",
        .short_name = "mx",
        .version = "1.4",
        .description = "Myricom MX (Myrinet Express) statistics",
        .num_mpx_cntrs = MX_MAX_COUNTERS,
        .num_cntrs = MX_MAX_COUNTERS,
        .default_domain = PAPI_DOM_ALL,
        .default_granularity = PAPI_GRN_SYS,
        .available_granularities = PAPI_GRN_SYS,
        .hardware_intr_sig = PAPI_INT_SIGNAL,

        /* component specific cmp_info initializations */
        .fast_real_timer = 0,
        .fast_virtual_timer = 0,
        .attach = 0,
        .attach_must_ptrace = 0,
        .available_domains = PAPI_DOM_ALL,
  },

    /* sizes of framework-opaque component-private structures */
    .size = {
        .context = sizeof ( MX_context_t ),
        .control_state = sizeof ( MX_control_state_t ),
        .reg_value = sizeof ( MX_register_t ),
        .reg_alloc = sizeof ( MX_reg_alloc_t ),
  },
        /* function pointers in this component */
    .init_thread =          _mx_init_thread,
    .init_component =       _mx_init_component,
    .init_control_state =   _mx_init_control_state,
    .start =                _mx_start,
    .stop =                 _mx_stop,
    .read =                 _mx_read,
    .shutdown_thread =      _mx_shutdown_thread,
    .shutdown_component =   _mx_shutdown_component,
    .ctl =                  _mx_ctl,
    .update_control_state = _mx_update_control_state,
    .set_domain =           _mx_set_domain,
    .reset =                _mx_reset,

    .ntv_enum_events =      _mx_ntv_enum_events,
    .ntv_code_to_name =     _mx_ntv_code_to_name,
    .ntv_code_to_descr =    _mx_ntv_code_to_descr,
};