cleaned up python components

TACC · Mar 31, 2016 · ac54eb6 · ac54eb6
1 parent 202d38b
commit ac54eb6
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 15 deletions.
diff --git a/setup.py b/setup.py
@@ -35,7 +35,9 @@
 
 with open("tacc_stats/cfg.py", 'w') as fd:
     for s in config.sections():
+        print s
         for key, val in dict(config.items(s)).iteritems():
+            print key,val
             fd.write(key + " = " + "\"" + val + "\"" + '\n')
 
 setup(

diff --git a/tacc_stats.ini b/tacc_stats.ini
@@ -3,9 +3,9 @@
 # server	= database and rmq server hostname
 # data_dir	= where data is stored
 [DEFAULT]
-machine		= stampede
-data_dir 	= /scratch/projects/tacc_stats
-server 		= tacc-stats03.tacc.utexas.edu
+machine		= ls5
+data_dir 	= /hpc/tacc_stats_site/%(machine)s
+server 		= tacc-stats02.tacc.utexas.edu
 
 ## RabbitMQ Configuration
 # RMQ_SERVER  	= RMQ server
@@ -18,7 +18,7 @@ rmq_queue 	= %(machine)s
 [PORTAL]
 acct_path       = %(data_dir)s/accounting/tacc_jobs_completed
 host_list_dir   = %(data_dir)s/hostfile_logs
-pickles_dir     = /corral-repl/tacc_stats_site/stampede/pickles
+pickles_dir     = %(data_dir)s/pickles
 archive_dir	= %(data_dir)s/archive
 host_name_ext   = %(machine)s.tacc.utexas.edu
 batch_system    = SLURM

diff --git a/tacc_stats/listend.py b/tacc_stats/listend.py
@@ -4,13 +4,10 @@
 import time
 import cfg
 
-stats_dir = "/hpc/tacc_stats_site/ls5/archive"
-
 def on_message(channel, method_frame, header_frame, body):
     if body[0] == '$': host = body.split('\n')[1].split()[1]       
     else: host = body.split()[2]
 
-    print host
     host_dir = os.path.join(cfg.archive_dir, host)
     if not os.path.exists(host_dir):
         os.makedirs(host_dir)
@@ -30,9 +27,10 @@ def on_message(channel, method_frame, header_frame, body):
 
     channel.basic_ack(delivery_tag=method_frame.delivery_tag)
 
-connection = pika.BlockingConnection()
+parameters = pika.ConnectionParameters(cfg.rmq_server)
+connection = pika.BlockingConnection(parameters)
 channel = connection.channel()
-channel.basic_consume(on_message, sys.argv[1])
+channel.basic_consume(on_message, cfg.rmq_queue)
 try:
     channel.start_consuming()
 except KeyboardInterrupt:

diff --git a/tacc_stats/pickler/job_pickles_slurm.py b/tacc_stats/pickler/job_pickles_slurm.py
@@ -44,10 +44,10 @@ def job_pickle(reader_inst,
 
 class JobPickles:
 
-    def __init__(self, start, end, processes, pickles_dir):
+    def __init__(self, start, end, processes, pickles_dir, jobids):
 
         self.pool   = multiprocessing.Pool(processes = processes)        
-
+        self.jobids = jobids
         self.start  = start
         self.end    = end
         if not pickles_dir:
@@ -78,6 +78,7 @@ def acct_reader(self, filename):
         acct = []
         with open(filename, "rb") as fd:
             for job in csv.DictReader(fd, delimiter = '|'):
+                if self.jobids and job['JobID'] not in self.jobids: continue
                 nodelist_str = job['NodeList']
                 if '[' in nodelist_str and ']' in nodelist_str:
                     nodelist = []
@@ -119,22 +120,26 @@ def acct_reader(self, filename):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Run pickler for jobs')
-    parser.add_argument('start', type = parse, default = datetime.now() - timedelta(days=1), 
+    parser.add_argument('start', type = parse, nargs = '?', default = datetime.now() - timedelta(days=1), 
                         help = 'Start (YYYY-mm-dd)')
     parser.add_argument('end',   type = parse, nargs = '?', default = False, 
                         help = 'End (YYYY-mm-dd)')
     parser.add_argument('-p', '--processes', type = int, default = 1,
                         help = 'number of processes')
     parser.add_argument('-d', '--directory', type = str, 
                         help='Directory to store data')
+    parser.add_argument('-jobids', help = 'Pickle this list of jobs', 
+                        type = str, nargs = '+')
+
     args = parser.parse_args()
     if not args.end:
-        args.end = args.start + timedelta(days=1)
+        args.end = args.start + timedelta(days=2)
 
     pickle_options = { 'processes'       : args.processes,
                        'start'           : args.start,
                        'end'             : args.end,
-                       'pickles_dir'      : args.directory,
+                       'pickles_dir'     : args.directory,
+                       'jobids'          : args.jobids
                        }
 
     pickler = JobPickles(**pickle_options)

diff --git a/tacc_stats/pickler/job_stats.py b/tacc_stats/pickler/job_stats.py
@@ -589,7 +589,9 @@ def error(fmt, *args):
                         # Rebase narrow counters and spurious resets.
                         # 64-bit overflows are correctly handled automatically                        
                         fudged = False
-                        if e.width:
+                        if 'intel_' in type_name: 
+                            r = numpy.uint(0) - A[i-1,j]
+                        elif e.width:
                             trace("time %d, counter `%s', rollover prev %d, curr %d\n",
                                   self.times[i], e.key, p, v)
                             r -= numpy.uint64(1L << e.width)