From 6c0442d664aa91200c8d0614d16afb5d92b93755 Mon Sep 17 00:00:00 2001 From: "Steve L. Nyemba" Date: Fri, 13 Oct 2017 12:03:09 -0500 Subject: [PATCH 1/2] CO - Bug fix/optimization --- src/api/index.py | 2 +- src/monitor.py | 125 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 92 insertions(+), 35 deletions(-) diff --git a/src/api/index.py b/src/api/index.py index 5999009..45f1e3a 100644 --- a/src/api/index.py +++ b/src/api/index.py @@ -92,7 +92,7 @@ def register(): global p body = request.get_json(silent=True) - + print body['uid'] auid = str(uuid.uuid4()) headers = {"uid":body['uid'],"pid":body['plan'],"auid":json.dumps([auid])} couchdb = factory.instance(type=class_read,args=p) diff --git a/src/monitor.py b/src/monitor.py index c973d2d..43e67e9 100755 --- a/src/monitor.py +++ b/src/monitor.py @@ -177,40 +177,45 @@ class DetailProcess(Analysis): def reboot(self,rows,conf=None) : return np.sum([int(item['label']=='crash') for item in rows]) > 0 def evaluate(self,name) : - cmd = "ps -eo pmem,pcpu,vsize,command|grep -E \":app\"" - handler = subprocess.Popen(cmd.replace(":app",name),shell=True,stdout=subprocess.PIPE) - ostream = handler.communicate()[0].split('\n') - #xstr = ostream - ostream = [ self.split(name,row) for row in ostream if row != '' and 'grep' not in row] - if len(ostream) == 0 or len(ostream[0]) < 4 : - ostream = [['0','0','0','0',name]] - r = [] - for row in ostream : - # - # Though the comm should only return the name as specified, - # On OSX it has been observed that the fully qualified path is sometimes returned (go figure) - # - row = [float(value) for value in row if value.strip() != '' and name not in value ] +[re.sub('\$|^','',name)] - r.append(row) - # + #cmd = "ps -eo pmem,pcpu,vsize,command|grep -E \":app\"" + fields = ['user','pid','start','time','memory_usage','cpu_usage','memory_available','status','label','command'] + + cmd = "ps -eo user,pid,stime,time,pmem,pcpu,vsize,stat,cmd,command|grep -E \":app\"".replace(":app",name) + handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) + m = handler.communicate()[0].split('\n') + r = self.format(fields,cmd,m) + ##xstr = ostream + ##ostream = [ self.split(name,row) for row in ostream if row != '' and 'grep' not in row] + #ostream = [ row.split() for row in ostream if row.strip() != '' and 'grep' not in row] + #if len(ostream) == 0 or len(ostream[0]) < 4 : + #ostream = [] #['0','0','0','0',name]] + #r = [] + #for row in ostream : + ## + ## Though the comm should only return the name as specified, + ## On OSX it has been observed that the fully qualified path is sometimes returned (go figure) + ## + #row = [float(value) for value in row if value.strip() != '' and name not in value ] +[re.sub('\$|^','',name)] + #r.append(row) + ## # At this point we should aggregate results # The aggregation is intended for applications with several processes (e.g: apache2) # - if len(r) > 1: - m = None - for row in r: - if m is None: - m = row - else: - m[3] += row[3] - m[0] += row[0] - m[1] += row[1] - m[2] += row[2] - m[0] = round((m[0] / m[3]),2) - m[1] = round((m[1] / m[3]),2) - m[2] = round((m[2] / m[3]),2) + #if len(r) > 1: + # m = None + # for row in r: + # if m is None: + # m = row + # else: + # m[3] += row[3] + # m[0] += row[0] + # m[1] += row[1] + # m[2] += row[2] + # m[0] = round((m[0] / m[3]),2) + # m[1] = round((m[1] / m[3]),2) + # m[2] = round((m[2] / m[3]),2) - r = [m] + # r = [m] return r def status(self,row): x = row['memory_usage'] @@ -222,20 +227,68 @@ class DetailProcess(Analysis): return "idle" else: return "crash" - def format(self,row): + def _format(self,row): r= {"memory_usage":row[0],"cpu_usage":row[1],"memory_available":row[2]/1000,"proc_count":row[3],"label":self.cleanup(row[4])} status = self.status(r) r['status'] = status return r + def _getObject(self,fields,terms): + r = {} + value = [] + id = None + for id in fields : + index = fields.index(id) + r[id] = terms[index].strip() + if re.match('^\d{0,}\.\d{1,}$',r[id]) : + r[id] = round(float(r[id]),4) + elif re.match('^[0-9]+$',r[id]) : + r[id] = int(r[id]) + r[id] = " ".join( terms[index:]).strip() + + + r['memory_available'] /= 1000 + r['status'] = self.status(r) + return r + + + def format(self,fields,cmd,matrix): + """ + This function will create an associative array given the fields (meta data) provided for the vector output in the matrix + + @param matrix matrix of outputs for the given command + @param fields field name for every column (standard for ps) + @param cmd name of the command that was executed, helps to remove noise + """ + r = [] + + xcmd = cmd.split() + N = len(xcmd) + for row in matrix : + n = len(set(row.split()) & set(xcmd)) + if n/N > .5 or len(row.strip()) == 0 or 'grep' in row.split(): + continue + + row = [col for col in row.split() if col.strip() != '' and 'grep' not in col] + r += [self._getObject(fields,row)] + if len(r) == 0 : + obj = {} + for id in fields : + obj[id] = 0 + obj['label'] = cmd.split('-E')[1].replace('"','') + obj['status']= 'crash' + r = [obj] + + return r + def composite(self): ma = [] now = self.getNow() for name in self.names: - matrix = self.evaluate(name) - - ma += [ dict(now, **self.format(row)) for row in matrix] + #matrix = self.evaluate(name) + ma += self.evaluate(name) + #ma += [ dict(now, **self.format(row)) for row in matrix] return ma """ @@ -394,3 +447,7 @@ class FileWatch(Analysis): # BEG = len(self.logs) - MAX_SIZE -1 # self.logs = self.logs[BEG:] +p = DetailProcess() +r = p.evaluate('firefox') +for item in r : + print item['label'],item['status'] \ No newline at end of file From ab7f312ea28a54a5362c22f3b5c1d75d44a0cb96 Mon Sep 17 00:00:00 2001 From: "Steve L. Nyemba" Date: Fri, 13 Oct 2017 12:04:06 -0500 Subject: [PATCH 2/2] CO - bug fix and updates --- config.json | 11 +---------- requirements.txt | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/config.json b/config.json index 02f1be0..1e219cb 100644 --- a/config.json +++ b/config.json @@ -1,17 +1,8 @@ { - "id":"michaels-MBP", - "key":"4q-h8r5-247&!570p=[0v8]x360", "api":"dev.the-phi.com", - "delay":10.0, "store":{ "class":{"read":"CouchdbReader","write":"CouchdbWriter"}, - "args":{"uri":"http://dev.the-phi.com:5984","dbname":"mike-db","uid":"logs"} - }, - "procs":["mail","safari", "chrome", "terminal"], - "folders":["/Users/michaelmead/Downloads"], - "actions":{ - "folders":{"threshold":"10mb","action":"archive","key":""}, - "apps":{"mail":"","safari":"","chrome":"" } + "args":{"uri":"http://localhost.com:5984","dbname":"monitor","uid":"logs"} } diff --git a/requirements.txt b/requirements.txt index c83f924..1ce03fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,34 @@ -Flask==0.11.1 -Flask-Session==0.3.0 -Flask-SocketIO==2.8.2 -Jinja2==2.8 -MarkupSafe==0.23 -Werkzeug==0.11.11 aniso8601==1.2.0 -argparse==1.2.1 +asn1crypto==0.23.0 +certifi==2017.7.27.1 +cffi==1.11.2 +chardet==3.0.4 click==6.6 couchdbkit==0.6.5 +cryptography==2.1.1 +enum34==1.1.6 +Flask==0.11.1 +Flask-Session==0.3.0 +Flask-SocketIO==2.8.2 http-parser==0.8.3 +idna==2.6 +ipaddress==1.0.18 itsdangerous==0.24 +Jinja2==2.8 +MarkupSafe==0.23 ngram==3.3.0 numpy==1.11.3 pika==0.10.0 +pkg-resources==0.0.0 +pycparser==2.18 +pyOpenSSL==17.3.0 python-dateutil==2.6.0 python-engineio==1.1.0 python-socketio==1.6.2 pytz==2016.10 +requests==2.18.4 restkit==4.2.2 six==1.10.0 socketpool==0.5.3 -wsgiref==0.1.2 +urllib3==1.22 +Werkzeug==0.11.11