|
|
|
|
@ -16,6 +16,8 @@ from threading import Thread, RLock
|
|
|
|
|
import time
|
|
|
|
|
import numpy as np
|
|
|
|
|
from utils.ml import ML
|
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
class Analysis:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.logs = []
|
|
|
|
|
@ -30,6 +32,11 @@ class Analysis:
|
|
|
|
|
return {"month":d.month,"year":d.year, "day":d.day,"hour":d.hour,"minute":d.minute}
|
|
|
|
|
def getName(self):
|
|
|
|
|
return self.__class__.__name__
|
|
|
|
|
def reboot(self,row,conf) :
|
|
|
|
|
return False
|
|
|
|
|
def cleanup(self,text):
|
|
|
|
|
return re.sub('[^a-zA-Z0-9\s:]',' ',str(text)).strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
This class is designed to analyze environment variables. Environment variables can either be folders, files or simple values
|
|
|
|
|
@ -106,6 +113,8 @@ class Sandbox(Analysis):
|
|
|
|
|
return [row.replace('-',' ').replace('_',' ') for row in r if row.strip() != '']
|
|
|
|
|
def evaluate(self):
|
|
|
|
|
pass
|
|
|
|
|
def reboot(self,rows,limit=None) :
|
|
|
|
|
return sum([ len(item['missing']) for item in rows ]) > 0
|
|
|
|
|
"""
|
|
|
|
|
This function returns the ratio of existing modules relative to the ones expected
|
|
|
|
|
"""
|
|
|
|
|
@ -148,14 +157,17 @@ class ProcessCounter(Analysis):
|
|
|
|
|
#n = sum(r)
|
|
|
|
|
#return n/N
|
|
|
|
|
return dict(self.getNow(),**r)
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
This class returns an application's both memory and cpu usage
|
|
|
|
|
"""
|
|
|
|
|
class DetailProcess(Analysis):
|
|
|
|
|
def __init__(self):
|
|
|
|
|
Analysis.__init__(self)
|
|
|
|
|
|
|
|
|
|
def init (self,names):
|
|
|
|
|
#Analysis.init(self)
|
|
|
|
|
|
|
|
|
|
self.names = names;
|
|
|
|
|
def getName(self):
|
|
|
|
|
return "apps"
|
|
|
|
|
@ -167,42 +179,52 @@ class DetailProcess(Analysis):
|
|
|
|
|
return list(g.groups())+['1']+[name]
|
|
|
|
|
else:
|
|
|
|
|
return ''
|
|
|
|
|
def evaluate(self,name) :
|
|
|
|
|
cmd = "ps -eo pmem,pcpu,vsize,command|grep -E \":app\""
|
|
|
|
|
handler = subprocess.Popen(cmd.replace(":app",name),shell=True,stdout=subprocess.PIPE)
|
|
|
|
|
ostream = handler.communicate()[0].split('\n')
|
|
|
|
|
#xstr = ostream
|
|
|
|
|
ostream = [ self.split(name,row) for row in ostream if row != '' and 'grep' not in row]
|
|
|
|
|
if len(ostream) == 0 or len(ostream[0]) < 4 :
|
|
|
|
|
ostream = [['0','0','0','0',name]]
|
|
|
|
|
r = []
|
|
|
|
|
for row in ostream :
|
|
|
|
|
#
|
|
|
|
|
# Though the comm should only return the name as specified,
|
|
|
|
|
# On OSX it has been observed that the fully qualified path is sometimes returned (go figure)
|
|
|
|
|
#
|
|
|
|
|
row = [float(value) for value in row if value.strip() != '' and name not in value ] +[re.sub('\$|^','',name)]
|
|
|
|
|
r.append(row)
|
|
|
|
|
def reboot(self,rows,conf=None) :
|
|
|
|
|
return np.sum([int(item['label']=='crash') for item in rows]) > 0
|
|
|
|
|
def parse(self,row,fields):
|
|
|
|
|
"""
|
|
|
|
|
The last field should be the command in its integrity
|
|
|
|
|
@pre len(fields) > len(row)
|
|
|
|
|
"""
|
|
|
|
|
r = {}
|
|
|
|
|
|
|
|
|
|
now = self.getNow()
|
|
|
|
|
r['date'] = now
|
|
|
|
|
row = [term for term in row.split() if term.strip() != '']
|
|
|
|
|
for name in fields :
|
|
|
|
|
index = fields.index(name)
|
|
|
|
|
|
|
|
|
|
r[name] = row[index] if row else 0
|
|
|
|
|
if name not in ['user','cmd','status','pid'] :
|
|
|
|
|
r[name] = float(r[name])
|
|
|
|
|
r[name] = row[index: ] if row else []
|
|
|
|
|
#
|
|
|
|
|
# At this point we should aggregate results
|
|
|
|
|
# The aggregation is intended for applications with several processes (e.g: apache2)
|
|
|
|
|
# Let's set the status give the data extracted
|
|
|
|
|
#
|
|
|
|
|
if len(r) > 1:
|
|
|
|
|
m = None
|
|
|
|
|
for row in r:
|
|
|
|
|
if m is None:
|
|
|
|
|
m = row
|
|
|
|
|
else:
|
|
|
|
|
m[3] += row[3]
|
|
|
|
|
m[0] += row[0]
|
|
|
|
|
m[1] += row[1]
|
|
|
|
|
m[2] += row[2]
|
|
|
|
|
m[0] = round((m[0] / m[3]),2)
|
|
|
|
|
m[1] = round((m[1] / m[3]),2)
|
|
|
|
|
m[2] = round((m[2] / m[3]),2)
|
|
|
|
|
|
|
|
|
|
r = [m]
|
|
|
|
|
if r['status'] == 0 :
|
|
|
|
|
r['status'] = 'crash'
|
|
|
|
|
elif 'Z' in r['status'] :
|
|
|
|
|
r['status'] = 'zombie'
|
|
|
|
|
elif r['memory_usage'] > 0 and r['cpu_usage'] > 0:
|
|
|
|
|
r['status'] = 'running'
|
|
|
|
|
else:
|
|
|
|
|
r['status'] = 'idle'
|
|
|
|
|
return r
|
|
|
|
|
|
|
|
|
|
def evaluate(self,name=None) :
|
|
|
|
|
if name is None :
|
|
|
|
|
name = ".*"
|
|
|
|
|
fields = ["user","pid","memory_usage","cpu_usage","memory_available","status","cmd"]
|
|
|
|
|
cmd = "ps -eo user,pid,pmem,pcpu,vsize,stat,command|grep -Ei \":app\"".replace(":app",name)
|
|
|
|
|
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
|
|
|
|
|
logs = handler.communicate()[0].split('\n')
|
|
|
|
|
logs = [row for row in logs if (row.strip() != '') and ('grep -Ei' in row )== False ]
|
|
|
|
|
|
|
|
|
|
if len(logs) == 0:
|
|
|
|
|
return [dict(self.parse('',fields),**{'label':name}) ]
|
|
|
|
|
else :
|
|
|
|
|
return [dict(self.parse(row,fields),**{'label':name}) for row in logs if row.strip() != '' and 'grep' not in row and '-Ei' not in row]
|
|
|
|
|
|
|
|
|
|
def status(self,row):
|
|
|
|
|
x = row['memory_usage']
|
|
|
|
|
y = row['cpu_usage']
|
|
|
|
|
@ -213,20 +235,17 @@ class DetailProcess(Analysis):
|
|
|
|
|
return "idle"
|
|
|
|
|
else:
|
|
|
|
|
return "crash"
|
|
|
|
|
def format(self,row):
|
|
|
|
|
r= {"memory_usage":row[0],"cpu_usage":row[1],"memory_available":row[2]/1000,"proc_count":row[3],"label":row[4]}
|
|
|
|
|
status = self.status(r)
|
|
|
|
|
r['status'] = status
|
|
|
|
|
return r
|
|
|
|
|
#def format(self,row):
|
|
|
|
|
# r= {"memory_usage":row[0],"cpu_usage":row[1],"memory_available":row[2]/1000,"proc_count":row[3],"label":self.cleanup(row[4])}
|
|
|
|
|
# status = self.status(r)
|
|
|
|
|
# r['status'] = status
|
|
|
|
|
# return r
|
|
|
|
|
|
|
|
|
|
def composite(self):
|
|
|
|
|
ma = []
|
|
|
|
|
now = self.getNow()
|
|
|
|
|
for name in self.names:
|
|
|
|
|
|
|
|
|
|
matrix = self.evaluate(name)
|
|
|
|
|
|
|
|
|
|
ma += [ dict(now, **self.format(row)) for row in matrix]
|
|
|
|
|
row = self.evaluate(name)
|
|
|
|
|
ma += row
|
|
|
|
|
|
|
|
|
|
return ma
|
|
|
|
|
"""
|
|
|
|
|
@ -237,6 +256,7 @@ class FileWatch(Analysis):
|
|
|
|
|
def __init__(self):
|
|
|
|
|
pass
|
|
|
|
|
def init(self,folders):
|
|
|
|
|
print folders
|
|
|
|
|
self.folders = folders;
|
|
|
|
|
def getName(self):
|
|
|
|
|
return "folders"
|
|
|
|
|
@ -277,12 +297,22 @@ class FileWatch(Analysis):
|
|
|
|
|
|
|
|
|
|
def evaluate(self,path):
|
|
|
|
|
cmd = "find :path -print0|xargs -0 ls -ls |awk '{print $6,$7,$8,$9,$10}'".replace(":path",path)
|
|
|
|
|
|
|
|
|
|
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
|
|
|
|
|
ostream = handler.communicate()[0].split('\n')
|
|
|
|
|
|
|
|
|
|
ostream = [row for row in ostream if row.strip() != '']
|
|
|
|
|
print cmd
|
|
|
|
|
print ostream[0]
|
|
|
|
|
print ostream[1]
|
|
|
|
|
#return [self.split(stream) for stream in ostream if stream.strip() != '' and '.DS_Store' not in stream and 'total' not in stream]
|
|
|
|
|
return [self.split(stream) for stream in ostream if path not in stream and not set(['','total','.DS_Store']) & set(stream.split(' '))]
|
|
|
|
|
#return [self.split(stream) for stream in ostream if path not in stream and not set(['','total','.DS_Store']) & set(stream.split(' '))]
|
|
|
|
|
return []
|
|
|
|
|
def toMB(self,size):
|
|
|
|
|
m = {'GB':1000,'TB':1000000}
|
|
|
|
|
v,u = size.split(' ')
|
|
|
|
|
return round(float(v)* m[u.upper()],2)
|
|
|
|
|
|
|
|
|
|
def reboot(self,rows,limit) :
|
|
|
|
|
return np.sum([ int(self.toMB(item['size']) > self.toMB(limit)) for item in rows]) > 0
|
|
|
|
|
def composite(self):
|
|
|
|
|
d = [] #-- vector of details (age,size)
|
|
|
|
|
|
|
|
|
|
|
Steve Nyemba
7 years ago