library
/
smart-top
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug fix, added date, node name for easy filter and rendering

Browse Source
data-collector
Steve L. Nyemba 6 years ago
parent 8af1094563
commit 4e3061a37c
2 changed files with 42 additions and 20 deletions
Show Stats Download Patch File Download Diff File

13
src/data-collector.py
Unescape View File

@ -41,8 +41,8 @@ class Collector :
# -- let's get the list of features we are interested .
url = SYS_ARGS['api']+'/1/client/login'
key = SYS_ARGS['key']
id = SYS_ARGS['id'] if 'id' in SYS_ARGS else os.environ['HOSTNAME']
headers = {"key":key,"id":id}
self.id = SYS_ARGS['id'] if 'id' in SYS_ARGS else os.environ['HOSTNAME']
headers = {"key":key,"id":self.id}
#
#-- what features are allowed
@ -75,7 +75,9 @@ class Collector :
key = SYS_ARGS['key']
id = SYS_ARGS['id'] if 'id' in SYS_ARGS else os.environ['HOSTNAME']
headers = {"key":key,"id":id,"context":args['context'],"content-type":"application/json"}
body = args['data'].to_json(orient='records')
body = args['data'].fillna('').to_json(orient='records')
if args['data'].shape[0] > 0 :
r = self.httpclient.post(url,headers=headers,data=body)
Logger.log(action="post."+args['context'],value=r.status_code)
@ -93,9 +95,9 @@ class Collector :
if self.config and self.features :
ELAPSED_TIME = 60* int(self.features['schedule'].replace("min","").strip())
if 'apps' in self.config :
self.post( data=(Apps()).get(filter=self.config['apps']),context="apps")
self.post( data=(Apps(node=self.id)).get(filter=self.config['apps']),context="apps")
if 'folders' in self.config and self.config['folders'] :
folder = Folders()
folder = Folders(node=self.id)
f = folder.get(path=self.config['folders'])
self.post(data = f ,context="folders")
@ -106,6 +108,7 @@ class Collector :
#
time.sleep(ELAPSED_TIME)
except Exception,e:
Logger.log(action='error',value=e.message)
print e

45
src/monitor.py
Unescape View File

@ -16,17 +16,21 @@ import sys
import pandas as pd
import datetime
class SmartTop:
def __init__(self,**args):
self.node = args['node']
def get(self,**args):
return None
class Apps(SmartTop) :
def __init__(self):
def __init__(self,**args):
"""
This class will process a system command and parse the outpout accordingly given a parser
@param parse is a parser pointer
"""
SmartTop.__init__(self,**args)
self.cmd = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'"
self.xchar = ';'
def get_app(self,stream):
index = 1 if os.path.exists(" ".join(stream[:1])) else len(stream)-1
@ -46,13 +50,18 @@ class Apps(SmartTop) :
@param m raw matrix i.e list of values like a csv
"""
d = datetime.datetime.now().strftime('%m-%d-%Y')
t = datetime.datetime.now().strftime('%H:%M:%S')
m = [item for item in m if len(item) != len (m[0])]
m = "\n".join(m[1:])
df = pd.read_csv(pd.compat.StringIO(m),sep=self.xchar)
df.columns =['pid','user','mem','cpu','status','started','name','cmd','args']
df['date'] = np.repeat(d,df.shape[0])
df['time'] = np.repeat(t,df.shape[0])
df['node'] = np.repeat(self.node,df.shape[0])
df.columns =['pid','user','mem','cpu','status','started','name','cmd','args','date','time','node']
return df
def empty(self,name):
return pd.DataFrame([{"pid":None,"user":None,"mem":0,"cpu":0,"status":"X","started":None,"name":name,"cmd":None,"args":None}])
return pd.DataFrame([{"pid":None,"user":None,"mem":0,"cpu":0,"status":"X","started":None,"name":name,"cmd":None,"args":None,"date":None,"time":None,"node":self.node}])
def parse(self,rows):
m = []
TIME_INDEX = 5
@ -87,20 +96,28 @@ class Apps(SmartTop) :
if 'filter' in args :
pattern = "|".join(args['filter'])
i = df.cmd.str.contains(pattern)
r = df[i]
r = df[i].copy()
r.index = np.arange(0,r.shape[0])
ii= (1 + np.array(i)*-1) == 1
other = pd.DataFrame(df[ii].sum()).T
other = pd.DataFrame(df[ii].sum()).T.copy()
other.index = np.arange(0,other.shape[0])
other.user = other.name = other.status = other.cmd = other.args = 'other'
other.started = other.pid = -1
other = other[other.columns[1:]]
for name in args['filter'] :
i = r.cmd.str.contains(str(name),case=False,na=False)
i = r.cmd.str.contains(str(name.strip()),case=False,na=False)
if i.sum() == 0:
r = r.append(self.empty(name),sort=False)
else :
r.loc[i,'name'] = name
pass
# r[i].update (pd.DataFrame({"name":np.repeat(name,r.shape[0])}))
r.loc[i, 'name'] = np.repeat(name,i.sum())
# r.loc[i].name = name
r = r.append(other,sort=False)
r.index = np.arange(r.shape[0])
return r
except Exception,e:
print (e)
@ -111,8 +128,9 @@ class Folders(SmartTop):
"""
This class will assess a folder and produce a report in a data-frame that can be later on used for summary statistics
"""
def __init__(self):
pass
def __init__(self,**args):
SmartTop.__init__(self,**args)
def _get(self,dir_path,r=[]):
for child in os.listdir(dir_path):
path = os.path.join(dir_path, child)
@ -125,9 +143,8 @@ class Folders(SmartTop):
file_date = datetime.datetime.fromtimestamp(file_date)
now = datetime.datetime.now()
age = (now - file_date ).days
name = os.path.basename(path)
r.append({"name":name,"path":path,"size":size,"age":age})
r.append({"name":name,"path":path,"size":size,"age":age,"date":now.strftime('%m-%d-%Y'),"time":now.strftime('%H:%M:%S'),"node":self.node })
return r
def get(self,**args):
@ -140,6 +157,7 @@ class Folders(SmartTop):
paths = paths
_out = pd.DataFrame()
for path in paths :
name = os.path.basename(path)
if os.path.exists(path) :
#
# If the folder does NOT exists it should not be treated.
@ -147,9 +165,10 @@ class Folders(SmartTop):
rows = self._get(path)
if len(rows) > 0 :
r = pd.DataFrame(rows)
r = pd.DataFrame([{"name":path,"files":r.shape[0],"age_in_days":r.age.mean(),"size_in_kb":r['size'].sum()}])
r = pd.DataFrame([{"name":name,"path":path,"files":r.shape[0],"age_in_days":r.age.mean(),"size_in_kb":r['size'].sum(),"date":r.date.max(),"time":r.time.max(),"node":r.node.max()}])
_out = _out.append(r,sort=False)
#
# @TODO: The state of the hard-drive would be a good plus
# os.system('df -h /')
_out.index = np.arange(0,_out.shape[0])
return _out

Write Preview
Loading…
Cancel
Save