enabled cli mode

data-collector
Steve L. Nyemba 4 years ago
parent 7151ee4726
commit 5734fedeb6

@ -7,13 +7,15 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = { args = {
"name":"smart-top", "name":"smart-top",
"version":"1.0.4", "version":"1.0.6",
"author":"The Phi Technology LLC","author_email":"info@the-phi.com", "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":"MIT", "license":"MIT",
"packages":["smart","smart.top","smart.folder","smart.logger"]} "packages":["smart","smart.top","smart.folder","smart.logger"]}
args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite'] args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite']
args["install_requires"] = ['pandas','numpy','requests','data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git'] args["install_requires"] = ['pandas','numpy','requests','data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git']
args["url"] = "https://dev.the-phi.com/git/steve/smart-top.git" args["url"] = "https://dev.the-phi.com/git/steve/smart-top.git"
args['scripts'] = ['bin/smart-top']
# args['entry_point'] = {'console-scripts':['smart-top=smart-top:main']}
# #
#@TODO: #@TODO:
# How to run this from the command line (and submit the data to [file,url,mongodb,couchdb,s3]) # How to run this from the command line (and submit the data to [file,url,mongodb,couchdb,s3])

@ -13,6 +13,7 @@ import datetime
# from transport import factory # from transport import factory
import sys import sys
import hashlib import hashlib
import re
from io import StringIO from io import StringIO
class Util: class Util:
@ -41,6 +42,7 @@ class Util:
ARGS_INDEX = 6 ARGS_INDEX = 6
for item in rows : for item in rows :
if rows.index(item) != 0 : if rows.index(item) != 0 :
parts = item.split(xchar) parts = item.split(xchar)
row = parts[:TIME_INDEX] row = parts[:TIME_INDEX]
@ -62,6 +64,7 @@ def read(**args) :
cmd = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'" cmd = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'"
xchar = ";" xchar = ";"
try: try:
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
stream = handler.communicate()[0] stream = handler.communicate()[0]
if sys.version_info[0] > 2 : if sys.version_info[0] > 2 :
@ -82,7 +85,7 @@ def read(**args) :
df['time'] = np.repeat(t,df.shape[0]) df['time'] = np.repeat(t,df.shape[0])
df['node'] = np.repeat(os.uname()[1],df.shape[0]) df['node'] = np.repeat(os.uname()[1],df.shape[0])
df.columns =['pid','user','mem','cpu','status','started','name','cmd','args','date','time','node'] df.columns =['pid','user','mem','cpu','status','started','name','cmd','args','date','time','node']
# #
# We should filter the name of the apps we are interested in here (returning the full logs ) # We should filter the name of the apps we are interested in here (returning the full logs )
@ -93,17 +96,23 @@ def read(**args) :
names = args['name'].split(',') names = args['name'].split(',')
r = pd.DataFrame() r = pd.DataFrame()
for name in names : for name in names :
tmp = df[df.name == name.strip()] # tmp = df[df.name == name.strip() ]
if not tmp.shape[0] : ii = df.apply(lambda row: row['name'] == name.strip() or (name.strip() in str(row['name'])),axis=1).tolist()
tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":name,"cmd":None,"args":None,"date":d,"time":t,"node":n} tmp= df[ii]
r = r.append(tmp,ignore_index=True) # tmp.index = np.arange(tmp.shape[0])
if tmp.empty:
df = r tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":_name,"cmd":None,"args":None,"date":d,"time":t,"node":n}
else:
r = r.append(tmp,ignore_index=False)
if not r.empty :
# r.index = np.arange(r.shape[0])
df = r.copy()
# #
# For security reasons lets has the args columns with an MD5 or sha256 # For security reasons lets has the args columns with an MD5 or sha256
# #
if 'args' in df : if not df.empty and 'args' in df :
df.args = [hashlib.md5(str(value).encode('utf-8')).hexdigest() for value in df.args.tolist()] df.args = [hashlib.md5(str(value).encode('utf-8')).hexdigest() for value in df.args.tolist()]
STATUS = {'R':'RUNNING','Z':'DEAD','D':'STASIS','S':'SLEEP','Sl':'SLEEP','Ss':'SLEEP','W':'PAGING','T':'DEAD'} STATUS = {'R':'RUNNING','Z':'DEAD','D':'STASIS','S':'SLEEP','Sl':'SLEEP','Ss':'SLEEP','W':'PAGING','T':'DEAD'}
df.status = df.status.apply(lambda value: STATUS.get(value,'UNKNOWN')) df.status = df.status.apply(lambda value: STATUS.get(value,'UNKNOWN'))
@ -116,13 +125,15 @@ def read(**args) :
if 'logger' in args and args['logger'] != None : if 'logger' in args and args['logger'] != None :
logger = args['logger'] logger = args['logger']
logger(data=df) logger(data=df)
df.index = np.arange(df.shape[0])
return df.to_dict(orient='records') return df.to_dict(orient='records')
except Exception as e: except Exception as e:
print (e) print (e)
pass pass
if __name__ == '__main__' : # if __name__ == '__main__' :
# # #
# Being directly called (external use of the ) # # Being directly called (external use of the )
print(read()) # print(read())

@ -1,5 +1,71 @@
#!/usr/bin/env python
"""
smart-top, The Phi Technology LLC
Steve L. Nyemba & Michael Meade
The smart-top is a utility that enables to monitor processes (among other things) and use the data for:
- detecting anomalies
-
"""
import smart.top import smart.top
import pandas as pd import pandas as pd
df = pd.DataFrame (smart.top.read(name='firefox,code')) import sys
import os
print (df.groupby(['user'])['cpu','mem'].sum()) import re
import time
# df = pd.DataFrame (smart.top.read(name='firefox,code'))
SYS_ARGS = {}
if len(sys.argv) > 1:
N = len(sys.argv)
for i in range(1,N):
value = None
if sys.argv[i].startswith('--'):
key = sys.argv[i][2:] #.replace('-','')
SYS_ARGS[key] = 1
if i + 1 < N:
value = sys.argv[i + 1] = sys.argv[i+1].strip()
if key and value and not value.startswith('--'):
SYS_ARGS[key] = value
i += 2
if __name__ == '__main__' :
try:
if 'help' in SYS_ARGS:
print (help_me)
sys.exit(0)
if 'watch' in SYS_ARGS :
SYS_ARGS['watch'] = int(SYS_ARGS['watch'])
if SYS_ARGS['watch'] <= 1 :
SYS_ARGS['watch'] = 10
log = pd.DataFrame()
while True:
os.system('clear')
print ()
print ("================================= SMART TOP ================================= ")
print ()
df = pd.DataFrame()
if 'name' in SYS_ARGS :
df = df.append(pd.DataFrame(smart.top.read(name=SYS_ARGS['name'])))
else:
df = pd.DataFrame(smart.top.read())
# df = pd.DataFrame(smart.top.read(name='fire'))
log = log.append(df)
print (df[['pid','name','user','cpu','mem','started','date','time','status']])
if 'watch' in SYS_ARGS :
time.sleep(SYS_ARGS['watch'])
else:
break
except KeyboardInterrupt:
if 'log' in SYS_ARGS :
file = 'smart-top.csv' if SYS_ARGS['log'] == 1 else SYS_ARGS['log']
log[['name','cmd','cpu','mem','started','date','time','status','node']].to_csv(file,index=False)
print ()
print ("... Exiting, Thanks for using smart-top")
# pass
# print (df.groupby(['user'])['cpu','mem'].sum())
Loading…
Cancel
Save