diff --git a/setup.py b/setup.py index c794e24..a4ab639 100644 --- a/setup.py +++ b/setup.py @@ -7,13 +7,15 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() args = { "name":"smart-top", - "version":"1.0.4", + "version":"1.0.6", "author":"The Phi Technology LLC","author_email":"info@the-phi.com", "license":"MIT", "packages":["smart","smart.top","smart.folder","smart.logger"]} args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite'] args["install_requires"] = ['pandas','numpy','requests','data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git'] args["url"] = "https://dev.the-phi.com/git/steve/smart-top.git" +args['scripts'] = ['bin/smart-top'] +# args['entry_point'] = {'console-scripts':['smart-top=smart-top:main']} # #@TODO: # How to run this from the command line (and submit the data to [file,url,mongodb,couchdb,s3]) diff --git a/smart/top/__init__.py b/smart/top/__init__.py index f769cf2..483e23c 100644 --- a/smart/top/__init__.py +++ b/smart/top/__init__.py @@ -13,6 +13,7 @@ import datetime # from transport import factory import sys import hashlib +import re from io import StringIO class Util: @@ -41,6 +42,7 @@ class Util: ARGS_INDEX = 6 for item in rows : + if rows.index(item) != 0 : parts = item.split(xchar) row = parts[:TIME_INDEX] @@ -62,6 +64,7 @@ def read(**args) : cmd = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'" xchar = ";" try: + handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) stream = handler.communicate()[0] if sys.version_info[0] > 2 : @@ -82,7 +85,7 @@ def read(**args) : df['time'] = np.repeat(t,df.shape[0]) df['node'] = np.repeat(os.uname()[1],df.shape[0]) df.columns =['pid','user','mem','cpu','status','started','name','cmd','args','date','time','node'] - + # # We should filter the name of the apps we are interested in here (returning the full logs ) @@ -93,17 +96,23 @@ def read(**args) : names = args['name'].split(',') r = pd.DataFrame() for name in names : - tmp = df[df.name == name.strip()] - if not tmp.shape[0] : - tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":name,"cmd":None,"args":None,"date":d,"time":t,"node":n} - r = r.append(tmp,ignore_index=True) - - df = r + # tmp = df[df.name == name.strip() ] + ii = df.apply(lambda row: row['name'] == name.strip() or (name.strip() in str(row['name'])),axis=1).tolist() + tmp= df[ii] + # tmp.index = np.arange(tmp.shape[0]) + if tmp.empty: + tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":_name,"cmd":None,"args":None,"date":d,"time":t,"node":n} + + else: + r = r.append(tmp,ignore_index=False) + if not r.empty : + # r.index = np.arange(r.shape[0]) + df = r.copy() # # For security reasons lets has the args columns with an MD5 or sha256 # - if 'args' in df : + if not df.empty and 'args' in df : df.args = [hashlib.md5(str(value).encode('utf-8')).hexdigest() for value in df.args.tolist()] STATUS = {'R':'RUNNING','Z':'DEAD','D':'STASIS','S':'SLEEP','Sl':'SLEEP','Ss':'SLEEP','W':'PAGING','T':'DEAD'} df.status = df.status.apply(lambda value: STATUS.get(value,'UNKNOWN')) @@ -116,13 +125,15 @@ def read(**args) : if 'logger' in args and args['logger'] != None : logger = args['logger'] logger(data=df) + df.index = np.arange(df.shape[0]) + return df.to_dict(orient='records') except Exception as e: print (e) pass -if __name__ == '__main__' : - # - # Being directly called (external use of the ) - print(read()) +# if __name__ == '__main__' : +# # +# # Being directly called (external use of the ) +# print(read()) diff --git a/smart/top/__main__.py b/smart/top/__main__.py index 4871de6..047f507 100644 --- a/smart/top/__main__.py +++ b/smart/top/__main__.py @@ -1,5 +1,71 @@ +#!/usr/bin/env python +""" + smart-top, The Phi Technology LLC + Steve L. Nyemba & Michael Meade + + The smart-top is a utility that enables to monitor processes (among other things) and use the data for: + - detecting anomalies + - +""" import smart.top import pandas as pd -df = pd.DataFrame (smart.top.read(name='firefox,code')) - -print (df.groupby(['user'])['cpu','mem'].sum()) \ No newline at end of file +import sys +import os +import re +import time +# df = pd.DataFrame (smart.top.read(name='firefox,code')) +SYS_ARGS = {} +if len(sys.argv) > 1: + + N = len(sys.argv) + for i in range(1,N): + value = None + if sys.argv[i].startswith('--'): + key = sys.argv[i][2:] #.replace('-','') + SYS_ARGS[key] = 1 + if i + 1 < N: + value = sys.argv[i + 1] = sys.argv[i+1].strip() + if key and value and not value.startswith('--'): + SYS_ARGS[key] = value + + + i += 2 +if __name__ == '__main__' : + try: + if 'help' in SYS_ARGS: + print (help_me) + sys.exit(0) + if 'watch' in SYS_ARGS : + SYS_ARGS['watch'] = int(SYS_ARGS['watch']) + if SYS_ARGS['watch'] <= 1 : + SYS_ARGS['watch'] = 10 + log = pd.DataFrame() + while True: + os.system('clear') + print () + print ("================================= SMART TOP ================================= ") + print () + df = pd.DataFrame() + if 'name' in SYS_ARGS : + df = df.append(pd.DataFrame(smart.top.read(name=SYS_ARGS['name']))) + else: + df = pd.DataFrame(smart.top.read()) + # df = pd.DataFrame(smart.top.read(name='fire')) + + log = log.append(df) + + print (df[['pid','name','user','cpu','mem','started','date','time','status']]) + if 'watch' in SYS_ARGS : + time.sleep(SYS_ARGS['watch']) + + else: + break + except KeyboardInterrupt: + if 'log' in SYS_ARGS : + file = 'smart-top.csv' if SYS_ARGS['log'] == 1 else SYS_ARGS['log'] + log[['name','cmd','cpu','mem','started','date','time','status','node']].to_csv(file,index=False) + print () + print ("... Exiting, Thanks for using smart-top") + + # pass +# print (df.groupby(['user'])['cpu','mem'].sum()) \ No newline at end of file