library
/
smart-top
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug fix with data and layout

Browse Source
community
Steve L. Nyemba 6 years ago
parent 0a0de5a0a4
commit f07611d141
3 changed files with 114 additions and 43 deletions
Show Stats Download Patch File Download Diff File

21
src/api/index.py
Unescape View File

@ -201,7 +201,7 @@ def active_plan():
return ('',403) return ('',403)
@app.route("/1/board",methods=["GET"]) @app.route("/1/board",methods=["GET"])
def get_board(): def get_board():
session['key'] = 'cus_D2x3ItYNfWjSY3' # session['key'] = 'cus_D2x3ItYNfWjSY3'
if 'key' in session : if 'key' in session :
args = {"type":SYS_STORE['class']['read'],"args":SYS_STORE['args']} args = {"type":SYS_STORE['class']['read'],"args":SYS_STORE['args']}
@ -361,10 +361,27 @@ def client_log():
args = dict({"type":SYS_STORE['class']['write'],"args":SYS_STORE['args']}) args = dict({"type":SYS_STORE['class']['write'],"args":SYS_STORE['args']})
args['args']['uid'] = session['key'] args['args']['uid'] = session['key']
row = {"node":request.headers['id'],"date":date,"log":body} row = {"node":request.headers['id'],"date":date,"log":body}
# #
# We should make sure that we don't have to archive the data # We should make sure that we don't have to archive the data
writer = factory.instance(**args) writer = factory.instance(**args)
writer.write (label=context,row=row) document = writer.dbase.get(writer.uid)
node = request.headers['id']
if context not in document :
document[context] = {}
if node in document[context] :
logs = document[context][node]['logs']
logs += [body]
else:
document[context][node] = {}
logs = [body]
LIMIT = 40
if len(logs) > LIMIT :
log = logs[LIMIT:]
document[context][node] = {"date":date,"logs":logs}
writer.set(**document)
# writer.write (label=context,data=row)
return "",200 return "",200
else : else :
return "",403 return "",403

4
src/api/templates/dashboard/apps/summary.html
Unescape View File

@ -205,8 +205,8 @@
<div class="unit small">Folders</div> <div class="unit small">Folders</div>
</div> </div>
<div class="number border"> <div class="number border">
<div class="value">{{ row.age}}</div> <div class="value">{{ row.files}}</div>
<div class="unit small">Days</div> <div class="unit small">Files</div>
</div> </div>
<div class="number border"> <div class="number border">
<div class="value">{{ row.size}}</div> <div class="value">{{ row.size}}</div>

142
src/utils/ml/analytics.py
Unescape View File

@ -28,16 +28,24 @@ class analytics :
key = self.cache['key'] key = self.cache['key']
r = self.handler.view('clients/latest_logs',key=key) r = self.handler.view('clients/latest_logs',key=key)
id = self.get('name') id = self.get('name')
if id in r : logs = r[id]
nodes = r[id].keys() nodes = logs.keys()
self.set('nodes',nodes) self.set('nodes',nodes)
logs = {} self.set('logs',logs)
for name in nodes : self.set('summary',self.summary(logs))
logs[name] = r[id][name]['log'] self.format(logs)
self.set('logs',logs) # if id in r :
self.set('summary',self.summary(r[id])) # nodes = r[id].keys()
self.format(self.summary(r[id])) # print nodes
# self.set('nodes',nodes)
# logs = {}
# for name in nodes :
# logs[name] = r[id][name]['log']
# self.set('logs',logs)
# self.set('summary',self.summary(r[id]))
# self.format(self.summary(r[id]))
def summary(self,logs) : def summary(self,logs) :
raise Exception("needs to be implemented") raise Exception("needs to be implemented")
@ -78,38 +86,76 @@ class apps(analytics) :
grid['fields'] = [{"name":"name","title":"Process","headercss":"small"},{"name":"cpu","title":"CPU Usage","headercss":"small","type":"number"},{"name":"mem","title":"RAM Usage","headercss":"small","type":"number"},{"name":"status","title":"Status","headercss":"small","align":"center"}] grid['fields'] = [{"name":"name","title":"Process","headercss":"small"},{"name":"cpu","title":"CPU Usage","headercss":"small","type":"number"},{"name":"mem","title":"RAM Usage","headercss":"small","type":"number"},{"name":"status","title":"Status","headercss":"small","align":"center"}]
self.set('grid',grid) self.set('grid',grid)
def summary(self,logs): def summary(self,data):
""" """
The This function will organize the summary of the logs per node
Status count and load assessment In terms of {crash,idle,running} counts
""" """
# logs = pd.DataFrame(self.get('logs'))
r = [] r = []
for id in logs : for node in data :
row = pd.DataFrame(logs[id]['log']) logs = data[node]['logs']
date = data[node]['date']['long']
crash = np.sum(row.status == 'X') df = pd.DataFrame(logs)
idle = np.sum(row.status == 'S') + np.sum(row.status == 'S+') df = df[df.name.str.contains('other',na=False)==False]
cpu = row.cpu.sum() crash = df.status.str.contains('X').sum()
mem = row.mem.sum() #{"sum":row.mem.sum(),"mean":row.mem.mean(),"sd":np.sqrt(row.mem.var())} idle = df.status.str.contains('S').sum()
running = row.shape[0] - crash - idle running = df.shape[0] - crash - idle
r.append({"date":date,"node":node,"running":running,"idle":idle,"crash":crash})
r.append({"node":id,"date":logs[id]['date']['long'],"running":running,"idle":idle,"crash":crash,"mem":mem,"cpu":cpu})
return r return r
# logs = pd.DataFrame(self.get('logs'))
# r = []
# for id in logs :
# row = pd.DataFrame(logs[id]['log'])
# crash = np.sum(row.status == 'X')
# idle = np.sum(row.status == 'S') + np.sum(row.status == 'S+')
# cpu = row.cpu.sum()
# mem = row.mem.sum() #{"sum":row.mem.sum(),"mean":row.mem.mean(),"sd":np.sqrt(row.mem.var())}
# running = row.shape[0] - crash - idle
# r.append({"node":id,"date":logs[id]['date']['long'],"running":running,"idle":idle,"crash":crash,"mem":mem,"cpu":cpu})
# return r
# self.set("summary",) # self.set("summary",)
def format(self,slogs) : def format(self,data) :
"""
This function adds other somewhat important statistics :
- resources used for the node
r = [] """
r = []
q = [] q = []
for row in slogs : labels = ['Other','Monitored']
title = ""
r.append( {"x":[row['crash'],row['idle'],row['running']],"labels":['Crash','Idle','Running'],"title":"","date":row['date']}) series = ['CPU','RAM']
# q.append({"x":[[row['cpu'],row['mem']]],"labels":["CPU","RAM"],"title":"Resources","date":row['date'],"title":row['node'],"series":[ 'CPU','RAM' ]}) ylabel = " Resource Used"
for node in data :
q.append({"x":[[row['cpu'],0],[0,row['mem']]],"labels":["",""],"date":row['date'],"title":"","series":[ 'CPU','RAM' ],"ylabel":"% Resource Used"}) df = pd.DataFrame(data[node]['logs'])
N = df.shape[0] - 1
self.set('status',r) other_df = pd.DataFrame(df[df.name.str.contains('other',na=False)])
self.set('resource',q) watch_df = pd.DataFrame(df[df.name.str.contains('other',na=False)==False])
X = [[other_df.cpu.sum(),other_df.mem.sum()],[watch_df.cpu.sum(),watch_df.mem.sum()]]
date= data[node]['date']['long']
q.append({"node":node, "x":X,"labels":labels, "title":title,"series":series,"ylabel":ylabel})
crash = watch_df.status.str.contains('X').sum()
idle = watch_df.status.str.contains('S').sum()
running = N - crash - idle
r.append( {"x":[crash,idle,running],"labels":['Crash','Idle','Running'],"title":"","date":date})
self.set("resource",q)
self.set("status",r)
# r = []
# q = []
# for row in slogs :
# r.append( {"x":[row['crash'],row['idle'],row['running']],"labels":['Crash','Idle','Running'],"title":"","date":row['date']})
# # q.append({"x":[[row['cpu'],row['mem']]],"labels":["CPU","RAM"],"title":"Resources","date":row['date'],"title":row['node'],"series":[ 'CPU','RAM' ]})
# q.append({"x":[[row['cpu'],0],[0,row['mem']]],"labels":["",""],"date":row['date'],"title":"","series":[ 'CPU','RAM' ],"ylabel":"% Resource Used"})
# self.set('status',r)
# self.set('resource',q)
class folders(analytics): class folders(analytics):
@ -126,7 +172,7 @@ class folders(analytics):
grid['fields'] = [{"name":"name","name":"Process"},{"name":"size","title":"Size (MB)"},{"name":"mem","title":"RAM Usage"}] grid['fields'] = [{"name":"name","name":"Process"},{"name":"size","title":"Size (MB)"},{"name":"mem","title":"RAM Usage"}]
self.set('grid',grid) self.set('grid',grid)
def summary(self,logs): def summary(self,data):
r = [] r = []
features = self.handler.view('clients/features',key=self.get('key')) features = self.handler.view('clients/features',key=self.get('key'))
@ -140,14 +186,22 @@ class folders(analytics):
else: else:
max_size = 0 max_size = 0
self.set('max_size',max_size) self.set('max_size',max_size)
for id in logs : for node in data :
df = pd.DataFrame(data[node]['logs'])
row = pd.DataFrame(logs[id]['log']) N = df.shape[0]
size = row.size_in_kb.mean() * .001 print ' --- ',df.size_in_kb.values
N = row.shape[0] df = pd.DataFrame(df.mean()[['size_in_kb','files','age_in_days']]).T
age = np.round(row.age_in_days.mean(),2)
files=row.files.mean() r.append({"node":node,"folders":N, "max_size":max_size,"size":np.round(df.size_in_kb.values[0]*.000001,2),"age":df.age_in_days.values[0].round(2),"files":df.files.values[0].round(2)})
r.append({"node":id,"size":size,"max_size":max_size,"age":age,"folders":N,"files":files}) return r
# for id in logs :
# row = pd.DataFrame(logs[id]['log'])
# size = row.size_in_kb.mean() * .001
# N = row.shape[0]
# age = np.round(row.age_in_days.mean(),2)
# files=row.files.mean()
# r.append({"node":id,"size":size,"max_size":max_size,"age":age,"folders":N,"files":files})
return r return r
class protocol(analytics): class protocol(analytics):
pass pass

Write Preview
Loading…
Cancel
Save