From f07611d1411290ae276db91fba5e096555fe8b84 Mon Sep 17 00:00:00 2001 From: "Steve Nyemba, The Architect" Date: Fri, 12 Oct 2018 10:55:40 -0500 Subject: [PATCH] Bug fix with data and layout --- src/api/index.py | 21 ++- src/api/templates/dashboard/apps/summary.html | 4 +- src/utils/ml/analytics.py | 132 ++++++++++++------ 3 files changed, 114 insertions(+), 43 deletions(-) diff --git a/src/api/index.py b/src/api/index.py index 7b0e325..0266064 100755 --- a/src/api/index.py +++ b/src/api/index.py @@ -201,7 +201,7 @@ def active_plan(): return ('',403) @app.route("/1/board",methods=["GET"]) def get_board(): - session['key'] = 'cus_D2x3ItYNfWjSY3' + # session['key'] = 'cus_D2x3ItYNfWjSY3' if 'key' in session : args = {"type":SYS_STORE['class']['read'],"args":SYS_STORE['args']} @@ -361,10 +361,27 @@ def client_log(): args = dict({"type":SYS_STORE['class']['write'],"args":SYS_STORE['args']}) args['args']['uid'] = session['key'] row = {"node":request.headers['id'],"date":date,"log":body} + # # We should make sure that we don't have to archive the data writer = factory.instance(**args) - writer.write (label=context,row=row) + document = writer.dbase.get(writer.uid) + node = request.headers['id'] + if context not in document : + document[context] = {} + if node in document[context] : + logs = document[context][node]['logs'] + logs += [body] + + else: + document[context][node] = {} + logs = [body] + LIMIT = 40 + if len(logs) > LIMIT : + log = logs[LIMIT:] + document[context][node] = {"date":date,"logs":logs} + writer.set(**document) + # writer.write (label=context,data=row) return "",200 else : return "",403 diff --git a/src/api/templates/dashboard/apps/summary.html b/src/api/templates/dashboard/apps/summary.html index 8c6a862..fe3c7dc 100644 --- a/src/api/templates/dashboard/apps/summary.html +++ b/src/api/templates/dashboard/apps/summary.html @@ -205,8 +205,8 @@
Folders
-
{{ row.age}}
-
Days
+
{{ row.files}}
+
Files
{{ row.size}}
diff --git a/src/utils/ml/analytics.py b/src/utils/ml/analytics.py index 6c91149..2d3bfc9 100644 --- a/src/utils/ml/analytics.py +++ b/src/utils/ml/analytics.py @@ -28,16 +28,24 @@ class analytics : key = self.cache['key'] r = self.handler.view('clients/latest_logs',key=key) id = self.get('name') - if id in r : - nodes = r[id].keys() - self.set('nodes',nodes) - logs = {} - for name in nodes : - logs[name] = r[id][name]['log'] + logs = r[id] + nodes = logs.keys() + self.set('nodes',nodes) + self.set('logs',logs) + self.set('summary',self.summary(logs)) + self.format(logs) + + # if id in r : + # nodes = r[id].keys() + # print nodes + # self.set('nodes',nodes) + # logs = {} + # for name in nodes : + # logs[name] = r[id][name]['log'] - self.set('logs',logs) - self.set('summary',self.summary(r[id])) - self.format(self.summary(r[id])) + # self.set('logs',logs) + # self.set('summary',self.summary(r[id])) + # self.format(self.summary(r[id])) def summary(self,logs) : raise Exception("needs to be implemented") @@ -78,38 +86,76 @@ class apps(analytics) : grid['fields'] = [{"name":"name","title":"Process","headercss":"small"},{"name":"cpu","title":"CPU Usage","headercss":"small","type":"number"},{"name":"mem","title":"RAM Usage","headercss":"small","type":"number"},{"name":"status","title":"Status","headercss":"small","align":"center"}] self.set('grid',grid) - def summary(self,logs): + def summary(self,data): """ - The - Status count and load assessment + This function will organize the summary of the logs per node + In terms of {crash,idle,running} counts """ - # logs = pd.DataFrame(self.get('logs')) r = [] - for id in logs : - row = pd.DataFrame(logs[id]['log']) + for node in data : + logs = data[node]['logs'] + date = data[node]['date']['long'] + df = pd.DataFrame(logs) + df = df[df.name.str.contains('other',na=False)==False] + crash = df.status.str.contains('X').sum() + idle = df.status.str.contains('S').sum() + running = df.shape[0] - crash - idle + r.append({"date":date,"node":node,"running":running,"idle":idle,"crash":crash}) + return r + + # logs = pd.DataFrame(self.get('logs')) + # r = [] + # for id in logs : + # row = pd.DataFrame(logs[id]['log']) - crash = np.sum(row.status == 'X') - idle = np.sum(row.status == 'S') + np.sum(row.status == 'S+') - cpu = row.cpu.sum() - mem = row.mem.sum() #{"sum":row.mem.sum(),"mean":row.mem.mean(),"sd":np.sqrt(row.mem.var())} - running = row.shape[0] - crash - idle + # crash = np.sum(row.status == 'X') + # idle = np.sum(row.status == 'S') + np.sum(row.status == 'S+') + # cpu = row.cpu.sum() + # mem = row.mem.sum() #{"sum":row.mem.sum(),"mean":row.mem.mean(),"sd":np.sqrt(row.mem.var())} + # running = row.shape[0] - crash - idle - r.append({"node":id,"date":logs[id]['date']['long'],"running":running,"idle":idle,"crash":crash,"mem":mem,"cpu":cpu}) - return r + # r.append({"node":id,"date":logs[id]['date']['long'],"running":running,"idle":idle,"crash":crash,"mem":mem,"cpu":cpu}) + # return r # self.set("summary",) - def format(self,slogs) : - - r = [] + def format(self,data) : + """ + This function adds other somewhat important statistics : + - resources used for the node + + """ + r = [] q = [] - for row in slogs : + labels = ['Other','Monitored'] + title = "" + series = ['CPU','RAM'] + ylabel = " Resource Used" + for node in data : + df = pd.DataFrame(data[node]['logs']) + N = df.shape[0] - 1 + other_df = pd.DataFrame(df[df.name.str.contains('other',na=False)]) + watch_df = pd.DataFrame(df[df.name.str.contains('other',na=False)==False]) + X = [[other_df.cpu.sum(),other_df.mem.sum()],[watch_df.cpu.sum(),watch_df.mem.sum()]] + date= data[node]['date']['long'] + q.append({"node":node, "x":X,"labels":labels, "title":title,"series":series,"ylabel":ylabel}) + crash = watch_df.status.str.contains('X').sum() + idle = watch_df.status.str.contains('S').sum() + running = N - crash - idle + r.append( {"x":[crash,idle,running],"labels":['Crash','Idle','Running'],"title":"","date":date}) + + self.set("resource",q) + self.set("status",r) + + # r = [] + # q = [] + # for row in slogs : - r.append( {"x":[row['crash'],row['idle'],row['running']],"labels":['Crash','Idle','Running'],"title":"","date":row['date']}) - # q.append({"x":[[row['cpu'],row['mem']]],"labels":["CPU","RAM"],"title":"Resources","date":row['date'],"title":row['node'],"series":[ 'CPU','RAM' ]}) + # r.append( {"x":[row['crash'],row['idle'],row['running']],"labels":['Crash','Idle','Running'],"title":"","date":row['date']}) + # # q.append({"x":[[row['cpu'],row['mem']]],"labels":["CPU","RAM"],"title":"Resources","date":row['date'],"title":row['node'],"series":[ 'CPU','RAM' ]}) - q.append({"x":[[row['cpu'],0],[0,row['mem']]],"labels":["",""],"date":row['date'],"title":"","series":[ 'CPU','RAM' ],"ylabel":"% Resource Used"}) + # q.append({"x":[[row['cpu'],0],[0,row['mem']]],"labels":["",""],"date":row['date'],"title":"","series":[ 'CPU','RAM' ],"ylabel":"% Resource Used"}) - self.set('status',r) - self.set('resource',q) + # self.set('status',r) + # self.set('resource',q) class folders(analytics): @@ -126,7 +172,7 @@ class folders(analytics): grid['fields'] = [{"name":"name","name":"Process"},{"name":"size","title":"Size (MB)"},{"name":"mem","title":"RAM Usage"}] self.set('grid',grid) - def summary(self,logs): + def summary(self,data): r = [] features = self.handler.view('clients/features',key=self.get('key')) @@ -140,14 +186,22 @@ class folders(analytics): else: max_size = 0 self.set('max_size',max_size) - for id in logs : + for node in data : + df = pd.DataFrame(data[node]['logs']) + N = df.shape[0] + print ' --- ',df.size_in_kb.values + df = pd.DataFrame(df.mean()[['size_in_kb','files','age_in_days']]).T + + r.append({"node":node,"folders":N, "max_size":max_size,"size":np.round(df.size_in_kb.values[0]*.000001,2),"age":df.age_in_days.values[0].round(2),"files":df.files.values[0].round(2)}) + return r + # for id in logs : - row = pd.DataFrame(logs[id]['log']) - size = row.size_in_kb.mean() * .001 - N = row.shape[0] - age = np.round(row.age_in_days.mean(),2) - files=row.files.mean() - r.append({"node":id,"size":size,"max_size":max_size,"age":age,"folders":N,"files":files}) + # row = pd.DataFrame(logs[id]['log']) + # size = row.size_in_kb.mean() * .001 + # N = row.shape[0] + # age = np.round(row.age_in_days.mean(),2) + # files=row.files.mean() + # r.append({"node":id,"size":size,"max_size":max_size,"age":age,"folders":N,"files":files}) return r class protocol(analytics): pass