library
/
smart-top
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

dependency virtuenv

Browse Source
community
Steve L. Nyemba 6 years ago
parent b0c6cb31fb
commit 20f1f74ff3
6 changed files with 201 additions and 378 deletions
Show Stats Download Patch File Download Diff File

1
requirements.txt
Unescape View File

@ -31,3 +31,4 @@ socketpool==0.5.3
subprocess32==3.5.2 subprocess32==3.5.2
urllib3==1.23 urllib3==1.23
Werkzeug==0.11.11 Werkzeug==0.11.11
XlsxWriter==1.1.2

463
src/api/index.py
Unescape View File

@ -13,20 +13,21 @@
- Add socketio, so that each section of the dashboard updates independently - Add socketio, so that each section of the dashboard updates independently
""" """
from flask import Flask, session, request, redirect, Response, stream_with_context from flask import Flask, session, request, redirect, Response, stream_with_context,send_file
from flask.templating import render_template from flask.templating import render_template
import requests import requests
from flask_session import Session from flask_session import Session
from datetime import datetime from datetime import datetime
import time import time
from werkzeug.datastructures import FileStorage
import tarfile
from io import BytesIO
import sys import sys
import os import os
import json import json
import re import re
# import monitor
import uuid import uuid
from cStringIO import StringIO
# from utils.register import Register # from utils.register import Register
from utils.charting import * from utils.charting import *
@ -233,22 +234,6 @@ def get_clients():
# features = json.loads(session['user-plan']['metadata']['features']) # features = json.loads(session['user-plan']['metadata']['features'])
args['client_count'] = features['clients'] args['client_count'] = features['clients']
return render_template('dashboard/client-keys.html',**args) return render_template('dashboard/client-keys.html',**args)
# elif request.method == 'POST' :
# clients = request.get_json(silent=True)
# #
# # We can NOT have uncontrolled writes
# #
# # N = session['user-plan'][0]['metadata']['features']
# # clients = client[:N] if len(clients) > N else clients
# args = {"type":SYS_STORE['class']['write'],"args":SYS_STORE['args']}
# args['args']['uid'] = session['key']
# writer = factory.instance(**args)
# # writer.set({"clients":clients})
# writer.set(clients=clients)
# return ('',200)
# pass
else: else:
return ('',403) return ('',403)
@app.route('/1/clients',methods=['POST']) @app.route('/1/clients',methods=['POST'])
@ -375,6 +360,103 @@ def client_log():
return "",200 return "",200
else : else :
return "",403 return "",403
@app.route("/1/data/<action>/<format>",methods=['POST'])
def manage_data(action,format) :
if 'key' not in session :
return ('',403)
elif action in ['archive','download']:
args = dict({"type":SYS_STORE['class']['read'],"args":SYS_STORE['args']})
args['args']['uid'] = session['key']
reader = factory.instance(**args)
data = reader.view('clients/logs',key=session['key'])
logs = [ {"name":id,"data-frame":pd.DataFrame(data[id])} for id in data]
# return ('',200)
#
# We need to convert the logs into a giant spreadsheet and archive it
if format in ['csv','xls','xlsx'] :
#
# This will go straight to the cloud ... we get the token and push to the cloud
#
headers = {"key":session['cloud-info']['user']['access_token']}
if format in ['xls','xlsx'] :
filename = "smart-top-logs-" + "-".join([str(time.localtime().tm_year),str(time.localtime().tm_mon),str(time.localtime().tm_mday)])
stream = StringIO()
writer = pd.ExcelWriter(filename, engine='xlsxwriter')
writer.book.filename = stream
mimetype="application/octet-stream"
else:
mimetype="text/plain; charset=dropbox-cors-hack"
files = {}
ext = '.'+format
for row in logs :
df = row['data-frame']
id = row['name']
if format in ['xls','xlsx'] :
#
# creating sheets in the workbook (excel terminology)
df.to_excel(writer,sheet_name=id)
else:
filename = "smart-top-logs-"+id+"-"+ "-".join([str(time.localtime().tm_year),str(time.localtime().tm_mon),str(time.localtime().tm_mday)])
filename = filename + ext
stream = StringIO()
df.to_csv(stream)
stream.seek(0)
files[id] = (FileStorage(stream=stream,filename=filename,name=filename,content_type='text/plain; charset=dropbox-cors-hack'))
if format in ['xls','xlsx'] :
writer.save()
stream.seek(0)
ext = ".xlsx"
filename = filename+ext
files = {}
files[filename] = FileStorage(stream=stream,filename=filename,name=filename,content_type='application/octet-stream; charset=utf8')
if action == 'archive' :
service = session['cloud-info']['user']['sid']
# datetime = "-".join([str(time.localtime().tm_year),str(time.localtime().tm_mon),str(time.localtime().tm_mday)])
headers['mimetype'] = mimetype # 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
# headers['filename'] = "".join(['logs-',datetime])+".xlsx"
headers['folder'] = 'smart-top'
url = "/".join(['http://localhost/cloud-view',service,'upload'])
r = requests.post(url,files=files,headers=headers)
return ('',200)
else:
#
# Let's download the files (tarball or workbook)
if len(files.keys()) > 1 :
ostream = BytesIO()
tar = tarfile.open(mode='w:gz',fileobj = ostream)
filename = "smart-top-logs-" + "-".join([str(time.localtime().tm_year),str(time.localtime().tm_mon),str(time.localtime().tm_mday)])
for id in files :
stream = files[id].stream
info = tarfile.TarInfo(name=files[id].filename)
info.size = len(stream.getvalue())
tar.addfile(tarinfo=info,fileobj=BytesIO(stream.getvalue()))
tar.close()
ostream.seek(0)
return send_file(ostream,as_attachment=True,attachment_filename=filename+'.gz',mimetype='application/tgz')
else:
#
# returning the file as is spreadsheet
file = files[filename]
return send_file(
file.stream,
as_attachment=True,
attachment_filename=filename,
mimetype=mimetype
)
return ('',404)
@app.route("/1/log",methods=['POST']) @app.route("/1/log",methods=['POST'])
def log(): def log():
key = request.headers['key'] key = request.headers['key']
@ -529,48 +611,6 @@ def set_logs() :
key = session['key'] key = session['key']
pass pass
# @app.route('/1/get/logs')
# def get_logs() :
# """
# @deprecate
# """
# r = {}
# #session['key'] = 'c259e8b1-e2fb-40df-bf03-f521f8ee352d'
# key = session['key'] if 'key' in session else None
# if key is None and 'key' in request.headers:
# key = request.headers['key']
# if key is None :
# return json.dumps(r)
# else:
# try:
# #gReader = factory.instance(type=class_read,args=p)
# #plan = gReader.view('plans/active',key=key)
# plan = session['plan']
# if plan :
# dbname = plan['name']
# args = str(json.dumps(p))
# args = json.loads(args)
# args['dbname'] = dbname
# args['uid'] = key
# #
# # Let us persis this information (as well as the key)
# #session['plan'] = plan['name']
# session['store']= args
# session['key'] = key
# scope = ['app_resources','folder_info','app_status_details','app_resource_usage_details'] #,'emails','log_size']
# gReader = factory.instance(type=class_read,args=args)
# for id in scope :
# view = ('summary/'+id).strip()
# r[id] = gReader.view(view,key=key)
# if 'logs' in session :
# for id in session['logs'] :
# r[id] = session['logs'][id]
# # r[id] = r[node_id]
# except Exception,e:
# print (e)
# return json.dumps(r)
@app.route("/1/set/logs",methods=['PUT']) @app.route("/1/set/logs",methods=['PUT'])
def update_profile(): def update_profile():
try: try:
@ -625,119 +665,6 @@ def get_usage_trend():
return json.dumps(r) return json.dumps(r)
# @app.route("/1/app/usage/trend")
# def get_usage_detail():
# """
# @deprecate
# This function returns detailed information about usage per application monitored. It will return the 24 most recent observations in the logs
# @param node node identifier e.g: apps@zulu.io
# @return {node_x:{app_1:{memory_usage:[],cpu_usage:[]}},...}
# """
# r = {}
# try:
# if 'id' not in request.args and 'node' not in request.args :
# id = session['default.node']
# else:
# id = request.args['id'] if 'id' in request.args else request.args.get('node')
# if 'app' not in request.args :
# if 'default.app' in session :
# app_id = session['default.app']
# else:
# app_id = None
# else:
# app_id = request.args.get('app')
# #
# # removing trailing white spaces
# gReader = factory.instance(type=class_read,args=p)
# r = gReader.view('summary/app_resource_usage_details',key=p['uid'])
# id = id.strip()
# if app_id is not None :
# app_id = app_id.strip()
# r = r[id][app_id]
# else :
# r = r[id]
# except Exception,e:
# print ' *** ',(e)
# return json.dumps(r)
# @app.route('/1/app/status')
# def app_status() :
# """
# @deprecate
# This function aggregates the number of crashes/running/idle instances found in the past 24 log entries
# for a particular application
# @param nid node identifier e.g: app@zulu.io
# @param app application identifier e.g: kate, firefox, chrome ... specified in the configuraiton
# """
# r = []
# try:
# nid = request.args.get('node') # Node identifier
# aid = request.args.get('app') # application identifier
# gReader = factory.instance(type=class_read,args=p)
# r = gReader.view('summary/app_status_details',key=p['uid'])
# #
# #@TODO: Once the back-end enables the nodes in which the application is running, uncomment the line below
# #
# print[nid,aid]
# r = r[nid][aid]
# except Exception,e:
# print e
# return json.dumps(r)
#@app.route('/get/<id>')
#def procs(id):
#try:
#gReader = factory.instance(type=class_read,args=p)
#data = gReader.read()
#ahandler = AnalyzeAnomaly()
#learn = {}
#if 'learn' in data :
#for row in data['learn'] :
#label = row['label']
#learn[label] = row
#r = {}
#for label in data :
#if label not in ['learn','folders'] :
#index = len(data[label]) - 1
#row = data[label][index]
#r[label] = row
##
## Let us determine if this is a normal operation or not
## We will update the status of the information ...
##
#for row in r[label] :
#index = r[label].index(row)
#if row['label'] in learn:
#id = row['label']
#px = ahandler.predict([row],learn[id])
#if px :
## row['anomaly'] = px[1]==1
#print ""
#print label,' *** ',index
#row = dict(row,**px)
#r[label][index] =row
##
## @TODO:
## Compile a report here that will be sent to the mailing list
##
#except Exception, e:
#print e
#r = []
#return json.dumps(r)
@app.route('/init/collector',methods=['POST']) @app.route('/init/collector',methods=['POST'])
def init_collector(): def init_collector():
""" """
@ -835,65 +762,6 @@ def InitCollector():
print ' init (error) ',str(e) print ' init (error) ',str(e)
return json.dumps(r) return json.dumps(r)
"""
This function/endpoint will assess n-virtual environments and return the results
@TODO: Should this be stored for future mining (I don't think so but could be wrong)
"""
# @app.route('/sandbox')
# def sandbox():
# global CONFIG
# if 'sandbox' in CONFIG: #CONFIG['monitor']:
# #handler = HANDLERS['sandbox']['class']
# #conf = HANDLERS['sandbox']['config']
# r = []
# # p = Factory.instance('sandbox',CONFIG)
# handler = monitor.Sandbox()
# conf = CONFIG['sandbox']
# for id in conf:
# try:
# handler.init(conf[id])
# r.append (dict(handler.composite(),**{"label":id}))
# except Exception as e:
# pass
# else:
# r = []
# return json.dumps(r)
#@app.route('/trends')
#def trends ():
#id = request.args.get('id')
#app = request.args.get('app').strip()
#p = CONFIG['store']['args']
#class_read = CONFIG['store']['class']['read']
#gReader = factory.instance(type=class_read,args=p)
#r = gReader.read()
#if id in r:
#r = r[id] #--matrix
#series = []
#for row in r:
#series += [item for item in row if str(item['label'])== app]
#if len(series) > 12 :
#beg = len(series) - 8
#series = series[beg:]
#return json.dumps(series)
#else:
#return "[]"
#@app.route('/download',methods=['POST'])
#def requirements():
#stream = request.form['missing']
#print stream
#stream = "\n".join(json.loads(stream))
#headers = {"content-disposition":"attachment; filename=requirements.txt"}
#return Response(stream,mimetype='text/plain',headers=headers)
@app.route('/dashboard/<node>') @app.route('/dashboard/<node>')
def dashboard(): def dashboard():
context = PARAMS['context'] context = PARAMS['context']
@ -927,133 +795,6 @@ def user():
title = 'Upgrade' title = 'Upgrade'
return render_template('user.html',context=context,title=title) return render_template('user.html',context=context,title=title)
#"""
#This function is designed to trigger learning for anomaly detection
#@TODO: forward this to a socket i.e non-blocking socket
#"""
#@app.route('/anomalies/get')
#def learn():
#global CONFIG
#p = CONFIG['store']['args']
#class_read = CONFIG['store']['class']['read']
#gReader = factory.instance(type=class_read,args=p)
#d = gReader.read()
#if 'learn' in d :
#info = d['learn']
#del d['learn']
#else :
#info = []
#r = []
#if 'id' in request.args:
#id = request.args['id']
#d = d[id]
#params = {}
#for item in info:
#label = item['label']
#params[label] = item
##apps = list(set(ML.Extract(['label'],d)))
#r = []
#if params :
##
## If we have parameters available
#p = AnomalyDetection()
#apps = params.keys()
#for name in apps :
#if name not in params:
#continue
#_info = params[name]
#try:
#xo = ML.Filter('label',name,d)
#except Exception as e:
#xo = []
##print name,e
#if len(xo) == 0:
#continue
#xo = [xo[ len(xo) -1]]
#value = p.predict(xo,_info)[0]
#if len(value):
#report = dict(_info,**{'predicton':value})
#r.append(report)
##print app,value
##if value is not None:
## r.append(value)
#return json.dumps(r)
"""
This function returns anomalies for a given context or group of processes
The information returned is around precision/recall and f-score and parameters
"""
#@app.route('/anomalies/status')
#def anomalies_status():
#global CONFIG
#p = CONFIG['store']['args']
#class_read = CONFIG['store']['class']['read']
#gReader = factory.instance(type=class_read,args=p)
#d = gReader.read()
#if 'learn' in d :
#info = d['learn']
#del d['learn']
#else :
#info = []
#print info
#r = []
#if 'id' in request.args:
#id = request.args['id']
#r = info
#return json.dumps(r)
#@app.route('/folders')
#def get_folders():
#global CONFIG
#p = CONFIG['store']['args']
#class_read = CONFIG['store']['class']['read']
#gReader = factory.instance(type=class_read,args=p)
#d = gReader.read()
#if 'folders' in d:
#d = d['folders']
#hosts = set([row[0]['id'] for row in d])
#m = {}
#for id in hosts:
#for row in d:
#if id == row[0]['id'] :
#m[id] = row
#d = m.values()
#for row in d:
#print row[0]['id']
## index = len(d) - 1
## d = d[index]
## m = {}
## for row in d :
## key = row.keys()[0]
## row = row[key]
## if key not in m:
## r.append(row)
## m[key] = len(r) -1
## else:
## index = m[key]
## r[index] = row
## d = r
#else:
#d = []
#return json.dumps(d)
@app.route("/1/plot/<id>",methods=['POST']) @app.route("/1/plot/<id>",methods=['POST'])
def prepare_plot(id): def prepare_plot(id):
if 'key' in session : if 'key' in session :

2
src/api/static/css/default.css
Unescape View File

@ -7,7 +7,7 @@
} }
@media only screen and (max-width: 1024px){ @media only screen and (max-width: 1024px){
body {font-size:14px; font-weight:lighter; color:green} body {font-size:14px; font-weight:lighter;}
} }
body { body {
/* margin-left:1%; */ /* margin-left:1%; */

94
src/api/templates/account/data.html
Unescape View File

@ -14,6 +14,7 @@
<script src="{{context}}/static/js/jx/dom.js"></script> <script src="{{context}}/static/js/jx/dom.js"></script>
<script src="{{context}}/static/js/jx/utils.js"></script> <script src="{{context}}/static/js/jx/utils.js"></script>
<script src="{{context}}/static/js/jx/rpc.js"></script> <script src="{{context}}/static/js/jx/rpc.js"></script>
<script src="{{context}}/static/js/jx/ext/slides.js"></script>
<link href="{{context}}/static/css/default.css" rel="stylesheet" type="text/css"> <link href="{{context}}/static/css/default.css" rel="stylesheet" type="text/css">
<style> <style>
@ -22,6 +23,19 @@
grid-template-columns:33% 33% 33% ; grid-template-columns:33% 33% 33% ;
grid-gap:2px; grid-gap:2px;
} }
.action-pane {display:grid;
grid-template-columns: 63px auto;
grid-gap:8px;
padding:4px;
height:40px;
width:100%;
overflow:hidden;
}
.toggle {width:62px; height:38px; overflow:hidden; cursor:pointer;}
.toggle .slider {width:125px; overflow:hidden; display:grid; grid-template-columns: 49% 49%; grid-gap:1px;}
.toggle .slider div { overflow:hidden; font-weight:bold; background-color:#f3f3f3; text-align:center;}
.border-round { padding:8px ; border-radius:6px;} .border-round { padding:8px ; border-radius:6px;}
@ -31,32 +45,80 @@
</style> </style>
<script> <script>
$(document).ready(function(){
})
var slide = function(id,format){
id = '#'+id
var offset = 0
if (format.match(/xls|xlsx/i)){
offset = 62
}
$(id).attr('format',format)
$(id+' > .slider').animate({'margin-left':-offset})
}
var post = function(id){
id = '#'+id
var format = $(id).attr('format') != null ? $(id).attr('format') : 'csv'
if (id.match(/down/i)){
var url = (['/monitor/1/data/download',format]).join('/')
var form = jx.dom.get.instance('dform')
form.action = url
form.method = 'POST'
form.submit()
}else{
$(document).ready(function(){ var url = (['/monitor/1/data/archive',format]).join('/')
}) ; console.log(url)
var http = HttpClient.instance()
http.post(url,function(x){
if(x.status == 200){
}
})
}
}
</script> </script>
<div class="title-bar"><div class="title" align="left">Manage My Data</div> </div> <div class="title-bar border-bottom"><div class="title" align="left">Manage My Data</div> </div>
<div class="data-frame"> <div class="data-frame">
<div class="pane"> <div class="pane">
<div class=" " style="padding:8px"> <div class=" " style="padding:4px">
<div class="title-bar bold border-bottom"><div class="title" align="center">Archive</div> </div> <div class="title-bar bold border-bottom"><div class="title" align="center">Archive</div> </div>
<div class="" style="min-height:40%"> <div class="" style="min-height:40%; padding-top:15px">
Archive to your cloud account in a <span class="bold">smart-top</span> folder: Archive to your cloud account in a <span class="bold">smart-top</span> folder:
<p style="margin-left:15px"> <p style="margin-left:15px">
<br><i class="fa fa-check"></i> Using cloud storage as a device <br><i class="fa fa-check"></i> Using cloud storage as a device
<br><i class="fa fa-check"></i> Share and control access to your data <br><i class="fa fa-check"></i> Share and control access to your data
</p> </p>
</div> </div>
<div class="button action border-round border" style="width:50%; margin-left:25%"> <div class="border-bottom"><i class="fa fa-file-excel-o" style="font-size:18px; color:green"></i> Format</div>
<div class="action-pane">
<div id="arch_format" class="toggle">
<div class="slider">
<div class="border border-round" onclick="slide('arch_format','xlsx')">CSV</div> <div class="border border-round" onclick="slide('arch_format','csv')">XLSX</div>
</div>
</div>
<div class="">
<div class="button action border-round border" style="width:50%" onclick="post('arch_format')">
<i class="fa fa-cloud"></i> <span class="bold"> Archive Now</span> <i class="fa fa-cloud"></i> <span class="bold"> Archive Now</span>
</div> </div>
</div>
</div>
</div> </div>
<div class=" border-left" style="padding:8px"> <div class=" border-left" style="padding:8px">
<div class="title-bar bold border-bottom"><div class="title" align="center">Download</div> </div> <div class="title-bar bold border-bottom"><div class="title" align="center">Download</div> </div>
<div class="" style="min-height:40%"> <div class="" style="min-height:40%; padding-top:15px">
Download your logs in CSV or excel format: Download your logs in CSV or excel format:
<p style="margin-left:15px"> <p style="margin-left:15px">
<br><i class="fa fa-check"></i> Conduct personal analysis <br><i class="fa fa-check"></i> Conduct personal analysis
@ -65,13 +127,27 @@ $(document).ready(function(){
</p> </p>
</div> </div>
<div class="button action border-round border" style="width:50%; margin-left:25%"> <div class="border-bottom"><i class="fa fa-file-excel-o" style="font-size:18px; color:green"></i> Format</div>
<div class="action-pane">
<div id="down_format" class="toggle">
<form id="dform" style="display:none"></form>
<div class="slider">
<div class="border border-round" onclick="slide('down_format','xlsx')">CSV</div> <div class="border border-round" onclick="slide('down_format','csv')">XLSX</div>
</div>
</div>
<div class="">
<div class="button action border-round border left" style="width:50%;" onclick="post('down_format')">
<i class="fa fa-download"></i> <span class="bold"> Download Now</span> <i class="fa fa-download"></i> <span class="bold"> Download Now</span>
</div> </div>
</div> </div>
</div>
</div>
<div class=" border-left" style="padding:8px"> <div class=" border-left" style="padding:8px">
<div class="title-bar bold border-bottom" style="color:maroon"><div class="title" align="center">Delete</div> </div> <div class="title-bar bold border-bottom" style="color:maroon"><div class="title" align="center">Delete</div> </div>
<div style="min-height:40%"> <div style="min-height:40%; padding-top:15px">
Delete all of your logs from our servers. We recommend : Delete all of your logs from our servers. We recommend :
<p style="margin-left:15px"> <p style="margin-left:15px">
<br><i class="fa fa-exclamation "></i> Either downloading or archiving first <br><i class="fa fa-exclamation "></i> Either downloading or archiving first

5
src/models/basic.py
Unescape View File

@ -4,11 +4,16 @@
"""" """"
class model : class model :
"""
This model provides an overview of the raw data provided a list of variables. If one variable is provided a regression line will be added.
The intent of this model is to allow the user to visualize the distribution and trend of the data as is
"""
def __init__(**args): def __init__(**args):
self.data = args['data'] self.data = args['data']
self.node = args['node'] self.node = args['node']
self.months = {1:"Jan",2:"Feb",3:"Mar",4:"Apr",5:"May",6:"Jun",7:"Jul",8:"Aug",9:"Sep",10:"Oct",11:"Nov",12:"Dec"} self.months = {1:"Jan",2:"Feb",3:"Mar",4:"Apr",5:"May",6:"Jun",7:"Jul",8:"Aug",9:"Sep",10:"Oct",11:"Nov",12:"Dec"}
self.cache = {} self.cache = {}
self.set("type","scatter")
def can_do(self): def can_do(self):
return False return False
def format_date(self,row): def format_date(self,row):

6
src/models/free/apps.py
Unescape View File

@ -63,8 +63,8 @@ class trend(model):
pass pass
model = keras.Sequential([keras.layers.Dense(2, activation=tf.nn.relu,input_shape=(x.shape[1],)),keras.layers.Dense(2, activation=tf.nn.relu),keras.layers.Dense(1)]) # model = keras.Sequential([keras.layers.Dense(2, activation=tf.nn.relu,input_shape=(x.shape[1],)),keras.layers.Dense(2, activation=tf.nn.relu),keras.layers.Dense(1)])
optimizer = tf.train.RMSPropOptimizer(0.001) # optimizer = tf.train.RMSPropOptimizer(0.001)
model.compile(loss='mse', optimizer=optimizer,metrics=['mae']) # model.compile(loss='mse', optimizer=optimizer,metrics=['mae'])
Write Preview
Loading…
Cancel
Save