|
|
|
@ -48,6 +48,21 @@ class_write= CONFIG['store']['class']['write']
|
|
|
|
|
factory = DataSourceFactory()
|
|
|
|
|
# gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
|
|
|
|
|
@app.route('/')
|
|
|
|
|
def dashboard():
|
|
|
|
|
context = PARAMS['context']
|
|
|
|
|
if 'title' in PARAMS :
|
|
|
|
|
title = PARAMS['title']
|
|
|
|
|
else:
|
|
|
|
|
title = 'Dashboard'
|
|
|
|
|
apps = []
|
|
|
|
|
try:
|
|
|
|
|
gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
apps = gReader.view('summary/app_names',key=p['uid'])
|
|
|
|
|
except Exception, e:
|
|
|
|
|
print (e)
|
|
|
|
|
return render_template('dashboard.html',context=context,title=title,app_names=apps)
|
|
|
|
|
|
|
|
|
|
@app.route('/1/get/nodes')
|
|
|
|
|
def get_apps():
|
|
|
|
|
"""
|
|
|
|
@ -159,50 +174,50 @@ def app_status() :
|
|
|
|
|
except Exception,e:
|
|
|
|
|
print e
|
|
|
|
|
return r
|
|
|
|
|
@app.route('/get/<id>')
|
|
|
|
|
def procs(id):
|
|
|
|
|
try:
|
|
|
|
|
gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
data = gReader.read()
|
|
|
|
|
ahandler = AnalyzeAnomaly()
|
|
|
|
|
learn = {}
|
|
|
|
|
if 'learn' in data :
|
|
|
|
|
for row in data['learn'] :
|
|
|
|
|
label = row['label']
|
|
|
|
|
learn[label] = row
|
|
|
|
|
r = {}
|
|
|
|
|
for label in data :
|
|
|
|
|
if label not in ['learn','folders'] :
|
|
|
|
|
index = len(data[label]) - 1
|
|
|
|
|
row = data[label][index]
|
|
|
|
|
r[label] = row
|
|
|
|
|
#
|
|
|
|
|
# Let us determine if this is a normal operation or not
|
|
|
|
|
# We will update the status of the information ...
|
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
for row in r[label] :
|
|
|
|
|
index = r[label].index(row)
|
|
|
|
|
if row['label'] in learn:
|
|
|
|
|
id = row['label']
|
|
|
|
|
px = ahandler.predict([row],learn[id])
|
|
|
|
|
if px :
|
|
|
|
|
|
|
|
|
|
# row['anomaly'] = px[1]==1
|
|
|
|
|
print ""
|
|
|
|
|
print label,' *** ',index
|
|
|
|
|
row = dict(row,**px)
|
|
|
|
|
r[label][index] =row
|
|
|
|
|
#
|
|
|
|
|
# @TODO:
|
|
|
|
|
# Compile a report here that will be sent to the mailing list
|
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
except Exception, e:
|
|
|
|
|
print e
|
|
|
|
|
r = []
|
|
|
|
|
|
|
|
|
|
return json.dumps(r)
|
|
|
|
|
#@app.route('/get/<id>')
|
|
|
|
|
#def procs(id):
|
|
|
|
|
#try:
|
|
|
|
|
#gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
#data = gReader.read()
|
|
|
|
|
#ahandler = AnalyzeAnomaly()
|
|
|
|
|
#learn = {}
|
|
|
|
|
#if 'learn' in data :
|
|
|
|
|
#for row in data['learn'] :
|
|
|
|
|
#label = row['label']
|
|
|
|
|
#learn[label] = row
|
|
|
|
|
#r = {}
|
|
|
|
|
#for label in data :
|
|
|
|
|
#if label not in ['learn','folders'] :
|
|
|
|
|
#index = len(data[label]) - 1
|
|
|
|
|
#row = data[label][index]
|
|
|
|
|
#r[label] = row
|
|
|
|
|
##
|
|
|
|
|
## Let us determine if this is a normal operation or not
|
|
|
|
|
## We will update the status of the information ...
|
|
|
|
|
##
|
|
|
|
|
|
|
|
|
|
#for row in r[label] :
|
|
|
|
|
#index = r[label].index(row)
|
|
|
|
|
#if row['label'] in learn:
|
|
|
|
|
#id = row['label']
|
|
|
|
|
#px = ahandler.predict([row],learn[id])
|
|
|
|
|
#if px :
|
|
|
|
|
|
|
|
|
|
## row['anomaly'] = px[1]==1
|
|
|
|
|
#print ""
|
|
|
|
|
#print label,' *** ',index
|
|
|
|
|
#row = dict(row,**px)
|
|
|
|
|
#r[label][index] =row
|
|
|
|
|
##
|
|
|
|
|
## @TODO:
|
|
|
|
|
## Compile a report here that will be sent to the mailing list
|
|
|
|
|
##
|
|
|
|
|
|
|
|
|
|
#except Exception, e:
|
|
|
|
|
#print e
|
|
|
|
|
#r = []
|
|
|
|
|
|
|
|
|
|
#return json.dumps(r)
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
This function/endpoint will assess n-virtual environments and return the results
|
|
|
|
@ -232,36 +247,36 @@ def sandbox():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return json.dumps(r)
|
|
|
|
|
@app.route('/trends')
|
|
|
|
|
def trends ():
|
|
|
|
|
id = request.args.get('id')
|
|
|
|
|
app = request.args.get('app').strip()
|
|
|
|
|
p = CONFIG['store']['args']
|
|
|
|
|
class_read = CONFIG['store']['class']['read']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
r = gReader.read()
|
|
|
|
|
if id in r:
|
|
|
|
|
r = r[id] #--matrix
|
|
|
|
|
series = []
|
|
|
|
|
|
|
|
|
|
for row in r:
|
|
|
|
|
|
|
|
|
|
series += [item for item in row if str(item['label'])== app]
|
|
|
|
|
if len(series) > 12 :
|
|
|
|
|
beg = len(series) - 8
|
|
|
|
|
series = series[beg:]
|
|
|
|
|
return json.dumps(series)
|
|
|
|
|
else:
|
|
|
|
|
return "[]"
|
|
|
|
|
@app.route('/download',methods=['POST'])
|
|
|
|
|
def requirements():
|
|
|
|
|
stream = request.form['missing']
|
|
|
|
|
print stream
|
|
|
|
|
stream = "\n".join(json.loads(stream))
|
|
|
|
|
headers = {"content-disposition":"attachment; filename=requirements.txt"}
|
|
|
|
|
return Response(stream,mimetype='text/plain',headers=headers)
|
|
|
|
|
#@app.route('/trends')
|
|
|
|
|
#def trends ():
|
|
|
|
|
#id = request.args.get('id')
|
|
|
|
|
#app = request.args.get('app').strip()
|
|
|
|
|
#p = CONFIG['store']['args']
|
|
|
|
|
#class_read = CONFIG['store']['class']['read']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
#r = gReader.read()
|
|
|
|
|
#if id in r:
|
|
|
|
|
#r = r[id] #--matrix
|
|
|
|
|
#series = []
|
|
|
|
|
|
|
|
|
|
#for row in r:
|
|
|
|
|
|
|
|
|
|
#series += [item for item in row if str(item['label'])== app]
|
|
|
|
|
#if len(series) > 12 :
|
|
|
|
|
#beg = len(series) - 8
|
|
|
|
|
#series = series[beg:]
|
|
|
|
|
#return json.dumps(series)
|
|
|
|
|
#else:
|
|
|
|
|
#return "[]"
|
|
|
|
|
#@app.route('/download',methods=['POST'])
|
|
|
|
|
#def requirements():
|
|
|
|
|
#stream = request.form['missing']
|
|
|
|
|
#print stream
|
|
|
|
|
#stream = "\n".join(json.loads(stream))
|
|
|
|
|
#headers = {"content-disposition":"attachment; filename=requirements.txt"}
|
|
|
|
|
#return Response(stream,mimetype='text/plain',headers=headers)
|
|
|
|
|
|
|
|
|
|
@app.route('/dashboard')
|
|
|
|
|
def dashboard():
|
|
|
|
@ -297,132 +312,132 @@ def user():
|
|
|
|
|
return render_template('user.html',context=context,title=title)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
This function is designed to trigger learning for anomaly detection
|
|
|
|
|
@TODO: forward this to a socket i.e non-blocking socket
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
@app.route('/anomalies/get')
|
|
|
|
|
def learn():
|
|
|
|
|
global CONFIG
|
|
|
|
|
p = CONFIG['store']['args']
|
|
|
|
|
class_read = CONFIG['store']['class']['read']
|
|
|
|
|
gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
d = gReader.read()
|
|
|
|
|
|
|
|
|
|
if 'learn' in d :
|
|
|
|
|
info = d['learn']
|
|
|
|
|
|
|
|
|
|
del d['learn']
|
|
|
|
|
else :
|
|
|
|
|
info = []
|
|
|
|
|
r = []
|
|
|
|
|
if 'id' in request.args:
|
|
|
|
|
id = request.args['id']
|
|
|
|
|
d = d[id]
|
|
|
|
|
params = {}
|
|
|
|
|
for item in info:
|
|
|
|
|
|
|
|
|
|
label = item['label']
|
|
|
|
|
params[label] = item
|
|
|
|
|
|
|
|
|
|
#apps = list(set(ML.Extract(['label'],d)))
|
|
|
|
|
r = []
|
|
|
|
|
if params :
|
|
|
|
|
#
|
|
|
|
|
# If we have parameters available
|
|
|
|
|
p = AnomalyDetection()
|
|
|
|
|
apps = params.keys()
|
|
|
|
|
for name in apps :
|
|
|
|
|
if name not in params:
|
|
|
|
|
continue
|
|
|
|
|
_info = params[name]
|
|
|
|
|
try:
|
|
|
|
|
xo = ML.Filter('label',name,d)
|
|
|
|
|
except Exception,e:
|
|
|
|
|
xo = []
|
|
|
|
|
#print name,e
|
|
|
|
|
if len(xo) == 0:
|
|
|
|
|
continue
|
|
|
|
|
xo = [xo[ len(xo) -1]]
|
|
|
|
|
|
|
|
|
|
value = p.predict(xo,_info)[0]
|
|
|
|
|
|
|
|
|
|
if len(value):
|
|
|
|
|
report = dict(_info,**{'predicton':value})
|
|
|
|
|
r.append(report)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#print app,value
|
|
|
|
|
#if value is not None:
|
|
|
|
|
# r.append(value)
|
|
|
|
|
|
|
|
|
|
return json.dumps(r)
|
|
|
|
|
#"""
|
|
|
|
|
#This function is designed to trigger learning for anomaly detection
|
|
|
|
|
#@TODO: forward this to a socket i.e non-blocking socket
|
|
|
|
|
#"""
|
|
|
|
|
|
|
|
|
|
#@app.route('/anomalies/get')
|
|
|
|
|
#def learn():
|
|
|
|
|
#global CONFIG
|
|
|
|
|
#p = CONFIG['store']['args']
|
|
|
|
|
#class_read = CONFIG['store']['class']['read']
|
|
|
|
|
#gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
#d = gReader.read()
|
|
|
|
|
|
|
|
|
|
#if 'learn' in d :
|
|
|
|
|
#info = d['learn']
|
|
|
|
|
|
|
|
|
|
#del d['learn']
|
|
|
|
|
#else :
|
|
|
|
|
#info = []
|
|
|
|
|
#r = []
|
|
|
|
|
#if 'id' in request.args:
|
|
|
|
|
#id = request.args['id']
|
|
|
|
|
#d = d[id]
|
|
|
|
|
#params = {}
|
|
|
|
|
#for item in info:
|
|
|
|
|
|
|
|
|
|
#label = item['label']
|
|
|
|
|
#params[label] = item
|
|
|
|
|
|
|
|
|
|
##apps = list(set(ML.Extract(['label'],d)))
|
|
|
|
|
#r = []
|
|
|
|
|
#if params :
|
|
|
|
|
##
|
|
|
|
|
## If we have parameters available
|
|
|
|
|
#p = AnomalyDetection()
|
|
|
|
|
#apps = params.keys()
|
|
|
|
|
#for name in apps :
|
|
|
|
|
#if name not in params:
|
|
|
|
|
#continue
|
|
|
|
|
#_info = params[name]
|
|
|
|
|
#try:
|
|
|
|
|
#xo = ML.Filter('label',name,d)
|
|
|
|
|
#except Exception,e:
|
|
|
|
|
#xo = []
|
|
|
|
|
##print name,e
|
|
|
|
|
#if len(xo) == 0:
|
|
|
|
|
#continue
|
|
|
|
|
#xo = [xo[ len(xo) -1]]
|
|
|
|
|
|
|
|
|
|
#value = p.predict(xo,_info)[0]
|
|
|
|
|
|
|
|
|
|
#if len(value):
|
|
|
|
|
#report = dict(_info,**{'predicton':value})
|
|
|
|
|
#r.append(report)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
##print app,value
|
|
|
|
|
##if value is not None:
|
|
|
|
|
## r.append(value)
|
|
|
|
|
|
|
|
|
|
#return json.dumps(r)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
This function returns anomalies for a given context or group of processes
|
|
|
|
|
The information returned is around precision/recall and f-score and parameters
|
|
|
|
|
"""
|
|
|
|
|
@app.route('/anomalies/status')
|
|
|
|
|
def anomalies_status():
|
|
|
|
|
global CONFIG
|
|
|
|
|
p = CONFIG['store']['args']
|
|
|
|
|
class_read = CONFIG['store']['class']['read']
|
|
|
|
|
gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
d = gReader.read()
|
|
|
|
|
if 'learn' in d :
|
|
|
|
|
info = d['learn']
|
|
|
|
|
|
|
|
|
|
del d['learn']
|
|
|
|
|
else :
|
|
|
|
|
info = []
|
|
|
|
|
print info
|
|
|
|
|
r = []
|
|
|
|
|
if 'id' in request.args:
|
|
|
|
|
id = request.args['id']
|
|
|
|
|
r = info
|
|
|
|
|
return json.dumps(r)
|
|
|
|
|
@app.route('/folders')
|
|
|
|
|
def get_folders():
|
|
|
|
|
global CONFIG
|
|
|
|
|
p = CONFIG['store']['args']
|
|
|
|
|
class_read = CONFIG['store']['class']['read']
|
|
|
|
|
gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
d = gReader.read()
|
|
|
|
|
if 'folders' in d:
|
|
|
|
|
d = d['folders']
|
|
|
|
|
hosts = set([row[0]['id'] for row in d])
|
|
|
|
|
m = {}
|
|
|
|
|
for id in hosts:
|
|
|
|
|
for row in d:
|
|
|
|
|
if id == row[0]['id'] :
|
|
|
|
|
m[id] = row
|
|
|
|
|
d = m.values()
|
|
|
|
|
for row in d:
|
|
|
|
|
print row[0]['id']
|
|
|
|
|
# index = len(d) - 1
|
|
|
|
|
# d = d[index]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# m = {}
|
|
|
|
|
# for row in d :
|
|
|
|
|
|
|
|
|
|
# key = row.keys()[0]
|
|
|
|
|
|
|
|
|
|
# row = row[key]
|
|
|
|
|
# if key not in m:
|
|
|
|
|
# r.append(row)
|
|
|
|
|
# m[key] = len(r) -1
|
|
|
|
|
# else:
|
|
|
|
|
# index = m[key]
|
|
|
|
|
# r[index] = row
|
|
|
|
|
# d = r
|
|
|
|
|
else:
|
|
|
|
|
d = []
|
|
|
|
|
|
|
|
|
|
return json.dumps(d)
|
|
|
|
|
#@app.route('/anomalies/status')
|
|
|
|
|
#def anomalies_status():
|
|
|
|
|
#global CONFIG
|
|
|
|
|
#p = CONFIG['store']['args']
|
|
|
|
|
#class_read = CONFIG['store']['class']['read']
|
|
|
|
|
#gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
#d = gReader.read()
|
|
|
|
|
#if 'learn' in d :
|
|
|
|
|
#info = d['learn']
|
|
|
|
|
|
|
|
|
|
#del d['learn']
|
|
|
|
|
#else :
|
|
|
|
|
#info = []
|
|
|
|
|
#print info
|
|
|
|
|
#r = []
|
|
|
|
|
#if 'id' in request.args:
|
|
|
|
|
#id = request.args['id']
|
|
|
|
|
#r = info
|
|
|
|
|
#return json.dumps(r)
|
|
|
|
|
#@app.route('/folders')
|
|
|
|
|
#def get_folders():
|
|
|
|
|
#global CONFIG
|
|
|
|
|
#p = CONFIG['store']['args']
|
|
|
|
|
#class_read = CONFIG['store']['class']['read']
|
|
|
|
|
#gReader = factory.instance(type=class_read,args=p)
|
|
|
|
|
#d = gReader.read()
|
|
|
|
|
#if 'folders' in d:
|
|
|
|
|
#d = d['folders']
|
|
|
|
|
#hosts = set([row[0]['id'] for row in d])
|
|
|
|
|
#m = {}
|
|
|
|
|
#for id in hosts:
|
|
|
|
|
#for row in d:
|
|
|
|
|
#if id == row[0]['id'] :
|
|
|
|
|
#m[id] = row
|
|
|
|
|
#d = m.values()
|
|
|
|
|
#for row in d:
|
|
|
|
|
#print row[0]['id']
|
|
|
|
|
## index = len(d) - 1
|
|
|
|
|
## d = d[index]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
## m = {}
|
|
|
|
|
## for row in d :
|
|
|
|
|
|
|
|
|
|
## key = row.keys()[0]
|
|
|
|
|
|
|
|
|
|
## row = row[key]
|
|
|
|
|
## if key not in m:
|
|
|
|
|
## r.append(row)
|
|
|
|
|
## m[key] = len(r) -1
|
|
|
|
|
## else:
|
|
|
|
|
## index = m[key]
|
|
|
|
|
## r[index] = row
|
|
|
|
|
## d = r
|
|
|
|
|
#else:
|
|
|
|
|
#d = []
|
|
|
|
|
|
|
|
|
|
#return json.dumps(d)
|
|
|
|
|
|
|
|
|
|
if __name__== '__main__':
|
|
|
|
|
|
|
|
|
|