refactoring learning and enabling learning api

master
Steve L. Nyemba 8 years ago
parent aaa6cf9a08
commit cbed6dbb70

@ -43,7 +43,7 @@ p = CONFIG['store']['args']
class_read = CONFIG['store']['class']['read'] class_read = CONFIG['store']['class']['read']
class_write= CONFIG['store']['class']['write'] class_write= CONFIG['store']['class']['write']
factory = DataSourceFactory() factory = DataSourceFactory()
gReader = factory.instance(type=class_read,args=p) # gReader = factory.instance(type=class_read,args=p)
atexit.register(ThreadManager.stop) atexit.register(ThreadManager.stop)
@app.route('/get/<id>') @app.route('/get/<id>')
@ -100,7 +100,6 @@ def trends ():
gReader = factory.instance(type=class_read,args=p) gReader = factory.instance(type=class_read,args=p)
r = gReader.read() r = gReader.read()
if id in r: if id in r:
r = r[id] #--matrix r = r[id] #--matrix
@ -131,15 +130,17 @@ def dashboard():
This function is designed to trigger learning for anomaly detection This function is designed to trigger learning for anomaly detection
@TODO: forward this to a socket i.e non-blocking socket @TODO: forward this to a socket i.e non-blocking socket
""" """
@app.route('/learn') @app.route('/anomalies/get')
def learn(): def learn():
global CONFIG global CONFIG
p = CONFIG['store']['args'] p = CONFIG['store']['args']
class_read = CONFIG['store']['class']['read'] class_read = CONFIG['store']['class']['read']
gReader = factory.instance(type=class_read,args=p) gReader = factory.instance(type=class_read,args=p)
d = gReader.read() d = gReader.read()
if 'learn' in d : if 'learn' in d :
info = d['learn'] info = d['learn']
del d['learn'] del d['learn']
else : else :
info = [] info = []
@ -147,27 +148,45 @@ def learn():
if 'id' in request.args: if 'id' in request.args:
id = request.args['id'] id = request.args['id']
d = d[id] d = d[id]
apps = CONFIG['monitor']['processes']['config'][id]
#print (apps)
params = {} params = {}
for item in info: for item in info:
id = item['label']
params[id] = item label = item['label']
params[label] = item
#apps = list(set(ML.Extract(['label'],d))) #apps = list(set(ML.Extract(['label'],d)))
r = []
if params :
#
# If we have parameters available
p = AnomalyDetection() p = AnomalyDetection()
apps = params.keys()
for name in apps : for name in apps :
xo = ML.Filter('label',name,d) if name not in params:
continue
_info = params[name] _info = params[name]
#info = ML.Filter('label',app,logs) try:
value = p.predict(xo,_info) xo = ML.Filter('label',name,d)
print [row[1] for row in value] except Exception,e:
break xo = []
#print name,e
if len(xo) == 0:
continue
xo = [xo[ len(xo) -1]]
value = p.predict(xo,_info)[0]
if len(value):
report = dict(_info,**{'predicton':value})
r.append(report)
#print app,value #print app,value
#if value is not None: #if value is not None:
# r.append(value) # r.append(value)
print r
return json.dumps([]) return json.dumps(r)
@ -175,10 +194,6 @@ def learn():
def anomalies_status(): def anomalies_status():
pass pass
@app.route('/anomalies/get')
def anomalies_get():
pass
if __name__== '__main__': if __name__== '__main__':
#ThreadManager.start(CONFIG) #ThreadManager.start(CONFIG)

@ -246,12 +246,12 @@ class MessageQueue:
self.qid = params['qid'] self.qid = params['qid']
def isready(self): def isready(self):
self.init() #self.init()
resp = self.connection is not None and self.connection.is_open resp = self.connection is not None and self.connection.is_open
self.close() self.close()
return resp return resp
def close(self): def close(self):
print "closing ..."
self.channel.close() self.channel.close()
self.connection.close() self.connection.close()
""" """
@ -463,7 +463,10 @@ class CouchdbReader(Couchdb,Reader):
# #
# setting the basic parameters for # setting the basic parameters for
Couchdb.__init__(self,**args) Couchdb.__init__(self,**args)
if 'filename' in args :
self.filename = args['filename'] self.filename = args['filename']
else:
self.filename = None
def isready(self): def isready(self):
# #
@ -487,8 +490,7 @@ class CouchdbReader(Couchdb,Reader):
r = False r = False
return r return r
def stream(self):
def read(self,size=-1):
content = self.dbase.fetch_attachment(self.uid,self.filename).split('\n') ; content = self.dbase.fetch_attachment(self.uid,self.filename).split('\n') ;
i = 1 i = 1
for row in content: for row in content:
@ -497,6 +499,11 @@ class CouchdbReader(Couchdb,Reader):
break break
i = i + 1 i = i + 1
def read(self,size=-1):
if self.filename is not None:
self.stream()
else:
return self.basic_read()
def basic_read(self): def basic_read(self):
document = self.dbase.get(self.uid) document = self.dbase.get(self.uid)
del document['_id'], document['_rev'] del document['_id'], document['_rev']

Loading…
Cancel
Save