Improved transport with flush, archive functions. Put Regression on hold

master
Steve L. Nyemba 8 years ago
parent 81bd6bd658
commit 14bfd8ce3f

@ -85,7 +85,7 @@ class ICollector(Thread) :
row = data row = data
self.lock.acquire() self.lock.acquire()
store = self.factory.instance(type=write_class,args=read_args) store = self.factory.instance(type=write_class,args=read_args)
store.flush(size=200)
store.write(label=label,row=row) store.write(label=label,row=row)
self.lock.release() self.lock.release()
if 'MONITOR_CONFIG_PATH' in os.environ : if 'MONITOR_CONFIG_PATH' in os.environ :

@ -3,6 +3,7 @@
""" """
from __future__ import division from __future__ import division
import numpy as np import numpy as np
from sklearn import linear_model
from threading import Thread,RLock from threading import Thread,RLock
from utils.transport import * from utils.transport import *
from utils.ml import AnomalyDetection,ML from utils.ml import AnomalyDetection,ML
@ -13,10 +14,17 @@ class BaseLearner(Thread):
Thread.__init__(self) Thread.__init__(self)
path = PARAMS['path'] path = PARAMS['path']
self.name = self.__class__.__name__.lower() self.name = self.__class__.__name__.lower()
self.rclass= None
self.wclass= None
self.rw_args=None
if os.path.exists(path) : if os.path.exists(path) :
f = open(path) f = open(path)
self.config = json.loads(f.read()) self.config = json.loads(f.read())
f.close() f.close()
self.rclass = self.config['store']['class']['read']
self.wclass = self.config['store']['class']['write']
self.rw_args = self.config['store']['args']
else: else:
self.config = None self.config = None
self.lock = lock self.lock = lock
@ -98,7 +106,7 @@ class Anomalies(BaseLearner) :
""" """
class Regression(BaseLearner): class Regression(BaseLearner):
def __init__(self,lock): def __init__(self,lock):
BaseLearner.__init__(self) BaseLearner.__init__(self,lock)
self.folders = self.config['folders'] self.folders = self.config['folders']
self.id = self.config['id'] self.id = self.config['id']
def run(self): def run(self):
@ -109,7 +117,12 @@ class Regression(BaseLearner):
data = ML.Filter('id',self.id,data['folders']) data = ML.Filter('id',self.id,data['folders'])
xo = ML.Extract(['date'],data) xo = ML.Extract(['date'],data)
yo = ML.Extract(['count'],data) yo = ML.Extract(['count'],data)
numpy.linalg.lstsq(xo, yo, rcond=-1)
pass
# print np.var(xo,yo)

@ -42,7 +42,12 @@ class ML:
def Extract(lattr,data): def Extract(lattr,data):
if isinstance(lattr,basestring): if isinstance(lattr,basestring):
lattr = [lattr] lattr = [lattr]
return [[row[id] for id in lattr] for row in data] # return [[row[id] for id in lattr] for row in data]
r = [[row[id] for id in lattr] for row in data]
if len(lattr) == 1 :
return [x[0] for x in r]
else:
return r
@staticmethod @staticmethod
def CleanupName(value) : def CleanupName(value) :
return value.replace('$','').replace('.+','') return value.replace('$','').replace('.+','')

@ -562,8 +562,25 @@ class CouchdbWriter(Couchdb,Writer):
document[label] = [] document[label] = []
document[label].append(row) document[label].append(row)
self.dbase.save_doc(document) self.dbase.save_doc(document)
def flush(self,params) :
def flush(self,params=None): size = params['size']
document = self.dbase.get(self.uid)
for key in documment:
if key not in ['_id','_rev','_attachments'] :
content = document[key]
else:
content = []
if isinstance(content,list):
index = len(content) - size
content = content[index:]
document[key] = content
else:
document[key] = {}
self.dbase.save_doc(document)
def archive(self,params=None):
document = self.dbase.get(self.uid) document = self.dbase.get(self.uid)
content = {} content = {}
_doc = {} _doc = {}

Loading…
Cancel
Save