diff --git a/src/api/static/js/dashboard.js b/src/api/static/js/dashboard.js index 6c5b42c..4825d74 100644 --- a/src/api/static/js/dashboard.js +++ b/src/api/static/js/dashboard.js @@ -143,7 +143,7 @@ monitor.processes.trend.render = function (logs, key,label) { type: 'time', gridLines: {display:false}, - unitStepSize:25, + time: { format:'DD-MMM HH:mm' } diff --git a/src/monitor.py b/src/monitor.py index 5da2e03..3429cc3 100755 --- a/src/monitor.py +++ b/src/monitor.py @@ -269,7 +269,8 @@ class FileWatch(Analysis): file_date = datetime.datetime(year,month,day,int(hour),int(minute)) # size = round(size,2) #file_date = datetime.datetime(year,month,day,hour,minute) - age = (datetime.datetime.now() - file_date ).days + now = datetime.datetime.now() + age = (now - file_date ).days return {"size":size,"age":age} return None @@ -319,11 +320,15 @@ class FileWatch(Analysis): age = age units = ' Days' age = str(age)+units - xo = {"label":folder,"details":xo_raw,"summary":{"size":size,"age":age,"count":len(xo[:,1])}} + N = len(xo[:,1]) + xo = {"label":folder} #,"details":xo_raw,"summary":{"size":size,"age":age,"count":len(xo[:,1])}} + xo = dict(xo,**{"size":size,"age":age,"count":N}) xo["name"] = name xo['day'] = now.day xo['month'] = now.month xo['year'] = now.year + xo['date'] = time.mktime(now.timetuple()) + d.append(xo) return d diff --git a/src/utils/agents/learner.py b/src/utils/agents/learner.py index 1c7bb36..4dbb15b 100644 --- a/src/utils/agents/learner.py +++ b/src/utils/agents/learner.py @@ -8,6 +8,28 @@ from utils.transport import * from utils.ml import AnomalyDetection,ML from utils.params import PARAMS import time +class BaseLearner(Thread): + def __init__(self,lock) : + Thread.__init__(self) + path = PARAMS['path'] + self.name = self.__class__.__name__.lower() + if os.path.exists(path) : + f = open(path) + self.config = json.loads(f.read()) + f.close() + else: + self.config = None + self.lock = lock + self.factory = DataSourceFactory() + self.quit = False + """ + This function is designed to stop processing gracefully + + """ + def stop(self): + self.quit = True + + """ This class is intended to apply anomaly detection to various areas of learning The areas of learning that will be skipped are : @@ -16,16 +38,10 @@ import time @TODO: - Find a way to perform dimensionality reduction if need be """ -class Anomalies(Thread) : +class Anomalies(BaseLearner) : def __init__(self,lock): - Thread.__init__(self) - path = PARAMS['path'] - self.name = self.__class__.__name__.lower() - if os.path.exists(path) : - f = open(path) - self.config = json.loads(f.read()) - f.close() - + BaseLearner.__init__(self,lock) + if self.config : # # Initializing data store & factory class # @@ -34,9 +50,9 @@ class Anomalies(Thread) : self.rclass = self.config['store']['class']['read'] self.wclass = self.config['store']['class']['write'] self.rw_args = self.config['store']['args'] - self.factory = DataSourceFactory() + # self.factory = DataSourceFactory() self.quit = False - self.lock = lock + # self.lock = lock def format(self,stream): pass def stop(self): @@ -46,39 +62,57 @@ class Anomalies(Thread) : DELAY = self.config['delay'] * 60 reader = self.factory.instance(type=self.rclass,args=self.rw_args) data = reader.read() - key = 'apps' - rdata = data[key] - features = ['memory_usage','cpu_usage'] - yo = {"1":["running"],"name":"status"} - while self.quit == False : - print ' *** ',self.name, ' ' , str(datetime.today()) - for app in self.apps: - print '\t',app,str(datetime.today()),' ** ',app - logs = ML.Filter('label',app,rdata) - - if logs : - handler = AnomalyDetection() - value = handler.learn(logs,'label',app,features,yo) - if value is not None: - value = dict(value,**{"features":features}) - value = dict({"id":self.id},**value) - #r[id][app] = value - self.lock.acquire() - writer = self.factory.instance(type=self.wclass,args=self.rw_args) - writer.write(label='learn',row=value) - self.lock.release() - # - if 'MONITOR_CONFIG_PATH' in os.environ : - break - time.sleep(DELAY) + key = 'apps@'+self.id + if key in data: + rdata = data[key] + features = ['memory_usage','cpu_usage'] + yo = {"1":["running"],"name":"status"} + while self.quit == False : + print ' *** ',self.name, ' ' , str(datetime.today()) + for app in self.apps: + print '\t',app,str(datetime.today()),' ** ',app + logs = ML.Filter('label',app,rdata) + + if logs : + handler = AnomalyDetection() + value = handler.learn(logs,'label',app,features,yo) + if value is not None: + value = dict(value,**{"features":features}) + value = dict({"id":self.id},**value) + #r[id][app] = value + self.lock.acquire() + writer = self.factory.instance(type=self.wclass,args=self.rw_args) + writer.write(label='learn',row=value) + self.lock.release() + # + if 'MONITOR_CONFIG_PATH' in os.environ : + break + time.sleep(DELAY) print ' *** Exiting ',self.name.replace('a','A') +""" + Let's estimate how many files we will have for a given date + y = ax + b with y: number files, x: date, y: Number of files +""" +class Regression(BaseLearner): + def __init__(self,lock): + BaseLearner.__init__(self) + self.folders = self.config['folders'] + self.id = self.config['id'] + def run(self): + DELAY = self.config['delay'] * 60 + reader = self.factory.instance(type=self.rclass,args=self.rw_args) + data = reader.read() + if 'folders' in data : + data = ML.Filter('id',self.id,data['folders']) + xo = ML.Extract(['date'],data) + yo = ML.Extract(['count'],data) + numpy.linalg.lstsq(xo, yo, rcond=-1) -class Regression(Thread): - def __init__(self,params): - pass + + if __name__ == '__main__' : lock = RLock() thread = Anomalies(lock) diff --git a/src/utils/ml.py b/src/utils/ml.py index 287919e..ddae91c 100644 --- a/src/utils/ml.py +++ b/src/utils/ml.py @@ -117,6 +117,7 @@ class AnomalyDetection: yo = self.split(yo) p = self.gParameters(xo['train']) has_cov = np.linalg.det(p['cov']) if p else False #-- making sure the matrix is invertible + if xo['train'] and has_cov : E = 0.001 ACCEPTABLE_FSCORE = 0.6 @@ -142,7 +143,7 @@ class AnomalyDetection: __operf__ = self.gPerformance(px,yo['test']) - + print value,__operf__ if __operf__['fscore'] == 1 : continue if perf is None : @@ -227,8 +228,8 @@ class AnomalyDetection: fp += 1 if (test[i][1] != labels[i] and test[i][1] == 1) else 0 fn += 1 if (test[i][1] != labels[i] and test[i][1] == 0) else 0 tn += 1 if (test[i][1] == labels[i] and test[i][1] == 0) else 0 - precision = tp / (tp + fp) if tp + fp > 0 else 1 - recall = tp / (tp + fn) if tp + fp > 0 else 1 + precision = tp /( (tp + fp) if tp + fp > 0 else 1) + recall = tp / ((tp + fn) if tp + fn > 0 else 1) fscore = (2 * precision * recall)/ ((precision + recall) if (precision + recall) > 0 else 1) return {"precision":precision,"recall":recall,"fscore":fscore} diff --git a/src/utils/transport.py b/src/utils/transport.py index bea8b61..eee6265 100644 --- a/src/utils/transport.py +++ b/src/utils/transport.py @@ -448,6 +448,8 @@ class Couchdb: dbname = args['dbname'] self.server = Server(uri=uri) self.dbase = self.server.get_db(dbname) + if self.dbase.doc_exist(self.uid) == False: + self.dbase.save_doc({"_id":self.uid}) """ Insuring the preconditions are met for processing """ @@ -542,6 +544,10 @@ class CouchdbWriter(Couchdb,Writer): dbname = args['dbname'] self.server = Server(uri=uri) self.dbase = self.server.get_db(dbname) + # + # If the document doesn't exist then we should create it + # + """ write a given attribute to a document database @param label scope of the row repair|broken|fixed|stats