From b011b2ce6e2703965af832f410cbdb48d04aa4e8 Mon Sep 17 00:00:00 2001 From: "Steve L. Nyemba" Date: Tue, 7 Mar 2017 00:16:52 -0600 Subject: [PATCH 01/72] Fixing layout & shadow as well as search in folder grid --- src/api/static/js/dashboard.js | 34 ++++++++++++++++++++------------ src/api/templates/dashboard.html | 31 ++++++++++++++--------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/api/static/js/dashboard.js b/src/api/static/js/dashboard.js index 26a271d..972f6c3 100644 --- a/src/api/static/js/dashboard.js +++ b/src/api/static/js/dashboard.js @@ -45,8 +45,8 @@ monitor.processes.init = function (x) { // jx.dom.hide('apps') } - - setTimeout(monitor.sandbox.init,3000) + monitor.sandbox.init() + // setTimeout(monitor.sandbox.init,1000) } /** @@ -231,23 +231,29 @@ monitor.processes.summary.init = function(logs){ } var data = {labels:['Running','Crash','Idle'],datasets:[{data:[xr,xc,xi],backgroundColor:[RUNNING_COLOR,CRASH_COLOR,IDLE_COLOR/**COLORS[11],COLORS[2],COLORS[100]*/]}]} var context = jx.dom.get.instance('CANVAS') - jx.dom.set.value('total-running', xr) - jx.dom.set.value('total-crash', xc) - jx.dom.set.value('total-idle', xi) + // jx.dom.set.value('total-running', xr) + // jx.dom.set.value('total-crash', xc) + // jx.dom.set.value('total-idle', xi) jx.dom.set.value('total-apps', xr + xi + xc) jx.dom.set.value('app-summary-date', date) jx.dom.set.value('summary_chart','') jx.dom.append('summary_chart',context) - var conf = {}//width:50,height:40} + var conf = {}//width:'auto',height:$('#process_summary').height} conf.type = 'doughnut' conf.responsive = true conf.data = data - conf.options = {legend:{ position:'right'},repsonsive:true} - var chart = new Chart(context,conf) - + conf.options = { legend: { position: 'right' }, repsonsive: true } + var _chart = new Chart(context,conf) + + $('#summary_chart').click(function (evt) { + console.log(_chart) + console.log($(_chart)) + var activePoints = $(_chart).getSegmentsAtEvent(evt); + console.log(activePoints) + }) jx.dom.set.value('summary_ranking','') context = jx.dom.get.instance('CANVAS') jx.dom.append('summary_ranking',context) @@ -405,13 +411,15 @@ monitor.folders.search.init = function(){ var term = jx.dom.get.value('folder_search') var data = jx.dom.get.attribute('folder_search', 'data') - term = term.replace(/\x32/g,'') + term = term.replace(/ /g,'') if (term.length == 0) { monitor.folders.render.summary(data) } else if (term.length > 0) { data = jx.utils.patterns.visitor(data, function (row) { - if (row.id.match(term)) { + pattern = "(.*" + term + ".*)" + + if (row.id.match(pattern)) { return row } }) @@ -513,11 +521,11 @@ monitor.folders.render.summary = function (data) { jx.dom.set.value('gridfolders', '') var options = { - width: $('#gridfolders').width(), height:'auto' + width: $('#gfolderframe').width()-8, height:'auto' } options.paging = true options.pageSize = 4 - options.pageIndex = 2 + options.pageIndex = 1 options.pageButtonCount = 4 options.pagerContainer = '#folderspager' options.pagerFormat= "{prev} Page {pageIndex} of {pageCount} {next}" diff --git a/src/api/templates/dashboard.html b/src/api/templates/dashboard.html index ccc3162..fcb848e 100644 --- a/src/api/templates/dashboard.html +++ b/src/api/templates/dashboard.html @@ -17,7 +17,7 @@ -iMonitor +{{title}} @@ -35,30 +35,30 @@ - +
-
+
-
Monitoring +
Monitoring
Last Lookup
- +
- -
+ +
-
+
Application Summary By Status
Latest Lookup
- +
@@ -72,27 +72,27 @@
- -
+ +
- -
+ +
Application Summary By Groups
-
+
-
+
CPU & Memory Usage Trend for
Last Lookup
- +
- +
@@ -101,26 +101,26 @@
Python Virtual Environment Analysis
- +
Last Lookup
- -
+ +
-
+
- +
Folder Analysis/Monitoring
Powered By Machine Learning
- +
@@ -135,50 +135,50 @@ - - - + + +
- - + + From a11c609d1d922cc3df03cfb85d48bdde953ba472 Mon Sep 17 00:00:00 2001 From: steve Date: Mon, 1 May 2017 15:21:17 -0500 Subject: [PATCH 32/72] Adding/Enhancing actors to respond to events --- src/api/static/img/uml-design | 194 ++++++++++++------------ src/api/static/js/jx | 2 +- src/utils/agents/actor.py | 231 ++++++++++++++++------------- src/utils/agents/data-collector.py | 21 ++- src/utils/transport.py | 8 +- 5 files changed, 243 insertions(+), 213 deletions(-) diff --git a/src/api/static/img/uml-design b/src/api/static/img/uml-design index d3b699d..d68bed6 100644 --- a/src/api/static/img/uml-design +++ b/src/api/static/img/uml-design @@ -1,50 +1,50 @@ - + umbrello uml modeller http://umbrello.kde.org 1.6.9 UnicodeUTF8 - + - + - - - - - + + + + + - + - - - - - - - - - - - + + + + + + + + + + + - + - - - - + + + + - + - + @@ -52,129 +52,129 @@ - + - + - - - - - - - - - - - + + + + + + + + + + + - - + + - - - - - + + + + + - - + + - - + + - - + + - + - + - - - - - - - - - - - + + + + + + + + + + + - - - - - + + + + + - + - - + + - + - - + + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + @@ -183,17 +183,17 @@ - + - + - + diff --git a/src/api/static/js/jx b/src/api/static/js/jx index 36f9f10..3d00f5a 160000 --- a/src/api/static/js/jx +++ b/src/api/static/js/jx @@ -1 +1 @@ -Subproject commit 36f9f10ff44406cc0418cc5934eb475bc77ebb1e +Subproject commit 3d00f5a126574f2277cdac25d60008ee35dc8740 diff --git a/src/utils/agents/actor.py b/src/utils/agents/actor.py index 24be011..51707d0 100644 --- a/src/utils/agents/actor.py +++ b/src/utils/agents/actor.py @@ -16,18 +16,23 @@ import os import subprocess from monitor import ProcessCounter from utils.transport import QueueListener, QueueWriter, QueueReader +from utils.params import PARAMS class Actor: - def __init__(self,config): - self.config = config + def __init__(self): + pass def getIdentifier(self): return self.__class__.__name__.lower() - - def init(self,litems): - self.items = litems + """ + Initializing the class with configuration. The configuration will be specific to each subclass + + """ + def init(self,config): + self.config = config def process(self,item): pass - def isValid(self,item): - return False + def isValid(self,item): + return False + def execute(self,cmd): stream = None try: @@ -41,24 +46,27 @@ class Actor: Sending a message to a queue with parameters to,from,content """ def post(self,**args): - to = args['to'] - content = args['content'] - message = {"from":self.getIdentifier(),"to":to,"content":content} - host = self.config['api'] - uid = self.config['key'] - qid = to#self.conorfig['id'] - - qwriter = QueueWriter(host=host,uid=uid,qid=qid) - qwriter.init(qid) - qwriter.write(label=qid,row=content) - #qwriter.close() - pass - + # to = args['to'] + # content = args['content'] + # message = {"from":self.getIdentifier(),"to":to,"content":content} + # host = self.config['api'] + # uid = self.config['key'] + # qid = to#self.conorfig['id'] + + # qwriter = QueueWriter(host=host,uid=uid,qid=qid) + # qwriter.init(qid) + # qwriter.write(label=qid,row=content) + # #qwriter.close() + pass +class Folders(Actor): + def isvalid(self,item): + print self.conf + def process(self,item): + print item class Kill(Actor): - def isValid(self,item): - print self.config - return (item is not None) and (item in self.config) + def isValid(self,item): + return (item is not None) and (item in self.config) def process(self,item): cmd = "".join(["ps -eo pid,command|grep ",item,'|grep -E "^ {0,1}[0-9]+" -o|xargs kill -9']) self.execute(cmd) @@ -66,17 +74,31 @@ class Kill(Actor): # We need to make sure we can get assess the process on this server # class Start(Actor): - def __init__(self,config): - Actor.__init__(self,config) + def __init__(self): + Actor.__init__(self) def isValid(self,name): - return name in self.config + return name in self.config def process(self,name): - item = self.config[name] - path = item['path'] - args = item['args'] if 'args' in item else '' - cmd = " ".join([path,args]) - self.execute(cmd) + if name in self.config : + item = self.config['actions']['apps']['start'][name] + path = item['path'] + args = item['args'] if 'args' in item else '' + cmd = " ".join([path,args]) + self.execute(cmd) +class Apps(Actor): + def __init__(self): + Actor.__init__(self) + def isValid(self,rows): + status = [row['status'] for row in rows] + return 'crash' in status + + def process(self,rows): + rows = [row for row in rows if row['status'] == 'crash'] : + handler = Start() + for app in rows: + handler.process(app['label']) + """ The orchestrator class is designed to aggregate actions and communicate back to the caller Mesage passing is structured as follows {from,to,content} The content is designed to be understood by the actor @@ -85,91 +107,95 @@ class Start(Actor): @TODO: action specifications should be provided remotely """ class Orchestrator(Actor,Thread): - def __init__(self,config): - Thread.__init__(self) - Actor.__init__(self,config) - self.actors = {} - self.is_master_node = False - self.items = [] + def __init__(self,config=None): + Thread.__init__(self) + if config is None: + f = open(PARAMS['path']) + config = json.loads(f.read()) + f.close() + self.config = config + Actor.__init__(self) + self.actors = {"apps":Apps(),"folders":Folders()} + self.is_master_node = False + self.items = [] + # + # If the configuration only has id,key then this is NOT the maestro + # + host = config['api'] + qid = config['id'] + if 'actions' in config : # - # If the configuration only has id,key then this is NOT the maestro + # We are to assume this is the maestro/main node, and it will make the configuration available to other nodes + # NOTE: This is has a predefined master thus is subject to known short comings (zombies) # - host = config['api'] - qid = config['id'] - if 'actions' in config : - # - # We are to assume this is the maestro/main node, and it will make the configuration available to other nodes - # NOTE: This is has a predefined master thus is subject to known short comings (zombies) - # - - q = QueueWriter(host=host,uid=config['key'],qid=qid) - q.flush(config['id']) - q.write(label=qid,row=json.dumps(config['actions'])) - q.close() - self.is_master_node = True - - - else: - qid = config['master'] - q = QueueReader(host=host,uid=config['key'],qid=qid) - r = q.read() - q.close() - info = r[qid][0] - self.init(info) - print "Initialized ***** ",self.getIdentifier(), " as ",config['id'] - # - # This object will have to request for the configuration - # - #for id in config['actions'] : - #conf = config['actions'][id] - #item = eval("".join([id,"(",json.dumps(conf),")"])) - #self.actors[id.lower()] = item + q = QueueWriter(host=host,uid=config['key'],qid=qid) + q.flush(config['id']) + q.write(label=qid,row=json.dumps(config['actions'])) + q.close() + self.is_master_node = True + + + else: + qid = config['master'] + q = QueueReader(host=host,uid=config['key'],qid=qid) + r = q.read() + q.close() + info = r[qid][0] + self.init(info) + print "Initialized ***** ",self.getIdentifier(), " as ",config['id'] + + # + # This object will have to request for the configuration + # + #for id in config['actions'] : + #conf = config['actions'][id] + #item = eval("".join([id,"(",json.dumps(conf),")"])) + #self.actors[id.lower()] = item """ This function is designed to provide the orchestrator a configuration @pre """ def init(self,config): - for id in config: - - setup_info = config[id] - item = eval("".join([id,"(",json.dumps(setup_info),")"])) - self.actors[id.lower()] = item + for id in config: + + setup_info = config[id] + item = eval("".join([id,"(",json.dumps(setup_info),")"])) + self.actors[id.lower()] = item def callback(self,channel,method,header,stream): - message = json.loads(stream) - if 'content' in message : - content = message['content'] - sender = message['from'] - to = message['to'] - if content.lower() == 'quit' or to == 'quit': - channel.close() - - elif content.lower() == 'ping' or to == 'ping': - self.post(to=sender,content="1") - else: - id = to.lower() - actor = self.actors[id] - print "\tPerforming ",actor.getIdentifier()," on ",content," status ",actor.isValid(content) - if actor.isValid(content) : - actor.process(content) - else: - content = {"status":"invalid","content":content} - - self.post(to=sender,content=content) + message = json.loads(stream) + if 'content' in message : + content = message['content'] + + #sender = message['from'] + to = message['to'] + if isinstance(content,basestring) and content.lower() in ['quit'] or to=='quit': + if content.lower() == 'quit' or to == 'quit': + channel.close() + else: + id = to.lower() + actor = self.actors[id] + print [id,actor.isValid(content)] + if actor is not None and actor.isValid(content) : + actor.process(content) + else: + content = {"status":"invalid","content":content} + + #self.post(to=sender,content=content) def run(self): info = {} host = self.config['api'] uid = self.config['key'] qid = self.config['id'] - if not self.is_master_node : - qlistener = QueueListener(qid=qid,uid=uid,host=host) - qlistener.callback = self.callback - qlistener.read() - r = [self.process(item) for item in self.items] + + qlistener = QueueListener(qid=qid,uid=uid,host=host) + qlistener.callback = self.callback + qlistener.read() + r = [self.process(item) for item in self.items] """ This class is designed to send a message to a given AMQP enpoint @@ -193,14 +219,7 @@ config = { #thread = Orchestrator(config) #thread.start() -config = { - "master":"demo-000", - "id":"demo-001", - "key":"[0v8]-247&7!v3","api":"localhost" - - - } -thread = Orchestrator(config) +thread = Orchestrator() thread.start() #config = {"id":"demo","key":"[0v8]-247&7!v3","api":"localhost"} diff --git a/src/utils/agents/data-collector.py b/src/utils/agents/data-collector.py index 2704d5d..636dcf3 100644 --- a/src/utils/agents/data-collector.py +++ b/src/utils/agents/data-collector.py @@ -74,6 +74,7 @@ class ICollector(Thread) : write_class = self.config['store']['class']['write'] read_args = self.config['store']['args'] DELAY = self.config['delay'] * 60 + while self.quit == False: for thread in self.pool : @@ -88,10 +89,20 @@ class ICollector(Thread) : else: label = id row = data - # - # At this point we should check for the status and if it prompts an action - # @TODO Use a design pattern for this ... - # + # + # At this point we should check for the status and if it prompts an action + # @TODO Use a design pattern for this ... + # - submit the row to Event for analysis + # - The event orchestrator will handle things from this point on + # + message = {} + + message['to'] = thread.getName() + message['content'] = row + qwriter = QueueWriter(host=self.config['api'],uid=self.config['key'],qid=self.id) + qwriter.write(label=self.id,row = message) + qwriter.close() + self.lock.acquire() store = self.factory.instance(type=write_class,args=read_args) store.flush(size=200) @@ -112,4 +123,4 @@ class ICollector(Thread) : if __name__ == '__main__': thread = ICollector() # thread.daemon = True - thread.start() \ No newline at end of file + thread.start() diff --git a/src/utils/transport.py b/src/utils/transport.py index 13b6406..724975c 100644 --- a/src/utils/transport.py +++ b/src/utils/transport.py @@ -587,7 +587,7 @@ class CouchdbWriter(Couchdb,Writer): self.dbase.save_doc(document) def flush(self,**params) : - size = params['size'] + size = params['size'] if 'size' in params else 0 has_changed = False document = self.dbase.get(self.uid) for key in document: @@ -595,7 +595,7 @@ class CouchdbWriter(Couchdb,Writer): content = document[key] else: continue - if isinstance(content,list): + if isinstance(content,list) and size > 0: index = len(content) - size content = content[index:] document[key] = content @@ -603,8 +603,8 @@ class CouchdbWriter(Couchdb,Writer): else: document[key] = {} has_changed = True - if has_changed: - self.dbase.save_doc(document) + + self.dbase.save_doc(document) def archive(self,params=None): document = self.dbase.get(self.uid) From e4a80bf9ff3c2b5095789374e7d659e019a78497 Mon Sep 17 00:00:00 2001 From: steve Date: Wed, 3 May 2017 15:13:38 -0500 Subject: [PATCH 33/72] service script to manage components (simple interface) --- init.sh | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 init.sh diff --git a/init.sh b/init.sh new file mode 100644 index 0000000..4c1df69 --- /dev/null +++ b/init.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# This script is designed to handle various operations related to setting up, starting, stopping the python application +# It will assume the requirements file is at the root (not with the source code) +# + +export PYTHONPATH=$PWD/src +pip_upgrade='sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o' +install(){ + + virtualenv sandbox + sandbox/bin/pip install -r requirements.txt + `sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade` + + +} +upgrade(){ + git pull + count=`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|wc -l` + if [ ! "$count" = "0" ]; then + `sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade` + else + echo "No Upgrade required for sandbox" + fi + +} +start(){ + + if [ "$1" = "collector" ]; then + sandbox/bin/python src/utils/agents/data-collector.py --path $PWD/config.json + else + + sandbox/bin/python src/api/index.py --path $PWD/config.json & + fi + +} +stop(){ + ps -eo pid,command|grep python|grep -E "$PWD"|grep index.py|grep -E "^ {0,}[0-9]+" -o |xargs kill -9 + ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -o |xargs kill -9 +} + +status(){ + pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep index.py|grep -E "^ {0,}[0-9]+" -m 1 -o` + if [ "$pid" = "" ]; then + echo "API IS OFFLINE" + else + echo "API IS ONLINE $pid" + fi + pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -m 1 -o` + if [ "$pid" = "" ]; then + echo "DATA-COLLECTOR IS OFFLINE" + else + echo "DATA-COLLECTOR IS ONLINE $pid" + fi + +} + +if [ "$1" = "start" ]; then + if [ "$2" = "collector" ]; then + start "collector" + else + start + fi +else + $1 + +fi From 5083ea7c9099dbd94280f3aeb59ce5b37c7a889a Mon Sep 17 00:00:00 2001 From: steve Date: Wed, 3 May 2017 15:14:45 -0500 Subject: [PATCH 34/72] Handling of actions @TODO: Folder clean/archive --- requirements.txt | 15 ++- src/utils/agents/actor.py | 264 +++++++++++++++++++------------------- 2 files changed, 139 insertions(+), 140 deletions(-) diff --git a/requirements.txt b/requirements.txt index b7bb58c..c83f924 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,16 @@ -aniso8601==1.2.0 -click==6.6 -couchdbkit==0.6.5 Flask==0.11.1 Flask-Session==0.3.0 Flask-SocketIO==2.8.2 -http-parser==0.8.3 -itsdangerous==0.24 Jinja2==2.8 MarkupSafe==0.23 +Werkzeug==0.11.11 +aniso8601==1.2.0 +argparse==1.2.1 +click==6.6 +couchdbkit==0.6.5 +http-parser==0.8.3 +itsdangerous==0.24 +ngram==3.3.0 numpy==1.11.3 pika==0.10.0 python-dateutil==2.6.0 @@ -17,4 +20,4 @@ pytz==2016.10 restkit==4.2.2 six==1.10.0 socketpool==0.5.3 -Werkzeug==0.11.11 +wsgiref==0.1.2 diff --git a/src/utils/agents/actor.py b/src/utils/agents/actor.py index 51707d0..7103bb8 100644 --- a/src/utils/agents/actor.py +++ b/src/utils/agents/actor.py @@ -17,52 +17,46 @@ import subprocess from monitor import ProcessCounter from utils.transport import QueueListener, QueueWriter, QueueReader from utils.params import PARAMS -class Actor: - def __init__(self): - pass - def getIdentifier(self): - return self.__class__.__name__.lower() +from ngram import NGram as ng +class Actor(Thread): + def __init__(self): + Thread.__init__(self) + pass + def getIdentifier(self): + return self.__class__.__name__.lower() """ Initializing the class with configuration. The configuration will be specific to each subclass """ - def init(self,config): + def init(self,config,item=None): self.config = config + self.item = item def process(self,item): pass def isValid(self,item): return False - def execute(self,cmd): - stream = None - try: - handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) - stream = handler.communicate()[0] - except Exception,e: - pass - return stream - - """ - Sending a message to a queue with parameters to,from,content - """ - def post(self,**args): - # to = args['to'] - # content = args['content'] - # message = {"from":self.getIdentifier(),"to":to,"content":content} - # host = self.config['api'] - # uid = self.config['key'] - # qid = to#self.conorfig['id'] - - # qwriter = QueueWriter(host=host,uid=uid,qid=qid) - # qwriter.init(qid) - # qwriter.write(label=qid,row=content) - # #qwriter.close() + def execute(self,cmd): + stream = None + try: + subprocess.call (cmd,shell=False) + #stream = handler.communicate()[0] + except Exception,e: pass + + def run(self): + if self.item is not None: + self.process(self.item) + """ + Sending a message to a queue with parameters to,from,content + """ + def post(self,**args): + pass class Folders(Actor): - def isvalid(self,item): - print self.conf - def process(self,item): - print item + def isvalid(self,item): + print self.conf + def process(self,item): + print item class Kill(Actor): def isValid(self,item): @@ -73,31 +67,68 @@ class Kill(Actor): # # We need to make sure we can get assess the process on this server # + class Start(Actor): - def __init__(self): - Actor.__init__(self) - def isValid(self,name): - return name in self.config + def __init__(self): + Actor.__init__(self) + self.ng = None + + def init(self,config,item): + Actor.init(self,config,item) + self.config = config['start'] + self.ng = ng(self.config.keys()) + + def isValid(self,name): + items = self.ng.search(name) + if len(items) == 0 : + return False + else: + return items[0][1] > 0.1 - def process(self,name): - if name in self.config : - item = self.config['actions']['apps']['start'][name] - path = item['path'] - args = item['args'] if 'args' in item else '' - cmd = " ".join([path,args]) - self.execute(cmd) + def process(self,row): + name = row['label'] + items = self.ng.search(name)[0] + app = items[0] + + + args = self.config[app] + cmd = " ".join([app,args]) + + self.execute([app,args]) +""" + This class is designed to handle applications i.e start/stopping applications + @TODO: Assess if a reboot is required, by looking at the variance/anomaly detection +""" class Apps(Actor): def __init__(self): Actor.__init__(self) + self.crashes = [] + self.running = [] + def isValid(self,rows): status = [row['status'] for row in rows] return 'crash' in status - + + def classify(self,rows): + self.crashes = [] + self.running = [] + for row in rows: + if row['status'] == 'crash' : + self.crashes.append(row) + else: + self.running.append(row) + def process(self,rows): - rows = [row for row in rows if row['status'] == 'crash'] : - handler = Start() - for app in rows: - handler.process(app['label']) + #rows = [row for row in rows if row['status'] == 'crash'] : + self.classify(rows) + #handler = Start() + #handler.init(self.config) + #[handler.process(row_crash) for row_crash in self.crashes ] + for row_crash in self.crashes: + thread = Start() + thread.init(self.config,row_crash) + thread.daemon = True + thread.start() """ The orchestrator class is designed to aggregate actions and communicate back to the caller @@ -106,65 +137,46 @@ class Apps(Actor): The orchestrator is implemented using a simple iterator design-pattern @TODO: action specifications should be provided remotely """ -class Orchestrator(Actor,Thread): - def __init__(self,config=None): - Thread.__init__(self) - if config is None: - f = open(PARAMS['path']) - config = json.loads(f.read()) - f.close() - self.config = config - Actor.__init__(self) - self.actors = {"apps":Apps(),"folders":Folders()} - self.is_master_node = False - self.items = [] +class Orchestrator(Actor): + + def __init__(self,config=None): + Actor.__init__(self) + if config is None: + f = open(PARAMS['path']) + config = json.loads(f.read()) + f.close() + self.config = config + Actor.__init__(self) + self.actors = {"apps":Apps(),"folders":Folders()} + self.is_master_node = False + self.items = [] + # + # If the configuration only has id,key then this is NOT the maestro + # + host = config['api'] + qid = config['id'] + print "Initialized ***** ",self.getIdentifier(), " as ",config['id'] + # - # If the configuration only has id,key then this is NOT the maestro + # This object will have to request for the configuration # - host = config['api'] - qid = config['id'] - if 'actions' in config : - # - # We are to assume this is the maestro/main node, and it will make the configuration available to other nodes - # NOTE: This is has a predefined master thus is subject to known short comings (zombies) - # - - q = QueueWriter(host=host,uid=config['key'],qid=qid) - q.flush(config['id']) - q.write(label=qid,row=json.dumps(config['actions'])) - q.close() - self.is_master_node = True - - - else: - qid = config['master'] - q = QueueReader(host=host,uid=config['key'],qid=qid) - r = q.read() - q.close() - info = r[qid][0] - self.init(info) - print "Initialized ***** ",self.getIdentifier(), " as ",config['id'] - - # - # This object will have to request for the configuration - # - #for id in config['actions'] : - #conf = config['actions'][id] - #item = eval("".join([id,"(",json.dumps(conf),")"])) - #self.actors[id.lower()] = item + #for id in config['actions'] : + #conf = config['actions'][id] + #item = eval("".join([id,"(",json.dumps(conf),")"])) + #self.actors[id.lower()] = item """ This function is designed to provide the orchestrator a configuration @pre """ - def init(self,config): - - for id in config: - - setup_info = config[id] - item = eval("".join([id,"(",json.dumps(setup_info),")"])) - self.actors[id.lower()] = item + def init(self,config): + + for id in config: + + setup_info = config[id] + item = eval("".join([id,"(",json.dumps(setup_info),")"])) + self.actors[id.lower()] = item - def callback(self,channel,method,header,stream): + def callback(self,channel,method,header,stream): message = json.loads(stream) if 'content' in message : @@ -174,23 +186,26 @@ class Orchestrator(Actor,Thread): to = message['to'] if isinstance(content,basestring) and content.lower() in ['quit'] or to=='quit': if content.lower() == 'quit' or to == 'quit': + print '**** closing ',self.getIdentifier() channel.close() else: id = to.lower() actor = self.actors[id] - print [id,actor.isValid(content)] - if actor is not None and actor.isValid(content) : + + if actor is not None and actor.isValid(content) : + actor.init(self.config['actions']) actor.process(content) else: content = {"status":"invalid","content":content} #self.post(to=sender,content=content) - def run(self): - info = {} - host = self.config['api'] - uid = self.config['key'] - qid = self.config['id'] + def run(self): + + info = {} + host = self.config['api'] + uid = self.config['key'] + qid = self.config['id'] qlistener = QueueListener(qid=qid,uid=uid,host=host) qlistener.callback = self.callback @@ -201,29 +216,10 @@ class Orchestrator(Actor,Thread): This class is designed to send a message to a given AMQP enpoint The AMQP endpoint is implemented by QueueWriter class """ -class Alert(Actor): - def process(self,item): - - pass - -config = { - "id":"demo-000", - "key":"[0v8]-247&7!v3","api":"localhost", - "actions":{ - "Kill":["firefox"], - "Alert":[] - } - - - } -#thread = Orchestrator(config) -#thread.start() - -thread = Orchestrator() -thread.start() - -#config = {"id":"demo","key":"[0v8]-247&7!v3","api":"localhost"} -#actor = Kill(config) -#actor.start() +# class Alert(Actor): +# def process(self,item): +# pass -#config = {"id":"demo-100","key":"[0v8]-247&7!v3","api":"localhost"} +if __name__ == '__main__': + thread = Orchestrator() + thread.start() From fc413cd239b281462eab690365a4b8ddfab19d23 Mon Sep 17 00:00:00 2001 From: steve Date: Wed, 3 May 2017 15:30:40 -0500 Subject: [PATCH 35/72] @TODO: Folder handling clean/archive --- src/utils/agents/actor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/agents/actor.py b/src/utils/agents/actor.py index 7103bb8..237bd0f 100644 --- a/src/utils/agents/actor.py +++ b/src/utils/agents/actor.py @@ -62,8 +62,8 @@ class Kill(Actor): def isValid(self,item): return (item is not None) and (item in self.config) def process(self,item): - cmd = "".join(["ps -eo pid,command|grep ",item,'|grep -E "^ {0,1}[0-9]+" -o|xargs kill -9']) - self.execute(cmd) + args = "".join(["-eo pid,command|grep ",item,'|grep -E "^ {0,1}[0-9]+" -o|xargs kill -9']) + self.execute(["ps",args]) # # We need to make sure we can get assess the process on this server # From 76acbb7bd6b90ff3cd1db13fa9fb3e93286a1955 Mon Sep 17 00:00:00 2001 From: Michael Mead Date: Sat, 13 May 2017 18:31:46 -0500 Subject: [PATCH 36/72] some style changes --- src/api/index.py | 93 +++++----- src/api/static/css/default.css | 229 ++++++++++++++++++++++-- src/api/static/css/reflect.css | 14 ++ src/api/static/js/dashboard.js | 176 +++++++++---------- src/api/templates/dashboard.html | 288 +++++++++++++++++-------------- 5 files changed, 521 insertions(+), 279 deletions(-) create mode 100644 src/api/static/css/reflect.css diff --git a/src/api/index.py b/src/api/index.py index c858d7d..1bcb736 100644 --- a/src/api/index.py +++ b/src/api/index.py @@ -1,7 +1,7 @@ """ This is a RESTful interface implemented using Flask micro framework. The API is driven by configuration that is organized in terms of the monitoring classes - + The API is both restful and websocket/socketio enabled. We designed the classes to be reusable (and powered by labels): @@ -47,7 +47,7 @@ factory = DataSourceFactory() atexit.register(ThreadManager.stop) @app.route('/get/') -def procs(id): +def procs(id): try: gReader = factory.instance(type=class_read,args=p) data = gReader.read() @@ -67,14 +67,14 @@ def procs(id): # Let us determine if this is a normal operation or not # We will update the status of the information ... # - - for row in r[label] : - index = r[label].index(row) + + for row in r[label] : + index = r[label].index(row) if row['label'] in learn: id = row['label'] px = ahandler.predict([row],learn[id]) if px : - + # row['anomaly'] = px[1]==1 print "" print label,' *** ',index @@ -83,12 +83,12 @@ def procs(id): # # @TODO: # Compile a report here that will be sent to the mailing list - # - + # + except Exception, e: print e r = [] - + return json.dumps(r) """ @@ -98,7 +98,7 @@ def procs(id): @app.route('/sandbox') def sandbox(): global CONFIG - + if 'sandbox' in CONFIG: #CONFIG['monitor']: #handler = HANDLERS['sandbox']['class'] #conf = HANDLERS['sandbox']['config'] @@ -106,7 +106,7 @@ def sandbox(): # p = Factory.instance('sandbox',CONFIG) handler = monitor.Sandbox() conf = CONFIG['sandbox'] - + for id in conf: try: handler.init(conf[id]) @@ -114,27 +114,27 @@ def sandbox(): except Exception,e: pass else: - + r = [] return json.dumps(r) -@app.route('/trends') +@app.route('/trends') def trends (): id = request.args.get('id') app = request.args.get('app').strip() p = CONFIG['store']['args'] class_read = CONFIG['store']['class']['read'] - - gReader = factory.instance(type=class_read,args=p) + + gReader = factory.instance(type=class_read,args=p) r = gReader.read() if id in r: r = r[id] #--matrix series = [] for row in r: - + series += [item for item in row if str(item['label'])== app] if len(series) > 12 : beg = len(series) - 8 @@ -162,17 +162,18 @@ def dashboard(): This function is designed to trigger learning for anomaly detection @TODO: forward this to a socket i.e non-blocking socket """ + @app.route('/anomalies/get') def learn(): global CONFIG p = CONFIG['store']['args'] - class_read = CONFIG['store']['class']['read'] + class_read = CONFIG['store']['class']['read'] gReader = factory.instance(type=class_read,args=p) d = gReader.read() - + if 'learn' in d : info = d['learn'] - + del d['learn'] else : info = [] @@ -182,44 +183,44 @@ def learn(): d = d[id] params = {} for item in info: - + label = item['label'] params[label] = item - + #apps = list(set(ML.Extract(['label'],d))) r = [] if params : # - # If we have parameters available + # If we have parameters available p = AnomalyDetection() - apps = params.keys() + apps = params.keys() for name in apps : if name not in params: continue - _info = params[name] + _info = params[name] try: - xo = ML.Filter('label',name,d) + xo = ML.Filter('label',name,d) except Exception,e: xo = [] #print name,e - if len(xo) == 0: - continue + if len(xo) == 0: + continue xo = [xo[ len(xo) -1]] - + value = p.predict(xo,_info)[0] - + if len(value): report = dict(_info,**{'predicton':value}) r.append(report) - - - + + + #print app,value #if value is not None: # r.append(value) - + return json.dumps(r) - + """ This function returns anomalies for a given context or group of processes @@ -229,12 +230,12 @@ def learn(): def anomalies_status(): global CONFIG p = CONFIG['store']['args'] - class_read = CONFIG['store']['class']['read'] + class_read = CONFIG['store']['class']['read'] gReader = factory.instance(type=class_read,args=p) d = gReader.read() if 'learn' in d : info = d['learn'] - + del d['learn'] else : info = [] @@ -248,7 +249,7 @@ def anomalies_status(): def get_folders(): global CONFIG p = CONFIG['store']['args'] - class_read = CONFIG['store']['class']['read'] + class_read = CONFIG['store']['class']['read'] gReader = factory.instance(type=class_read,args=p) d = gReader.read() if 'folders' in d: @@ -256,7 +257,7 @@ def get_folders(): hosts = set([row[0]['id'] for row in d]) m = {} for id in hosts: - for row in d: + for row in d: if id == row[0]['id'] : m[id] = row d = m.values() @@ -264,11 +265,11 @@ def get_folders(): print row[0]['id'] # index = len(d) - 1 # d = d[index] - - + + # m = {} # for row in d : - + # key = row.keys()[0] # row = row[key] @@ -281,15 +282,13 @@ def get_folders(): # d = r else: d = [] - + return json.dumps(d) - + if __name__== '__main__': - -# ThreadManager.start(CONFIG) + +# ThreadManager.start(CONFIG) if 'port' not in SYS_ARGS.PARAMS : SYS_ARGS.PARAMS['port'] = 8484 PORT = int(SYS_ARGS.PARAMS['port']) app.run(host='0.0.0.0' ,port=PORT,debug=True,threaded=True) - - diff --git a/src/api/static/css/default.css b/src/api/static/css/default.css index 8c2b95e..0da1c4f 100644 --- a/src/api/static/css/default.css +++ b/src/api/static/css/default.css @@ -3,7 +3,7 @@ body, .default { font-family:sans-serif; font-weight:lighter; padding:10px; - + } .jsgrid-grid-header{ background: #f9f9f9; @@ -40,7 +40,7 @@ body, .default { height:30px; font-family:sans-serif; font-weight:lighter; - + } .button { padding:8px; @@ -53,7 +53,7 @@ body, .default { .button:hover{ background-color:#4682B4 ; color:#ffffff; - + } .no-border{ border:1px solid transparent} .border { border:1px solid #CAD5E0} @@ -64,7 +64,7 @@ body, .default { .grid { font-family:sans-serif; font-weight:lighter; - + margin:4px; padding:4px; } @@ -78,7 +78,7 @@ body, .default { .grid .fa-times {color:maroon; } .menu { - margin:4px; + margin:5px; padding:4px; } .menu .fa-chevron-right {color:transparent; margin:4px; } @@ -104,11 +104,11 @@ input[type=text]:focus{ .padding-2x{padding:4px;} .margin-2x {margin:4px;} .info { - + margin:4px; width:43%; - - + + } .height-quarter{height:24%;} .height-half{height:47%} @@ -121,7 +121,7 @@ input[type=text]:focus{ .fa-warning, .warning{color:orange} .number {font-size:42px; font-weight:lighter; padding:2px; margin:2px;} -.action {cursor:pointer; padding:2px; margin:2px; border:1px solid transparent} +.action {cursor:pointer; padding:1px; margin:1px; border:1px solid transparent} .action:hover { border-bottom-color:#4682B4} .shadow{box-shadow: 7px 7px 5px #888888;} .gradient { @@ -142,5 +142,214 @@ background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0, background-image: -webkit-linear-gradient(top, #CAD5E0 0%, #F3F3F3 40%); background-image: linear-gradient(to bottom, #CAD5E0 0%, #F3F3F3 40%); - + +} + +/* ----------- Non-Retina Screens ----------- */ +@media screen + and (min-device-width: 1200px) + and (max-device-width: 1600px) + and (-webkit-min-device-pixel-ratio: 1) { +} + +/* ----------- Retina Screens ----------- */ +@media screen + and (min-device-width: 1200px) + and (max-device-width: 1600px) + and (-webkit-min-device-pixel-ratio: 2) + and (min-resolution: 192dpi) { +} + +/* ----------- iPad mini ----------- */ + +/* Portrait and Landscape */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (-webkit-min-device-pixel-ratio: 1) { + } + +} + +/* Portrait */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (orientation: portrait) + and (-webkit-min-device-pixel-ratio: 1) { + +} + +/* Landscape */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (orientation: landscape) + and (-webkit-min-device-pixel-ratio: 1) { + +} + +/* ----------- iPad 1 and 2 ----------- */ +/* Portrait and Landscape */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (-webkit-min-device-pixel-ratio: 1) { + .block {display: inline-block;} + +} + +/* Portrait */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (orientation: portrait) + and (-webkit-min-device-pixel-ratio: 1) { + +} + +/* Landscape */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (orientation: landscape) + and (-webkit-min-device-pixel-ratio: 1) { + +} + +/* ----------- iPad 3 and 4 ----------- */ +/* Portrait and Landscape */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (-webkit-min-device-pixel-ratio: 2) { + +} + +/* Portrait */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (orientation: portrait) + and (-webkit-min-device-pixel-ratio: 2) { + +} + +/* Landscape ////////////////////////////////////////////////////////////////////////////////////////////////////// */ +@media only screen + and (min-device-width: 768px) + and (max-device-width: 1024px) + and (orientation: landscape) + and (-webkit-min-device-pixel-ratio: 2) { + +} + +/* ----------- iPhone 4 and 4S ----------- */ + +/* Portrait and Landscape */ +@media only screen + and (min-device-width: 320px) + and (max-device-width: 480px) + and (-webkit-min-device-pixel-ratio: 2) { + +} + +/* Portrait */ +@media only screen + and (min-device-width: 320px) + and (max-device-width: 480px) + and (-webkit-min-device-pixel-ratio: 2) + and (orientation: portrait) { +} + +/* Landscape */ +@media only screen + and (min-device-width: 320px) + and (max-device-width: 480px) + and (-webkit-min-device-pixel-ratio: 2) + and (orientation: landscape) { + +} + +/* ----------- iPhone 5 and 5S ----------- */ + +/* Portrait and Landscape */ +@media only screen + and (min-device-width: 320px) + and (max-device-width: 568px) + and (-webkit-min-device-pixel-ratio: 2) { + +} + +/* Portrait */ +@media only screen + and (min-device-width: 320px) + and (max-device-width: 568px) + and (-webkit-min-device-pixel-ratio: 2) + and (orientation: portrait) { +} + +/* Landscape */ +@media only screen + and (min-device-width: 320px) + and (max-device-width: 568px) + and (-webkit-min-device-pixel-ratio: 2) + and (orientation: landscape) { + +} + +/* ----------- iPhone 6 ----------- */ + +/* Portrait and Landscape */ +@media only screen + and (min-device-width: 375px) + and (max-device-width: 667px) + and (-webkit-min-device-pixel-ratio: 2) { + +} + +/* Portrait */ +@media only screen + and (min-device-width: 375px) + and (max-device-width: 667px) + and (-webkit-min-device-pixel-ratio: 2) + and (orientation: portrait) { + +} + +/* Landscape */ +@media only screen + and (min-device-width: 375px) + and (max-device-width: 667px) + and (-webkit-min-device-pixel-ratio: 2) + and (orientation: landscape) { + +} + +/* ----------- iPhone 6+ ----------- */ + +/* Portrait and Landscape */ +@media only screen + and (min-device-width: 414px) + and (max-device-width: 736px) + and (-webkit-min-device-pixel-ratio: 3) { + +} + +/* Portrait */ +@media only screen + and (min-device-width: 414px) + and (max-device-width: 736px) + and (-webkit-min-device-pixel-ratio: 3) + and (orientation: portrait) { + +} + +/* Landscape */ +@media only screen + and (min-device-width: 414px) + and (max-device-width: 736px) + and (-webkit-min-device-pixel-ratio: 3) + and (orientation: landscape) { + } diff --git a/src/api/static/css/reflect.css b/src/api/static/css/reflect.css new file mode 100644 index 0000000..743743a --- /dev/null +++ b/src/api/static/css/reflect.css @@ -0,0 +1,14 @@ +.block { + border: 2px solid black; + display: inline-block; +} + +.jsgrid-cell { overflow:hidden; } + + +/* ----------- working media query (laptop) ----------- */ + +@media screen + and (max-device-width: 900px) { + + } diff --git a/src/api/static/js/dashboard.js b/src/api/static/js/dashboard.js index 6916f3b..54ee060 100644 --- a/src/api/static/js/dashboard.js +++ b/src/api/static/js/dashboard.js @@ -3,38 +3,38 @@ var monitor = {} monitor.utils = {} monitor.utils.get = {} monitor.utils.get.time = function (item) { - // date = ([item.day + '-' + MONTHS[item.month] + '-' + item.year, hour + ':' + item.minute]).join(' ') + // date = ([item.day + '-' + MONTHS[item.month] + '-' + item.year, hour + ':' + item.minute]).join(' ') var hour = item.hour > 9 ? item.hour : ('0' + item.hour) if (hour > 12) { var units = 'PM' } else { var units = 'AM' } - return ([hour+':'+item.minute,units]).join(' ') + return ([hour+':'+item.minute,units]).join(' ') } monitor.utils.get.dateTime = function (item) { var time = monitor.utils.get.time(item) - return ([item.day+'-'+MONTHS[item.month]+'-'+item.year,time]).join(' ') -} + return ([item.day+'-'+MONTHS[item.month]+'-'+item.year,time]).join(' ') +} monitor.processes = {} monitor.processes.fetch = function(){ var httpclient = HttpClient.instance() httpclient.get(HTTP_CONTEXT+'/get/processes',monitor.processes.init); - + } monitor.processes.init = function (x) { - + var r = JSON.parse(x.responseText) - monitor.processes.summary.init(r) - var keys = jx.utils.keys(r) + monitor.processes.summary.init(r) + var keys = jx.utils.keys(r) jx.dom.set.value('menu','') jx.utils.patterns.visitor(keys,function(label){ var div = jx.dom.get.instance('DIV') var frame= jx.dom.get.instance('DIV') var i = jx.dom.get.instance('I') i.className = 'fa fa-chevron-right left' - + div.innerHTML = label frame.data = r[label] frame.label = label @@ -49,13 +49,13 @@ monitor.processes.init = function (x) { jx.dom.append('menu',frame) }) // - // Auto start the first item in the menu + // Auto start the first item in the menu // This is designed not to let the user wander or wonder what is going on // var nodes = jx.dom.get.children('menu') if (nodes.length > 0) { nodes[0].click() - } else { + } else { // // We should hide the panes for this // @@ -73,19 +73,19 @@ monitor.processes.init = function (x) { monitor.processes.render = function(label,data) { data = jx.utils.patterns.visitor(data,function(row){ - var status = {"idle":'',"running":'',"crash":''} + var status = {"idle":'',"running":'',"crash":''} if (!row.status.match(/class/)) { row.status_id = row.status - row.status = status[row.status] - + row.status = status[row.status] + } return row }) jx.dom.set.value('latest_processes','') ; jx.dom.set.value('latest_processes_label', label) - + var options = { - width: $('#latest_processes').width(), height:'auto' + width: "90%", height:'auto', autoload:true } options.paging = true options.pageSize = 4 @@ -95,10 +95,10 @@ monitor.processes.render = function(label,data) { options.pagerFormat= "{prev} Page {pageIndex} of {pageCount} {next}" options.pagePrevText= '' options.pageNextText= " " - + options.data = data options.rowClass = function (item, index,evt) { - + return 'small' } options.rowClick = function(args){ @@ -106,19 +106,19 @@ monitor.processes.render = function(label,data) { var id = jx.dom.get.value('latest_processes_label') var app = item.label monitor.processes.trend.init(id, app) - + if (item.anomaly == true) { jx.dom.show('has_anomaly') } else { jx.dom.hide('has_anomaly') } // var hour = item.hour < 10? ('0'+item.hour): item.hour - // date = ([item.day + '-' + MONTHS[item.month] + '-' + item.year, hour + ':' + item.minute]).join(' ') - + // date = ([item.day + '-' + MONTHS[item.month] + '-' + item.year, hour + ':' + item.minute]).join(' ') + jx.dom.set.value('node_last_lookup',monitor.utils.get.dateTime(item)) } - + options.autoload = true options.fields = [ { name: 'label', type: 'text', title: "Process", headercss: "small bold", css: "small"}, @@ -141,7 +141,7 @@ monitor.processes.trend.init = function (label,app) { var logs = JSON.parse(x.responseText) var dom = jx.dom.get.instance('trend_info'); dom.logs = logs - jx.dom.set.value('trend_info',app.trim()) + jx.dom.set.value('trend_info',app.trim()) // jx.dom.set.attribute(label,'logs',logs) monitor.processes.trend.render(logs,null,app) }) @@ -151,7 +151,7 @@ monitor.processes.trend.render = function (logs, key,label) { // key = 'memory_usage' // } // if (logs == null || label == null){ - + // logs = jx.dom.get.instance('trend_info').logs // label= jx.dom.get.value('trend_info') ; // } @@ -165,21 +165,21 @@ monitor.processes.trend.render = function (logs, key,label) { conf.data = {} conf.options = { legend: { position: 'bottom' } } conf.options.scales = {} - conf.options.scales.yAxes = [ + conf.options.scales.yAxes = [ {id:'0',scaleLabel:{display:true,labelString:'CPU & MEMORY %'},ticks:{min:0,max:100,beginAtZero:true},gridLines: {display:false}} // {id:'1',position:'right',scaleLabel:{display:true,labelString:'PROCESS COUNT'},ticks:{min:0,stepSize:1,beginAtZero:true},gridLines: {display:false}} ] conf.options.scales.xAxes = [ { - + gridLines: {display:false}, - + time: { format:'HH:mm' } - + } - + ] conf.data.datasets = [ ] var x_axis = [] @@ -199,26 +199,26 @@ monitor.processes.trend.render = function (logs, key,label) { if (_x[x] == null ){//||(_x[x] == null && _y[y] == null)) { _x[x] = 1 // _y[y] = 1 - x_axis.push(x) + x_axis.push(x) cpu.data.push({ x: x, y: item.cpu_usage }) mem.data.push({x:x,y:item.memory_usage}) // proc.data.push({x:x,y:item.proc_count}) // return {x:x,y:y} - + } - + }) - + var item = logs[logs.length - 1] jx.dom.set.value('trend_last_lookup',monitor.utils.get.dateTime(item)) - + conf.data.datasets = [cpu,mem] x_axis = jx.utils.unique(x_axis) conf.data.labels = x_axis // console.log(conf) jx.dom.append('trends_chart',context) var chart = new Chart(context,conf) - + } monitor.processes.summary = {} @@ -236,10 +236,10 @@ monitor.processes.summary.init = function(logs){ for( label in logs ){ var rows = logs[label] series[label] = {data:[0,0,0],label:label} - + jx.utils.patterns.visitor(rows,function(item){ if (date == null) { - + // date = new Date(item.year,item.month-1,item.day,item.hour,item.minute) // date = ([item.day + '-' + MONTHS[item.month] + '-' + item.year, item.hour + ':' + item.minute]).join(' ') date = monitor.utils.get.dateTime(item) @@ -252,7 +252,7 @@ monitor.processes.summary.init = function(logs){ xc += 1 } }) - + } var data = {labels:['Running','Crash','Idle'],datasets:[{data:[xr,xc,xi],backgroundColor:[RUNNING_COLOR,CRASH_COLOR,IDLE_COLOR/**COLORS[11],COLORS[2],COLORS[100]*/]}]} var context = jx.dom.get.instance('CANVAS') @@ -261,22 +261,22 @@ monitor.processes.summary.init = function(logs){ jx.dom.set.value('total-crash', xc) jx.dom.set.value('total-idle', xi) // jx.dom.set.value('total-apps', xr + xi + xc) - - + + jx.dom.set.value('app-summary-date', date) jx.dom.set.value('summary_chart','') jx.dom.append('summary_chart', context) $("#doughnut").attr('width', 50) $("#doughnut").attr('height', 50) - + var conf = {}//width:100,height:100}//width:'auto',height:$('#process_summary').height} - + conf.type = 'doughnut' conf.responsive = true conf.data = data conf.options = { legend: { position: 'right' }, repsonsive: true } var _chart = new Chart(context,conf) - + $('#summary_chart').click(function (evt) { console.log(_chart) console.log($(_chart)) @@ -286,17 +286,17 @@ monitor.processes.summary.init = function(logs){ jx.dom.set.value('summary_ranking','') context = jx.dom.get.instance('CANVAS') jx.dom.append('summary_ranking',context) - + conf = { type: 'bar', responsive: true } - + conf.options={scales:{xAxes:[{gridLines: {display:false}}],yAxes:[{gridLines: {display:false},scaleLabel:{display:true,labelString:'PROCESS COUNTS'} }] }} conf.options.legend ={position:'right'} /* conf.data = {labels:['Running','Idle','Crash']} var labels = jx.utils.keys(series) - + var i = 0 - conf.data.datasets = jx.utils.patterns.visitor(labels,function(id){ + conf.data.datasets = jx.utils.patterns.visitor(labels,function(id){ series[id].backgroundColor = COLORS[i++] return series[id]}) chart = new Chart(context,conf); @@ -305,7 +305,7 @@ monitor.processes.summary.init = function(logs){ conf.data = { labels: labels, backgroundColor:colors } var xr = [], xi = [], xc = [],xr_bg = [],xc_bg = [],xi_bg = [] jx.utils.patterns.visitor(labels, function (id) { - + var rows = logs[id] var index = xr.length xr_bg[index] = RUNNING_COLOR @@ -316,25 +316,25 @@ monitor.processes.summary.init = function(logs){ xc[index] = 0 xi[index] = 0 } - + jx.utils.patterns.visitor(logs[id], function (row) { - + if (row.status.match(/running/i)) { xr[index] += 1 - + } else if (row.status.match(/idle/i)) { xi[index] += 1 - + } else { xc[index] += 1 - + } }) }) - + conf.data.datasets = [{ label: 'Running', data:xr,backgroundColor:xr_bg},{label:'Crash',data:xc,backgroundColor:xc_bg},{label:'Idle',data:xi,backgroundColor:xi_bg} ] chart = new Chart(context, conf) - + } monitor.sandbox = {} @@ -342,9 +342,9 @@ monitor.sandbox.init = function () { jx.dom.hide('inspect_sandbox') var httpclient = HttpClient.instance() httpclient.get(HTTP_CONTEXT+'/sandbox', function (x) { - + var r = JSON.parse(x.responseText) - + if (r.length > 0) { jx.dom.show('sandbox') monitor.sandbox.render(r); @@ -355,19 +355,19 @@ monitor.sandbox.init = function () { } monitor.sandbox.render = function (logs) { // months = { 1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec' } - + // var d = ([logs[0].day, '-', MONTHS[logs[0].month], '-', logs[0].year, ' ', logs[0].hour, ':', logs[0].minute]).join('') var item = logs[logs.length -1] jx.dom.set.value('sandbox_date', monitor.utils.get.dateTime(item)) - + var options = { width: $('#sandbox_status').width()-8, height: 'auto' } - options.data = jx.utils.patterns.visitor(logs, function (item) { + options.data = jx.utils.patterns.visitor(logs, function (item) { if (item.value == 100) { item.status = '' } else { item.status = '' } - return item + return item }) options.paging = true options.pageSize = 4 @@ -378,7 +378,7 @@ monitor.sandbox.render = function (logs) { options.pagePrevText = '' options.pageNextText = " "; options.rowClass = function (item) { - + if (item.value < 70) { return 'bad' } else if (item.value < 100) { @@ -395,7 +395,7 @@ monitor.sandbox.render = function (logs) { dom.type = 'hidden' dom.name = 'missing' dom.value = JSON.stringify(item.missing) - + form.action = HTTP_CONTEXT+'/download' form.method = 'POST' form.appendChild(dom) @@ -407,11 +407,11 @@ monitor.sandbox.render = function (logs) { {name:"status",title:"",width:20}, { name: 'label',title:'Virtual Environment Label',type:'text',css:'small',headercss:'small bold' }, { name: 'value', title:'Completeness %',type: 'number', css: 'small', headercss: 'small bold' } - + ] var grid = $('#sandbox_status').jsGrid(options) jx.dom.show('inspect_sandbox') - + } @@ -427,7 +427,7 @@ monitor.folders.init = function () { // item.id = id data = data.concat(item) } - + monitor.folders.render.init(data) }) } @@ -436,20 +436,20 @@ monitor.folders.search.reset = function () { jx.dom.set.value('folder_search', '') var data = jx.dom.get.attribute('folder_search', 'data') monitor.folders.render.summary(data) - + } monitor.folders.search.init = function(){ var term = jx.dom.get.value('folder_search') var data = jx.dom.get.attribute('folder_search', 'data') - + term = term.replace(/ /g,'') if (term.length == 0) { monitor.folders.render.summary(data) } else if (term.length > 0) { - + data = jx.utils.patterns.visitor(data, function (row) { pattern = "(.*" + term + ".*)" - + if (row.id.match(pattern)) { return row } @@ -473,7 +473,7 @@ monitor.folders.show.grid = function () { $('#folder_plan').slideUp(function () { $('#folder_summary').slideDown() }) - + } /*** * This function is designed to establish a folder clean up strategy i.e : @@ -491,7 +491,7 @@ monitor.folders.render.details = function (folder,data) { var xy = r[i] var mode = jx.math.mode(jx.utils.vector('x', xy)) var yvalues = jx.utils.patterns.visitor(xy, function (row) { - if (row.x == mode) { + if (row.x == mode) { return row.y } }) @@ -514,7 +514,7 @@ monitor.folders.render.details = function (folder,data) { prefix = 'size' var mean = jx.math.sum(yvalues) var max = 0// (mean + (1.5 * sd)) - if (mean > 1000) { + if (mean > 1000) { divide_by = 1000 units = 'GB' } else { @@ -541,15 +541,15 @@ monitor.folders.render.details = function (folder,data) { monitor.folders.show.plan() } - + jx.dom.set.value('folder_name', folder) - - - - + + + + } monitor.folders.render.summary = function (data) { - + jx.dom.set.value('gridfolders', '') var options = { width: $('#gfolderframe').width()-8, height:'auto' @@ -562,15 +562,15 @@ monitor.folders.render.summary = function (data) { options.pagerFormat= "{prev} Page {pageIndex} of {pageCount} {next}" options.pagePrevText= '' options.pageNextText= " " - + options.data = data options.rowClass = function (item, index,evt) { - + return 'small' } options.rowClick = function(args){ // var item = args.item - + // age = jx.utils.patterns.visitor(item.details.age, function (row) { // return {y:row[0],x:row[1]} // }) @@ -586,26 +586,26 @@ monitor.folders.render.summary = function (data) { options.fields = [ { name: 'id', type: 'text', title: "Host", headercss: "small bold", css: "small"}, { name: 'name', type: 'text', title: "Folder Name", headercss: "small bold", css: "small"}, - + { name: "size", type: "number", title: "Folder Size", type: "number", headercss: "small bold" }, { name: "count", type: "number", title: "File Count", type: "number", headercss: "small bold" } ] var grid = $('#gridfolders').jsGrid(options) ; - + } monitor.menu = {} monitor.menu.event = {} monitor.menu.event.toggle = function () { var dom = jx.dom.get.instance('menuframe') var value = dom.style.marginLeft.trim() - + if (value==0 || value == "0px" || value == "") { - var width = -$(dom).width() - 10 - $('#menuframe').animate({marginLeft:"-12%"}) + var width = -$(dom).width() - 20 + $('#menuframe').animate({marginLeft:"-20%"}) } else { $('#menuframe').animate({marginLeft:"0"}) } - + } /** * Socket handler, check for learning status diff --git a/src/api/templates/dashboard.html b/src/api/templates/dashboard.html index 795f799..6fb127c 100644 --- a/src/api/templates/dashboard.html +++ b/src/api/templates/dashboard.html @@ -4,8 +4,11 @@ + + + @@ -31,146 +34,163 @@
{{title}}
The Phi Technology LLC
- -