library
/
smart-top
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Handling of actions @TODO: Folder clean/archive

Browse Source
community
Steve L. Nyemba 8 years ago
parent e4a80bf9ff
commit 5083ea7c90
2 changed files with 139 additions and 140 deletions
Show Stats Download Patch File Download Diff File

15
requirements.txt
Unescape View File

@ -1,13 +1,16 @@
aniso8601==1.2.0
click==6.6
couchdbkit==0.6.5
Flask==0.11.1 Flask==0.11.1
Flask-Session==0.3.0 Flask-Session==0.3.0
Flask-SocketIO==2.8.2 Flask-SocketIO==2.8.2
http-parser==0.8.3
itsdangerous==0.24
Jinja2==2.8 Jinja2==2.8
MarkupSafe==0.23 MarkupSafe==0.23
Werkzeug==0.11.11
aniso8601==1.2.0
argparse==1.2.1
click==6.6
couchdbkit==0.6.5
http-parser==0.8.3
itsdangerous==0.24
ngram==3.3.0
numpy==1.11.3 numpy==1.11.3
pika==0.10.0 pika==0.10.0
python-dateutil==2.6.0 python-dateutil==2.6.0
@ -17,4 +20,4 @@ pytz==2016.10
restkit==4.2.2 restkit==4.2.2
six==1.10.0 six==1.10.0
socketpool==0.5.3 socketpool==0.5.3
Werkzeug==0.11.11 wsgiref==0.1.2

258
src/utils/agents/actor.py
Unescape View File

@ -17,52 +17,46 @@ import subprocess
from monitor import ProcessCounter from monitor import ProcessCounter
from utils.transport import QueueListener, QueueWriter, QueueReader from utils.transport import QueueListener, QueueWriter, QueueReader
from utils.params import PARAMS from utils.params import PARAMS
class Actor: from ngram import NGram as ng
def __init__(self): class Actor(Thread):
pass def __init__(self):
def getIdentifier(self): Thread.__init__(self)
return self.__class__.__name__.lower() pass
def getIdentifier(self):
return self.__class__.__name__.lower()
""" """
Initializing the class with configuration. The configuration will be specific to each subclass Initializing the class with configuration. The configuration will be specific to each subclass
""" """
def init(self,config): def init(self,config,item=None):
self.config = config self.config = config
self.item = item
def process(self,item): def process(self,item):
pass pass
def isValid(self,item): def isValid(self,item):
return False return False
def execute(self,cmd): def execute(self,cmd):
stream = None stream = None
try: try:
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) subprocess.call (cmd,shell=False)
stream = handler.communicate()[0] #stream = handler.communicate()[0]
except Exception,e: except Exception,e:
pass
return stream
"""
Sending a message to a queue with parameters to,from,content
"""
def post(self,**args):
# to = args['to']
# content = args['content']
# message = {"from":self.getIdentifier(),"to":to,"content":content}
# host = self.config['api']
# uid = self.config['key']
# qid = to#self.conorfig['id']
# qwriter = QueueWriter(host=host,uid=uid,qid=qid)
# qwriter.init(qid)
# qwriter.write(label=qid,row=content)
# #qwriter.close()
pass pass
def run(self):
if self.item is not None:
self.process(self.item)
"""
Sending a message to a queue with parameters to,from,content
"""
def post(self,**args):
pass
class Folders(Actor): class Folders(Actor):
def isvalid(self,item): def isvalid(self,item):
print self.conf print self.conf
def process(self,item): def process(self,item):
print item print item
class Kill(Actor): class Kill(Actor):
def isValid(self,item): def isValid(self,item):
@ -73,31 +67,68 @@ class Kill(Actor):
# #
# We need to make sure we can get assess the process on this server # We need to make sure we can get assess the process on this server
# #
class Start(Actor): class Start(Actor):
def __init__(self): def __init__(self):
Actor.__init__(self) Actor.__init__(self)
def isValid(self,name): self.ng = None
return name in self.config
def init(self,config,item):
def process(self,name): Actor.init(self,config,item)
if name in self.config : self.config = config['start']
item = self.config['actions']['apps']['start'][name] self.ng = ng(self.config.keys())
path = item['path']
args = item['args'] if 'args' in item else '' def isValid(self,name):
cmd = " ".join([path,args]) items = self.ng.search(name)
self.execute(cmd) if len(items) == 0 :
return False
else:
return items[0][1] > 0.1
def process(self,row):
name = row['label']
items = self.ng.search(name)[0]
app = items[0]
args = self.config[app]
cmd = " ".join([app,args])
self.execute([app,args])
"""
This class is designed to handle applications i.e start/stopping applications
@TODO: Assess if a reboot is required, by looking at the variance/anomaly detection
"""
class Apps(Actor): class Apps(Actor):
def __init__(self): def __init__(self):
Actor.__init__(self) Actor.__init__(self)
self.crashes = []
self.running = []
def isValid(self,rows): def isValid(self,rows):
status = [row['status'] for row in rows] status = [row['status'] for row in rows]
return 'crash' in status return 'crash' in status
def classify(self,rows):
self.crashes = []
self.running = []
for row in rows:
if row['status'] == 'crash' :
self.crashes.append(row)
else:
self.running.append(row)
def process(self,rows): def process(self,rows):
rows = [row for row in rows if row['status'] == 'crash'] : #rows = [row for row in rows if row['status'] == 'crash'] :
handler = Start() self.classify(rows)
for app in rows: #handler = Start()
handler.process(app['label']) #handler.init(self.config)
#[handler.process(row_crash) for row_crash in self.crashes ]
for row_crash in self.crashes:
thread = Start()
thread.init(self.config,row_crash)
thread.daemon = True
thread.start()
""" """
The orchestrator class is designed to aggregate actions and communicate back to the caller The orchestrator class is designed to aggregate actions and communicate back to the caller
@ -106,65 +137,46 @@ class Apps(Actor):
The orchestrator is implemented using a simple iterator design-pattern The orchestrator is implemented using a simple iterator design-pattern
@TODO: action specifications should be provided remotely @TODO: action specifications should be provided remotely
""" """
class Orchestrator(Actor,Thread): class Orchestrator(Actor):
def __init__(self,config=None):
Thread.__init__(self)
if config is None:
f = open(PARAMS['path'])
config = json.loads(f.read())
f.close()
self.config = config
Actor.__init__(self)
self.actors = {"apps":Apps(),"folders":Folders()}
self.is_master_node = False
self.items = []
#
# If the configuration only has id,key then this is NOT the maestro
#
host = config['api']
qid = config['id']
if 'actions' in config :
#
# We are to assume this is the maestro/main node, and it will make the configuration available to other nodes
# NOTE: This is has a predefined master thus is subject to known short comings (zombies)
#
q = QueueWriter(host=host,uid=config['key'],qid=qid)
q.flush(config['id'])
q.write(label=qid,row=json.dumps(config['actions']))
q.close()
self.is_master_node = True
def __init__(self,config=None):
Actor.__init__(self)
if config is None:
f = open(PARAMS['path'])
config = json.loads(f.read())
f.close()
self.config = config
Actor.__init__(self)
self.actors = {"apps":Apps(),"folders":Folders()}
self.is_master_node = False
self.items = []
#
# If the configuration only has id,key then this is NOT the maestro
#
host = config['api']
qid = config['id']
print "Initialized ***** ",self.getIdentifier(), " as ",config['id']
else: #
qid = config['master'] # This object will have to request for the configuration
q = QueueReader(host=host,uid=config['key'],qid=qid) #
r = q.read() #for id in config['actions'] :
q.close() #conf = config['actions'][id]
info = r[qid][0] #item = eval("".join([id,"(",json.dumps(conf),")"]))
self.init(info) #self.actors[id.lower()] = item
print "Initialized ***** ",self.getIdentifier(), " as ",config['id']
#
# This object will have to request for the configuration
#
#for id in config['actions'] :
#conf = config['actions'][id]
#item = eval("".join([id,"(",json.dumps(conf),")"]))
#self.actors[id.lower()] = item
""" """
This function is designed to provide the orchestrator a configuration This function is designed to provide the orchestrator a configuration
@pre @pre
""" """
def init(self,config): def init(self,config):
for id in config: for id in config:
setup_info = config[id] setup_info = config[id]
item = eval("".join([id,"(",json.dumps(setup_info),")"])) item = eval("".join([id,"(",json.dumps(setup_info),")"]))
self.actors[id.lower()] = item self.actors[id.lower()] = item
def callback(self,channel,method,header,stream): def callback(self,channel,method,header,stream):
message = json.loads(stream) message = json.loads(stream)
if 'content' in message : if 'content' in message :
@ -174,23 +186,26 @@ class Orchestrator(Actor,Thread):
to = message['to'] to = message['to']
if isinstance(content,basestring) and content.lower() in ['quit'] or to=='quit': if isinstance(content,basestring) and content.lower() in ['quit'] or to=='quit':
if content.lower() == 'quit' or to == 'quit': if content.lower() == 'quit' or to == 'quit':
print '**** closing ',self.getIdentifier()
channel.close() channel.close()
else: else:
id = to.lower() id = to.lower()
actor = self.actors[id] actor = self.actors[id]
print [id,actor.isValid(content)]
if actor is not None and actor.isValid(content) : if actor is not None and actor.isValid(content) :
actor.init(self.config['actions'])
actor.process(content) actor.process(content)
else: else:
content = {"status":"invalid","content":content} content = {"status":"invalid","content":content}
#self.post(to=sender,content=content) #self.post(to=sender,content=content)
def run(self): def run(self):
info = {}
host = self.config['api'] info = {}
uid = self.config['key'] host = self.config['api']
qid = self.config['id'] uid = self.config['key']
qid = self.config['id']
qlistener = QueueListener(qid=qid,uid=uid,host=host) qlistener = QueueListener(qid=qid,uid=uid,host=host)
qlistener.callback = self.callback qlistener.callback = self.callback
@ -201,29 +216,10 @@ class Orchestrator(Actor,Thread):
This class is designed to send a message to a given AMQP enpoint This class is designed to send a message to a given AMQP enpoint
The AMQP endpoint is implemented by QueueWriter class The AMQP endpoint is implemented by QueueWriter class
""" """
class Alert(Actor): # class Alert(Actor):
def process(self,item): # def process(self,item):
# pass
pass
config = {
"id":"demo-000",
"key":"[0v8]-247&7!v3","api":"localhost",
"actions":{
"Kill":["firefox"],
"Alert":[]
}
}
#thread = Orchestrator(config)
#thread.start()
thread = Orchestrator()
thread.start()
#config = {"id":"demo","key":"[0v8]-247&7!v3","api":"localhost"}
#actor = Kill(config)
#actor.start()
#config = {"id":"demo-100","key":"[0v8]-247&7!v3","api":"localhost"} if __name__ == '__main__':
thread = Orchestrator()
thread.start()

Write Preview
Loading…
Cancel
Save