h=""" This is a data-collector client, that is intended to perform data-collection operations and submit them to an endpoint @required: - key application/service key - id node identifier usage : python data-collector.py --path config.json The configuration file is structured as JSON object as follows : { id: node identifier key: customer's identification key apps:"app_1,app_2,...", folders:"path_1,path_2, ..." } """ from utils.params import PARAMS as SYS_ARGS, Logger import requests import pickle import json from threading import Thread, RLock import monitor import utils.agents.actor as actor from utils.agents.manager import Manager SYS_ARGS['host']='localhost' ENDPOINT="http://:host/monitor".replace(":host",SYS_ARGS['host']) class Collector(Thread) : def __init__(self): Thread.__init__(self) """ This function initializes the data collector with critical information The process will include validating the user's account and plan @param key customer's key @param id node identifier """ # # Let's open the file with the key (nothing else should be in the file #f = open(SYS_ARGS['key']) #SYS_ARGS['key'] = f.read() #f.close() headers = {"key":SYS_ARGS["key"],"id":SYS_ARGS["id"]} #,"scope":json.dumps(scope)} self.plan = None self.store= None #headers['content-type'] = 'application/json' try: self.key = SYS_ARGS['key'] Logger.log(subject='Collector',object='api',action='request',value=ENDPOINT) url = "/".join([ENDPOINT,"init/collector"]) r = requests.post(url,headers=headers) r = json.loads(r.text) if r : # # Persisting plan and data-store ... self.plan = r['plan'] self.store = r['store'] info = {"store":self.store,"plan":self.plan} if info['plan'] is not None and info['store'] is not None: info['plan'] = self.plan['name'] info['store'] = self.store['args']['dbname'] _action = 'init' self.initialize() else: info['plan'] = self.plan is not None info['store']= self.store is not None _action = 'init.error' Logger.log(subject='collector',object='api',action=_action,value=info) except Exception as e: print(e) Logger.log(subject='collector',object='api',action='init.error',value=str(e)) self.monitor = None def initialize(self): """ This function creates a monitoring object with the associated parameters from the plan plan.metadata = {"agents":...,"folder_size":...,"delay":...,"limit":...,"actors":...} """ _agents = [monitor.DetailProcess(),monitor.FileWatch()] _actors = [actor.Apps(),actor.Folders(),actor.Mailer()] # Initialiing the agents with the parameter values we know of r = [] for agent in _agents : if agent.getName() in SYS_ARGS : values = SYS_ARGS[agent.getName()] # print (["init ",agent.getName(),values]) Logger.log(subject='Collector',object=agent.getName(),action='init.agent',value=values ) agent.init(values) r.append(agent) _agents = r config = {"store":self.store,"plan":self.plan} self.manager = Manager() self.manager.init(node=SYS_ARGS['id'],agents=_agents,actors=_actors,config=config,key=self.key,host=SYS_ARGS['host']) def run(self): """ This funtion runs the authorized features and """ #self.monitor.start() # Logger.log(subject='Collector',object='monitor',action='rpc',value=(self.manager is None) ) thread = Thread(target=self.manager.run) thread.start() # print self.manager # print self.monitor.config['store'] # for agent in self.pool : # try: # agent = pickle.loads(agent) # agent.start() # #p = pickle.loads(self.pool[key]) # #p.init(SYS_ARGS[key].split(',')) # #p.start() # except Exception,e: # print e if __name__ == '__main__' : # # if 'path' in SYS_ARGS : path = SYS_ARGS['path'] f = open(path) p = json.loads(f.read()) f.close() else: for id in ['apps','folders']: if id in SYS_ARGS : SYS_ARGS[id] = SYS_ARGS[id].split(',') p = dict(SYS_ARGS) Logger.init('data-collector') SYS_ARGS = dict(SYS_ARGS,** p) if 'apps' in SYS_ARGS : SYS_ARGS['apps'] += [__file__] thread = Collector() thread.start() else: print (h)