smart-top/src/data-collector.py

h="""
	This is a data-collector client, that is intended to perform data-collection operations and submit them to an endpoint
	@required:
		- key	application/service key
		- id	node identifier
	usage :
		python data-collector.py --path config.json
	The configuration file is structured as JSON object as follows :
	{
		id: node identifier
		key: customer's identification key
		apps:"app_1,app_2,...",
		folders:"path_1,path_2, ..."
	}
"""
from utils.params import PARAMS as SYS_ARGS, Logger
import requests
import pickle
import json
from threading import Thread, RLock
import monitor
import utils.agents.actor as actor
from utils.agents.manager import Manager
SYS_ARGS['host']='localhost'
ENDPOINT="http://:host/monitor".replace(":host",SYS_ARGS['host'])
class Collector(Thread) :
	def __init__(self):
		Thread.__init__(self)
		"""
			This function initializes the data collector with critical information
			The process will include validating the user's account and plan
			@param key	customer's key
			@param id	node identifier
			
		"""
		#
		# Let's open the file with the key (nothing else should be in the file
		#f = open(SYS_ARGS['key'])
		#SYS_ARGS['key'] = f.read()
		#f.close()
		
		headers = {"key":SYS_ARGS["key"],"id":SYS_ARGS["id"]} #,"scope":json.dumps(scope)}
		self.plan = None
		self.store= None
		
		#headers['content-type'] = 'application/json'
		try:
			self.key = SYS_ARGS['key']
			Logger.log(subject='Collector',object='api',action='request',value=ENDPOINT)
			url 	= "/".join([ENDPOINT,"init/collector"])
			
			r	= requests.post(url,headers=headers)
			
			r 	= json.loads(r.text)
			
			if r :
				#
				# Persisting plan and data-store ...
				self.plan = r['plan']
				self.store = r['store']
			info = {"store":self.store,"plan":self.plan}
			
			if info['plan'] is not None and info['store'] is not None:
				info['plan'] = self.plan['name']			
				info['store'] = self.store['args']['dbname']
				_action = 'init'
				self.initialize()
			else:
				info['plan'] = self.plan is not None
				info['store']= self.store is not None
				_action = 'init.error'
			Logger.log(subject='collector',object='api',action=_action,value=info)
		except Exception as e:			
			print(e)
			Logger.log(subject='collector',object='api',action='init.error',value=str(e))
			self.monitor = None
	def initialize(self):
		"""
			This function creates a monitoring object with the associated parameters from the plan
			plan.metadata = {"agents":...,"folder_size":...,"delay":...,"limit":...,"actors":...}
		"""
		_agents = [monitor.DetailProcess(),monitor.FileWatch()]
		_actors = [actor.Apps(),actor.Folders(),actor.Mailer()]
		# Initialiing the agents with the parameter values we know of
		r = []
		for agent in _agents :
			
			if agent.getName() in SYS_ARGS :
				values = SYS_ARGS[agent.getName()]
				# print (["init ",agent.getName(),values])
				Logger.log(subject='Collector',object=agent.getName(),action='init.agent',value=values )
				agent.init(values)
				r.append(agent)
		_agents = r 

		
		
		config = {"store":self.store,"plan":self.plan}
		self.manager = Manager()		
		self.manager.init(node=SYS_ARGS['id'],agents=_agents,actors=_actors,config=config,key=self.key,host=SYS_ARGS['host'])
		
	def run(self):
		"""
			This funtion runs the authorized features and 
		"""
		#self.monitor.start()
		
		# Logger.log(subject='Collector',object='monitor',action='rpc',value=(self.manager is None) )
		thread = Thread(target=self.manager.run)
		thread.start()
		# print self.manager
		# print self.monitor.config['store']
		# for agent in self.pool :
		# 	try:
		# 		agent =  pickle.loads(agent)
		# 		agent.start()

		# 		#p = pickle.loads(self.pool[key])
		# 		#p.init(SYS_ARGS[key].split(','))
		# 		#p.start()
		# 	except Exception,e:
		# 		print e

if __name__ == '__main__' :
	#
	#
	if 'path' in SYS_ARGS :	
		path = SYS_ARGS['path']
		f = open(path)
		p = json.loads(f.read())
		f.close()
		
	else:
		for id in ['apps','folders']:
			if id in SYS_ARGS :
				SYS_ARGS[id] = SYS_ARGS[id].split(',')
	
		p = dict(SYS_ARGS)
	Logger.init('data-collector')
	SYS_ARGS = dict(SYS_ARGS,** p)
	if 'apps' in SYS_ARGS :
		SYS_ARGS['apps'] += [__file__]
	
	thread = Collector()
	thread.start()
else:
	print (h)