data-maker/pipeline.py

#!/usr/bin/env python3
			_args['num_gpu'] = int(args['num_gpu']) if 'num_gpu' in args else 1 
			os.environ['CUDA_VISIBLE_DEVICES'] = str(args['gpu']) if 'gpu' in args else '0'
			
			_args['data'] = df
			#
			# @log :
			#	Logging information about the training process for this partition (or not)
			#
			info = {"rows":df.shape[0],"cols":df.shape[1], "partition":partition,"logs":_args['logs']}
			logger.write({"module":"train","action":"train","input":info})
			data.maker.train(**_args)

		pass
		
	# @staticmethod
	def generate(self,args):
		"""
		This function will generate data and store it to a given,
		"""
		logger = factory.instance(type='mongo.MongoWriter',args={'dbname':'aou','doc':args['context']})
		log_folder = args['logs'] if 'logs' in args else 'logs'
		partition = args['partition'] if 'partition' in args else ''
			columns = args['columns']
		#
		args['columns'] = args['column']		
		(Components()).train(**args)		
		logger.write({"module":"process","action":"exit","info":info["info"]})
		channel.close()
		channel.connection.close()
		pass
	
		# This will start a worker just in case to listen to a queue