You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
|
|
_args['data'] = df
|
|
|
|
|
#
|
|
|
|
|
# @log :
|
|
|
|
|
# Logging information about the training process for this partition (or not)
|
|
|
|
|
#
|
|
|
|
|
info = {"rows":df.shape[0],"cols":df.shape[1], "partition":partition,"logs":_args['logs']}
|
|
|
|
|
logger.write({"module":"train","action":"train","input":info})
|
|
|
|
|
data.maker.train(**_args)
|
|
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
# @staticmethod
|
|
|
|
|
def generate(self,args):
|
|
|
|
|
"""
|
|
|
|
|
This function will generate data and store it to a given,
|
|
|
|
|
"""
|
|
|
|
|
logger = factory.instance(type='mongo.MongoWriter',args={'dbname':'aou','doc':args['context']})
|
|
|
|
|
log_folder = args['logs'] if 'logs' in args else 'logs'
|
|
|
|
|
partition = args['partition'] if 'partition' in args else ''
|
|
|
|
|
f = open (filename)
|
|
|
|
|
PIPELINE = json.loads(f.read())
|
|
|
|
|
f.close()
|
|
|
|
|
index = int(SYS_ARGS['index']) if 'index' in SYS_ARGS else 0
|
|
|
|
|
|
|
|
|
|
args = (PIPELINE[index])
|
|
|
|
|
QUEUE_TYPE = 'queue.QueueReader'
|
