You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
SQL = SQL.replace(':dataset',args['dataset']) #+ " LIMIT 1000 "
|
|
|
|
|
_args['gpu'] = args['gpu'] if 'gpu' in args else 0
|
|
|
|
|
|
|
|
|
|
# In case we want slow-mode, we can store the partitions in mongodb and process (Yes|No)?
|
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
os.environ['CUDA_VISIBLE_DEVICES'] = str(args['gpu']) if 'gpu' in args else '0'
|
|
|
|
|
_args['no_value']= args['no_value']
|
|
|
|
|
# filename = os.sep.join([log_folder,'output',name+'.csv'])
|
|
|
|
|
reader = lambda: _df
|
|
|
|
|
qhandlers = [factory.instance(type=QUEUE_TYPE,args={'queue':'aou.io'}) for i in np.arange(N)]
|
|
|
|
|
jobs = []
|
|
|
|
|
for qhandler in qhandlers :
|
|
|
|
|
qhandler.callback = Components.callback
|
|
|
|
|
job = Process(target=pointer,args=(qhandler,))
|
|
|
|
|
job.start()
|
|
|
|
|
jobs.append(job)
|
|
|
|
|
#
|
|
|
|
|
# let us wait for the jobs
|
|
|
|
|
print (["Started ",len(jobs)," trainers"])
|
|
|
|
|
while len(jobs) > 0 :
|
|
|
|
|
|
|
|
|
|
jobs = [job for job in jobs if job.is_alive()]
|
