You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
SQL = SQL.replace(':dataset',args['dataset']) #+ " LIMIT 1000 "
|
|
|
|
|
_args['gpu'] = args['gpu'] if 'gpu' in args else 0
|
|
|
|
|
|
|
|
|
|
logger.write({"module":"train","action":"setup-partition","input":info})
|
|
|
|
|
|
|
|
|
|
# credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
|
|
|
|
|
# _args['data'] = pd.read_gbq(SQL,credentials=credentials,dialect='standard').dropna()
|
|
|
|
|
reader = args['reader']
|
|
|
|
|
df = reader()
|
|
|
|
|
prefix = args['notify']+'.'+_args['context']
|
|
|
|
|
# if int(args['num_gpu']) > 1 and args['gpu'] > 0:
|
|
|
|
|
jobs = [job for job in jobs if job.is_alive()]
|
