|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
class KEYS :
|
|
|
|
|
logger.write({"module":"bigquery","action":"read","input":{"sql":SQL}})
|
|
|
|
|
|
|
|
|
|
for name in columns:
|
|
|
|
|
"ON y.row_number = x.row_number"
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
|
|
|
|
|
# _pname = os.sep.join([folder,table+'.csv'])
|
|
|
|
|
# _fname = table.replace('_io','_full_io')
|
|
|
|
|
# partial = '.'.join(['io',args['context']+'_partial_io'])
|
|
|
|
|
# complete= '.'.join(['io',args['context']+'_full_io'])
|
|
|
|
|
# data_comp.to_csv(_pname,index=False)
|
|
|
|
|
# if 'dump' in args :
|
|
|
|
|
# print (_args['data'].head())
|
|
|
|
|
# else:
|
|
|
|
|
# Components.lock.acquire()
|
|
|
|
|
# data_comp.to_gbq(if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)
|
|
|
|
|
# _args['data'].to_gbq(if_exists='append',destination_table=complete,credentials=credentials,chunksize=90000)
|
|
|
|
|
# Components.lock.release()
|
|
|
|
|
# _id = 'dataset'
|
|
|
|
|
# info = {"full":{_id:_fname,"rows":_args['data'].shape[0]},"partial":{"path":_pname,"rows":data_comp.shape[0]} }
|
|
|
|
|
# if partition :
|
|
|
|
|
# info ['partition'] = int(partition)
|
|
|
|
|
# logger.write({"module":"generate","action":"write","input":info} )
|
|
|
|
|
if key == 'pipeline' or key in args:
|
|
|
|
|
for _gpu in GPU_CHIPS :
|
|
|
|
|
_args['partition'] = int(_gpu) #index
|
|
|
|
|
index += 1
|
|
|
|
|
make = lambda _params: (Components()).train(**_params)
|
|
|
|
|
# continue
|
