|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
@staticmethod
|
|
|
|
|
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
|
|
|
|
|
else :
|
|
|
|
|
if 'ignore' in args and 'columns' in args['ignore'] :
|
|
|
|
|
else :
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# _pname = os.sep.join([folder,table+'.csv'])
|
|
|
|
|
# _fname = table.replace('_io','_full_io')
|
|
|
|
|
# partial = '.'.join(['io',args['context']+'_partial_io'])
|
|
|
|
|
# complete= '.'.join(['io',args['context']+'_full_io'])
|
|
|
|
|
# data_comp.to_csv(_pname,index=False)
|
|
|
|
|
# if 'dump' in args :
|
|
|
|
|
# print (_args['data'].head())
|
|
|
|
|
# else:
|
|
|
|
|
# Components.lock.acquire()
|
|
|
|
|
# data_comp.to_gbq(if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)
|
|
|
|
|
# _args['data'].to_gbq(if_exists='append',destination_table=complete,credentials=credentials,chunksize=90000)
|
|
|
|
|
# Components.lock.release()
|
|
|
|
|
# _id = 'dataset'
|
|
|
|
|
# info = {"full":{_id:_fname,"rows":_args['data'].shape[0]},"partial":{"path":_pname,"rows":data_comp.shape[0]} }
|
|
|
|
|
# if partition :
|
|
|
|
|
# info ['partition'] = int(partition)
|
|
|
|
|
# logger.write({"module":"generate","action":"write","input":info} )
|
|
|
|
|
generator.generate(args)
|
