|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
@staticmethod
|
|
|
|
|
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
|
|
|
|
|
"""
|
|
|
|
|
return np.array(x)
|
|
|
|
|
args['data'] = df
|
|
|
|
|
#
|
|
|
|
|
# The columns that are continuous should also be skipped because they don't need to be synthesied (like-that)
|
|
|
|
|
if 'continuous' in args :
|
|
|
|
|
x_cols = args['continuous']
|
|
|
|
|
else:
|
|
|
|
|
x_cols = []
|
|
|
|
|
|
|
|
|
|
# skip_columns.append(_name)
|
|
|
|
|
|
|
|
|
|
# data_comp.to_csv(_pname,index=False)
|
|
|
|
|
# if 'dump' in args :
|
|
|
|
|
# print (_args['data'].head())
|
|
|
|
|
# else:
|
|
|
|
|
# Components.lock.acquire()
|
|
|
|
|
# data_comp.to_gbq(if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)
|
|
|
|
|
# _args['data'].to_gbq(if_exists='append',destination_table=complete,credentials=credentials,chunksize=90000)
|
|
|
|
|
# Components.lock.release()
|
|
|
|
|
# _id = 'dataset'
|
|
|
|
|
# info = {"full":{_id:_fname,"rows":_args['data'].shape[0]},"partial":{"path":_pname,"rows":data_comp.shape[0]} }
|
|
|
|
|
# if partition :
|
|
|
|
|
# info ['partition'] = int(partition)
|
|
|
|
|
# logger.write({"module":"generate","action":"write","input":info} )
|
|
|
|
|
if 'matrix_size' in args :
|
|
|
|
|
# jobs = []
|
