You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
@staticmethod
|
|
|
|
|
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
|
|
|
|
|
else :
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"io":{"min":_df[_col].min(),"max":_df[_col].max(),"mean":_df[_col].mean(),"sd":_df[_col].values.std(),"missing": _df[_col].where(_df[_col] == -1).dropna().count(),"zeros":_df[_col].where(_df[_col] == 0).dropna().count()},
|
|
|
|
|
# Components.lock.acquire()
|
|
|
|
|
# data_comp.to_gbq(if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)
|
|
|
|
|
# _args['data'].to_gbq(if_exists='append',destination_table=complete,credentials=credentials,chunksize=90000)
|
|
|
|
|
# Components.lock.release()
|
|
|
|
|
# _id = 'dataset'
|
|
|
|
|
# info = {"full":{_id:_fname,"rows":_args['data'].shape[0]},"partial":{"path":_pname,"rows":data_comp.shape[0]} }
|
|
|
|
|
# if partition :
|
|
|
|
|
# info ['partition'] = int(partition)
|
|
|
|
|
# logger.write({"module":"generate","action":"write","input":info} )
|
|
|
|
|
if 'matrix_size' in args :
|
