You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
data-maker/pipeline.py

693 lines
23 KiB
Python

#!/usr/bin/env python3
5 years ago
class KEYS :
logger.write({"module":"bigquery","action":"read","input":{"sql":SQL}})
4 years ago
#
data.maker.train(**_args)
# randomly shifting the measurements
columns = args['columns'] if 'columns' in args else df.columns
def finalize(self,args):
#
otable = ".".join([args['store']['source']['args']['dataset'],args['from']])
dtable = ".".join([args['store']['target']['args']['dataset'],args['from']])
for _item in schema :
4 years ago
#
#
_params = {'data':_df,'store' : ostore,'from':args['from']}
if _schema :
_params ['schema'] = _schema
_info = {"module":"gan-prep","action":"write","input":{"rows":_df.shape[0],"cols":_df.shape[1]}}
logger.write(_info)
self.post(**_params)
# print (['_______ posting _________________',_df.shape])
break
if 'matrix_size' in args :
print (_args['store'])
else: