You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
data-maker/pipeline.py

509 lines
16 KiB
Python

#!/usr/bin/env python3
5 years ago
@staticmethod
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
else :
4 years ago
if 'ignore' in args and 'columns' in args['ignore'] :
4 years ago
skip_columns = []
_schema = schema
pass
if _schema :
writer.write(_df[cols],schema=_schema,table=args['from'])
else:
writer.write(_df[cols],table=args['from'])
# folder = os.sep.join([args['logs'],args['context'],partition,'output'])
# if 'file' in args :
5 years ago
#process.start()