You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
data-maker/pipeline.py

446 lines
15 KiB
Python

#!/usr/bin/env python3
5 years ago
SQL_FILTER = 'filter'
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
if 'batch_size' in args :
log_folder = os.sep.join([log_folder,args['context'],str(partition)])
5 years ago
# reader = args['reader']
5 years ago
if partition :
if 'dataset' not in args :
args['gpu'] = index