You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
data-maker/pipeline.py

702 lines
23 KiB
Python

#!/usr/bin/env python3
5 years ago
class KEYS :
logger.write({"module":"bigquery","action":"read","input":{"sql":SQL}})
if 'autopilot' in ( list(args.keys())) :
else:
if df[name].unique().size > 0 :
4 years ago
writer.write(_df,table=args['from'])
_cast = {}
4 years ago
for _df in candidates :
if 'batch_size' not in args :
for index in GPU_CHIPS :
jobs = [job for job in jobs if job.is_alive()]