|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
pipeline = args['pipeline']
|
|
|
|
|
return [ item for item in pipeline if item['context'] in contexts]
|
|
|
|
|
@staticmethod
|
|
|
|
|
def sql(**args) :
|
|
|
|
|
"""
|
|
|
|
|
This function is intended to build SQL query for the remainder of the table that was not synthesized
|
|
|
|
|
:config configuration entries
|
|
|
|
|
:from source of the table name
|
|
|
|
|
:dataset name of the source dataset
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
SQL = ["SELECT * FROM :from "]
|
|
|
|
|
SQL_FILTER = []
|
|
|
|
|
NO_FILTERS_FOUND = True
|
|
|
|
|
pipeline = Utils.get.config(**args)
|
|
|
|
|
REVERSE_QUALIFIER = {'IN':'NOT IN','NOT IN':'IN','=':'<>','<>':'='}
|
|
|
|
|
for item in pipeline :
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if 'filter' in item :
|
|
|
|
|
if NO_FILTERS_FOUND :
|
|
|
|
|
NO_FILTERS_FOUND = False
|
|
|
|
|
SQL += ['WHERE']
|
|
|
|
|
#
|
|
|
|
|
# Let us load the filter in the SQL Query
|
|
|
|
|
FILTER = item['filter']
|
|
|
|
|
QUALIFIER = REVERSE_QUALIFIER[FILTER['qualifier'].upper()]
|
|
|
|
|
#
|
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
schema = client.get_table(client.dataset(args['dataset']).table(args['from'])).schema
|
|
|
|
|
fields = [" ".join(["CAST (",item.name,"AS",item.field_type.replace("INTEGER","INT64").replace("FLOAT","FLOAT64"),") ",item.name]) for item in schema]
|
|
|
|
|
SQL = SQL.replace("*"," , ".join(fields))
|
|
|
|
|
# print (SQL)
|
|
|
|
|
out = client.query(SQL,location='US',job_config=config)
|
|
|
|
|
tables = args['from'].split(',')
|
