@ -101,10 +101,15 @@ def move (**args):
client = bq . Client . from_service_account_json ( private_key )
config = Utils . get . config ( * * args )
dataset = args [ ' dataset ' ]
if ' contexts ' in args :
SQL = [ ' ' . join ( [ " SELECT * FROM io. " , item [ ' context ' ] , ' _full_io ' ] ) for item in config ]
SQL + = [ Utils . get . sql ( * * args ) ]
SQL = ( ' \n UNION ALL \n ' . join ( SQL ) . replace ( ' :dataset ' , ' io ' ) )
else :
#
# moving a table to a designated location
tablename = args [ ' from ' ]
SQL = " SELECT * FROM :dataset.:table " . replace ( " :dataset " , dataset ) . replace ( " :table " , tablename )
#
# At this point we have gathered all the tables in the io folder and we should now see if we need to merge with the remainder from the original table
@ -128,7 +133,7 @@ def move (**args):
# print (SQL)
out = client . query ( SQL , location = ' US ' , job_config = config )
print ( )
print ( out . job_id )
return ( out . job_id )
@ -154,11 +159,24 @@ if __name__ == '__main__' :
finalize - - < move | stats > - - contexts < c1 , c2 , . . . c3 > - - from < table >
"""
if ' move ' in SYS_ARGS :
table = SYS_ARGS [ ' from ' ]
contexts = [ item [ ' context ' ] for item in config [ ' pipeline ' ] if item [ ' from ' ] == SYS_ARGS [ ' from ' ] ]
args = dict ( config , * * { " private_key " : " ../curation-prod.json " } )
# table = SYS_ARGS['from']
# args = dict(config,**{"private_key":"../curation-prod.json"})
args = dict ( args , * * SYS_ARGS )
contexts = [ item [ ' context ' ] for item in config [ ' pipeline ' ] if item [ ' from ' ] == SYS_ARGS [ ' from ' ] ]
log = [ ]
if contexts :
args [ ' contexts ' ] = contexts
move ( * * args )
log = move ( * * args )
else :
tables = args [ ' from ' ] . split ( ' , ' )
for name in tables :
name = name . strip ( )
args [ ' from ' ] = name
log + = [ move ( * * args ) ]
print ( " \n " . join ( log ) )
else :
print ( " NOT YET READY ! " )
