adding autopilot/drone mode for training and automatic data generation

dev
Steve L. Nyemba 5 years ago
parent e4b164a34b
commit 0016aec576

@ -101,10 +101,15 @@ def move (**args):
client = bq.Client.from_service_account_json(private_key) client = bq.Client.from_service_account_json(private_key)
config = Utils.get.config(**args) config = Utils.get.config(**args)
dataset = args['dataset'] dataset = args['dataset']
if 'contexts' in args :
SQL = [ ''.join(["SELECT * FROM io.",item['context'],'_full_io']) for item in config] SQL = [ ''.join(["SELECT * FROM io.",item['context'],'_full_io']) for item in config]
SQL += [Utils.get.sql(**args)] SQL += [Utils.get.sql(**args)]
SQL = ('\n UNION ALL \n'.join(SQL).replace(':dataset','io')) SQL = ('\n UNION ALL \n'.join(SQL).replace(':dataset','io'))
else:
#
# moving a table to a designated location
tablename = args['from']
SQL = "SELECT * FROM :dataset.:table".replace(":dataset",dataset).replace(":table",tablename)
# #
# At this point we have gathered all the tables in the io folder and we should now see if we need to merge with the remainder from the original table # At this point we have gathered all the tables in the io folder and we should now see if we need to merge with the remainder from the original table
@ -128,7 +133,7 @@ def move (**args):
# print (SQL) # print (SQL)
out = client.query(SQL,location='US',job_config=config) out = client.query(SQL,location='US',job_config=config)
print () print ()
print (out.job_id) return (out.job_id)
@ -154,11 +159,24 @@ if __name__ == '__main__' :
finalize --<move|stats> --contexts <c1,c2,...c3> --from <table> finalize --<move|stats> --contexts <c1,c2,...c3> --from <table>
""" """
if 'move' in SYS_ARGS : if 'move' in SYS_ARGS :
table = SYS_ARGS['from'] # table = SYS_ARGS['from']
contexts = [item['context'] for item in config['pipeline'] if item['from'] == SYS_ARGS['from']] # args = dict(config,**{"private_key":"../curation-prod.json"})
args = dict(config,**{"private_key":"../curation-prod.json"})
args = dict(args,**SYS_ARGS) args = dict(args,**SYS_ARGS)
contexts = [item['context'] for item in config['pipeline'] if item['from'] == SYS_ARGS['from']]
log = []
if contexts :
args['contexts'] = contexts args['contexts'] = contexts
move(**args) log = move(**args)
else:
tables = args['from'].split(',')
for name in tables :
name = name.strip()
args['from'] = name
log += [move(**args)]
print ("\n".join(log))
else: else:
print ("NOT YET READY !") print ("NOT YET READY !")

@ -117,6 +117,9 @@ class Components :
logger.write({"module":"train","action":"train","input":info}) logger.write({"module":"train","action":"train","input":info})
data.maker.train(**_args) data.maker.train(**_args)
if set(['drone','autopilot']) in set( list(args.keys())) :
print (['drone mode enabled ....'])
data.maker.generate(**args)
pass pass

@ -4,7 +4,7 @@ import sys
def read(fname): def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = {"name":"data-maker","version":"1.2.7","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT", args = {"name":"data-maker","version":"1.2.8","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT",
"packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]} "packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]}
args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo'] args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo']
args['url'] = 'https://hiplab.mc.vanderbilt.edu/git/aou/data-maker.git' args['url'] = 'https://hiplab.mc.vanderbilt.edu/git/aou/data-maker.git'

Loading…
Cancel
Save