From 0016aec576004c562b9901cfd8478f850360bec2 Mon Sep 17 00:00:00 2001
From: Steve Nyemba <steve@the-phi.com>
Date: Sat, 11 Apr 2020 13:10:01 -0500
Subject: [PATCH] adding autopilot/drone mode for training and automatic data
 generation

---
 finalize.py | 38 ++++++++++++++++++++++++++++----------
 pipeline.py |  3 +++
 setup.py    |  2 +-
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/finalize.py b/finalize.py
index b163b0d..079830d 100644
--- a/finalize.py
+++ b/finalize.py
@@ -101,10 +101,15 @@ def move (**args):
     client      = bq.Client.from_service_account_json(private_key)
     config      = Utils.get.config(**args)
     dataset     = args['dataset']
-    SQL         = [ ''.join(["SELECT * FROM io.",item['context'],'_full_io']) for item in config]
-    SQL         += [Utils.get.sql(**args)]
-    SQL         =  ('\n UNION ALL \n'.join(SQL).replace(':dataset','io'))
-
+    if 'contexts' in args :
+        SQL         = [ ''.join(["SELECT * FROM io.",item['context'],'_full_io']) for item in config]
+        SQL         += [Utils.get.sql(**args)]
+        SQL         =  ('\n UNION ALL \n'.join(SQL).replace(':dataset','io'))
+    else:
+        #
+        # moving a table to a designated location
+        tablename = args['from']
+        SQL = "SELECT * FROM :dataset.:table".replace(":dataset",dataset).replace(":table",tablename)
 
     #
     # At this point we have gathered all the tables in the io folder and we should now see if we need to merge with the remainder from the original table
@@ -128,7 +133,7 @@ def move (**args):
     # print (SQL)
     out = client.query(SQL,location='US',job_config=config)
     print ()
-    print (out.job_id)
+    return (out.job_id)
     
     
 
@@ -154,11 +159,24 @@ if __name__ == '__main__' :
         finalize --<move|stats> --contexts <c1,c2,...c3> --from <table>
     """
     if 'move' in SYS_ARGS :
-        table = SYS_ARGS['from']
+        # table = SYS_ARGS['from']
+        # args = dict(config,**{"private_key":"../curation-prod.json"})
+        args = dict(args,**SYS_ARGS)        
         contexts = [item['context'] for item in config['pipeline'] if item['from'] == SYS_ARGS['from']]
-        args = dict(config,**{"private_key":"../curation-prod.json"})
-        args = dict(args,**SYS_ARGS)
-        args['contexts'] = contexts
-        move(**args)
+        log = []
+        if contexts :
+            args['contexts'] = contexts
+            log = move(**args)
+            
+        else:
+            tables = args['from'].split(',')
+            for name in tables :
+                name = name.strip()
+                args['from'] = name
+                log += [move(**args)]
+        print ("\n".join(log))
+        
+        
+        
     else:
         print ("NOT YET READY !")
\ No newline at end of file
diff --git a/pipeline.py b/pipeline.py
index 3950942..e54e746 100644
--- a/pipeline.py
+++ b/pipeline.py
@@ -117,6 +117,9 @@ class Components :
 		
 		logger.write({"module":"train","action":"train","input":info})
 		data.maker.train(**_args)
+		if set(['drone','autopilot']) in set( list(args.keys())) :
+			print (['drone mode enabled ....'])
+			data.maker.generate(**args)
 
 		pass
 		
diff --git a/setup.py b/setup.py
index 939c241..71e14e0 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@ import sys
 
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read() 
-args = {"name":"data-maker","version":"1.2.7","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT",
+args = {"name":"data-maker","version":"1.2.8","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT",
         "packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]}
 args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo']
 args['url'] = 'https://hiplab.mc.vanderbilt.edu/git/aou/data-maker.git'