adding autopilot/drone mode for training and automatic data generation

finalize.py

@@ -101,10 +101,15 @@ def move (**args):
     client      = bq.Client.from_service_account_json(private_key)
     config      = Utils.get.config(**args)
     dataset     = args['dataset']
-    SQL         = [ ''.join(["SELECT * FROM io.",item['context'],'_full_io']) for item in config]
+    if 'contexts' in args :
-    SQL         += [Utils.get.sql(**args)]
+        SQL         = [ ''.join(["SELECT * FROM io.",item['context'],'_full_io']) for item in config]
-    SQL         =  ('\n UNION ALL \n'.join(SQL).replace(':dataset','io'))
+        SQL         += [Utils.get.sql(**args)]
-
+        SQL         =  ('\n UNION ALL \n'.join(SQL).replace(':dataset','io'))
+    else:
+        #
+        # moving a table to a designated location
+        tablename = args['from']
+        SQL = "SELECT * FROM :dataset.:table".replace(":dataset",dataset).replace(":table",tablename)
 
     #
     # At this point we have gathered all the tables in the io folder and we should now see if we need to merge with the remainder from the original table
@@ -128,7 +133,7 @@ def move (**args):
     # print (SQL)
     out = client.query(SQL,location='US',job_config=config)
     print ()
-    print (out.job_id)
+    return (out.job_id)
     
     
 
@@ -154,11 +159,24 @@ if __name__ == '__main__' :
         finalize --<move|stats> --contexts <c1,c2,...c3> --from <table>
     """
     if 'move' in SYS_ARGS :
-        table = SYS_ARGS['from']
+        # table = SYS_ARGS['from']
-        contexts = [item['context'] for item in config['pipeline'] if item['from'] == SYS_ARGS['from']]
+        # args = dict(config,**{"private_key":"../curation-prod.json"})
-        args = dict(config,**{"private_key":"../curation-prod.json"})
         args = dict(args,**SYS_ARGS)        
-        args['contexts'] = contexts
+        contexts = [item['context'] for item in config['pipeline'] if item['from'] == SYS_ARGS['from']]
-        move(**args)
+        log = []
+        if contexts :
+            args['contexts'] = contexts
+            log = move(**args)
+            
+        else:
+            tables = args['from'].split(',')
+            for name in tables :
+                name = name.strip()
+                args['from'] = name
+                log += [move(**args)]
+        print ("\n".join(log))
+        
+        
+        
     else:
         print ("NOT YET READY !")

pipeline.py

@@ -117,6 +117,9 @@ class Components :
 		
 		logger.write({"module":"train","action":"train","input":info})
 		data.maker.train(**_args)
+		if set(['drone','autopilot']) in set( list(args.keys())) :
+			print (['drone mode enabled ....'])
+			data.maker.generate(**args)
 
 		pass
 		

setup.py

@@ -4,7 +4,7 @@ import sys
 
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read() 
-args = {"name":"data-maker","version":"1.2.7","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT",
+args = {"name":"data-maker","version":"1.2.8","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT",
         "packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]}
 args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo']
 args['url'] = 'https://hiplab.mc.vanderbilt.edu/git/aou/data-maker.git'