@ -9,7 +9,7 @@ import pandas as pd
from google . oauth2 import service_account from google . oauth2 import service_account
from google . cloud import bigquery as bq from google . cloud import bigquery as bq
import data . maker import data . maker
import copy
from data . params import SYS_ARGS from data . params import SYS_ARGS
# #
@ -69,53 +69,45 @@ class Components :
This function will perform training on the basis of a given pointer that reads data
This function will perform training on the basis of a given pointer that reads data
"""
"""
#
schema = None
# @TODO: we need to log something here about the parameters being passed
if ' file ' in args :
# pointer = args['reader'] if 'reader' in args else lambda: Components.get(**args)
df = args [ ' data ' ]
#
# Now we can parse the arguments and submit the entire thing to training
#
logger = factory . instance ( type = ' mongo.MongoWriter ' , args = { ' dbname ' : ' aou ' , ' doc ' : args [ ' context ' ] } )
df = pd . read_csv ( args [ ' file ' ] )
log_folder = args [ ' logs ' ] if ' logs ' in args else ' logs '
del args [ ' file ' ]
PART_SIZE = int ( args [ ' part_size ' ] )
elif ' data ' not in args :
reader = factory . instance ( * * args [ ' store ' ] [ ' source ' ] )
partition = args [ ' partition ' ]
if ' row_limit ' in args :
log_folder = os . sep . join ( [ log_folder , args [ ' context ' ] , str ( partition ) ] )
df = reader . read ( sql = args [ ' sql ' ] , limit = args [ ' row_limit ' ] )
_args = { " batch_size " : 2000 , " logs " : log_folder , " context " : args [ ' context ' ] , " max_epochs " : 150 , " column " : args [ ' columns ' ] , " id " : " person_id " , " logger " : logger }
else :
_args [ ' max_epochs ' ] = 150 if ' max_epochs ' not in args else int ( args [ ' max_epochs ' ] )
df = reader . read ( sql = args [ ' sql ' ] )
if ' batch_size ' in args :
schema = reader . meta ( table = args [ ' from ' ] ) if hasattr ( reader , ' meta ' ) and ' from ' in args else None
_args [ ' batch_size ' ] = int ( args [ ' batch_size ' ] )
_args [ ' matrix_size ' ] = args [ ' matrix_size ' ] if ' matrix_size ' in args else 128 #
# We ask the process to assume 1 gpu given the system number of GPU and that these tasks can run in parallel
#
if int ( args [ ' num_gpu ' ] ) > 1 :
_args [ ' gpu ' ] = int ( args [ ' gpu ' ] ) if int ( args [ ' gpu ' ] ) < 8 else np . random . choice ( np . arange ( 8 ) ) . astype ( int )
else :
else :
_args [ ' gpu ' ] = 0
df = args [ ' data ' ]
_args [ ' num_gpu ' ] = 1
os . environ [ ' CUDA_VISIBLE_DEVICES ' ] = str ( args [ ' gpu ' ] )
_args [ ' partition ' ] = int ( partition )
# df = df.fillna('')
_args [ ' continuous ' ] = args [ ' continuous ' ] if ' continuous ' in args else [ ]
if schema :
_args [ ' store ' ] = { ' type ' : ' mongo.MongoWriter ' , ' args ' : { ' dbname ' : ' aou ' , ' doc ' : args [ ' context ' ] } }
_schema = { }
_args [ ' data ' ] = args [ ' data ' ]
for _item in schema :
_type = int
# print (['partition ',partition,df.value_source_concept_id.unique()])
_value = 0
#
if _item . field_type == ' FLOAT ' :
# @log :
_type = float
# Logging information about the training process for this partition (or not)
elif _item . field_type != ' INTEGER ' :
#
_type = str
_value = ' '
info = { " rows " : df . shape [ 0 ] , " cols " : df . shape [ 1 ] , " partition " : int ( partition ) , " logs " : _args [ ' logs ' ] }
_schema [ _item . name ] = _type
df [ _item . name ] = df [ _item . name ] . fillna ( _value ) . astype ( _type )
args [ ' schema ' ] = _schema
# df[_item.name] = df[_item.name].astype(_type)
_args = copy . deepcopy ( args )
# _args['store'] = args['store']['source']
_args [ ' data ' ] = df
logger . write ( { " module " : " train " , " action " : " train " , " input " : info } )
data . maker . train ( * * _args )
data . maker . train ( * * _args )
if ' autopilot ' in ( list ( args . keys ( ) ) ) :
if ' autopilot ' in ( list ( args . keys ( ) ) ) :
print ( [ ' autopilot mode enabled .... ' ] )
print ( [ ' autopilot mode enabled .... ' ,args [ ' context ' ] ])
self . generate ( args )
self . generate ( args )
pass
pass
@ -129,141 +121,167 @@ class Components :
"""
"""
This function will generate data and store it to a given ,
This function will generate data and store it to a given ,
"""
"""
logger = factory . instance ( type = ' mongo.MongoWriter ' , args = { ' dbname ' : ' aou ' , ' doc ' : args [ ' context ' ] } )
store = args [ ' store ' ] [ ' logs ' ]
log_folder = args [ ' logs ' ] if ' logs ' in args else ' logs '
store [ ' doc ' ] = args [ ' context ' ]
partition = args [ ' partition ' ] if ' partition ' in args else ' '
logger = factory . instance ( * * store ) #type='mongo.MongoWriter',args={'dbname':'aou','doc':args['context']})
log_folder = os . sep . join ( [ log_folder , args [ ' context ' ] , str ( partition ) ] )
ostore = args [ ' store ' ] [ ' target ' ]
_args = { " batch_size " : 2000 , " logs " : log_folder , " context " : args [ ' context ' ] , " max_epochs " : 150 , " column " : args [ ' columns ' ] , " id " : " person_id " , " logger " : logger }
writer = factory . instance ( * * ostore )
_args [ ' max_epochs ' ] = 150 if ' max_epochs ' not in args else int ( args [ ' max_epochs ' ] )
# log_folder = args['logs'] if 'logs' in args else 'logs'
# partition = args['partition'] if 'partition' in args else ''
# log_folder = os.sep.join([log_folder,args['context'],str(partition)])
# _args = {"batch_size":2000,"logs":log_folder,"context":args['context'],"max_epochs":150,"column":args['columns'],"id":"person_id","logger":logger}
# _args['max_epochs'] = 150 if 'max_epochs' not in args else int(args['max_epochs'])
# _args['num_gpu'] = int(args['num_gpu']) if 'num_gpu' in args else 1
# _args['num_gpu'] = int(args['num_gpu']) if 'num_gpu' in args else 1
if ' batch_size ' in args :
# if 'batch_size' in args
_args [ ' batch_size ' ] = int ( args [ ' batch_size ' ] )
#
if int ( args [ ' num_gpu ' ] ) > 1 :
# if int(args['num_gpu']) > 1
#
else
# else
#
_args [ ' num_gpu ' ] =
# _args['num_gpu'] =
os . environ [ ' CUDA_VISIBLE_DEVICES ' ] = str ( args [ ' gpu ' ] )
# os.environ['CUDA_VISIBLE_DEVICES'] = str(args['gpu'])
# _args['no_value']= args['no_value']
# # _args['no_value']= args['no_value']
_args [ ' matrix_size ' ] = args [ ' matrix_size ' ] if ' matrix_size ' in args else
# _args['matrix_size'] = args['matrix_size'] if 'matrix_size' in args else
# MAX_ROWS = args['max_rows'] if 'max_rows' in args else 0
# # MAX_ROWS = args['max_rows'] if 'max_rows' in args else 0
PART_SIZE = int ( args [ ' part_size ' ] ) if ' part_size ' in args else
# PART_SIZE = int(args['part_size']) if 'part_size' in args else
# credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
# credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
# _args['data'] = pd.read_gbq(SQL,credentials=credentials,dialect='standard').dropna()
# _args['data'] = pd.read_gbq(SQL,credentials=credentials,dialect='standard').dropna()
# reader = args['reader']
# reader = args['reader']
# df = reader()
# df = reader()
df = args [ ' reader ' ] ( ) if ' reader ' in args else args [ ' data ' ]
schema = args [ ' schema ' ] if ' schema ' in args else None
if ' file ' in args :
# if 'slice' in args and 'max_rows' in args['slice']:
# max_rows = args['slice']['max_rows']
# if df.shape[0] > max_rows :
# print (".. slicing ")
# i = np.random.choice(df.shape[0],max_rows,replace=False)
# df = df.iloc[i]
# bounds = Components.split(df,MAX_ROWS,PART_SIZE)
# if partition != '' :
# columns = args['columns']
# df = np.array_split(df[columns].values,PART_SIZE)
# df = pd.DataFrame(df[ int (partition) ],columns = columns)
# max_rows = int(args['partition_max_rows']) if 'partition_max_rows' in args else 1000000
# N = np.divide(df.shape[0],max_rows).astype(int) + 1
info = { " name " : args [ ' columns ' ] , " parition " : int ( partition ) , " gpu " : _args [ " gpu " ] , " rows " : int ( df . shape [ 0 ] ) , " cols " : int ( df . shape [ 1 ] ) , " space " : df [ args [ ' columns ' ] [ 0 ] ] . unique ( ) . size , " part_size " : int ( PART_SIZE ) }
logger . write ( { " module " : " generate " , " action " : " partition " , " input " : info } )
_args [ ' partition ' ] = int ( partition )
_args [ ' continuous ' ] = args [ ' continuous ' ] if ' continuous ' in args else [ ]
#
# How many rows sub-partition must we divide this into ?
# let us fix the data types here every _id field will be an np.int64...
#
schema = args [ ' schema ' ]
for item in schema :
if item . field_type == ' INTEGER ' and df [ item . name ] . dtype != np . int64 :
df [ item . name ] = np . array ( df [ item . name ] . values , dtype = np . int64 )
elif item . field_type == ' STRING ' and df [ item . name ] . dtype != object :
df [ item . name ] = np . array ( df [ item . name ] , dtype = object )
df = pd . read_csv ( args [ ' file ' ] )
else :
if ' data ' not in args :
reader = factory . instance ( * * args [ ' store ' ] [ ' source ' ] )
if ' row_limit ' in args :
df = reader . read ( sql = args [ ' sql ' ] , limit = args [ ' row_limit ' ] )
else :
df = reader . read ( sql = args [ ' sql ' ] )
if ' schema ' not in args and hasattr ( reader , ' meta ' ) :
schema = reader . meta ( table = args [ ' from ' ] )
# for name in df.columns.tolist():
else :
#
# This will account for autopilot mode ...
df = args [ ' data ' ]
# if name.endswith('_id') :
_info = { " module " : " gan-prep " , " action " : " read " , " shape " : { " rows " : df . shape [ 0 ] , " columns " : df . shape [ 0 ] } }
# if df[name].isnull().sum() > 0 and name not in ['unique_device_id']:
# df[name].fillna(np.nan_to_num(np.nan),inplace=True)
# df[name] = df[name].astype(int)
_dc = pd . DataFrame ( )
_dc = pd . DataFrame ( )
# for mdf in df :
# for mdf in df :
_args [ ' data ' ] = df
args [ ' data ' ] = df
args [ ' candidates ' ] = 1 if ' candidates ' not in args else int ( args [ ' candidates ' ] )
_dc = _dc . append ( data . maker . generate ( * * _args ) )
candidates = ( data . maker . generate ( * * args ) )
#
if ' sql.BQWriter ' in ostore [ ' type ' ] :
# We need to post the generate the data in order to :
#table = ".".join([ostore['['dataset'],args['context']])
# 1. compare immediately
# writer = factory.instance(**ostore)
# 2. synthetic copy
_columns = None
#
skip_columns = [ ]
_schema = [ { " name " : field . name , " type " : field . field_type , " description " : field . description } for field in schema ] if schema else [ ]
cols = _dc . columns . tolist ( )
for _df in candidates :
#
data_comp = _args [ ' data ' ] [ args [ ' columns ' ] ] . join ( _dc [ args [ ' columns ' ] ] , rsuffix = ' _io ' ) #-- will be used for comparison (store this in big query)
# we need to format the fields here to make sure we have something cohesive
#
#
# performing basic analytics on the synthetic data generated (easy to quickly asses)
#
if not skip_columns :
info = { " module " : " generate " , " action " : " io.metrics " , " input " : { " rows " : data_comp . shape [ 0 ] , " partition " : partition , " logs " : [ ] } }
# _columns = set(df.columns) - set(_df.columns)
if ' ignore ' in args and ' columns ' in args [ ' ignore ' ] :
#
# @TODO: Send data over to a process for analytics
for name in args [ ' ignore ' ] [ ' columns ' ] :
for _name in _df . columns :
base_cols = list ( set ( _args [ ' data ' ] . columns ) - set ( args [ ' columns ' ] ) ) #-- rebuilt the dataset (and store it)
if _name in name :
cols = _dc . columns . tolist ( )
skip_columns . append ( _name )
for name in cols :
#
_args [ ' data ' ] [ name ] = _dc [ name ]
# We perform a series of set operations to insure that the following conditions are met:
# - the synthetic dataset only has fields that need to be synthesized
#
# - The original dataset has all the fields except those that need to be synthesized
#-- Let us store all of this into bigquery
#
prefix = args [ ' notify ' ] + ' . ' + _args [ ' context ' ]
partition = str ( partition )
_df = _df [ list ( set ( _df . columns ) - set ( skip_columns ) ) ]
table = ' _ ' . join ( [ prefix , partition , ' io ' ] ) . replace ( ' __ ' , ' _ ' )
folder = os . sep . join ( [ args [ ' logs ' ] , args [ ' context ' ] , partition , ' output ' ] )
if set ( df . columns ) & set ( _df . columns ) :
if ' file ' in args :
_columns = set ( df . columns ) - set ( _df . columns )
df = df [ _columns ]
_fname = os . sep . join ( [ folder , table . replace ( ' _io ' , ' _full_io.csv ' ) ] )
_pname = os . sep . join ( [ folder , table ] ) + ' .csv '
#
data_comp . to_csv ( _pname , index = False )
# Let us merge the dataset here and and have a comprehensive dataset
_args [ ' data ' ] . to_csv ( _fname , index = False )
_df = pd . DataFrame . join ( df , _df )
_id = ' path '
writer . write ( _df , schema = _schema , table = args [ ' from ' ] )
# writer.write(df,table=table)
pass
else :
else :
pass
credentials = service_account . Credentials . from_service_account_file ( ' /home/steve/dev/aou/accounts/curation-prod.json ' )
_pname = os . sep . join ( [ folder , table + ' .csv ' ] )
_fname = table . replace ( ' _io ' , ' _full_io ' )
# #
partial = ' . ' . join ( [ ' io ' , args [ ' context ' ] + ' _partial_io ' ] )
# # We need to post the generate the data in order to :
complete = ' . ' . join ( [ ' io ' , args [ ' context ' ] + ' _full_io ' ] )
# # 1. compare immediately
data_comp . to_csv ( _pname , index = False )
# # 2. synthetic copy
if ' dump ' in args :
# #
print ( _args [ ' data ' ] . head ( ) )
else :
# cols = _dc.columns.tolist()
Components . lock . acquire ( )
data_comp . to_gbq ( if_exists = ' append ' , destination_table = partial , credentials = credentials , chunksize = 90000 )
# data_comp = _args['data'][args['columns']].join(_dc[args['columns']],rsuffix='_io') #-- will be used for comparison (store this in big query)
_args [ ' data ' ] . to_gbq ( if_exists = ' append ' , destination_table = complete , credentials = credentials , chunksize = 90000 )
# #
Components . lock . release ( )
# # performing basic analytics on the synthetic data generated (easy to quickly asses)
_id = ' dataset '
# #
info = { " full " : { _id : _fname , " rows " : _args [ ' data ' ] . shape [ 0 ] } , " partial " : { " path " : _pname , " rows " : data_comp . shape [ 0 ] } }
# info = {"module":"generate","action":"io.metrics","input":{"rows":data_comp.shape[0],"partition":partition,"logs":[]}}
if partition :
info [ ' partition ' ] = int ( partition )
# #
logger . write ( { " module " : " generate " , " action " : " write " , " input " : info } )
# # @TODO: Send data over to a process for analytics
# base_cols = list(set(_args['data'].columns) - set(args['columns'])) #-- rebuilt the dataset (and store it)
# cols = _dc.columns.tolist()
# for name in cols :
# _args['data'][name] = _dc[name]
# #
# #-- Let us store all of this into bigquery
# prefix = args['notify']+'.'+_args['context']
# partition = str(partition)
# table = '_'.join([prefix,partition,'io']).replace('__','_')
# folder = os.sep.join([args['logs'],args['context'],partition,'output'])
# if 'file' in args :
# _fname = os.sep.join([folder,table.replace('_io','_full_io.csv')])
# _pname = os.sep.join([folder,table])+'.csv'
# data_comp.to_csv( _pname,index=False)
# _args['data'].to_csv(_fname,index=False)
# _id = 'path'
# else:
# credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
# _pname = os.sep.join([folder,table+'.csv'])
# _fname = table.replace('_io','_full_io')
# partial = '.'.join(['io',args['context']+'_partial_io'])
# complete= '.'.join(['io',args['context']+'_full_io'])
# data_comp.to_csv(_pname,index=False)
# if 'dump' in args :
# print (_args['data'].head())
# else:
# Components.lock.acquire()
# data_comp.to_gbq(if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)
# _args['data'].to_gbq(if_exists='append',destination_table=complete,credentials=credentials,chunksize=90000)
# Components.lock.release()
# _id = 'dataset'
# info = {"full":{_id:_fname,"rows":_args['data'].shape[0]},"partial":{"path":_pname,"rows":data_comp.shape[0]} }
# if partition :
# info ['partition'] = int(partition)
# logger.write({"module":"generate","action":"write","input":info} )
@ -308,98 +326,95 @@ if __name__ == '__main__' :
# Log what was initiated so we have context of this processing ...
# Log what was initiated so we have context of this processing ...
#
#
# if 'listen' not in SYS_ARGS :
# if 'listen' not in SYS_ARGS :
if ' file ' in args
# if 'file' in args
#
#
else
# else
#
#
#
COLUMNS = DATA .
# COLUMNS = DATA.
DATA = np . array_split ( DATA , PART_SIZE
# DATA = np.array_split(DATA,PART_SIZE
args [ ' schema ' ] =
# args['schema'] =
if ' generate ' in SYS_ARGS :
if ' generate ' in SYS_ARGS :
#
#
# Let us see if we have partitions given the log folder
# Let us see if we have partitions given the log folder
content = os . listdir ( os . sep . join ( [ args [ ' logs ' ] , args [ ' context ' ] ] ) )
content = os . listdir ( os . sep . join ( [ args [ ' logs ' ] , ' train ' , args [ ' context ' ] ] ) )
generator = Components ( )
generator = Components ( )
if ' ' . join ( content ) . isnumeric ( )
# if ''.join(content).isnumeric()
# #
# # we have partitions we are working with
#
# # columns = DATA.columns.tolist()
# # DATA = np.array_split(DATA,PART_SIZE)
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
else
# else
#
# Components.generate(args)
# Components.generate(args)
elif ' shuffle ' in SYS_ARGS :
generator . generate ( args )
for data in DATA :
args [ ' data ' ] = data
_df = ( Components ( ) ) . shuffle ( args )
else :
else :
# DATA = np.array_split(DATA,PART_SIZE)
# DATA = np.array_split(DATA,PART_SIZE)
agent = Components ( )
jobs = [ ]
agent . train ( * * args )
for index in range ( 0 , PART_SIZE ) :
# jobs = []
if ' focus ' in args and int ( args [ ' focus ' ] ) != index :
# for index in range(0,PART_SIZE) :
continue
# if 'focus' in args and int(args['focus']) != index :
args [ ' part_size ' ] = PART_SIZE
# continue
args [ ' partition ' ] = index
# args['part_size'] = PART_SIZE
args [ ' data ' ] = DATA [ index ]
# args['partition'] = index
if int ( args [ ' num_gpu ' ] ) > 1 :
# args['data'] = DATA[index]
args [ ' gpu ' ] = index
# if int(args['num_gpu']) > 1 :
else :
# args['gpu'] = index
args [ ' gpu ' ] = 0
# else:
# args['gpu']=0
make = lambda _args : ( Components ( ) ) . train ( * * _args )
job = Process ( target = make , args = ( dict ( args ) , ) )
# make = lambda _args: (Components()).train(**_args)
job . name = ' Trainer # ' + str ( index )
# job = Process(target=make,args=( dict(args),))
job . start ( )
# job.name = 'Trainer # ' + str(index)
jobs . append ( job )
# job.start()
# args['gpu']
# jobs.append(job)
print ( [ " Started " , len ( jobs ) , " trainers " if len ( jobs ) > 1 else " trainer " ] )
# # args['gpu']
while len ( jobs ) > 0 :
# print (["Started ",len(jobs),"trainers" if len(jobs)>1 else "trainer" ])
jobs = [ job for job in jobs if job . is_alive ( ) ]
# while len(jobs)> 0 :
time . sleep ( 2 )
# jobs = [job for job in jobs if job.is_alive()]
# time.sleep(2)
# trainer = Components()
# trainer = Components()
# trainer.train(**args)
# trainer.train(**args)
