From a73e186f77d174056063760af65fbd25ad29ee44 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Thu, 1 Apr 2021 13:20:35 -0500 Subject: [PATCH] gpu indexing --- data/gan.py | 4 ++- data/maker/__init__.py | 59 +++--------------------------------------- pipeline.py | 2 ++ 3 files changed, 9 insertions(+), 56 deletions(-) diff --git a/data/gan.py b/data/gan.py index 985e706..dd8ea6a 100644 --- a/data/gan.py +++ b/data/gan.py @@ -64,7 +64,9 @@ class GNet : self.GPU_CHIPS = [0] if 'CUDA_VISIBLE_DEVICES' in os.environ : os.environ.pop('CUDA_VISIBLE_DEVICES') - self.NUM_GPUS = len(self.GPU_CHIPS) + self.NUM_GPUS = 0 + else: + self.NUM_GPUS = len(self.GPU_CHIPS) self.PARTITION = args['partition'] # if self.NUM_GPUS > 1 : diff --git a/data/maker/__init__.py b/data/maker/__init__.py index 3e42419..bfd6a5f 100644 --- a/data/maker/__init__.py +++ b/data/maker/__init__.py @@ -86,18 +86,6 @@ def train (**_args): :params sql :params store """ - # - # Let us prepare the data by calling the utility function - # - # if 'file' in _args : - # # - # # We are reading data from a file - # _args['data'] = pd.read_csv(_args['file']) - # else: - # # - # # data will be read from elsewhere (a data-store)... - # pass - # if 'ignore' in _args and 'columns' in _args['ignore']: _inputhandler = prepare.Input(**_args) values,_matrix = _inputhandler.convert() @@ -125,6 +113,8 @@ def train (**_args): args['matrix_size'] = _matrix.shape[0] args['batch_size'] = 2000 args['partition'] = 0 if 'partition' not in _args else _args['partition'] + if 'gpu' in _args : + args['gpu'] = _args['gpu'] # os.environ['CUDA_VISIBLE_DEVICES'] = str(args['gpu']) if 'gpu' in args else '0' trainer = gan.Train(**args) @@ -137,50 +127,7 @@ def train (**_args): trainer.apply() pass -def _train (**args) : - """ - This function is intended to train the GAN in order to learn about the distribution of the features - :column columns that need to be synthesized (discrete) - :logs where the output of the (location on disk) - :id identifier of the dataset - :data data-frame to be synthesized - :context label of what we are synthesizing - """ - column = args['column'] if (isinstance(args['column'],list)) else [args['column']] - # CONTINUOUS = args['continuous'] if 'continuous' in args else [] - # column_id = args['id'] - df = args['data'] if not isinstance(args['data'],str) else pd.read_csv(args['data']) - df.columns = [name.lower() for name in df.columns] - # - # @TODO: - # Consider sequential training of sub population for extremely large datasets - # - - # - # If we have several columns we will proceed one at a time (it could be done in separate threads) - # @TODO : Consider performing this task on several threads/GPUs simulataneously - # - for col in column : - msize = args['matrix_size'] if 'matrix_size' in args else -1 - args['real'] = (Binary()).apply(df[col],msize) - context = args['context'] - if 'store' in args : - args['store']['args']['doc'] = context - logger = factory.instance(**args['store']) - args['logger'] = logger - info = {"rows":args['real'].shape[0],"cols":args['real'].shape[1],"name":col,"partition":args['partition']} - logger.write({"module":"gan-train","action":"data-prep","input":info}) - - else: - logger = None - args['column'] = col - args['context'] = col - - # - # If the s - trainer = gan.Train(**args) - trainer.apply() def get(**args): """ This function will restore a checkpoint from a persistant storage on to disk @@ -214,6 +161,8 @@ def generate(**_args): _inputhandler = prepare.Input(**_args) values,_matrix = _inputhandler.convert() args['values'] = np.array(values) + if 'gpu' in _args : + args['gpu'] = _args['gpu'] handler = gan.Predict (**args) handler.load_meta(None) diff --git a/pipeline.py b/pipeline.py index 6f39b55..e2bbbec 100644 --- a/pipeline.py +++ b/pipeline.py @@ -87,6 +87,8 @@ class Components : _index = str(gpu[0]) os.environ['CUDA_VISIBLE_DEVICES'] = _index return gpu + else : + return None def train(self,**args): """ This function will perform training on the basis of a given pointer that reads data