diff --git a/data/gan.py b/data/gan.py index e7ab6cf..c61d1b1 100644 --- a/data/gan.py +++ b/data/gan.py @@ -58,7 +58,14 @@ class GNet : self.layers.normalize = self.normalize self.logs = {} - self.NUM_GPUS = 1 if 'num_gpu' not in args else args['num_gpu'] + # self.NUM_GPUS = 1 if 'num_gpu' not in args else args['num_gpu'] + self.GPU_CHIPS = None if 'gpu' not in args else args['gpu'] + if self.GPU_CHIPS is None: + self.GPU_CHIPS = [0] + if 'CUDA_VISIBLE_DEVICES' in os.environ : + os.environ.pop('CUDA_VISIBLE_DEVICES') + self.NUM_GPUS = len(self.GPU_CHIPS) + self.PARTITION = args['partition'] # if self.NUM_GPUS > 1 : # os.environ['CUDA_VISIBLE_DEVICES'] = "4" @@ -150,6 +157,7 @@ class GNet : "D_STRUCTURE":self.D_STRUCTURE, "G_STRUCTURE":self.G_STRUCTURE, "NUM_GPUS":self.NUM_GPUS, + "GPU_CHIPS":self.GPU_CHIPS, "NUM_LABELS":self.NUM_LABELS, "MAX_EPOCHS":self.MAX_EPOCHS, "ROW_COUNT":self.ROW_COUNT @@ -443,7 +451,7 @@ class Train (GNet): # - abstract hardware specification # - determine if the GPU/CPU are busy # - for i in range(self.NUM_GPUS): + for i in self.GPU_CHIPS : #range(self.NUM_GPUS): with tf.device('/gpu:%d' % i): with tf.name_scope('%s_%d' % ('TOWER', i)) as scope: if self._LABEL is not None : diff --git a/data/maker/__init__.py b/data/maker/__init__.py index cfdd8e2..fbdf208 100644 --- a/data/maker/__init__.py +++ b/data/maker/__init__.py @@ -90,16 +90,16 @@ def train (**_args): # # Let us prepare the data by calling the utility function # - if 'file' in _args : - # - # We are reading data from a file - _args['data'] = pd.read_csv(_args['file']) - else: - # - # data will be read from elsewhere (a data-store)... - pass + # if 'file' in _args : + # # + # # We are reading data from a file + # _args['data'] = pd.read_csv(_args['file']) + # else: + # # + # # data will be read from elsewhere (a data-store)... + # pass # if 'ignore' in _args and 'columns' in _args['ignore']: - + _inputhandler = prepare.Input(**_args) values,_matrix = _inputhandler.convert() args = {"real":_matrix,"context":_args['context']} @@ -107,6 +107,7 @@ def train (**_args): if 'store' in _args : # # This + args['store'] = copy.deepcopy(_args['store']['logs']) args['store']['args']['doc'] = _args['context'] logger = factory.instance(**args['store']) diff --git a/data/maker/prepare/__init__.py b/data/maker/prepare/__init__.py index 2c773de..381dfc0 100644 --- a/data/maker/prepare/__init__.py +++ b/data/maker/prepare/__init__.py @@ -13,7 +13,7 @@ import transport import json import pandas as pd import numpy as np -import cupy as cp +# import cupy as cp import sys import os # from multiprocessing import Process, Queue @@ -62,7 +62,7 @@ class Input : self._schema = _args['schema'] if 'schema' in _args else {} self.df = _args['data'] if 'sql' not in _args : - # self._initdata(**_args) + self._initdata(**_args) # pass else: @@ -70,12 +70,12 @@ class Input : self._map = {} if 'map' not in _args else _args['map'] # self._metadf = pd.DataFrame(self.df[self._columns].dtypes.values.astype(str)).T #,self._columns] # self._metadf.columns = self._columns - if 'gpu' in _args and 'GPU' in os.environ: + # if 'gpu' in _args and 'GPU' in os.environ: - np = cp - index = int(_args['gpu']) - np.cuda.Device(index).use() - print(['..:: GPU ',index]) + # np = cp + # index = int(_args['gpu']) + # np.cuda.Device(index).use() + # print(['..:: GPU ',index]) def _initsql(self,**_args): """ @@ -107,6 +107,8 @@ class Input : row_count = self.df.shape[0] cols = None if 'columns' not in _args else _args['columns'] self.columns = self.df.columns.tolist() + self._io = [] + if 'columns' in _args : self._columns = _args['columns'] else: @@ -115,6 +117,8 @@ class Input : _df = pd.DataFrame(self.df.apply(lambda col: col.dropna().unique().size )).T MIN_SPACE_SIZE = 2 self._columns = cols if cols else _df.apply(lambda col:None if col[0] == row_count or col[0] < MIN_SPACE_SIZE else col.name).dropna().tolist() + self._io = _df.to_dict(orient='records') + def _initdata(self,**_args): """ This function will initialize the class with a data-frame and columns of interest (if any) diff --git a/setup.py b/setup.py index 7970f14..4b96d08 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import sys def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -args = {"name":"data-maker","version":"1.4.0","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT", +args = {"name":"data-maker","version":"1.4.1","author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT", "packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]} args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo'] args['url'] = 'https://hiplab.mc.vanderbilt.edu/git/aou/data-maker.git'