From 772d841ee80d9279c0348356ea268787d54ef44b Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Thu, 12 Mar 2020 14:37:01 -0500 Subject: [PATCH] bug fix ... --- data/maker/__init__.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/data/maker/__init__.py b/data/maker/__init__.py index 2b51670..5b4cb7e 100644 --- a/data/maker/__init__.py +++ b/data/maker/__init__.py @@ -24,24 +24,25 @@ class ContinuousToDiscrete : # BOUNDS = np.repeat(np.divide(X.max(),n),n).cumsum().tolist() # print ( X.values.astype(np.float32)) # print ("___________________________") - values = X.values.astype(np.float32) + values = np.array(X).astype(np.float32) BOUNDS = ContinuousToDiscrete.bounds(values,n) # _map = [{"index":BOUNDS.index(i),"ubound":i} for i in BOUNDS] _matrix = [] m = [] for value in X : x_ = np.zeros(n) - _matrix.append(x_) + for row in BOUNDS : if value>= row.left and value <= row.right : index = BOUNDS.index(row) x_[index] = 1 break + _matrix += x_.tolist() # # for items in BOUNDS : # index = BOUNDS.index(items) - return np.array(_matrix) + return np.array(_matrix).reshape(len(X),n) @staticmethod def bounds(x,n): @@ -92,7 +93,7 @@ def train (**args) : :context label of what we are synthesizing """ column = args['column'] if (isinstance(args['column'],list)) else [args['column']] - CONTINUOUS = args['continuous'] if 'continuous' in args else [] + # CONTINUOUS = args['continuous'] if 'continuous' in args else [] # column_id = args['id'] df = args['data'] if not isinstance(args['data'],str) else pd.read_csv(args['data']) df.columns = [name.lower() for name in df.columns] @@ -109,15 +110,16 @@ def train (**args) : # args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values # if 'float' not in df[col].dtypes.name : # args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values - if col in CONTINUOUS: - BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) - args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32) - # print ( pd.DataFrame(args['real']).head() ) - else: + # if col in CONTINUOUS: + # BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) + # args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32) + # # args['real'] = args['real'].reshape(df.shape[0],BIN_SIZE) + + # else: # df.to_csv('tmp-'+args['logs'].replace('/','_')+'-'+col+'.csv',index=False) # print (df[col].dtypes) # print (df[col].dropna/(axis=1).unique()) - args['real'] = pd.get_dummies(df[col].dropna()).astype(np.float32).values + args['real'] = pd.get_dummies(df[col].dropna()).astype(np.float32).values @@ -170,6 +172,7 @@ def generate(**args): #@TODO: # If the identifier is not present, we should fine a way to determine or make one # + BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) _df = df.copy() for col in column : args['context'] = col @@ -181,10 +184,15 @@ def generate(**args): # BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) # values = ContinuousToDiscrete.continuous(df[col].values,BIN_SIZE) # # values = np.unique(values).tolist() + # else: + # if col in CONTINUOUS : + # values = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32).T + # else: values = df[col].dropna().unique().tolist() + args['values'] = values args['row_count'] = df.shape[0] # @@ -192,10 +200,9 @@ def generate(**args): handler = gan.Predict (**args) handler.load_meta(col) r = handler.apply() - BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) - # _df[col] = ContinuousToDiscrete.continuous(r[col],BIN_SIZE) if col in CONTINUOUS else r[col] - _df[col] = r[col] + _df[col] = ContinuousToDiscrete.continuous(r[col],BIN_SIZE) if col in CONTINUOUS else r[col] + # _df[col] = r[col] # # @TODO: log basic stats about the synthetic attribute #