data-maker/data/gan.py

"""
import pickle
                        self.NUM_LABELS = args['label'].shape[1]
                self.mkdir(self.log_dir)
                #
                # 
                for key in ['train','output'] :
                        self.mkdir(os.sep.join([self.log_dir,key]))
                        self.mkdir (os.sep.join([self.log_dir,key,self.CONTEXT]))
                        
                self.train_dir  = os.sep.join([self.log_dir,'train',self.CONTEXT])                
                self.out_dir = os.sep.join([self.log_dir,'output',self.CONTEXT])
                if self.logger :
                        #
                        # We will clear the logs from the data-store 
                        #
                        column = self.ATTRIBUTES['synthetic']
                        db = self.logger.db
                        if db[column].count() > 0 :
                                db.backup.insert({'name':column,'logs':list(db[column].find()) })
                                db[column].drop()
                
        def load_meta(self,column):
                """
                This function is designed to accomodate the uses of the sub-classes outside of a strict dependency model.
                Because prediction and training can happen independently
                """
                # suffix = "-".join(column) if isinstance(column,list)else column
                suffix = self.get.suffix()
                _name = os.sep.join([self.out_dir,'meta-'+suffix+'.json'])
                if os.path.exists(_name) :
                        attr = json.loads((open(_name)).read())
                        for key in attr :
                                value = attr[key]
                                setattr(self,key,value)
                self.train_dir  = os.sep.join([self.log_dir,'train',self.CONTEXT])                
                self.out_dir = os.sep.join([self.log_dir,'output',self.CONTEXT])
                                
                        
        def log_meta(self,**args) :
                
                _object = {
                        # '_id':'meta',
                        'CONTEXT':self.CONTEXT,
                        'ATTRIBUTES':self.ATTRIBUTES,
                        'BATCHSIZE_PER_GPU':self.BATCHSIZE_PER_GPU,
                        'Z_DIM':self.Z_DIM,
                        "X_SPACE_SIZE":self.X_SPACE_SIZE,
                        "D_STRUCTURE":self.D_STRUCTURE,
                        "G_STRUCTURE":self.G_STRUCTURE,
                        "NUM_GPUS":self.NUM_GPUS,
                        "NUM_LABELS":self.NUM_LABELS,
                        "MAX_EPOCHS":self.MAX_EPOCHS,
                        "ROW_COUNT":self.ROW_COUNT
                }
                if args and 'key' in args and 'value' in args :
                        key = args['key']
                        value= args['value']
                        object[key] = value
                # suffix = "-".join(self.column) if isinstance(self.column,list) else self.column
                suffix = self.get.suffix()
                _name = os.sep.join([self.out_dir,'meta-'+suffix])
                
                f = open(_name+'.json','w')
                f.write(json.dumps(_object))
                return _object
        def mkdir (self,path):
                if not os.path.exists(path) :
                        os.mkdir(path)            
                        average_grads.append(grad_and_var)
                return average_grads            
                        i = len(self.D_STRUCTURE)
                        kernel = self.get.variables(name='W_' + str(i), shape=[self.D_STRUCTURE[-1], 1])
                        bias = self.get.variables(name='b_' + str(i), shape=[1])
                        y = tf.add(tf.matmul(x, kernel), bias)
                return y
                # label = label[:,1] * 4 + tf.squeeze( label[:,2]*[[0],[1],[2],[3]] )
                z = tf.random.normal(shape=[self.BATCHSIZE_PER_GPU, self.Z_DIM])
                
                fake = self.generator.network(inputs=z, label=label)
                if stage == 'D':
                        w, loss = self.discriminator.loss(real=real, fake=fake, label=label)
                        #losses = tf.get_collection('dlosses', scope)
                        flag = 'dlosses'
                        losses = tf.compat.v1.get_collection('dlosses', scope)
                else:
                        w, loss = self.generator.loss(fake=fake, label=label)
                        #losses = tf.get_collection('glosses', scope)
                        flag = 'glosses'
                        losses = tf.compat.v1.get_collection('glosses', scope)
                # losses = tf.compat.v1.get_collection(flag, scope)

                total_loss = tf.add_n(losses, name='total_loss')

                return total_loss, w
        def input_fn(self):
                """
                This function seems to produce 
                """
                features_placeholder = tf.compat.v1.placeholder(shape=self._REAL.shape, dtype=tf.float32)
                REAL = self._REAL
                LABEL= self._LABEL       
                if (self.logger):
                        pass
                
                with tf.device('/cpu:0'):
                        opt_d = tf.compat.v1.train.AdamOptimizer(1e-4)
                        opt_g = tf.compat.v1.train.AdamOptimizer(1e-4)
                        
                        train_d, w_distance, iterator_d, features_placeholder_d, labels_placeholder_d = self.network(stage='D', opt=opt_d)
                        train_g, _, iterator_g, features_placeholder_g, labels_placeholder_g = self.network(stage='G', opt=opt_g)
                        # saver = tf.train.Saver()
                        saver   = tf.compat.v1.train.Saver()
                        # init    = tf.global_variables_initializer()
                        init    = tf.compat.v1.global_variables_initializer()
                        logs = []
                        #with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
                        with tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
                                        assert not np.isnan(w_sum), 'Model diverged with loss = NaN'
                super().load_meta(column)
                self.generator.load_meta(column)
        def apply(self,**args):
                # print (self.train_dir)
                # suffix = "-".join(self.ATTRIBUTES['synthetic']) if isinstance(self.ATTRIBUTES['synthetic'],list) else self.ATTRIBUTES['synthetic']
                suffix = self.get.suffix()
                model_dir = os.sep.join([self.train_dir,suffix+'-'+str(self.MAX_EPOCHS)])
                demo = self._LABEL #np.zeros([self.ROW_COUNT,self.NUM_LABELS]) #args['de"shape":{"LABEL":list(self._LABEL.shape)} mo']
                tf.compat.v1.reset_default_graph()
                z = tf.random.normal(shape=[self.BATCHSIZE_PER_GPU, self.Z_DIM])
                y = tf.compat.v1.placeholder(shape=[self.BATCHSIZE_PER_GPU, self.NUM_LABELS], dtype=tf.int32)
                                else:
                                        f = sess.run(fake)
                tf.compat.v1.reset_default_graph()
                
                return df.to_dict(orient='list')
                        # return df.to_dict(orient='list')
                        # count = str(len(os.listdir(self.out_dir)))
                        # _name = os.sep.join([self.out_dir,self.CONTEXT+'-'+count+'.csv'])
                        # df.to_csv(_name,index=False)

                        
                        # output.extend(np.round(f))

                        # for m in range(2):
                        #         for n in range(2, self.NUM_LABELS):
                        #                 idx1 = (demo[:, m] == 1)
                        #                 idx2 = (demo[:, n] == 1)
                        #                 idx = [idx1[j] and idx2[j] for j in range(len(idx1))]
                        #                 num = np.sum(idx)
                        #                 print ("___________________list__")
                        #                 print (idx1)
                        #                 print (idx2)
                        #                 print (idx)
                        #                 print (num)
                        #                 print ("_____________________")
                        #                 nbatch = int(np.ceil(num / self.BATCHSIZE_PER_GPU))
                        #                 label_input = np.zeros((nbatch*self.BATCHSIZE_PER_GPU, self.NUM_LABELS))
                        #                 label_input[:, n] = 1
                        #                 label_input[:, m] = 1
                        #                 output = []
                        #                 for i in range(nbatch):
                        #                         f = sess.run(fake,feed_dict={y: label_input[i* self.BATCHSIZE_PER_GPU:(i+1)* self.BATCHSIZE_PER_GPU]})
                        #                         output.extend(np.round(f))
                        #                 output = np.array(output)[:num]
                                        # print ([m,n,output])
                                        
                                        # np.save(self.out_dir + str(m) + str(n), output)
        
if __name__ == '__main__' :
                print (df)
                print ()
                df[column] = r[column]
                print (df)