|
|
|
"""
|
|
|
|
usage :
|
|
|
|
optional :
|
|
|
|
--num_gpu number of gpus to use will default to 1
|
|
|
|
--epoch steps per epoch default to 256
|
|
|
|
"""
|
|
|
|
import tensorflow as tf
|
|
|
|
from tensorflow.contrib.layers import l2_regularizer
|
|
|
|
import numpy as np
|
|
|
|
import pandas as pd
|
|
|
|
import time
|
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
Because prediction and training can happen independently
|
|
|
|
"""
|
|
|
|
grad = tf.reduce_mean(grad, 0)
|
|
|
|
|
|
|
|
v = grad_and_vars[0][1]
|
|
|
|
grad_and_var = (grad, v)
|
|
|
|
average_grads.append(grad_and_var)
|
|
|
|
return average_grads
|
|
|
|
|
|
|
|
|
|
|
|
class Generator (GNet):
|
|
|
|
"""
|
|
|
|
This class is designed to handle generation of candidate datasets for this it will aggregate a discriminator, this allows the generator not to be random
|
|
|
|
|
|
|
|
"""
|
|
|
|
def __init__(self,**args):
|
|
|
|
GNet.__init__(self,**args)
|
|
|
|
self.discriminator = Discriminator(**args)
|
|
|
|
def loss(self,**args):
|
|
|
|
fake = args['fake']
|
|
|
|
label = args['label']
|
|
|
|
y_hat_fake = self.discriminator.network(inputs=fake, label=label)
|
|
|
|
all_regs = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
|
|
|
|
loss = -tf.reduce_mean(y_hat_fake) + sum(all_regs)
|
|
|
|
tf.add_to_collection('glosses', loss)
|
|
|
|
return loss, loss
|
|
|
|
def load_meta(self, column):
|
|
|
|
super().load_meta(column)
|
|
|
|
self.discriminator.load_meta(column)
|
|
|
|
def network(self,**args) :
|
|
|
|
"""
|
|
|
|
This function will build the network that will generate the synthetic candidates
|
|
|
|
:inputs matrix of data that we need
|
|
|
|
:dim dimensions of ...
|
|
|
|
"""
|
|
|
|
x = args['inputs']
|
|
|
|
tmp_dim = self.Z_DIM if 'dim' not in args else args['dim']
|
|
|
|
label = args['label']
|
|
|
|
|
|
|
|
with tf.compat.v1.variable_scope('G', reuse=tf.compat.v1.AUTO_REUSE , regularizer=l2_regularizer(0.00001)):
|
|
|
|
for i, dim in enumerate(self.G_STRUCTURE[:-1]):
|
|
|
|
kernel = self.get.variables(name='W_' + str(i), shape=[tmp_dim, dim])
|
|
|
|
h1 = self.normalize(inputs=tf.matmul(x, kernel),shift=0, name='cbn' + str(i), labels=label, n_labels=self.NUM_LABELS)
|
|
|
|
h2 = tf.nn.relu(h1)
|
|
|
|
x = x + h2
|
|
|
|
tmp_dim = dim
|
|
|
|
i = len(self.G_STRUCTURE) - 1
|
|
|
|
#
|
|
|
|
# This seems to be an extra hidden layer:
|
|
|
|
# It's goal is to map continuous values to discrete values (pre-trained to do this)
|
|
|
|
kernel = self.get.variables(name='W_' + str(i), shape=[tmp_dim, self.G_STRUCTURE[-1]])
|
|
|
|
h1 = self.normalize(inputs=tf.matmul(x, kernel), name='cbn' + str(i),
|
|
|
|
labels=label, n_labels=self.NUM_LABELS)
|
|
|
|
h2 = tf.nn.tanh(h1)
|
|
|
|
x = x + h2
|
|
|
|
# This seems to be the output layer
|
|
|
|
#
|
|
|
|
kernel = self.get.variables(name='W_' + str(i+1), shape=[self.Z_DIM, self.X_SPACE_SIZE])
|
|
|
|
bias = self.get.variables(name='b_' + str(i+1), shape=[self.X_SPACE_SIZE])
|
|
|
|
x = tf.nn.sigmoid(tf.add(tf.matmul(x, kernel), bias))
|
|
|
|
return x
|
|
|
|
|
|
|
|
class Discriminator(GNet):
|
|
|
|
def __init__(self,**args):
|
|
|
|
GNet.__init__(self,**args)
|
|
|
|
def network(self,**args):
|
|
|
|
"""
|
|
|
|
This function will apply a computational graph on a dataset passed in with the associated labels and the last layer must have a single output (neuron)
|
|
|
|
:inputs
|
|
|
|
:label
|
|
|
|
"""
|
|
|
|
x = args['inputs']
|
|
|
|
print ()
|
|
|
|
print (x[:3,:])
|
|
|
|
print()
|
|
|
|
label = args['label']
|
|
|
|
with tf.compat.v1.variable_scope('D', reuse=tf.compat.v1.AUTO_REUSE , regularizer=l2_regularizer(0.00001)):
|
|
|
|
for i, dim in enumerate(self.D_STRUCTURE[1:]):
|
|
|
|
kernel = self.get.variables(name='W_' + str(i), shape=[self.D_STRUCTURE[i], dim])
|
|
|
|
bias = self.get.variables(name='b_' + str(i), shape=[dim])
|
|
|
|
print (["\t",bias,kernel])
|
|
|
|
x = tf.nn.relu(tf.add(tf.matmul(x, kernel), bias))
|
|
|
|
x = self.normalize(inputs=x, name='cln' + str(i), shift=1,labels=label, n_labels=self.NUM_LABELS)
|
|
|
|
i = len(self.D_STRUCTURE)
|
|
|
|
kernel = self.get.variables(name='W_' + str(i), shape=[self.D_STRUCTURE[-1], 1])
|
|
|
|
bias = self.get.variables(name='b_' + str(i), shape=[1])
|
|
|
|
y = tf.add(tf.matmul(x, kernel), bias)
|
|
|
|
return y
|
|
|
|
|
|
|
|
def loss(self,**args) :
|
|
|
|
"""
|
|
|
|
This function compute the loss of
|
|
|
|
:real
|
|
|
|
:fake
|
|
|
|
:label
|
|
|
|
"""
|
|
|
|
real = args['real']
|
|
|
|
fake = args['fake']
|
|
|
|
label = args['label']
|
|
|
|
epsilon = tf.random.uniform(shape=[self.BATCHSIZE_PER_GPU,1],minval=0,maxval=1)
|
|
|
|
|
|
|
|
x_hat = real + epsilon * (fake - real)
|
|
|
|
y_hat_fake = self.network(inputs=fake, label=label)
|
|
|
|
|
|
|
|
y_hat_real = self.network(inputs=real, label=label)
|
|
|
|
y_hat = self.network(inputs=x_hat, label=label)
|
|
|
|
|
|
|
|
grad = tf.gradients(y_hat, [x_hat])[0]
|
|
|
|
slopes = tf.sqrt(tf.reduce_sum(tf.square(grad), 1))
|
|
|
|
gradient_penalty = tf.reduce_mean((slopes - 1.) ** 2)
|
|
|
|
all_regs = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
|
|
|
|
w_distance = -tf.reduce_mean(y_hat_real) + tf.reduce_mean(y_hat_fake)
|
|
|
|
loss = w_distance + 10 * gradient_penalty + sum(all_regs)
|
|
|
|
tf.add_to_collection('dlosses', loss)
|
|
|
|
|
|
|
|
return w_distance, loss
|
|
|
|
class Train (GNet):
|
|
|
|
def __init__(self,**args):
|
|
|
|
GNet.__init__(self,**args)
|
|
|
|
self.generator = Generator(**args)
|
|
|
|
self.discriminator = Discriminator(**args)
|
|
|
|
self._REAL = args['real']
|
|
|
|
self._LABEL= args['label']
|
|
|
|
self.generator.load_meta(column)
|
|
|
|
dataset = dataset.batch(batch_size=self.BATCHSIZE_PER_GPU)
|
|
|
|
dataset = dataset.prefetch(1)
|
|
|
|
iterator = dataset.make_initializable_iterator()
|
|
|
|
# next_element = iterator.get_next()
|
|
|
|
# init_op = iterator.initializer
|
|
|
|
return iterator, features_placeholder, labels_placeholder
|
|
|
|
|
|
|
|
def network(self,**args):
|
|
|
|
# def graph(stage, opt):
|
|
|
|
# global_step = tf.get_variable(stage+'_step', [], initializer=tf.constant_initializer(0), trainable=False)
|
|
|
|
stage = args['stage']
|
|
|
|
opt = args['opt']
|
|
|
|
tower_grads = []
|
|
|
|
per_gpu_w = []
|
|
|
|
iterator, features_placeholder, labels_placeholder = self.input_fn()
|
|
|
|
with tf.compat.v1.variable_scope(tf.compat.v1.get_variable_scope()):
|
|
|
|
for i in range(self.NUM_GPUS):
|
|
|
|
with tf.device('/gpu:%d' % i):
|
|
|
|
with tf.name_scope('%s_%d' % ('TOWER', i)) as scope:
|
|
|
|
(real, label) = iterator.get_next()
|
|
|
|
loss, w = self.loss(scope=scope, stage=stage, real=self._REAL, label=self._LABEL)
|
|
|
|
tf.get_variable_scope().reuse_variables()
|
|
|
|
vars_ = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=stage)
|
|
|
|
grads = opt.compute_gradients(loss, vars_)
|
|
|
|
tower_grads.append(grads)
|
|
|
|
per_gpu_w.append(w)
|
|
|
|
|
|
|
|
grads = self.average_gradients(tower_grads)
|
|
|
|
apply_gradient_op = opt.apply_gradients(grads)
|
|
|
|
|
|
|
|
mean_w = tf.reduce_mean(per_gpu_w)
|
|
|
|
train_op = apply_gradient_op
|
|
|
|
return train_op, mean_w, iterator, features_placeholder, labels_placeholder
|
|
|
|
def apply(self,**args):
|
|
|
|
# max_epochs = args['max_epochs'] if 'max_epochs' in args else 10
|
|
|
|
REAL = self._REAL
|
|
|
|
LABEL= self._LABEL
|
|
|
|
with tf.device('/cpu:0'):
|
|
|
|
opt_d = tf.compat.v1.train.AdamOptimizer(1e-4)
|
|
|
|
opt_g = tf.compat.v1.train.AdamOptimizer(1e-4)
|
|
|
|
|
|
|
|
train_d, w_distance, iterator_d, features_placeholder_d, labels_placeholder_d = self.network(stage='D', opt=opt_d)
|
|
|
|
train_g, _, iterator_g, features_placeholder_g, labels_placeholder_g = self.network(stage='G', opt=opt_g)
|
|
|
|
# saver = tf.train.Saver()
|
|
|
|
saver = tf.compat.v1.train.Saver()
|
|
|
|
init = tf.global_variables_initializer()
|
|
|
|
|
|
|
|
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
|
|
|
|
sess.run(init)
|
|
|
|
sess.run(iterator_d.initializer,
|
|
|
|
feed_dict={features_placeholder_d: REAL, labels_placeholder_d: LABEL})
|
|
|
|
sess.run(iterator_g.initializer,
|
|
|
|
feed_dict={features_placeholder_g: REAL, labels_placeholder_g: LABEL})
|
|
|
|
|
|
|
|
for epoch in range(1, self.MAX_EPOCHS + 1):
|
|
|
|
start_time = time.time()
|
|
|
|
w_sum = 0
|
|
|
|
for i in range(self.STEPS_PER_EPOCH):
|
|
|
|
for _ in range(2):
|
|
|
|
_, w = sess.run([train_d, w_distance])
|
|
|
|
w_sum += w
|
|
|
|
sess.run(train_g)
|
|
|
|
duration = time.time() - start_time
|
|
|
|
|
|
|
|
assert not np.isnan(w_sum), 'Model diverged with loss = NaN'
|
|
|
|
|
|
|
|
format_str = 'epoch: %d, w_distance = %f (%.1f)'
|
|
|
|
print(format_str % (epoch, -w_sum/(self.STEPS_PER_EPOCH*2), duration))
|
|
|
|
if epoch % self.MAX_EPOCHS == 0:
|
|
|
|
"""
|
|
|
|
print (__doc__)
|
|
|
|
pass
|
|
|
|
|