|
|
|
"""
|
|
|
|
import pickle
|
|
|
|
self.get.suffix = lambda : "-".join(self.ATTRIBUTES['synthetic']) if isinstance(self.ATTRIBUTES['synthetic'],list) else self.ATTRIBUTES['synthetic']
|
|
|
|
self.logger = args['logger'] if 'logger' in args and args['logger'] else None
|
|
|
|
self.init_logs(**args)
|
|
|
|
|
|
|
|
def init_logs(self,**args):
|
|
|
|
self.log_dir = args['logs'] if 'logs' in args else 'logs'
|
|
|
|
self.mkdir(self.log_dir)
|
|
|
|
#
|
|
|
|
#
|
|
|
|
for key in ['train','output'] :
|
|
|
|
self.mkdir(os.sep.join([self.log_dir,key]))
|
|
|
|
self.mkdir (os.sep.join([self.log_dir,key,self.CONTEXT]))
|
|
|
|
|
|
|
|
self.train_dir = os.sep.join([self.log_dir,'train',self.CONTEXT])
|
|
|
|
self.out_dir = os.sep.join([self.log_dir,'output',self.CONTEXT])
|
|
|
|
if self.logger :
|
|
|
|
#
|
|
|
|
# We will clear the logs from the data-store
|
|
|
|
#
|
|
|
|
column = self.ATTRIBUTES['synthetic']
|
|
|
|
db = self.logger.db
|
|
|
|
if db[column].count() > 0 :
|
|
|
|
db.backup.insert({'name':column,'logs':list(db[column].find()) })
|
|
|
|
db[column].drop()
|
|
|
|
|
|
|
|
def load_meta(self,column):
|
|
|
|
"""
|
|
|
|
This function is designed to accomodate the uses of the sub-classes outside of a strict dependency model.
|
|
|
|
Because prediction and training can happen independently
|
|
|
|
"""
|
|
|
|
# suffix = "-".join(column) if isinstance(column,list)else column
|
|
|
|
suffix = self.get.suffix()
|
|
|
|
_name = os.sep.join([self.out_dir,'meta-'+suffix+'.json'])
|
|
|
|
if os.path.exists(_name) :
|
|
|
|
attr = json.loads((open(_name)).read())
|
|
|
|
for key in attr :
|
|
|
|
value = attr[key]
|
|
|
|
setattr(self,key,value)
|
|
|
|
self.train_dir = os.sep.join([self.log_dir,'train',self.CONTEXT])
|
|
|
|
self.out_dir = os.sep.join([self.log_dir,'output',self.CONTEXT])
|
|
|
|
|
|
|
|
|
|
|
|
def log_meta(self,**args) :
|
|
|
|
|
|
|
|
_object = {
|
|
|
|
# '_id':'meta',
|
|
|
|
'CONTEXT':self.CONTEXT,
|
|
|
|
'ATTRIBUTES':self.ATTRIBUTES,
|
|
|
|
'BATCHSIZE_PER_GPU':self.BATCHSIZE_PER_GPU,
|
|
|
|
'Z_DIM':self.Z_DIM,
|
|
|
|
"X_SPACE_SIZE":self.X_SPACE_SIZE,
|
|
|
|
"D_STRUCTURE":self.D_STRUCTURE,
|
|
|
|
"G_STRUCTURE":self.G_STRUCTURE,
|
|
|
|
"NUM_GPUS":self.NUM_GPUS,
|
|
|
|
"NUM_LABELS":self.NUM_LABELS,
|
|
|
|
"MAX_EPOCHS":self.MAX_EPOCHS,
|
|
|
|
"ROW_COUNT":self.ROW_COUNT
|
|
|
|
}
|
|
|
|
if args and 'key' in args and 'value' in args :
|
|
|
|
key = args['key']
|
|
|
|
value= args['value']
|
|
|
|
object[key] = value
|
|
|
|
# suffix = "-".join(self.column) if isinstance(self.column,list) else self.column
|
|
|
|
suffix = self.get.suffix()
|
|
|
|
_name = os.sep.join([self.out_dir,'meta-'+suffix])
|
|
|
|
|
|
|
|
f = open(_name+'.json','w')
|
|
|
|
f.write(json.dumps(_object))
|
|
|
|
return _object
|
|
|
|
def mkdir (self,path):
|
|
|
|
if not os.path.exists(path) :
|
|
|
|
if not os.path.exists(os.sep.join(root)) :
|
|
|
|
# This seems to be the output layer
|
|
|
|
#
|
|
|
|
kernel = self.get.variables(name='W_' + str(i+1), shape=[self.Z_DIM, self.X_SPACE_SIZE])
|
|
|
|
bias = self.get.variables(name='b_' + str(i+1), shape=[self.X_SPACE_SIZE])
|
|
|
|
x = tf.nn.sigmoid(tf.add(tf.matmul(x, kernel), bias))
|
|
|
|
return x
|
|
|
|
column name
|
|
|
|
"""
|
|
|
|
super().load_meta(column)
|
|
|
|
self.generator.load_meta(column)
|
|
|
|
self.discriminator.load_meta(column)
|
|
|
|
def loss(self,**args):
|
|
|
|
"""
|
|
|
|
This function will compute a "tower" loss of the generated candidate against real data
|
|
|
|
Training will consist in having both generator and discriminators
|
|
|
|
:scope
|
|
|
|
:stage
|
|
|
|
:real
|
|
|
|
:label
|
|
|
|
"""
|
|
|
|
|
|
|
|
scope = args['scope']
|
|
|
|
stage = args['stage']
|
|
|
|
real = args['real']
|
|
|
|
label = args['label']
|
|
|
|
flag = 'glosses'
|
|
|
|
opt = args['opt']
|
|
|
|
tower_grads = []
|
|
|
|
pass
|
|
|
|
print(format_str % (epoch, -w_sum/(self.STEPS_PER_EPOCH*2), duration))
|
|
|
|
# print (dir (w_distance))
|
|
|
|
|
|
|
|
logs.append({"epoch":epoch,"distance":-w_sum/(self.STEPS_PER_EPOCH*2) })
|
|
|
|
|
|
|
|
This class uses synthetic data given a learned model
|
|
|
|
# df = pd.DataFrame(np.round(f)).astype(np.int32)
|
|
|
|
df = __x__
|
|
|
|
df = pd.DataFrame(df)
|
|
|
|
values.sort()
|