healthcareio
/
data-maker
mirror of https://hiplab.mc.vanderbilt.edu/git/aou/data-maker.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fixes

Browse Source
dev
Steve Nyemba 2 years ago
parent 7e92571d0a
commit d89daf76d6
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File

13
data/maker/__init__.py
Unescape View File

@ -59,7 +59,7 @@ class Learner(Process):
# @TODO: allow for verbose mode so we have a sens of what is going on within the newtork # @TODO: allow for verbose mode so we have a sens of what is going on within the newtork
# #
_log = {'action':'init','context':self.info['context'],'gpu':(self.gpu if self.gpu is not None else -1)} _log = {'action':'init','gpu':(self.gpu if self.gpu is not None else -1)}
self.log(**_log) self.log(**_log)
# self.logpath= _args['logpath'] if 'logpath' in _args else 'logs' # self.logpath= _args['logpath'] if 'logpath' in _args else 'logs'
@ -67,8 +67,10 @@ class Learner(Process):
def log(self,**_args): def log(self,**_args):
# self.lock.acquire() # self.lock.acquire()
try: try:
_context = self.info['context']
_label = self.info['info'] if 'info' in self.info else _context
logger = transport.factory.instance(**self.store['logger']) if 'logger' in self.store else transport.factory.instance(provider='console',context='write',lock=True) logger = transport.factory.instance(**self.store['logger']) if 'logger' in self.store else transport.factory.instance(provider='console',context='write',lock=True)
_args = dict({'ndx':self.ndx,'module':self.name,'table':self.info['from'],'info':self.info['context'],**_args}) _args = dict({'ndx':self.ndx,'module':self.name,'table':self.info['from'],'context':_context,'info':_info,**_args})
logger.write(_args) logger.write(_args)
self.ndx += 1 self.ndx += 1
if hasattr(logger,'close') : if hasattr(logger,'close') :
@ -345,7 +347,7 @@ class Generator (Learner):
_type = np.float _type = np.float
if _type : if _type :
_df[name] = _df[name].fillna(0).replace('',0).replace('NA',0).replace('nan',0).astype(_type) _df[name] = _df[name].fillna(0).replace(' ',0).replace('',0).replace('NA',0).replace('nan',0).astype(_type)
# else: # else:
# _df[name] = _df[name].astype(str) # _df[name] = _df[name].astype(str)
# _df = _df.replace('NaT','').replace('NA','') # _df = _df.replace('NaT','').replace('NA','')
@ -397,7 +399,10 @@ class Generator (Learner):
# print (_df[cols]) # print (_df[cols])
writer = transport.factory.instance(**_store) writer = transport.factory.instance(**_store)
writer.write(_df[:],schema=_schema) if _store['provider'] == 'bigquery':
writer.write(_df,schema=[],table=self.info['from'])
else:
writer.write(_df,table=self.info['from'])

Write Preview
Loading…
Cancel
Save