bug fix: fix with design pattern of the last observer

dev
Steve Nyemba 2 years ago
parent 6ddda98527
commit 41c6f72cb3

@ -12,6 +12,7 @@ from multiprocessing import Process, Lock
import numpy as np import numpy as np
import json import json
import pandas as pd import pandas as pd
from zmq import has
class Subject (Process): class Subject (Process):
cache = pd.DataFrame() cache = pd.DataFrame()
@ -94,8 +95,8 @@ class Worker :
except Exception as error: except Exception as error:
pass pass
finally: finally:
if hasattr(self,'caller') :
self.caller.notify() self.caller.notify()
def _apply(self): def _apply(self):
pass pass
def get(self): def get(self):
@ -176,11 +177,16 @@ class Reader(Worker):
def init(self,**_args): def init(self,**_args):
super().init(**_args) super().init(**_args)
self.rows = [] self.rows = []
def _apply(self): def _apply(self):
try: try:
if 'type' in self._info :
self._info['type'] = self._info['type'].replace('Writer','Reader')
if 'fields' in self._info['args'] :
del self._info['args']['fields']
else:
self._info['context'] = 'read'
self.reader = transport.factory.instance(**self._info) ; self.reader = transport.factory.instance(**self._info) ;
# self.rows = self.reader.read(mongo=self.pipeline) # self.rows = self.reader.read(mongo=self.pipeline)
@ -206,7 +212,7 @@ class Reader(Worker):
except Exception as e : except Exception as e :
_log['status'] = 0 _log['status'] = 0
_log['info'] = {"error":e.args[0]} _log['info'] = {"error":e.args[0]}
print (e) print ([e])
self.log(**_log) self.log(**_log)
@ -221,13 +227,13 @@ class Writer(Worker):
super().__init__(**_args) super().__init__(**_args)
if 'provider' in self._info : if 'provider' in self._info :
self._info['context'] = 'write' self._info['context'] = 'write'
def init(self,**_args): def init(self,**_args):
""" """
:param store output data-store needed for writing :param store output data-store needed for writing
:param invalues input values with to be written somewhere :param invalues input values with to be written somewhere
""" """
super().init(**_args)
self._invalues = _args['invalues'] self._invalues = _args['invalues']
@ -259,8 +265,8 @@ class Writer(Worker):
# Upon upgrade use the operator "$toString" in export.init function # Upon upgrade use the operator "$toString" in export.init function
# #
rows = [dict(item,**{"_id":str(item["_id"])}) for item in rows] rows = [dict(item,**{"_id":str(item["_id"])}) for item in rows]
_df = pd.DataFrame(rows)
writer.write(rows) writer.write(_df)
index += 1 index += 1
# for _e in rows : # for _e in rows :
# writer.write(_e) # writer.write(_e)

Loading…
Cancel
Save