|
|
|
|
@ -212,27 +212,10 @@ class IETL(BaseIO) :
|
|
|
|
|
This class performs an ETL operation by ineriting a read and adding writes as pipeline functions
|
|
|
|
|
"""
|
|
|
|
|
def __init__(self,**_args):
|
|
|
|
|
# _source = _args['source']
|
|
|
|
|
# _plugins = _source['plugins'] if 'plugins' in _source else None
|
|
|
|
|
|
|
|
|
|
# # super().__init__(transport.get.reader(**_args['source']))
|
|
|
|
|
# super().__init__(agent=transport.get.reader(**_source),plugins=_plugins)
|
|
|
|
|
# # _logger =
|
|
|
|
|
# if 'target' in _args:
|
|
|
|
|
# self._targets = _args['target'] if type(_args['target']) == list else [_args['target']]
|
|
|
|
|
# else:
|
|
|
|
|
# self._targets = []
|
|
|
|
|
# self.jobs = []
|
|
|
|
|
|
|
|
|
|
# #
|
|
|
|
|
# # If the parent is already multiprocessing
|
|
|
|
|
# if 'token' in _source :
|
|
|
|
|
# self._logToken = _source['token']
|
|
|
|
|
# self._sourceArgs = _source['args'] if 'args' in _source else None
|
|
|
|
|
# self._hasParentProcess = False if 'hasParentProcess' not in _args else _args['hasParentProcess']
|
|
|
|
|
|
|
|
|
|
super().__init__()
|
|
|
|
|
self._source = _args['source']
|
|
|
|
|
self._targets= _args['target'] if _args['target'] == list else [_args['target']]
|
|
|
|
|
self._targets= _args['target'] if type(_args['target']) == list else [_args['target']]
|
|
|
|
|
#
|
|
|
|
|
# ETL Initialization, we should provide some measure of context ...
|
|
|
|
|
#
|
|
|
|
|
@ -254,7 +237,8 @@ class IETL(BaseIO) :
|
|
|
|
|
_data = _reader.read(**self._source['args'])if 'args' in self._source else _reader.read()
|
|
|
|
|
_reader.close()
|
|
|
|
|
_writers = [transport.get.writer(**_kwargs) for _kwargs in self._targets]
|
|
|
|
|
_schema = [] if not getattr(_reader._agent,'_table') else _reader.meta()
|
|
|
|
|
# _schema = [] if not getattr(_reader._agent,'_table') else _reader.meta()
|
|
|
|
|
_schema = [] if not hasattr(_reader._agent,'_table') else _reader.meta()
|
|
|
|
|
if types.GeneratorType == type(_data):
|
|
|
|
|
_index = 0
|
|
|
|
|
for _segment in _data :
|
|
|
|
|
|