|
|
|
|
#!/usr/bin/env python
|
|
|
|
|
__doc__ = """
|
|
|
|
|
(c) 2018 - 2021 data-transport
|
|
|
|
|
steve@the-phi.com, The Phi Technology LLC
|
|
|
|
|
https://dev.the-phi.com/git/steve/data-transport.git
|
|
|
|
|
|
|
|
|
|
This program performs ETL between 9 supported data sources : Couchdb, Mongodb, Mysql, Mariadb, PostgreSQL, Netezza,Redshift, Sqlite, File
|
|
|
|
|
|
|
|
|
|
_config = json.loads (file.read() )
|
|
|
|
|
file.close()
|
|
|
|
|
if index :
|
|
|
|
|
_config = _config[ int(index)]
|
|
|
|
|
etl.instance(**_config)
|
|
|
|
|
else:
|
|
|
|
|
etl.instance(_config)
|
|
|
|
|
|
|
|
|
|
#
|
|
|
|
|
# if type(_config) == dict :
|
|
|
|
|
# _object = transport.etl.instance(**_config)
|
|
|
|
|
# _proxy(_object)
|
|
|
|
|
# else:
|
|
|
|
|
# #
|
|
|
|
|
# # here we are dealing with a list of objects (long ass etl job)
|
|
|
|
|
# jobs = []
|
|
|
|
|
# failed = []
|
|
|
|
|
# for _args in _config :
|
|
|
|
|
# if index and _config.index(_args) != index :
|
|
|
|
|
# continue
|
|
|
|
|
|
|
|
|
|
# _object=transport.etl.instance(**_args)
|
|
|
|
|
# thread = Process(target=_proxy,args=(_object,))
|
|
|
|
|
# thread.start()
|
|
|
|
|
# jobs.append(thread())
|
|
|
|
|
# if _config.index(_args) == 0 :
|
|
|
|
|
# thread.join()
|
|
|
|
|
wait(jobs)
|
|
|
|
|
|
|
|
|
|
@app.command()
|
|
|
|
|
def generate (path:str):
|
|
|
|
|
__doc__="""
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
_config = [{"source":{"provider":"http","url":"https://cdn.wsform.com/wp-content/uploads/2020/06/agreement.csv"},"target":{"provider":"file","path":"addresses.csv","delimiter":"csv"}}]
|
|
|
|
|
file = open(path,'w')
|
|
|
|
|
file.write(json.dumps(_config))
|
|
|
|
|
file.close()
|
|
|
|
|
|
|
|
|
|
# if __name__ == '__main__' :
|
|
|
|
|
# #
|
|
|
|
|
# # Load information from the file ...
|
|
|
|
|
# if 'help' in SYS_ARGS :
|
|
|
|
|
# print (__doc__)
|
|
|
|
|
# else:
|
|
|
|
|
# try:
|
|
|
|
|
# _info = json.loads(open(SYS_ARGS['config']).read())
|
|
|
|
|
# if 'index' in SYS_ARGS :
|
|
|
|
|
# _index = int(SYS_ARGS['index'])
|
|
|
|
|
# _info = [_item for _item in _info if _info.index(_item) == _index]
|
|
|
|
|
# pass
|
|
|
|
|
# elif 'id' in SYS_ARGS :
|
|
|
|
|
# _info = [_item for _item in _info if 'id' in _item and _item['id'] == SYS_ARGS['id']]
|
