From 6544bf852acdac578b78c03c3e7fb9e2a6b9f392 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 14 Jun 2024 14:14:12 -0500 Subject: [PATCH 01/13] feature: registry for security and enterprise use --- bin/transport | 94 ++++++++++++++++--------------------- transport/__init__.py | 7 +++ transport/other/callback.py | 4 ++ transport/other/console.py | 3 ++ transport/registry.py | 81 ++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 54 deletions(-) create mode 100644 transport/registry.py diff --git a/bin/transport b/bin/transport index fd5d41b..4f9a7e8 100755 --- a/bin/transport +++ b/bin/transport @@ -13,29 +13,6 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -Usage : - transport help -- will print this page - - transport move [index] - path to the configuration file - optional index within the configuration file - -e.g: configuration file (JSON formatted) - - single source to a single target - - {"source":{"provider":"http","url":"https://cdn.wsform.com/wp-content/uploads/2020/06/agreement.csv"} - "target":{"provider":"sqlite3","path":"transport-demo.sqlite","table":"agreement"} - } - - - single source to multiple targets - { - "source":{"provider":"http","url":"https://cdn.wsform.com/wp-content/uploads/2020/06/agreement.csv"}, - "target":[ - {"provider":"sqlite3","path":"transport-demo.sqlite","table":"agreement}, - {"provider":"mongodb","db":"transport-demo","collection":"agreement"} - ] - } - """ import pandas as pd import numpy as np @@ -53,9 +30,13 @@ import typer from typing_extensions import Annotated from typing import Optional import time +from termcolor import colored app = typer.Typer() - +REGISTRY_PATH=os.sep.join([os.environ['HOME'],'.data-transport']) +REGISTRY_FILE= 'transport-registry.json' +CHECK_MARK = ' '.join(['[',colored(u'\u2713', 'green'),']']) +TIMES_MARK= ' '.join(['[',colored(u'\u2717','red'),']']) # @app.command() def help() : print (__doc__) @@ -68,7 +49,7 @@ def wait(jobs): def apply (path:Annotated[str,typer.Argument(help="path of the configuration file")], index:int = typer.Option(help="index of the item of interest, otherwise everything in the file will be processed")): """ - This function applies data transport from one source to one or several others + This function applies data transport ETL feature to read data from one source to write it one or several others """ # _proxy = lambda _object: _object.write(_object.read()) if os.path.exists(path): @@ -124,35 +105,40 @@ def generate (path:Annotated[str,typer.Argument(help="path of the ETL configurat file = open(path,'w') file.write(json.dumps(_config)) file.close() + +@app.command(name="init") +def initregistry (email:Annotated[str,typer.Argument(help="email")], + path:str=typer.Option(default=REGISTRY_PATH,help="path or location of the configuration file"), + override:bool=typer.Option(default=False,help="override existing configuration or not")): + """ + This functiion will initialize the registry and have both application and calling code loading the database parameters by a label + + """ + try: + transport.registry.init(email=email, path=path, override=override) + _msg = f"""{CHECK_MARK} Successfully wrote configuration to {path} from {email}""" + except Exception as e: + _msg = f"{TIMES_MARK} {e}" + print (_msg) + print () +@app.command(name="register") +def register (label:Annotated[str,typer.Argument(help="unique label that will be used to load the parameters of the database")], + auth_file:Annotated[str,typer.Argument(help="path of the auth_file")], + default:bool=typer.Option(default=False,help="set the auth_file as default"), + path:str=typer.Option(default=REGISTRY_PATH,help="path of the data-transport registry file")): + """ + This function will register an auth-file i.e database connection and assign it a label, + Learn more about auth-file at https://healthcareio.the-phi.com/data-transport + """ + try: + transport.registry.set(label=label,auth_file=auth_file, default=default, path=path) + _msg = f"""{CHECK_MARK} Successfully added label "{label}" to data-transport registry""" + except Exception as e: + _msg = f"""{TIMES_MARK} {e}""" + print (_msg) + + pass if __name__ == '__main__' : app() -# # -# # Load information from the file ... -# if 'help' in SYS_ARGS : -# print (__doc__) -# else: -# try: -# _info = json.loads(open(SYS_ARGS['config']).read()) -# if 'index' in SYS_ARGS : -# _index = int(SYS_ARGS['index']) -# _info = [_item for _item in _info if _info.index(_item) == _index] -# pass -# elif 'id' in SYS_ARGS : -# _info = [_item for _item in _info if 'id' in _item and _item['id'] == SYS_ARGS['id']] - -# procs = 1 if 'procs' not in SYS_ARGS else int(SYS_ARGS['procs']) -# jobs = transport.factory.instance(provider='etl',info=_info,procs=procs) -# print ([len(jobs),' Jobs are running']) -# N = len(jobs) -# while jobs : -# x = len(jobs) -# jobs = [_job for _job in jobs if _job.is_alive()] -# if x != len(jobs) : -# print ([len(jobs),'... jobs still running']) -# time.sleep(1) -# print ([N,' Finished running']) -# except Exception as e: - -# print (e) - + diff --git a/transport/__init__.py b/transport/__init__.py index d7d4518..2f97c0f 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -26,8 +26,11 @@ from info import __version__,__author__,__email__,__license__,__app_name__ from transport.iowrapper import IWriter, IReader, IETL from transport.plugins import PluginLoader from transport import providers +import copy +from transport import registry PROVIDERS = {} + def init(): global PROVIDERS for _module in [cloud,sql,nosql,other] : @@ -45,6 +48,10 @@ def instance (**_args): kwargs These are arguments that are provider/vendor specific """ global PROVIDERS + if not registry.isloaded () : + registry.load() if 'path' not in _args else registry.load(_args['path']) + if 'label' in _args : + _info = registry.load(_args['label']) if 'auth_file' in _args: if os.path.exists(_args['auth_file']) : # diff --git a/transport/other/callback.py b/transport/other/callback.py index c56c175..aba2f02 100644 --- a/transport/other/callback.py +++ b/transport/other/callback.py @@ -1,3 +1,7 @@ +""" +This module uses callback architectural style as a writer to enable user-defined code to handle the output of a reader +The intent is to allow users to have control over the output of data to handle things like logging, encryption/decryption and other +""" import queue from threading import Thread, Lock # from transport.common import Reader,Writer diff --git a/transport/other/console.py b/transport/other/console.py index 16f589a..b2f374b 100644 --- a/transport/other/console.py +++ b/transport/other/console.py @@ -1,3 +1,6 @@ +""" +This class uses classback pattern to allow output to be printed to the console (debugging) +""" from . import callback diff --git a/transport/registry.py b/transport/registry.py new file mode 100644 index 0000000..9fe942d --- /dev/null +++ b/transport/registry.py @@ -0,0 +1,81 @@ +import os +import json +from info import __version__ +import copy +import transport + +""" +This class manages data from the registry and allows (read only) +@TODO: add property to the DATA attribute +""" + +REGISTRY_PATH=os.sep.join([os.environ['HOME'],'.data-transport']) +REGISTRY_FILE= 'transport-registry.json' + +DATA = {} + +def isloaded (): + return DATA not in [{},None] +def load (_path=REGISTRY_PATH): + global DATA + _path = os.sep.join([_path,REGISTRY_FILE]) + if os.path.exists(_path) : + f = open(_path) + DATA = json.loads(f.read()) + f.close() +def init (email,path=REGISTRY_PATH,override=False): + """ + Initializing the registry and will raise an exception in the advent of an issue + """ + p = '@' in email + q = False if '.' not in email else email.split('.')[-1] in ['edu','com','io','ai'] + if p and q : + _config = {"email":email,'version':__version__} + if not os.path.exists(path): + os.makedirs(path) + filename = os.sep.join([path,REGISTRY_FILE]) + if not os.path.exists(filename) or override == True : + + f = open(filename,'w') + f.write( json.dumps(_config)) + f.close() + # _msg = f"""{CHECK_MARK} Successfully wrote configuration to {path} from {email}""" + + else: + raise Exception (f"""Unable to write configuration, Please check parameters (or help) and try again""") + else: + raise Exception (f"""Invalid Input, {email} is not well formatted, provide an email with adequate format""") + +def get (label='default') : + global DATA + return copy.copy(DATA[label]) if label in DATA else {} + +def set (label, auth_file, default=False,path=REGISTRY_PATH) : + reg_file = os.sep.join([path,REGISTRY_FILE]) + if os.path.exists (auth_file) and os.path.exists(path) and os.path.exists(reg_file): + f = open(auth_file) + _info = json.loads(f.read()) + f.close() + f = open(reg_file) + _config = json.loads(f.read()) + f.close() + + # + # set the proposed label + _object = transport.factory.instance(**_info) + if _object : + _config[label] = _info + if default : + _config['default'] = _info + # + # now we need to write this to the location + f = open(reg_file,'w') + f.write(json.dumps(_config)) + f.close() + else: + _msg = f"""Unable to load file locate at {path},\nLearn how to generate auth-file with wizard found at https://healthcareio.the-phi.com/data-transport""" + pass + else: + pass + pass + From 8edb764d112c9a14b2c4c8a26733920de8513ffa Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 14 Jun 2024 14:16:06 -0500 Subject: [PATCH 02/13] documentation typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 528176d..42bc859 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Mostly data scientists that don't really care about the underlying database and 1. Familiarity with **pandas data-frames** 2. Connectivity **drivers** are included -3. Mining data from various sources +3. Reading/Writing data from various sources 4. Useful for data migrations or **ETL** From b9bc898161f6ee4810dc7c9d30360af2f39a07a1 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 14 Jun 2024 15:30:09 -0500 Subject: [PATCH 03/13] bug fix: registry (more usable) and added to factory method --- bin/transport | 33 ++++++++++++++++++++------------- transport/__init__.py | 29 +++++++++++++++++++++++++---- transport/registry.py | 26 +++++++++++++++++++++----- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/bin/transport b/bin/transport index 4f9a7e8..4053c4e 100755 --- a/bin/transport +++ b/bin/transport @@ -47,7 +47,7 @@ def wait(jobs): @app.command(name="apply") def apply (path:Annotated[str,typer.Argument(help="path of the configuration file")], - index:int = typer.Option(help="index of the item of interest, otherwise everything in the file will be processed")): + index:int = typer.Option(default= None, help="index of the item of interest, otherwise everything in the file will be processed")): """ This function applies data transport ETL feature to read data from one source to write it one or several others """ @@ -92,19 +92,23 @@ def version(): @app.command() def generate (path:Annotated[str,typer.Argument(help="path of the ETL configuration file template (name included)")]): - """ - This function will generate a configuration template to give a sense of how to create one - """ - _config = [ - { - "source":{"provider":"http","url":"https://raw.githubusercontent.com/codeforamerica/ohana-api/master/data/sample-csv/addresses.csv"}, - "target": + """ + This function will generate a configuration template to give a sense of how to create one + """ + _config = [ + { + "source":{"provider":"http","url":"https://raw.githubusercontent.com/codeforamerica/ohana-api/master/data/sample-csv/addresses.csv"}, + "target": [{"provider":"files","path":"addresses.csv","delimiter":","},{"provider":"sqlite","database":"sample.db3","table":"addresses"}] } ] - file = open(path,'w') - file.write(json.dumps(_config)) - file.close() + file = open(path,'w') + file.write(json.dumps(_config)) + file.close() + print (f"""{CHECK_MARK} Successfully generated a template ETL file at {path}""" ) + print ("""NOTE: Each line (source or target) is the content of an auth-file""") + + @app.command(name="init") def initregistry (email:Annotated[str,typer.Argument(help="email")], @@ -131,8 +135,11 @@ def register (label:Annotated[str,typer.Argument(help="unique label that will be Learn more about auth-file at https://healthcareio.the-phi.com/data-transport """ try: - transport.registry.set(label=label,auth_file=auth_file, default=default, path=path) - _msg = f"""{CHECK_MARK} Successfully added label "{label}" to data-transport registry""" + if transport.registry.exists(path) : + transport.registry.set(label=label,auth_file=auth_file, default=default, path=path) + _msg = f"""{CHECK_MARK} Successfully added label "{label}" to data-transport registry""" + else: + _msg = f"""{TIMES_MARK} Registry is not initialized, please initialize the registry (check help)""" except Exception as e: _msg = f"""{TIMES_MARK} {e}""" print (_msg) diff --git a/transport/__init__.py b/transport/__init__.py index 2f97c0f..b2ea543 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -48,10 +48,16 @@ def instance (**_args): kwargs These are arguments that are provider/vendor specific """ global PROVIDERS - if not registry.isloaded () : - registry.load() if 'path' not in _args else registry.load(_args['path']) - if 'label' in _args : - _info = registry.load(_args['label']) + # if not registry.isloaded () : + # if ('path' in _args and registry.exists(_args['path'] )) or registry.exists(): + # registry.load() if 'path' not in _args else registry.load(_args['path']) + # print ([' GOT IT']) + # if 'label' in _args and registry.isloaded(): + # _info = registry.get(_args['label']) + # if _info : + # # + # _args = dict(_args,**_info) + if 'auth_file' in _args: if os.path.exists(_args['auth_file']) : # @@ -67,6 +73,17 @@ def instance (**_args): else: filename = _args['auth_file'] raise Exception(f" {filename} was not found or is invalid") + if 'provider' not in _args and 'auth_file' not in _args : + if not registry.isloaded () : + if ('path' in _args and registry.exists(_args['path'] )) or registry.exists(): + registry.load() if 'path' not in _args else registry.load(_args['path']) + if 'label' in _args and registry.isloaded(): + _info = registry.get(_args['label']) + print(_info) + if _info : + # + _args = dict(_args,**_info) + if 'provider' in _args and _args['provider'] in PROVIDERS : _info = PROVIDERS[_args['provider']] _module = _info['module'] @@ -110,6 +127,8 @@ class get : """ @staticmethod def reader (**_args): + if not _args : + _args['label'] = 'default' _args['context'] = 'read' return instance(**_args) @staticmethod @@ -117,6 +136,8 @@ class get : """ This function is a wrapper that will return a writer to a database. It disambiguates the interface """ + if not _args : + _args['label'] = 'default' _args['context'] = 'write' return instance(**_args) @staticmethod diff --git a/transport/registry.py b/transport/registry.py index 9fe942d..f487b54 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -16,11 +16,20 @@ DATA = {} def isloaded (): return DATA not in [{},None] +def exists (path=REGISTRY_PATH) : + """ + This function determines if there is a registry at all + """ + p = os.path.exists(path) + q = os.path.exists( os.sep.join([path,REGISTRY_FILE])) + print ([p,q, os.sep.join([path,REGISTRY_FILE])]) + return p and q def load (_path=REGISTRY_PATH): global DATA - _path = os.sep.join([_path,REGISTRY_FILE]) - if os.path.exists(_path) : - f = open(_path) + + if exists(_path) : + path = os.sep.join([_path,REGISTRY_FILE]) + f = open(path) DATA = json.loads(f.read()) f.close() def init (email,path=REGISTRY_PATH,override=False): @@ -45,12 +54,19 @@ def init (email,path=REGISTRY_PATH,override=False): raise Exception (f"""Unable to write configuration, Please check parameters (or help) and try again""") else: raise Exception (f"""Invalid Input, {email} is not well formatted, provide an email with adequate format""") - +def lookup (label): + global DATA + return label in DATA def get (label='default') : global DATA return copy.copy(DATA[label]) if label in DATA else {} def set (label, auth_file, default=False,path=REGISTRY_PATH) : + """ + This function will add a label (auth-file data) into the registry and can set it as the default + """ + if label == 'default' : + raise Exception ("""Invalid label name provided, please change the label name and use the switch""") reg_file = os.sep.join([path,REGISTRY_FILE]) if os.path.exists (auth_file) and os.path.exists(path) and os.path.exists(reg_file): f = open(auth_file) @@ -73,7 +89,7 @@ def set (label, auth_file, default=False,path=REGISTRY_PATH) : f.write(json.dumps(_config)) f.close() else: - _msg = f"""Unable to load file locate at {path},\nLearn how to generate auth-file with wizard found at https://healthcareio.the-phi.com/data-transport""" + raise Exception( f"""Unable to load file locate at {path},\nLearn how to generate auth-file with wizard found at https://healthcareio.the-phi.com/data-transport""") pass else: pass From 24cdd9f8fe2a44caeb0bb65fc81429230272957b Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 14 Jun 2024 19:56:42 -0500 Subject: [PATCH 04/13] bug fix: print statement --- transport/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport/registry.py b/transport/registry.py index f487b54..b8d5b16 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -22,7 +22,7 @@ def exists (path=REGISTRY_PATH) : """ p = os.path.exists(path) q = os.path.exists( os.sep.join([path,REGISTRY_FILE])) - print ([p,q, os.sep.join([path,REGISTRY_FILE])]) + return p and q def load (_path=REGISTRY_PATH): global DATA From 8aa6f2c93de50d6b2b35e7aae83d20e1857de34a Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 14 Jun 2024 20:05:12 -0500 Subject: [PATCH 05/13] bug fix: improve handling in registry --- transport/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport/__init__.py b/transport/__init__.py index b2ea543..27f2efb 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -79,7 +79,7 @@ def instance (**_args): registry.load() if 'path' not in _args else registry.load(_args['path']) if 'label' in _args and registry.isloaded(): _info = registry.get(_args['label']) - print(_info) + if _info : # _args = dict(_args,**_info) From dde4767e37b2824cddb29cce1857ea5b95e464a6 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 14 Jun 2024 20:11:33 -0500 Subject: [PATCH 06/13] new version --- info/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/info/__init__.py b/info/__init__.py index f45fdcd..d34b2f4 100644 --- a/info/__init__.py +++ b/info/__init__.py @@ -1,6 +1,6 @@ __app_name__ = 'data-transport' __author__ = 'The Phi Technology' -__version__= '2.0.4' +__version__= '2.2.0' __email__ = "info@the-phi.com" __license__=f""" Copyright 2010 - 2024, Steve L. Nyemba From c443c6c953b1bf2ca7eae4cbcce6dde0e095c403 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Sat, 15 Jun 2024 00:50:53 -0500 Subject: [PATCH 07/13] duckdb support --- setup.py | 4 ++-- transport/sql/__init__.py | 2 +- transport/sql/duckdb.py | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 transport/sql/duckdb.py diff --git a/setup.py b/setup.py index 002feb8..9b46d71 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,8 @@ args = { # "packages":["transport","info","transport/sql"]}, "packages": find_packages(include=['info','transport', 'transport.*'])} -args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite'] -args["install_requires"] = ['pyncclient','pymongo','sqlalchemy','pandas','typer','pandas-gbq','numpy','cloudant','pika','nzpy','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python','numpy','pymssql'] +args["keywords"]=['mongodb','duckdb','couchdb','rabbitmq','file','read','write','s3','sqlite'] +args["install_requires"] = ['pyncclient','duckdb-engine','pymongo','sqlalchemy','pandas','typer','pandas-gbq','numpy','cloudant','pika','nzpy','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python','numpy','pymssql'] args["url"] = "https://healthcareio.the-phi.com/git/code/transport.git" args['scripts'] = ['bin/transport'] # if sys.version_info[0] == 2 : diff --git a/transport/sql/__init__.py b/transport/sql/__init__.py index 9d026bf..b5aaa98 100644 --- a/transport/sql/__init__.py +++ b/transport/sql/__init__.py @@ -3,7 +3,7 @@ This namespace/package wrap the sql functionalities for a certain data-stores - netezza, postgresql, mysql and sqlite - mariadb, redshift (also included) """ -from . import postgresql, mysql, netezza, sqlite, sqlserver +from . import postgresql, mysql, netezza, sqlite, sqlserver, duckdb # diff --git a/transport/sql/duckdb.py b/transport/sql/duckdb.py new file mode 100644 index 0000000..ab82bb2 --- /dev/null +++ b/transport/sql/duckdb.py @@ -0,0 +1,21 @@ +""" +This module implements the handler for duckdb (in memory or not) +""" +from transport.sql.common import Base, BaseReader, BaseWriter + +class Duck : + def __init__(self,**_args): + self.database = _args['database'] + def get_provider(self): + return "duckdb" + + def _get_uri(self,**_args): + return f"""duckdb:///{self.database}""" +class Reader(Duck,BaseReader) : + def __init__(self,**_args): + Duck.__init__(self,**_args) + BaseReader.__init__(self,**_args) +class Writer(Duck,BaseWriter): + def __init__(self,**_args): + Duck.__init__(self,**_args) + BaseWriter.__init__(self,**_args) From 037019c1d79b367daf1d4656d6ec47db8f3a8037 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Sat, 15 Jun 2024 01:12:29 -0500 Subject: [PATCH 08/13] bug fix --- transport/providers/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transport/providers/__init__.py b/transport/providers/__init__.py index 4a583f7..6422d74 100644 --- a/transport/providers/__init__.py +++ b/transport/providers/__init__.py @@ -10,8 +10,11 @@ HTTP='http' BIGQUERY ='bigquery' FILE = 'file' ETL = 'etl' + SQLITE = 'sqlite' SQLITE3= 'sqlite3' +DUCKDB = 'duckdb' + REDSHIFT = 'redshift' NETEZZA = 'netezza' MYSQL = 'mysql' @@ -42,5 +45,6 @@ PGSQL = POSTGRESQL AWS_S3 = 's3' RABBIT = RABBITMQ + # QLISTENER = 'qlistener' \ No newline at end of file From 235a44be66e26c8540e80d489d445e2d4dbd3f58 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Wed, 19 Jun 2024 08:38:46 -0500 Subject: [PATCH 09/13] bug fix: registry and parameter handling --- README.md | 10 +++++++++- info/__init__.py | 6 ++++++ transport/__init__.py | 11 ++++++++--- transport/registry.py | 5 +++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 42bc859..dd2beb1 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,14 @@ Within the virtual environment perform the following : pip install git+https://github.com/lnyemba/data-transport.git +## What's new + +Unlike older versions 2.0 and under, we focus on collaborative environments like jupyter-x servers; apache zeppelin: + + 1. Simpler syntax to create reader or writer + 2. auth-file registry that can be referenced using a label + + ## Learn More -We have available notebooks with sample code to read/write against mongodb, couchdb, Netezza, PostgreSQL, Google Bigquery, Databricks, Microsoft SQL Server, MySQL ... Visit [data-transport homepage](https://healthcareio.the-phi.com/data-transport) \ No newline at end of file +We have available notebooks with sample code to read/write against mongodb, couchdb, Netezza, PostgreSQL, Google Bigquery, Databricks, Microsoft SQL Server, MySQL ... Visit [data-transport homepage](https://healthcareio.the-phi.com/data-transport) diff --git a/info/__init__.py b/info/__init__.py index d34b2f4..d84150e 100644 --- a/info/__init__.py +++ b/info/__init__.py @@ -12,3 +12,9 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + +__whatsnew__=f"""version {__version__}, focuses on collaborative environments like jupyter-base servers (apache zeppelin; jupyter notebook, jupyterlab, jupyterhub) + + 1. simpler syntax to create readers/writers + 2. auth-file registry that can be referenced using a label +""" diff --git a/transport/__init__.py b/transport/__init__.py index 27f2efb..6062453 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -22,7 +22,7 @@ from transport import sql, nosql, cloud, other import pandas as pd import json import os -from info import __version__,__author__,__email__,__license__,__app_name__ +from info import __version__,__author__,__email__,__license__,__app_name__,__whatsnew__ from transport.iowrapper import IWriter, IReader, IETL from transport.plugins import PluginLoader from transport import providers @@ -38,7 +38,11 @@ def init(): if _provider_name.startswith('__') or _provider_name == 'common': continue PROVIDERS[_provider_name] = {'module':getattr(_module,_provider_name),'type':_module.__name__} - +def _getauthfile (path) : + f = open(path) + _object = json.loads(f.read()) + f.close() + return _object def instance (**_args): """ This function returns an object of to read or write from a supported database provider/vendor @@ -82,7 +86,8 @@ def instance (**_args): if _info : # - _args = dict(_args,**_info) + # _args = dict(_args,**_info) + _args = dict(_info,**_args) #-- we can override the registry parameters with our own arguments if 'provider' in _args and _args['provider'] in PROVIDERS : _info = PROVIDERS[_args['provider']] diff --git a/transport/registry.py b/transport/registry.py index b8d5b16..ad94481 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -10,6 +10,11 @@ This class manages data from the registry and allows (read only) """ REGISTRY_PATH=os.sep.join([os.environ['HOME'],'.data-transport']) +# +# This path can be overriden by an environment variable ... +# +if 'DATA_TRANSPORT_REGISTRY_PATH' in os.environ : + REGISTRY_PATH = os.environ['DATA_TRANSPORT_REGISTRY_PATH'] REGISTRY_FILE= 'transport-registry.json' DATA = {} From 2edce85aede5c92b4d70540a09ae838baf6184f0 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Wed, 19 Jun 2024 08:40:24 -0500 Subject: [PATCH 10/13] documentation duckdb support --- README.md | 1 + info/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index dd2beb1..bfa67d9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Unlike older versions 2.0 and under, we focus on collaborative environments like 1. Simpler syntax to create reader or writer 2. auth-file registry that can be referenced using a label + 3. duckdb support ## Learn More diff --git a/info/__init__.py b/info/__init__.py index d84150e..6379b6c 100644 --- a/info/__init__.py +++ b/info/__init__.py @@ -17,4 +17,5 @@ __whatsnew__=f"""version {__version__}, focuses on collaborative environments li 1. simpler syntax to create readers/writers 2. auth-file registry that can be referenced using a label + 3. duckdb support """ From 808378afdbec21144288889b476890143a751dcb Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Wed, 19 Jun 2024 09:22:56 -0500 Subject: [PATCH 11/13] bug fix: delegate (new feature) --- transport/__init__.py | 14 ++++++++------ transport/iowrapper.py | 7 +++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/transport/__init__.py b/transport/__init__.py index 6062453..16a2467 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -81,13 +81,15 @@ def instance (**_args): if not registry.isloaded () : if ('path' in _args and registry.exists(_args['path'] )) or registry.exists(): registry.load() if 'path' not in _args else registry.load(_args['path']) + _info = {} if 'label' in _args and registry.isloaded(): _info = registry.get(_args['label']) - - if _info : - # - # _args = dict(_args,**_info) - _args = dict(_info,**_args) #-- we can override the registry parameters with our own arguments + else: + _info = registry.get() + if _info : + # + # _args = dict(_args,**_info) + _args = dict(_info,**_args) #-- we can override the registry parameters with our own arguments if 'provider' in _args and _args['provider'] in PROVIDERS : _info = PROVIDERS[_args['provider']] @@ -132,7 +134,7 @@ class get : """ @staticmethod def reader (**_args): - if not _args : + if not _args or 'provider' not in _args: _args['label'] = 'default' _args['context'] = 'read' return instance(**_args) diff --git a/transport/iowrapper.py b/transport/iowrapper.py index d6cba1c..e3abf6c 100644 --- a/transport/iowrapper.py +++ b/transport/iowrapper.py @@ -52,6 +52,13 @@ class IO: if hasattr(self._agent,'apply') : return self._agent.apply(_query) return None + def submit(self,_query): + return self.delegate('submit',_query) + def delegate(self,_name,_query): + if hasattr(self._agent,_name) : + pointer = getattr(self._agent,_name) + return pointer(_query) + return None class IReader(IO): """ This is a wrapper for read functionalities From 6f6fd489821664692d56951e97d7c27ec14bab86 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Tue, 25 Jun 2024 11:48:57 -0500 Subject: [PATCH 12/13] bug fixes: environment variable usage --- transport/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport/registry.py b/transport/registry.py index ad94481..6764f1b 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -42,7 +42,7 @@ def init (email,path=REGISTRY_PATH,override=False): Initializing the registry and will raise an exception in the advent of an issue """ p = '@' in email - q = False if '.' not in email else email.split('.')[-1] in ['edu','com','io','ai'] + q = False if '.' not in email else email.split('.')[-1] in ['edu','com','io','ai','org'] if p and q : _config = {"email":email,'version':__version__} if not os.path.exists(path): From 3faee02fa26e0cef920ec9468fb6460dccdff869 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Sun, 7 Jul 2024 11:39:29 -0500 Subject: [PATCH 13/13] documentation ... --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index bfa67d9..7d8b414 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ Within the virtual environment perform the following : pip install git+https://github.com/lnyemba/data-transport.git +## Features + + - read/write from over a dozen databases + - run ETL jobs seamlessly + - scales and integrates into shared environments like apache zeppelin; jupyterhub; SageMaker; ... ## What's new