diff --git a/bin/transport b/bin/transport index 8cec73f..f6c511e 100755 --- a/bin/transport +++ b/bin/transport @@ -177,7 +177,25 @@ def register (label:Annotated[str,typer.Argument(help="unique label that will be except Exception as e: _msg = f"""{TIMES_MARK} {e}""" print (_msg) - +@app_r.command(name="template") +def template(name:Annotated[str,typer.Argument(help="database technology provider" ) ]): + """ + This function will generate a template entry for the registry (content of an auth file) + """ + # + # retrieve the provider and display the template if it has one + for _module in ['sql','cloud','warehouse','nosql','other'] : + ref = getattr(transport,_module) if hasattr(transport,_module) else None + _entry = {} + if ref : + if hasattr(ref,name) : + _pointer = getattr(ref,name) + _entry = dict({'provider':name},**_pointer.template()) if hasattr(_pointer,'template') else {} + break + # + # + print ( json.dumps(_entry)) + pass @app_r.command(name="list") def register_list (): """ diff --git a/pyproject.toml b/pyproject.toml index 0391412..ccdf679 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,9 +46,4 @@ include = [ "transport", "transport.*"] [tool.setuptools.dynamic] version = {attr = "transport.info.__version__"} -#authors = {attr = "transport.__author__"} - -# If you have a info.py file, you might also want to include the author dynamically: -# [tool.setuptools.dynamic] -# version = {attr = "info.__version__"} -# authors = {attr = "info.__author__"} +#authors = {attr = "info.__author__"} diff --git a/transport/cloud/bigquery.py b/transport/cloud/bigquery.py index ba720af..5c6e1d1 100644 --- a/transport/cloud/bigquery.py +++ b/transport/cloud/bigquery.py @@ -14,6 +14,8 @@ import numpy as np import time MAX_CHUNK = 2000000 +def template (): + return {'provider':'bigquery','private_key':'path-to-key','dataset':'name-of-dataset','table':'table'} class BigQuery: def __init__(self,**_args): path = _args['service_key'] if 'service_key' in _args else _args['private_key'] @@ -23,6 +25,7 @@ class BigQuery: self.dtypes = _args['dtypes'] if 'dtypes' in _args else None self.table = _args['table'] if 'table' in _args else None self.client = bq.Client.from_service_account_json(self.path) + def meta(self,**_args): """ This function returns meta data for a given table or query with dataset/table properly formatted diff --git a/transport/cloud/databricks.py b/transport/cloud/databricks.py index 5c1ee0d..088fb9e 100644 --- a/transport/cloud/databricks.py +++ b/transport/cloud/databricks.py @@ -17,6 +17,8 @@ import sqlalchemy # from transport.common import Reader,Writer import pandas as pd +def template (): + return {'provider':'databricks','host':'fqn-host','token':'token','cluster_path':'path-of-cluster','catalog':'name-of-catalog','database':'schema-or-database','table':'table'} class Bricks: """ diff --git a/transport/cloud/nextcloud.py b/transport/cloud/nextcloud.py index ebb44d3..071122a 100644 --- a/transport/cloud/nextcloud.py +++ b/transport/cloud/nextcloud.py @@ -8,6 +8,8 @@ import pandas as pd from io import StringIO import json import nextcloud_client as nextcloud +def template(): + return {"url":None,"token":None,"uid":None,"file":None} class Nextcloud : def __init__(self,**_args): diff --git a/transport/cloud/s3.py b/transport/cloud/s3.py index 81e2e69..7ecd091 100644 --- a/transport/cloud/s3.py +++ b/transport/cloud/s3.py @@ -20,6 +20,8 @@ from io import StringIO import pandas as pd import json +def template(): + return {'access_key':'access-key','secret_key':'secret-key','region':'region','bucket':'name-of-bucket','file':'file-name','chunksize':10000} class s3 : """ @TODO: Implement a search function for a file given a bucket?? diff --git a/transport/info.py b/transport/info.py index eb1453d..374d2bc 100644 --- a/transport/info.py +++ b/transport/info.py @@ -1,6 +1,6 @@ __app_name__ = 'data-transport' __author__ = 'The Phi Technology' -__version__= '2.2.30' +__version__= '2.2.42' __email__ = "info@the-phi.com" __edition__= 'community' __license__=f""" diff --git a/transport/iowrapper.py b/transport/iowrapper.py index 6fd1196..105d024 100644 --- a/transport/iowrapper.py +++ b/transport/iowrapper.py @@ -111,8 +111,8 @@ class IETL(IReader) : _data = super().read(**_args) _schema = super().meta() for _kwargs in self._targets : - if _schema : - _kwargs['schema'] = _schema + # if _schema : + # _kwargs['schema'] = _schema self.post(_data,**_kwargs) return _data diff --git a/transport/nosql/couchdb.py b/transport/nosql/couchdb.py index aa503fb..d90c016 100644 --- a/transport/nosql/couchdb.py +++ b/transport/nosql/couchdb.py @@ -11,6 +11,8 @@ import sys # from transport.common import Reader, Writer from datetime import datetime +def template(): + return {'dbname':'database','doc':'document','username':'username','password':'password','url':'url-with-port'} class Couch: """ diff --git a/transport/nosql/mongodb.py b/transport/nosql/mongodb.py index 503f821..3d4e209 100644 --- a/transport/nosql/mongodb.py +++ b/transport/nosql/mongodb.py @@ -19,7 +19,8 @@ import json import re from multiprocessing import Lock, RLock from transport.common import IEncoder - +def template(): + return {'provider':'mongodb','host':'localhost','port':27017,'db':'db-name','collection':'collection-name','username':'username','password':'password','mechanism':'SCRAM-SHA-256'} class Mongo : lock = RLock() """ diff --git a/transport/other/files.py b/transport/other/files.py index 57d3f25..31dfb90 100644 --- a/transport/other/files.py +++ b/transport/other/files.py @@ -4,6 +4,10 @@ This file is a wrapper around pandas built-in functionalities to handle characte import pandas as pd import numpy as np import os + +def template(): + return {'path':None,'delimiter':None} + class File : def __init__(self,**params): """ diff --git a/transport/other/http.py b/transport/other/http.py index d92e334..48a25a9 100644 --- a/transport/other/http.py +++ b/transport/other/http.py @@ -7,7 +7,8 @@ import requests from io import StringIO import pandas as pd - +def template(): + return {'url':None, 'headers':{'key':'value'}} class Reader: """ This class is designed to read data from an Http request file handler provided to us by flask diff --git a/transport/other/rabbitmq.py b/transport/other/rabbitmq.py index f56800d..0fa2b0c 100644 --- a/transport/other/rabbitmq.py +++ b/transport/other/rabbitmq.py @@ -17,6 +17,10 @@ import sys # from common import Reader, Writer import json from multiprocessing import RLock + +def template(): + return {'port':5672,'host':'localhost','queue':None,'vhost':None,'username':None,'password':None} + class MessageQueue: """ This class hierarchy is designed to handle interactions with a queue server using pika framework (our tests are based on rabbitmq) diff --git a/transport/registry.py b/transport/registry.py index 4fb96db..3a84902 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -51,6 +51,7 @@ def init (email,path=REGISTRY_PATH,override=False,_file=REGISTRY_FILE): p = '@' in email #q = False if '.' not in email else email.split('.')[-1] in ['edu','com','io','ai','org'] q = len(email.split('.')[-1]) in [2,3] + if p and q : _config = {"email":email,'version':__version__} if not os.path.exists(path): diff --git a/transport/sql/common.py b/transport/sql/common.py index 7cf303f..4695283 100644 --- a/transport/sql/common.py +++ b/transport/sql/common.py @@ -6,7 +6,8 @@ import sqlalchemy as sqa from sqlalchemy import text , MetaData, inspect import pandas as pd - +def template(): + return {'host':'localhost','database':'database','table':'table'} class Base: def __init__(self,**_args): self._host = _args['host'] if 'host' in _args else 'localhost' diff --git a/transport/sql/duckdb.py b/transport/sql/duckdb.py index 97fb3fa..ab5b0d2 100644 --- a/transport/sql/duckdb.py +++ b/transport/sql/duckdb.py @@ -2,7 +2,8 @@ This module implements the handler for duckdb (in memory or not) """ from transport.sql.common import Base, BaseReader, BaseWriter - +def template (): + return {'database':'path-to-database','table':'table'} class Duck : def __init__(self,**_args): # diff --git a/transport/sql/mysql.py b/transport/sql/mysql.py index 320eb68..b00ebf8 100644 --- a/transport/sql/mysql.py +++ b/transport/sql/mysql.py @@ -1,8 +1,13 @@ """ This file implements support for mysql and maria db (with drivers mysql+mysql) """ -from transport.sql.common import BaseReader, BaseWriter +from transport.sql.common import BaseReader, BaseWriter, template as _template +# import transport.sql.common.template as _template + # import mysql.connector as my +def template (): + return dict(_template(),**{'port':3306}) + class MYSQL: def get_provider(self): diff --git a/transport/sql/netezza.py b/transport/sql/netezza.py index 6d53164..f8cd466 100644 --- a/transport/sql/netezza.py +++ b/transport/sql/netezza.py @@ -1,6 +1,9 @@ import nzpy as nz -from transport.sql.common import BaseReader, BaseWriter +from transport.sql.common import BaseReader, BaseWriter, template as _template + +def template (): + return dict(_template(),**{'port':5480}) class Netezza: def get_provider(self): return 'netezza+nzpy' diff --git a/transport/sql/postgresql.py b/transport/sql/postgresql.py index 0831291..73bc431 100644 --- a/transport/sql/postgresql.py +++ b/transport/sql/postgresql.py @@ -1,10 +1,13 @@ -from transport.sql.common import BaseReader , BaseWriter +from transport.sql.common import BaseReader , BaseWriter, template as _template from psycopg2.extensions import register_adapter, AsIs import numpy as np register_adapter(np.int64, AsIs) +def template (): + return dict(_template(),**{'port':5432,'chunksize':10000}) + class PG: def __init__(self,**_args): super().__init__(**_args) diff --git a/transport/sql/sqlite.py b/transport/sql/sqlite.py index 734ab24..ecff530 100644 --- a/transport/sql/sqlite.py +++ b/transport/sql/sqlite.py @@ -1,6 +1,8 @@ import sqlalchemy import pandas as pd from transport.sql.common import Base, BaseReader, BaseWriter +def template(): + return {'database':'path-to-database','table':'table'} class SQLite (BaseReader): def __init__(self,**_args): super().__init__(**_args) diff --git a/transport/sql/sqlserver.py b/transport/sql/sqlserver.py index 6a53842..0cc6f85 100644 --- a/transport/sql/sqlserver.py +++ b/transport/sql/sqlserver.py @@ -3,7 +3,11 @@ Handling Microsoft SQL Server via pymssql driver/connector """ import sqlalchemy import pandas as pd -from transport.sql.common import Base, BaseReader, BaseWriter +from transport.sql.common import Base, BaseReader, BaseWriter, template as _template + + +def template (): + return dict(_template(),**{'port':1433}) class MsSQLServer: diff --git a/transport/warehouse/drill.py b/transport/warehouse/drill.py index 71f0e64..3e82070 100644 --- a/transport/warehouse/drill.py +++ b/transport/warehouse/drill.py @@ -3,6 +3,10 @@ import pandas as pd from .. sql.common import BaseReader , BaseWriter import sqlalchemy as sqa + +def template(): + return {'host':'localhost','port':8047,'ssl':False,'table':None,'database':None} + class Drill : __template = {'host':None,'port':None,'ssl':None,'table':None,'database':None} def __init__(self,**_args): diff --git a/transport/warehouse/iceberg.py b/transport/warehouse/iceberg.py index 3def181..808846c 100644 --- a/transport/warehouse/iceberg.py +++ b/transport/warehouse/iceberg.py @@ -11,6 +11,8 @@ from pyspark.sql.types import * from pyspark.sql.functions import col, to_date, to_timestamp import copy +def template(): + return {'catalog':None,'database':None,'table':None} class Iceberg : def __init__(self,**_args): """