From 66d881fdda720119d0fa94029c66fc926e66322b Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Mon, 30 Jun 2025 15:16:30 -0500 Subject: [PATCH 1/3] upgrade pyproject.toml, bug fix with registry --- info/__init__.py | 2 +- setup.py | 33 --------- transport/registry.py | 163 ++---------------------------------------- 3 files changed, 7 insertions(+), 191 deletions(-) delete mode 100644 setup.py diff --git a/info/__init__.py b/info/__init__.py index ef03a5b..501211f 100644 --- a/info/__init__.py +++ b/info/__init__.py @@ -1,6 +1,6 @@ __app_name__ = 'data-transport' __author__ = 'The Phi Technology' -__version__= '2.2.18' +__version__= '2.2.22' __email__ = "info@the-phi.com" __edition__= 'community' __license__=f""" diff --git a/setup.py b/setup.py deleted file mode 100644 index 119baab..0000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -This is a build file for the -""" -from setuptools import setup, find_packages -import os -import sys -# from version import __version__,__author__ -from info import __version__, __author__,__app_name__,__license__,__edition__ - - -def read(fname): - return open(os.path.join(os.path.dirname(__file__), fname)).read() -args = { - "name":__app_name__, - "version":__version__, - "author":__author__,"author_email":"info@the-phi.com", - "license":__license__, - # "packages":["transport","info","transport/sql"]}, - - "packages": find_packages(include=['info','transport', 'transport.*'])} -args["keywords"]=['mongodb','duckdb','couchdb','rabbitmq','file','read','write','s3','sqlite'] -args["install_requires"] = ['pyncclient','duckdb-engine','pymongo','sqlalchemy','pandas','typer','pandas-gbq','numpy','cloudant','pika','nzpy','termcolor','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python','numpy','pymssql','pyspark','pydrill','sqlalchemy_drill','plugin-ix@git+https://github.com/lnyemba/plugins-ix'] -args["url"] = "https://healthcareio.the-phi.com/git/code/transport.git" -args['scripts'] = ['bin/transport'] -args['classifiers'] = ['Programming Language :: Python :: 3', - 'License :: OSI Approved :: MIT License', - "Topic :: Utilities", - 'Operating System :: OS Independent' - ] -# if sys.version_info[0] == 2 : -# args['use_2to3'] = True -# args['use_2to3_exclude_fixers']=['lib2to3.fixes.fix_import'] -setup(**args) diff --git a/transport/registry.py b/transport/registry.py index 8853069..1c9443e 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -6,7 +6,7 @@ import transport import importlib import importlib.util import shutil - +from io import StringIO """ This class manages data from the registry and allows (read only) @@ -21,161 +21,7 @@ if 'DATA_TRANSPORT_REGISTRY_PATH' in os.environ : REGISTRY_PATH = os.environ['DATA_TRANSPORT_REGISTRY_PATH'] REGISTRY_FILE= 'transport-registry.json' DATA = {} -# class plugins: -# # -# # This is a utility function that should enable management of plugins-registry -# # The class allows to add/remove elements -# # -# # @TODO: add read/write properties to the class (better design practice) -# # -# _data = {} -# FOLDER = os.sep.join([REGISTRY_PATH,'plugins']) -# CODE = os.sep.join([REGISTRY_PATH,'plugins','code']) -# FILE = os.sep.join([REGISTRY_PATH,'plugin-registry.json']) -# @staticmethod -# def init(): - -# if not os.path.exists(plugins.FOLDER) : -# os.makedirs(plugins.FOLDER) -# if not os.path.exists(plugins.CODE): -# os.makedirs(plugins.CODE) -# if not os.path.exists(plugins.FILE): -# f = open(plugins.FILE,'w') -# f.write("{}") -# f.close() -# plugins._read() #-- will load data as a side effect - -# @staticmethod -# def copy (path) : - -# shutil.copy2(path,plugins.CODE) -# @staticmethod -# def _read (): -# f = open(plugins.FILE) -# try: -# _data = json.loads(f.read()) -# f.close() -# except Exception as e: -# print (f"Corrupted registry, resetting ...") -# _data = {} -# plugins._write(_data) - -# plugins._data = _data -# @staticmethod -# def _write (_data): -# f = open(plugins.FILE,'w') -# f.write(json.dumps(_data)) -# f.close() -# plugins._data = _data - -# @staticmethod -# def inspect (_path): -# _names = [] - -# if os.path.exists(_path) : -# _filename = _path.split(os.sep)[-1] -# spec = importlib.util.spec_from_file_location(_filename, _path) -# module = importlib.util.module_from_spec(spec) -# spec.loader.exec_module(module) -# # _names = [{'name':getattr(getattr(module,_name),'name'),'pointer':getattr(module,_name)} for _name in dir(module) if type( getattr(module,_name)).__name__ == 'function'] -# for _name in dir(module) : -# _pointer = getattr(module,_name) -# if hasattr(_pointer,'transport') : -# _item = {'real_name':_name,'name':getattr(_pointer,'name'),'pointer':_pointer,'version':getattr(_pointer,'version')} -# _names.append(_item) - - -# return _names -# @staticmethod -# def add (alias,path): -# """ -# Add overwrite the registry entries -# """ -# _names = plugins.inspect (path) -# _log = [] - -# if _names : -# # -# # We should make sure we have all the plugins with the attributes (transport,name) set -# _names = [_item for _item in _names if hasattr(_item['pointer'],'transport') ] -# if _names : -# plugins.copy(path) -# _content = [] - -# for _item in _names : -# _key = '@'.join([alias,_item['name']]) -# _log.append(_item['name']) -# # -# # Let us update the registry -# # -# plugins.update(alias,path,_log) -# return _log - -# @staticmethod -# def update (alias,path,_log) : -# """ -# updating the registry entries of the plugins (management data) -# """ -# # f = open(plugins.FILE) -# # _data = json.loads(f.read()) -# # f.close() -# _data = plugins._data -# # _log = plugins.add(alias,path) - -# if _log : -# _data[alias] = {'content':_log,'name':path.split(os.sep)[-1]} -# plugins._write(_data) #-- will update data as a side effect - -# return _log -# @staticmethod -# def get(**_args) : -# # f = open(plugins.FILE) -# # _data = json.loads(f.read()) -# # f.close() -# # if 'key' in _args : -# # alias,name = _args['key'].split('.') if '.' in _args['key'] else _args['key'].split('@') -# # else : -# # alias = _args['alias'] -# # name = _args['name'] - -# # if alias in _data : - -# # _path = os.sep.join([plugins.CODE,_data[alias]['name']]) -# # _item = [_item for _item in plugins.inspect(_path) if name == _item['name']] - -# # _item = _item[0] if _item else None -# # if _item : - -# # return _item['pointer'] -# # return None -# _item = plugins.has(**_args) -# return _item['pointer'] if _item else None - -# @staticmethod -# def has (**_args): -# f = open(plugins.FILE) -# _data = json.loads(f.read()) -# f.close() -# if 'key' in _args : -# alias,name = _args['key'].split('.') if '.' in _args['key'] else _args['key'].split('@') -# else : -# alias = _args['alias'] -# name = _args['name'] - -# if alias in _data : - -# _path = os.sep.join([plugins.CODE,_data[alias]['name']]) -# _item = [_item for _item in plugins.inspect(_path) if name == _item['name']] - -# _item = _item[0] if _item else None -# if _item : - -# return copy.copy(_item) -# return None -# @staticmethod -# def synch(): -# pass def isloaded (): return DATA not in [{},None] @@ -233,8 +79,11 @@ def set (label, auth_file, default=False,path=REGISTRY_PATH) : if label == 'default' : raise Exception ("""Invalid label name provided, please change the label name and use the switch""") reg_file = os.sep.join([path,REGISTRY_FILE]) - if os.path.exists (auth_file) and os.path.exists(path) and os.path.exists(reg_file): - f = open(auth_file) + if os.path.exists(path) and os.path.exists(reg_file): + if type(auth_file) == str and os.path.exists (auth_file) : + f = open(auth_file) + elif type(auth_file) == StringIO: + f = auth_file _info = json.loads(f.read()) f.close() f = open(reg_file) From 6e1c4209529c339aba0d20e565d517e981d770d5 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Mon, 30 Jun 2025 15:51:50 -0500 Subject: [PATCH 2/3] project file specification --- pyproject.toml | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b61e7e5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,62 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "data-transport" +dynamic = ["version"] +authors = [ + {name="Steve L. Nyemba" , email = "info@the-phi.com"}, +] +description = "" +readme = "README.md" +license = {text = "LICENSE"} +keywords = ["mongodb","duckdb","couchdb","rabbitmq","file","read","write","s3","sqlite"] +classifiers = [ + "License :: OSI Approved :: MIT License", + "Topic :: Utilities", +] +dependencies = [ + "termcolor","sqlalchemy", "aiosqlite","duckdb-engine", + "typer","pandas","numpy","sqlalchemy","pyarrow", + "plugin-ix@git+https://github.com/lnyemba/plugins-ix" +] +[project.optional-dependencies] +sql = ["mysql-connector-python","psycopg2-binary","nzpy","pymssql","duckdb-engine","aiosqlite"] +nosql = ["pymongo","cloudant"] +cloud = ["pandas-gbq","google-cloud-bigquery","google-cloud-bigquery-storage", "databricks-sqlalchemy","pyncclient","boto3","boto","botocore"] +warehouse = ["pydrill","pyspark","sqlalchemy_drill"] +rabbitmq = ["pika"] +sqlite = ["aiosqlite"] +aws3 = ["boto3","boto","botocore"] +nextcloud = ["pyncclient"] +mongodb = ["pymongo"] +netezza = ["nzpy"] +mysql = ["mysql-connector-python"] +postgresql = ["psycopg2-binary"] +sqlserver = ["pymssql"] +http = ["flask-session"] +all = ["mysql-connector-python","psycopg2-binary","nzpy","pymssql","duckdb-engine","aiosqlite","pymongo","cloudant","pandas-gbq","google-cloud-bigquery","google-cloud-bigquery-storage", "databricks-sqlalchemy","pyncclient","boto3","boto","botocore","pydrill","pyspark","sqlalchemy_drill", "pika","aiosqlite","boto3","boto","botocore", "pyncclient"] + +[project.urls] +Homepage = "https://healthcareio.the-phi.com/git/code/transport.git" + +#[project.scripts] +#transport = "transport:main" + +[tool.setuptools] +include-package-data = true +zip-safe = false +script-files = ["bin/transport"] + +[tool.setuptools.packages.find] +include = ["info","info.*", "transport", "transport.*"] + +[tool.setuptools.dynamic] +version = {attr = "info.__version__"} +#authors = {attr = "meta.__author__"} + +# If you have a info.py file, you might also want to include the author dynamically: +# [tool.setuptools.dynamic] +# version = {attr = "info.__version__"} +# authors = {attr = "info.__author__"} From fbdb4a493135f8a91275b98013d04f4207f87e7b Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 4 Jul 2025 16:54:52 -0500 Subject: [PATCH 3/3] bug fix: registry and emails --- transport/registry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/transport/registry.py b/transport/registry.py index 1c9443e..196b2f0 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -12,8 +12,11 @@ from io import StringIO This class manages data from the registry and allows (read only) @TODO: add property to the DATA attribute """ +if 'HOME' in os.environ : + REGISTRY_PATH=os.sep.join([os.environ['HOME'],'.data-transport']) +else: + REGISTRY_PATH=os.sep.join([os.environ['USERPROFILE'],'.data-transport']) -REGISTRY_PATH=os.sep.join([os.environ.get('HOME','USERPROFILE'),'.data-transport']) # # This path can be overriden by an environment variable ... #