data-transport
/
data-transport
1
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'v2.2.0' (#33) from v2.2.0 into master

Browse Source 
Reviewed-on: #33
master
Steve L. Nyemba 4 months ago
parent 6c26588462 fbdb4a4931
commit b461ce9d7b
4 changed files with 73 additions and 192 deletions
Show Stats Download Patch File Download Diff File

2
info/__init__.py
Unescape View File

@ -1,6 +1,6 @@
__app_name__ = 'data-transport'
__author__ = 'The Phi Technology'
__version__= '2.2.18'
__version__= '2.2.22'
__email__ = "info@the-phi.com"
__edition__= 'community'
__license__=f"""

62
pyproject.toml
Unescape View File

@ -0,0 +1,62 @@
[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "data-transport"
dynamic = ["version"]
authors = [
{name="Steve L. Nyemba" , email = "info@the-phi.com"},
]
description = ""
readme = "README.md"
license = {text = "LICENSE"}
keywords = ["mongodb","duckdb","couchdb","rabbitmq","file","read","write","s3","sqlite"]
classifiers = [
"License :: OSI Approved :: MIT License",
"Topic :: Utilities",
]
dependencies = [
"termcolor","sqlalchemy", "aiosqlite","duckdb-engine",
"typer","pandas","numpy","sqlalchemy","pyarrow",
"plugin-ix@git+https://github.com/lnyemba/plugins-ix"
]
[project.optional-dependencies]
sql = ["mysql-connector-python","psycopg2-binary","nzpy","pymssql","duckdb-engine","aiosqlite"]
nosql = ["pymongo","cloudant"]
cloud = ["pandas-gbq","google-cloud-bigquery","google-cloud-bigquery-storage", "databricks-sqlalchemy","pyncclient","boto3","boto","botocore"]
warehouse = ["pydrill","pyspark","sqlalchemy_drill"]
rabbitmq = ["pika"]
sqlite = ["aiosqlite"]
aws3 = ["boto3","boto","botocore"]
nextcloud = ["pyncclient"]
mongodb = ["pymongo"]
netezza = ["nzpy"]
mysql = ["mysql-connector-python"]
postgresql = ["psycopg2-binary"]
sqlserver = ["pymssql"]
http = ["flask-session"]
all = ["mysql-connector-python","psycopg2-binary","nzpy","pymssql","duckdb-engine","aiosqlite","pymongo","cloudant","pandas-gbq","google-cloud-bigquery","google-cloud-bigquery-storage", "databricks-sqlalchemy","pyncclient","boto3","boto","botocore","pydrill","pyspark","sqlalchemy_drill", "pika","aiosqlite","boto3","boto","botocore", "pyncclient"]
[project.urls]
Homepage = "https://healthcareio.the-phi.com/git/code/transport.git"
#[project.scripts]
#transport = "transport:main"
[tool.setuptools]
include-package-data = true
zip-safe = false
script-files = ["bin/transport"]
[tool.setuptools.packages.find]
include = ["info","info.*", "transport", "transport.*"]
[tool.setuptools.dynamic]
version = {attr = "info.__version__"}
#authors = {attr = "meta.__author__"}
# If you have a info.py file, you might also want to include the author dynamically:
# [tool.setuptools.dynamic]
# version = {attr = "info.__version__"}
# authors = {attr = "info.__author__"}

33
setup.py
Unescape View File

@ -1,33 +0,0 @@
"""
This is a build file for the
"""
from setuptools import setup, find_packages
import os
import sys
# from version import __version__,__author__
from info import __version__, __author__,__app_name__,__license__,__edition__
def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = {
"name":__app_name__,
"version":__version__,
"author":__author__,"author_email":"info@the-phi.com",
"license":__license__,
# "packages":["transport","info","transport/sql"]},
"packages": find_packages(include=['info','transport', 'transport.*'])}
args["keywords"]=['mongodb','duckdb','couchdb','rabbitmq','file','read','write','s3','sqlite']
args["install_requires"] = ['pyncclient','duckdb-engine','pymongo','sqlalchemy','pandas','typer','pandas-gbq','numpy','cloudant','pika','nzpy','termcolor','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python','numpy','pymssql','pyspark','pydrill','sqlalchemy_drill','plugin-ix@git+https://github.com/lnyemba/plugins-ix']
args["url"] = "https://healthcareio.the-phi.com/git/code/transport.git"
args['scripts'] = ['bin/transport']
args['classifiers'] = ['Programming Language :: Python :: 3',
'License :: OSI Approved :: MIT License',
"Topic :: Utilities",
'Operating System :: OS Independent'
]
# if sys.version_info[0] == 2 :
# args['use_2to3'] = True
# args['use_2to3_exclude_fixers']=['lib2to3.fixes.fix_import']
setup(**args)

168
transport/registry.py
Unescape View File

@ -6,14 +6,17 @@ import transport
import importlib
import importlib.util
import shutil
from io import StringIO
"""
This class manages data from the registry and allows (read only)
@TODO: add property to the DATA attribute
"""
if 'HOME' in os.environ :
REGISTRY_PATH=os.sep.join([os.environ['HOME'],'.data-transport'])
else:
REGISTRY_PATH=os.sep.join([os.environ['USERPROFILE'],'.data-transport'])
REGISTRY_PATH=os.sep.join([os.environ.get('HOME','USERPROFILE'),'.data-transport'])
#
# This path can be overriden by an environment variable ...
#
@ -21,161 +24,7 @@ if 'DATA_TRANSPORT_REGISTRY_PATH' in os.environ :
REGISTRY_PATH = os.environ['DATA_TRANSPORT_REGISTRY_PATH']
REGISTRY_FILE= 'transport-registry.json'
DATA = {}
# class plugins:
# #
# # This is a utility function that should enable management of plugins-registry
# # The class allows to add/remove elements
# #
# # @TODO: add read/write properties to the class (better design practice)
# #
# _data = {}
# FOLDER = os.sep.join([REGISTRY_PATH,'plugins'])
# CODE = os.sep.join([REGISTRY_PATH,'plugins','code'])
# FILE = os.sep.join([REGISTRY_PATH,'plugin-registry.json'])
# @staticmethod
# def init():
# if not os.path.exists(plugins.FOLDER) :
# os.makedirs(plugins.FOLDER)
# if not os.path.exists(plugins.CODE):
# os.makedirs(plugins.CODE)
# if not os.path.exists(plugins.FILE):
# f = open(plugins.FILE,'w')
# f.write("{}")
# f.close()
# plugins._read() #-- will load data as a side effect
# @staticmethod
# def copy (path) :
# shutil.copy2(path,plugins.CODE)
# @staticmethod
# def _read ():
# f = open(plugins.FILE)
# try:
# _data = json.loads(f.read())
# f.close()
# except Exception as e:
# print (f"Corrupted registry, resetting ...")
# _data = {}
# plugins._write(_data)
# plugins._data = _data
# @staticmethod
# def _write (_data):
# f = open(plugins.FILE,'w')
# f.write(json.dumps(_data))
# f.close()
# plugins._data = _data
# @staticmethod
# def inspect (_path):
# _names = []
# if os.path.exists(_path) :
# _filename = _path.split(os.sep)[-1]
# spec = importlib.util.spec_from_file_location(_filename, _path)
# module = importlib.util.module_from_spec(spec)
# spec.loader.exec_module(module)
# # _names = [{'name':getattr(getattr(module,_name),'name'),'pointer':getattr(module,_name)} for _name in dir(module) if type( getattr(module,_name)).__name__ == 'function']
# for _name in dir(module) :
# _pointer = getattr(module,_name)
# if hasattr(_pointer,'transport') :
# _item = {'real_name':_name,'name':getattr(_pointer,'name'),'pointer':_pointer,'version':getattr(_pointer,'version')}
# _names.append(_item)
# return _names
# @staticmethod
# def add (alias,path):
# """
# Add overwrite the registry entries
# """
# _names = plugins.inspect (path)
# _log = []
# if _names :
# #
# # We should make sure we have all the plugins with the attributes (transport,name) set
# _names = [_item for _item in _names if hasattr(_item['pointer'],'transport') ]
# if _names :
# plugins.copy(path)
# _content = []
# for _item in _names :
# _key = '@'.join([alias,_item['name']])
# _log.append(_item['name'])
# #
# # Let us update the registry
# #
# plugins.update(alias,path,_log)
# return _log
# @staticmethod
# def update (alias,path,_log) :
# """
# updating the registry entries of the plugins (management data)
# """
# # f = open(plugins.FILE)
# # _data = json.loads(f.read())
# # f.close()
# _data = plugins._data
# # _log = plugins.add(alias,path)
# if _log :
# _data[alias] = {'content':_log,'name':path.split(os.sep)[-1]}
# plugins._write(_data) #-- will update data as a side effect
# return _log
# @staticmethod
# def get(**_args) :
# # f = open(plugins.FILE)
# # _data = json.loads(f.read())
# # f.close()
# # if 'key' in _args :
# # alias,name = _args['key'].split('.') if '.' in _args['key'] else _args['key'].split('@')
# # else :
# # alias = _args['alias']
# # name = _args['name']
# # if alias in _data :
# # _path = os.sep.join([plugins.CODE,_data[alias]['name']])
# # _item = [_item for _item in plugins.inspect(_path) if name == _item['name']]
# # _item = _item[0] if _item else None
# # if _item :
# # return _item['pointer']
# # return None
# _item = plugins.has(**_args)
# return _item['pointer'] if _item else None
# @staticmethod
# def has (**_args):
# f = open(plugins.FILE)
# _data = json.loads(f.read())
# f.close()
# if 'key' in _args :
# alias,name = _args['key'].split('.') if '.' in _args['key'] else _args['key'].split('@')
# else :
# alias = _args['alias']
# name = _args['name']
# if alias in _data :
# _path = os.sep.join([plugins.CODE,_data[alias]['name']])
# _item = [_item for _item in plugins.inspect(_path) if name == _item['name']]
# _item = _item[0] if _item else None
# if _item :
# return copy.copy(_item)
# return None
# @staticmethod
# def synch():
# pass
def isloaded ():
return DATA not in [{},None]
@ -233,8 +82,11 @@ def set (label, auth_file, default=False,path=REGISTRY_PATH) :
if label == 'default' :
raise Exception ("""Invalid label name provided, please change the label name and use the switch""")
reg_file = os.sep.join([path,REGISTRY_FILE])
if os.path.exists (auth_file) and os.path.exists(path) and os.path.exists(reg_file):
f = open(auth_file)
if os.path.exists(path) and os.path.exists(reg_file):
if type(auth_file) == str and os.path.exists (auth_file) :
f = open(auth_file)
elif type(auth_file) == StringIO:
f = auth_file
_info = json.loads(f.read())
f.close()
f = open(reg_file)

Write Preview
Loading…
Cancel
Save