From 9d00a459eb6c5f8a3f529acc48f1f8293e346ac4 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Wed, 17 Dec 2025 11:07:56 -0600 Subject: [PATCH] version update, etl bug fixes --- README.md | 20 ++++++++++++++------ transport/info.py | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 577350e..21b224d 100644 --- a/README.md +++ b/README.md @@ -13,20 +13,28 @@ Data transport is a simple framework that: ## Installation -Within the virtual environment perform the following : +Within the virtual environment perform the following (the following will install everything): - pip install git+https://github.com/lnyemba/data-transport.git + pip install data-transport[all]@git+https://github.com/lnyemba/data-transport.git -Options to install components in square brackets +Options to install components in square brackets are **nosql**; **cloud**; **other** and **warehouse** - pip install data-transport[nosql,cloud,warehouse,all]@git+https://github.com/lnyemba/data-transport.git + pip install data-transport[nosql,cloud,other, warehouse,all]@git+https://github.com/lnyemba/data-transport.git +The components available: + + 0. sql by default netezza; mysql; postgresql; duckdb; sqlite3; sqlserver + 1. nosql mongodb/ferretdb; couchdb + 2. cloud s3; bigquery; databricks + 3. other files; http; rabbitmq + 4. warehouse apache drill; apache iceberg ## Additional features - - In addition to read/write, there is support for functions for pre/post processing + - Reads are separated from writes to avoid accidental writes. + - Streaming (for large volumes of data) by specifying chunksize - CLI interface to add to registry, run ETL - - scales and integrates into shared environments like apache zeppelin; jupyterhub; SageMaker; ... + - Implements best-pracices for collaborative environments like apache zeppelin; jupyterhub; SageMaker; ... ## Learn More diff --git a/transport/info.py b/transport/info.py index 037bc06..2b773d5 100644 --- a/transport/info.py +++ b/transport/info.py @@ -1,6 +1,6 @@ __app_name__ = 'data-transport' __author__ = 'Steve L. Nyemba' -__version__= '2.4.32' +__version__= '2.4.34' __edition__= 'enterprise' __email__ = "info@the-phi.com" __license__=f"""