From 95f2f640283e59271bd6a70ae918673280322c94 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Thu, 24 Sep 2026 21:48:04 -0500 Subject: [PATCH] created by gemma4:12b --- README.md | 35 ++++++++++++++------------ www/html/_plugins/parser.py | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 16 deletions(-) create mode 100644 www/html/_plugins/parser.py diff --git a/README.md b/README.md index a5e1e95..75f957c 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,31 @@ -# QCMS (Quality Control Management System) +# Data-Transport Studio -QCMS is a robust system built to streamline and manage quality control processes. +**Data-Transport Studio** is a front-end for the [data-transport](https://github.com/lnyemba/data-transport) framework and library. It lowers the barrier to entry to the capabilities of data-transport by providing an intuitive interface that makes it easy to connect to, query, and analyze data across diverse sources—without needing to write boilerplate code. -## Powered by data-transport +Powered by the data-transport framework and library, Data-Transport Studio adds **AI-powered querying** and an **integrated spreadsheet** to empower users to explore, transform, and better understand their data. -QCMS leverages **data-transport**, a powerful abstraction layer for seamless data communication across diverse systems. +## What data-transport Provides -### What is data-transport? - -**data-transport** provides a consistent interface to interact with a wide range of data sources, including: +At its core, Data-Transport Studio is powered by data-transport—a lightweight abstraction layer that enables seamless, unified access to a wide variety of data environments: - **SQL Databases:** PostgreSQL, MySQL, SQL Server, SQLite3+, DuckDB - **NoSQL Databases:** MongoDB, CouchDB -- **Cloud Storage & Warehouse:** S3, Nextcloud, Apache Iceberg, Apache Drill +- **Cloud & Warehouse:** S3, Nextcloud, Apache Iceberg, Apache Drill - **Other Integrations:** Files, RabbitMQ, HTTP -### Why use data-transport in QCMS? +## Key Features + +- **Simple Connection Management:** Easily register and manage connections to diverse data sources through a label-based interface, eliminating the need to handle raw connection strings. +- **Intuitive Querying:** Query your data directly through the interface without writing Python scripts or complex boilerplate. +- **AI-Powered Data Exploration:** Translate natural language prompts into precise SQL queries and generate multi-sheet analytical reports using integrated providers like **Ollama** and **Azure OpenAI**. +- **Integrated Spreadsheet:** View query results directly in an integrated spreadsheet, enabling ad-hoc analysis, data wrangling, augmentation, and export with ease. +- **Immediate Results:** Interact with your data in real-time and iterate on analysis quickly through a spreadsheet-driven workflow. -Integrating data-transport into the QCMS architecture provides several key advantages: +## Why Use Data-Transport Studio? -- **Unified Data Access:** QCMS can interact with multiple database types (SQL, NoSQL, Cloud) using a consistent code base, simplifying development and maintenance. -- **Enhanced Security:** It masks raw database connection details, preventing the leak of sensitive credentials within the application. -- **Seamless Data Manipulation:** By leveraging **Pandas** and **SQLAlchemy**, data-transport allows QCMS to perform complex data transformations and analysis efficiently. -- **Scalable ETL Pipelines:** It facilitates the creation of robust, reproducible data processing pipelines by treating different storage backends as a unified "transport" layer. -- **Portability:** It allows for easier sharing of scripts and notebooks across different environments without exposing specific connection strings. +- **Empowers Non-Technical Users:** Analysts and domain experts can explore and analyze data without needing to write code. +- **Accelerates Insights:** Natural language querying, automated analysis, and spreadsheet-based exploration help users understand their data faster. +- **Reduces Complexity:** A single interface replaces the need to juggle multiple tools, clients, or scripts to work with different data sources. +- **Focus on Analysis, Not Infrastructure:** Spend less time on connectivity and boilerplate, and more time uncovering actionable insights. -For more information on the underlying technology, visit the [data-transport repository](https://github.com/lnyemba/data-transport). +For more information on the underlying framework and library, visit the [data-transport repository](https://github.com/lnyemba/data-transport). diff --git a/www/html/_plugins/parser.py b/www/html/_plugins/parser.py new file mode 100644 index 0000000..7fac8fc --- /dev/null +++ b/www/html/_plugins/parser.py @@ -0,0 +1,49 @@ +""" +This file implements the parsing strategy for a response to an LLM. This has been tested on : + - ollama gemma4 + - azure openai gpt-6-astra +""" +import json +import pandas as pd +def format(_data) : + if type(_data) == str : + _data = json.loads(_data) + return pd.DataFrame(_data).fillna(0).to_dict(orient='records') +def gemma4_12b(_stream) : + try: + _data = json.loads(_stream.replace('```json','').replace('```','').strip()) + + _data['data'] = format(_data['data']) + if 'analysis' in _data: + for _item in _data['analysis'] : + if 'data' in _item : + _item['data'] = format(_item['data']) + # + # + return _data + except Exception as e: + # print ([' Error ',e]) + pass + return _stream + +def gpt6_astra(_stream): + try: + _data = _stream[0]['text'] if _stream[0]['type'] == 'text' else _stream[1]['text'] + + if type(_data) == str : + _data = json.loads(_data) + _data['data'] = format(_data['data']) + + if 'analysis' in _data: + for _item in _data['analysis'] : + if 'data' in _item : + _item['data'] = format(_item['data']) + # + # + return _data + + except Exception as e: + print ([' Error ',e]) + pass + + return _stream