diff --git a/www/html/_assets/css/studio.css b/www/html/_assets/css/studio.css
index a44a26a..7ddc040 100644
--- a/www/html/_assets/css/studio.css
+++ b/www/html/_assets/css/studio.css
@@ -123,10 +123,10 @@ input[type=text]:focus {
overflow:hidden;
height:20px;
}
-.main .menu {display:none}
-.main .content{grid-row:2; grid-column:1}
-.main .pane {grid-row:2; grid-column:2}
-.main .footer{grid-row:4; grid-column:1;}
+.qcms-main .menu {display:none}
+.qcms-main .content{grid-row:2; grid-column:1}
+/* .qcms-main .qcms-pane {grid-row:2; grid-column:2} */
+.qcms-main .qcms-footer{display:block; }
/**
* syncfusion spreadsheet ...
*/
diff --git a/www/html/_assets/images/http.png b/www/html/_assets/images/http.png
new file mode 100644
index 0000000..b58510d
Binary files /dev/null and b/www/html/_assets/images/http.png differ
diff --git a/www/html/_assets/js/studio.js b/www/html/_assets/js/studio.js
index faed3e8..5318632 100644
--- a/www/html/_assets/js/studio.js
+++ b/www/html/_assets/js/studio.js
@@ -3,6 +3,7 @@
ej.base.registerLicense('ORg4AjUWIQA/Gnt2XFhhQlJHfV5AQmBIYVp/TGpJfl96cVxMZVVBJAtUQF1hTH5VdkxhWnpWcHZcTmhfWkZ/')
ej.spreadsheet.Spreadsheet.Inject(ej.spreadsheet.ExcelExport);
var studio = {_context:''}
+r = [{name:'steve',age:46},{name:'elon',age:10}]
studio.defaults = {
'duckdb':'SELECT * \nFROM INFORMATION_SCHEMA.TABLES',
'sqlite':'SELECT * \nFROM sqlite_master',
@@ -70,6 +71,7 @@ function triggerDownload(blob, filename) {
URL.revokeObjectURL(url); // cleanup
}
studio.grid = function (){
+
this.columns = function (_data){
var _columns = []
if(_data.columns){
@@ -162,23 +164,25 @@ studio.grid = function (){
var rows = []
- _data.data.forEach(row=>{
- var _record = {} //-- formatted appropriately
- row.forEach((value,index)=>{
- _col = _data.columns[index]
- _record[_col] = value
- })
- rows.push(_record)
- })
-
+ // _data.data.forEach(row=>{
+ // var _record = {} //-- formatted appropriately
+ // row.forEach((value,index)=>{
+ // _col = _data.columns[index]
+ // _record[_col] = value
+ // })
+ // rows.push(_record)
+ // })
+
var spreadsheet = new ej.spreadsheet.Spreadsheet();
_sheet_name = _id.trim().replace(/[#,.]/g,'').replace(/ /g,'-').trim()
+
spreadsheet.sheets = [
{name:_sheet_name,
- ranges:[{dataSource:rows}],
+ ranges:[{dataSource:_data.data}],
}
]
+
if( $(_id)[0].spreadsheet ){
book = $(_id)[0].spreadsheet ;
book.sheets.forEach((sheet)=>{
@@ -246,15 +250,53 @@ studio.frame = function (_args){
$(_id).empty()
$(_id).append($(''))
if(x.status == 200 && x.readyState == 4){
+ console.log(x.responseText)
_r = JSON.parse(x.responseText)
// sessionStorage._data = x.responseText
+
if (_r.query == null){
_compute(_label,_r)
}else{
- _compute(_label,_r.data)
+
+ _ghandler = _compute(_label,_r)
var _id = '.code.'+_args.label.trim()
// $(_id).val('-- '+$(_id).val().replace(/\n/g,'-- ') + '\n'+_r['query'])
$(_id).val(_r['query'])
+ if (_r.analysis) {
+ //
+ // We need to append document
+ var spreadsheet = $(`.${_args.label.trim()} .output`)[0].spreadsheet
+ //
+ rows = []
+ _r.analysis.forEach((row)=>{
+ var _record = {} //-- formatted appropriately
+
+ // sheet.ranges: [{ dataSource: row.data }]
+
+ N = spreadsheet.sheets.length + 1
+ var sheet = {index:N-1}
+ sheet.name = row.title
+ sheet.ranges = [{dataSource:row.data,startCell:'B1'}]
+
+
+
+ spreadsheet.insertSheet([sheet],-1)
+ spreadsheet.refresh()
+ spreadsheet.updateCell({
+ value: 'Analysis',style:{fontWeight:'bold'}
+ },`${row.title}!A1`)
+ spreadsheet.updateCell({
+ value: row.explanation
+ },`${row.title}!A2`)
+
+ spreadsheet.refresh()
+ })
+
+ // spreadsheet.sheets.push(sheet)
+ // spreadsheet.insertSheet({range:{dataSource:rows}},0)
+
+
+ }
}
sessionStorage[_label] = _query
@@ -577,12 +619,13 @@ studio.frame = function (_args){
}
$(_id).empty()
+
_gHandler = new studio.grid()
_gHandler.syncfusion(_id,_data)
_rows = (_data.data)?_data.data.length : 0
$('.'+_label + ' .status').html(_rows + ' rows')
-
-
+ return _gHandler
+
}
}
diff --git a/www/html/_assets/themes/default/footer.css b/www/html/_assets/themes/default/footer.css
index ee9c89a..77f4124 100644
--- a/www/html/_assets/themes/default/footer.css
+++ b/www/html/_assets/themes/default/footer.css
@@ -2,13 +2,12 @@
* This file styles the footer of the page
*/
-#footer {
+.qcms-footer {
text-align:center;
display:grid;
grid-template-columns: repeat(3,1fr);
gap:4px;
-
font-size:12px;
color:black;
align-items: center;
diff --git a/www/html/_assets/themes/default/header.css b/www/html/_assets/themes/default/header.css
index f5e2d24..eb9ad80 100644
--- a/www/html/_assets/themes/default/header.css
+++ b/www/html/_assets/themes/default/header.css
@@ -1,12 +1,12 @@
-.main .header {
+.qcms-main .qcms-header {
grid-column:1 ;
display:grid; grid-template-columns:64px auto; gap:4px}
-.main .header .title {font-size:28px; font-weight:bold}
-.main .header .subtitle {font-size:12px;}
-.main .header img {width:55px; height:55px;}
+.qcms-main .qcms-header .qcms-title {font-size:28px; font-weight:bold}
+.qcms-main .qcms-header .qcms-subtitle {font-size:12px;}
+.qcms-main .qcms-header img {width:55px; height:55px;}
-.header, .footer {
+.qcms-header, .qcms-footer {
padding:4px;
align-content: center;
align-items:center;
diff --git a/www/html/_assets/themes/default/layout.css b/www/html/_assets/themes/default/layout.css
index d6bb234..ff97d09 100644
--- a/www/html/_assets/themes/default/layout.css
+++ b/www/html/_assets/themes/default/layout.css
@@ -1,6 +1,6 @@
-.main {
+.qcms-main {
display:grid;
grid-template-rows:70px calc(100% - 70px);
grid-template-columns:75% 25%;
@@ -9,7 +9,7 @@
height:96vh;
line-height: 1.5; font-size:16px; font-family: sans-serif; font-weight:lighter;
}
-.main #content{
+.qcms-main .qcms-content{
display:grid;
@@ -22,7 +22,7 @@
}
-/* .main .pane {border-left:1px solid transparent; height:100%;} */
+/* .qcms-main .pane {border-left:1px solid transparent; height:100%;} */
.bold {font-weight:bold}
.small-text {font-size:12px; font-weight:lighter; color :#000000;}
.active {cursor:pointer; padding:2px; margin:2px; border-bottom:2px solid transparent}
diff --git a/www/html/_assets/themes/default/menu.css b/www/html/_assets/themes/default/menu.css
index 0e47ae7..dd5cf8e 100644
--- a/www/html/_assets/themes/default/menu.css
+++ b/www/html/_assets/themes/default/menu.css
@@ -1,4 +1,4 @@
-.menu {
+.qcms-menu {
padding:8px;
border:1px solid #CAD5E0 ;
@@ -10,9 +10,9 @@
align-items: center;
}
-.menu .icon {padding:4px;}
-.menu .icon img {width:30px; height:30px;}
-.menu .item {
+.qcms-menu .qcms-icon {padding:4px;}
+.qcms-menu .qcms-icon img {width:30px; height:30px;}
+.qcms-menu .item {
font-weight:bold;
cursor:pointer;
padding:4px;
@@ -24,7 +24,7 @@
}
-.menu .sub-menu {
+.qcms-menu .qcms-sub-menu {
display:none;
position:absolute;
@@ -40,7 +40,7 @@
}
-.menu .item:hover .sub-menu{
+.qcms-menu .item:hover .qcms-sub-menu{
display:block;
height:auto;
diff --git a/www/html/_assets/themes/default/responsive.css b/www/html/_assets/themes/default/responsive.css
index cd91420..b3aadd3 100644
--- a/www/html/_assets/themes/default/responsive.css
+++ b/www/html/_assets/themes/default/responsive.css
@@ -1,19 +1,19 @@
/* Responsive Design */
@media (max-width: 1024px) {
- .main {
+ .qcms-main {
grid-template-columns: 60% 40%;
grid-template-rows: 70px auto 32px;
}
}
@media (max-width: 768px) {
- .main {
+ .qcms-main {
grid-template-columns: 100%;
grid-template-rows: auto;
height: auto;
}
- .main .pane {
+ .qcms-main .qcms-pane {
border-left: none;
border-top: 1px solid #CAD5E0;
padding: 8px;
@@ -21,7 +21,7 @@
}
@media (max-width: 480px) {
- .main {
+ .qcms-main {
padding: 5%;
font-size: 14px;
}
diff --git a/www/html/_plugins/agent.py b/www/html/_plugins/agent.py
index 836f501..62fe0d1 100644
--- a/www/html/_plugins/agent.py
+++ b/www/html/_plugins/agent.py
@@ -1,218 +1,214 @@
-"""
-This is code that will interact with an LLM (AzureOpenAI/Ollama) leveraging langchain
-"""
-# _CATALOG = {
-# 'sqlite':{'sql':'select tbl_name as table_name, json_group_array(y.name) as columns from sqlite_master x INNER JOIN PRAGMA_TABLE_INFO(tbl_name) y group by table_name'} ,
-# 'postgresql':{'sql':"SELECT table_name, to_json(array_agg(column_name)) as columns FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = 'public' GROUP BY table_name"},
-# 'bigquery':{'sql':'SELECT table_name, TO_JSON(ARRAY_AGG(column_name)) as columns FROM :dataset.INFORMATION_SCHEMA.COLUMNS','args':['dataset']},
-# 'duckdb':{'sql' :'SELECT table_name, TO_JSON(ARRAY_AGG(column_name)) as columns FROM INFORMATION_SCHEMA.COLUMNS GROUP BY table_name'}
-# }
-
-# _CATALOG['sqlite3'] = _CATALOG['sqlite']
-import transport
+from langchain_ollama import ChatOllama
+from langchain_openai import AzureChatOpenAI, ChatOpenAI
+import requests
+from typing import Optional
import json
-import os
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.tools import StructuredTool
+from langchain.agents import create_agent
+from pydantic import create_model
+
+from pydantic import BaseModel, Field, field_validator
+
+from typing import Dict, Any, List
+from langchain.agents.structured_output import ToolStrategy
+import transport
import cms
import pandas as pd
-from langchain_openai import AzureOpenAI, AzureChatOpenAI
-from langchain_ollama import OllamaEmbeddings, OllamaLLM
-from langchain_core.messages import HumanMessage, AIMessage
-from langchain_classic.chains import LLMChain
-from langchain_core.prompts import PromptTemplate
-from langchain_core.runnables import RunnablePassthrough
-from langchain_core.output_parsers import StrOutputParser
-
-class Agent :
- def __init__(self,**_args) :
+import plugin_ix as plug
+import os
+
+_map ={
+ 'postgresql':"Select table_schema||'.'||table_name as table_name, table_schema as schema_name, column_name, data_type as type from information_schema.columns",
+ 'duckdb':"SELECT table_schema||'.'||table_name as table_name , table_schema AS schema_name, column_name, data_type FROM information_schema.columns",
+ 'sqlite3':'SELECT * FROM sqlite_master',
+ 'mysql':'SELECT table_name, table_schema AS schema_name, column_name, data_type FROM information_schema.columns',
+ 'bigquery':'SELECT table_name, table_schema AS schema_name, column_name, data_type FROM :dataset.INFORMATION_SCHEMA.COLUMNS',
+ 'iceberg':'SHOW TABLES', 'mariadb':'SHOW TABLES'
+}
+class AnalysisItem(BaseModel):
+ title: str = Field(description="Short title for this piece of analysis")
+ explanation: str = Field(description="Explanation of the insight")
+ # Using a structured dict or strict List[Dict]
+ data: List[Dict[str, Any]] = Field(description="Relevant rows supporting this analysis point")
+
+ @field_validator('data', mode='before')
+ @classmethod
+ def ensure_list_of_dicts(cls, v):
+ if isinstance(v, dict):
+ return [v] # Wrap single dicts if the LLM forgot the list wrapper
+ return v
+
+class QueryOutput(BaseModel):
+ query: str = Field(description="The query that was generated and executed")
+ data: List[Dict[str, Any]] = Field(description="Rows returned from executing the query")
+ analysis: List[AnalysisItem] = Field(description="Analysis points derived from the data")
+
+ @field_validator('data', mode='before')
+ @classmethod
+ def ensure_list_of_dicts(cls, v):
+ if isinstance(v, dict):
+ return [v]
+ return v
+
+class Tool:
+ def __init__(self,_dbr) :
+ self._dbr = _dbr
+ self._provider = _dbr._agent._engine.name
+
+
+ def catalog(self):
"""
- :backend OpenAI or Ollama
- :kwargs The initialization parameters associated with the backend
- The arguments will contain temperature and model name to be used
+ This function returns the catalog of the current database
"""
- _instance = AzureChatOpenAI if _args['backend'].lower() in ['azureopenai', 'openai'] else OllamaLLM
- self._llm = _instance(**_args['kwargs'])
- def isSQL(self,_question):
- _template = """Is the provided text a valid sql statmement. Yes or No? Your answer is a properly formatted JSON object with three attributes.
- class (1 for valid sql statement and 0 for not a valid sql statement), explanation place a short explanation for the answer and original containing with the original text.
+ global _map
+ _query = _map.get(self._provider)
- text:
- {input_text}
+ return self.read(_query)
+ def schema(self,table:str) ->dict:
"""
- _prompt = PromptTemplate(temperature=0.1,input_variables=['input_text'],template=_template)
-
- r = self.apply(_prompt,input_text=_question)
- #
- # @TODO: Make sure the response is properly formatted (output not to be trusted)
- return json.loads(r)
- def apply(self,_prompt,**_args):
-
- chain = (
- RunnablePassthrough.assign()
- | _prompt
- | self._llm
- | StrOutputParser())
- _out = chain.invoke(_args)
-
- return _out #son.loads(_out)
- def toSQL(self,_question,_catalog,_about):
- _template="""Your task is to convert a question to an SQL query. The query will run on schema that will be provided in csv format.
-
-Output:
-The expected output will be a JSON object with two attributes sql and tables:
-- "sql": the SQL query to be executed.
-- "tables": list of relevant tables used.
+ This function returns the schema for a given database, if table is not provided the entire catalog will be returned with the list of tables
+ """
+ _query = f""" {_map[self._provider]} WHERE name = '{table}' """
+ if self._provider in ['sqlite','sqlite3'] :
+ _info = self.read(f"select * from {table} limit 20").dtypes
+ return pd.DataFrame({"column_name":_info.dtypes.index.tolist()[1:],"data_type": [x.name for x in _info.dtypes.tolist()[1:]]}).to_dict(orient='records')
+ pass
+ else:
+ return self.read(_query)
+ def read(self,query:str)->dict:
+ """
+ This function will execute a query against a given label/database. The query must adhere to the schema of the tables involved.
+ """
+ return self._dbr.read(sql=query).to_dict(orient='records')
-Guidelines:
-- If the question can not be answered with the provided schema return empty string in the sql attribute
-- Parse the question word by word so as to be able to identify tables, fields and operations associated (Joins, filters ...)
-- Under no circumstances will you provide an explanation of tables or reasoning detail.
-- Avoid using subqueries, and use field names as represented in the provided schema and their data types
+class TransportAgent:
+ """
+ This class is intended to allow an LLM to interact with data-transport MCP server.
+ """
+ def __init__(self, **kwargs):
+ """
+ :backend ollama or openai (for now)
+ :kwargs parameters that will be used for the openai,ollama classes
+ """
+ self.backend = kwargs.get('backend', 'ollama').lower()
+ self.kwargs = kwargs.get('kwargs', {}).copy()
+ # self.mcp_url = kwargs.get('mcp_url', 'http://localhost:8090')
+ self.chatbot = self.instance(self.backend, **self.kwargs)
+ # self.tools = self._init_mcp_tools()
+ # self.agent = self._init_agent()
+ registry = plug.Registry()
+ if registry.has(kwargs['parser']) :
+ self._parser = registry.get(kwargs['parser'])
+ else:
+ self._parser = None
-question:
-{question}
+ def instance(self, backend, **kwargs):
+ """
+ Returns a chat model instance for the given backend.
+ Supported: ollama, azureopenai, openai.
+ """
+ if backend == 'azureopenai':
+ return AzureChatOpenAI(**kwargs)
+ elif backend == 'openai':
+ return ChatOpenAI(**kwargs)
+ else:
+ return ChatOllama(**kwargs)
+ def get(self,label,query):
+ dbr = transport.get.reader(label=label)
+ tools = Tool(dbr)
+ # self.chatbot.bind_tools ([tools.catalog,tools.read])
-Database schema:
-{catalog}
+ _message = f"""
+ You are a specialized Data-Transport Agent designed to facilitate data access and analysis from registered
+ labels/databases. The user is working in {label} database ({dbr._agent._engine.name} database).
+
+ Your primary goal is to execute a requested operation based on the user's query. You MUST strictly adhere to
+ the following process and output format.
-additional information:
-{context}
+ **Process Flow:**
+ if and ONLY if the user's question is a database query execute the "read" function associated with the tools.
+ Otherwise perform the following:
+ 1. **Catalog Retrieval:** First, return the list of tables with a field describing the table.
+ schema) from the catalog.
+ 2. **Query Generation:** Construct an accurate SQL query that directly addresses the user's request using the
+ the information pulled from the catalog.
+ 3. Make sure the generated query adheres to constraints of {dbr._agent._engine.name} database
+ 4. **Execution:** Execute the SQL query against by calling the "read" function of the tools.
+ 5. **Analysis is Optional and should be provided ONLY if the user requested it:** If the user asks for an analysis, perform the necessary data processing and derive
+ meaningful insights based on the query data.
+
+ **Output Format Enforcement:**
+ You MUST respond ONLY with a single JSON object. Do not include any introductory text, explanations, or
+ conversational language outside of the JSON structure.
+
+ **JSON Schema Requirements:**
+ The output MUST strictly conform to the following JSON structure, utilizing the `QueryOutput` schema defined by
+ your execution context:
+ - **"query"**: The exact SQL query that was executed.
+ - **"data"**: A list of dictionaries containing the raw data rows returned from the query execution.
+ - **"analysis"**: An optional list of analysis points. If analysis was performed, this list MUST contain
+ objects with the following structure:
+ - "title": A concise title for the derived insight. It can ONLY be 30 characters at most.
+ - "data": The specific data rows supporting the analysis.
+ - "text": A detailed explanation of the insight derived from the data.
+
+ Ensure all generated data and analysis are directly traceable back to the executed query and the retrieved
+ table schemas.
"""
-
- _prompt = PromptTemplate(temperature=0.1,input_variables=['question','catalog','context'],template=_template)
- r = self.apply(_prompt,question=_question,catalog=_catalog,context=_about)
- # print (' ############### ------------- #####################')
- # print (r)
- # print (' #########################################')
- #
- # @TODO: Make sure the response is properly formatted (output not to be trusted)
- if '```json' in r :
- r = r.split('```json')[-1].replace('```','')
- print (r)
- return json.loads(r)
-
-
-
-#
-# We are building an interface to execute an sql query
-#
-
-def AIProxy (_label,_query,_path,_CATALOG) :
- _qreader = transport.get.reader(label=_label)
- _entry = transport.registry.get(_label)
- _provider =_entry['provider']
- _about = _entry.get('about','')
- _database = _entry['database'] if 'database' in _entry else ''
-
- if 'dataset' in _entry :
- _database = _entry['dataset']
- _about = f'{_about}, with dataset name {_database}'
- else:
- _about = f'{_about}, with the database name {_database}'
- _catalog = None
- _kwargs = None
- _data = pd.DataFrame()
- r = None
- try:
- #
- # we should run the command here, assuming a valid query
- #
-
- _data = _qreader.apply(_query)
- except Exception as e:
- #
- # here we are assuming we are running text to be translated as a query
- # we need to make sure that we have a JSON configurator
- #
- if _provider in _CATALOG and os.path.exists(_path):
- #
- # -- reading arguments from the LLM config file, {backend,kwargs}
- _kwargs = json.loads((open(_path)).read())
- _agent = Agent(**_kwargs)
- _qcat = _CATALOG[_provider]
- if 'args' in _CATALOG[_provider] :
- _entry = transport.registry.get(_label)
- for _var in _CATALOG[_provider]['args'] :
- if _var in _entry:
- _value = _entry[_var]
- _about = f'{_about}\n{_var} = {_value}'
- _qcat['sql'] = _qcat['sql'].replace(f':{_var}',_value)
-
- _catalog = _qreader.read(**_qcat).to_csv(index=0)
- _about = f"The queries will run on {_provider} database.\n{_about}"
- r = _agent.toSQL(_query,_catalog, _about)
-
- try:
- _data = _qreader.apply(r['sql'])
- except Exception as e1:
- _data = pd.DataFrame()
- pass
- #
- # returning the data and the information needed
- #
- _data = _data.astype(str).to_dict(orient='split')
- del _data['index']
- if r :
- return {'data':_data,'query':r['sql']}
- return _data
-
+ _caller = create_agent(self.chatbot,[tools.catalog,tools.schema,tools.read],system_prompt=_message,response_format=QueryOutput)
+ _output = _caller.invoke({"messages":[("user",query)]})["messages"][-1].content
+ if self._parser :
+ return self._parser(_output)
+ else:
+ return _output
+LLM_ARGS = {}
+def InitLLMArgs (_config):
+ global LLM_ARGS
- # if not _path :
- # #
- # # exececute the query as is !
- # pass
- # else:
-
- # if _provider in _CATALOG and os.path.exists(_path) :
-
- # #
- # # As the agent if it is an SQL Query
- # f = open(_path)
- # _kwargs = json.loads( f.read() )
- # f.close()
- # try:
-
- # print ([f"Running Model {_kwargs['kwargs']['model']}"])
- # _agent = Agent(**_kwargs)
- # # r = _agent.isSQL(_query)
-
- # # print (f"****** {_query}\n{r['class']}")
- # # if r and int(r['class']) == 0 :
- # _catalog = _qreader.read(**_CATALOG[_provider]).to_csv(index=0)
- # # print (['****** TABLES FOUND ', _catalog])
- # _about = _about if _about else ''
- # r = _agent.toSQL(_query,_catalog,f"This is a {_provider} database, and queries generated should account for this. {_about}")
- # _query = r['sql']
- # # else:
- # # #
- # # # provided an sql query
- # # pass
- # except Exception as e:
- # #
- # # AI Service is unavailable ... need to report this somehow
- # print (e)
- # else:
- # #
- # # Not in catalog ...
- # pass
- # _data = _qreader.apply(_query)
- # if _data.shape[0] :
- # _data = _data.astype(str).to_dict(orient='split')
- # if 'index' in _data :
- # del _data['index']
- # return _data
+ path = _config['system']['source']['ai']['config-path']
+ f = open(path)
+ LLM_ARGS = json.loads(f.read())
+ f.close()
+ #
+ # We also have to initialize the plugin path
+ os.environ['REGISTRY_FOLDER'] = _config['system']['source']['ai']['plugin-path']
+ return LLM_ARGS
-
@cms.Plugin(mimetype="application/json",method="POST")
-def apply (**_args):
+def apply(**_args):
+ global LLM_ARGS
_request = _args['request']
+ _config = _args['config']
+ #
+ # let's make sure the _label is whitelisted
_label = _request.json['label']
_query = _request.json['query']
- _source = _args['config']['system']['source']
- _path = _source.get('llm',None)
- _CATALOGS = _source.get('catalogs',{})
- return AIProxy(_label,_query,_path,_CATALOGS)
+ if not LLM_ARGS :
+ LLM_ARGS = InitLLMArgs(_args['config'])
+ if _query.strip().lower().startswith("with") or _query.strip().lower().startswith("select") :
+ dbr = transport.get.reader(label=_label)
+
+ return { "data":dbr.read(sql=_query).fillna(0).to_dict(orient='records') ,"analysis":[]}
+ elif _query.strip().lower().startswith("{") :
+ dbr = transport.get.reader(label=_label)
+ return { "data":dbr.read(json.loads(_query)).to_dict(orient='records'),"analysis":[] }
+
+ _agent = TransportAgent(**LLM_ARGS)
+
+ # we need an output parser for the models (should be loaded as a plugin)
+ print ([LLM_ARGS['backend'], LLM_ARGS['kwargs']['model']])
+ return _agent.get(_label,_query)
+
+ # _data = {"query": "SELECT condition_concept_id, COUNT(*) AS condition_count FROM tabular.condition_occurrence GROUP BY condition_concept_id ORDER BY condition_count DESC LIMIT 10;", "data": [{"condition_concept_id": "316866", "condition_count": 5635}, {"condition_concept_id": "4159131", "condition_count": 5462}, {"condition_concept_id": "2005200547", "condition_count": 5375}, {"condition_concept_id": "4291025", "condition_count": 5037}, {"condition_concept_id": "201826", "condition_count": 4026}, {"condition_concept_id": "433736", "condition_count": 4011}, {"condition_concept_id": "4036620", "condition_count": 3734}, {"condition_concept_id": "4317977", "condition_count": 3341}, {"condition_concept_id": "37018196", "condition_count": 2627}, {"condition_concept_id": "81902", "condition_count": 2458}], "analysis": [{"title": "Top 10 Most Frequent Conditions", "data": [{"condition_concept_id": "316866", "condition_count": 5635}, {"condition_concept_id": "4159131", "condition_count": 5462}, {"condition_concept_id": "2005200547", "condition_count": 5375}, {"condition_concept_id": "4291025", "condition_count": 5037}, {"condition_concept_id": "201826", "condition_count": 4026}, {"condition_concept_id": "433736", "condition_count": 4011}, {"condition_concept_id": "4036620", "condition_count": 3734}, {"condition_concept_id": "4317977", "condition_count": 3341}, {"condition_concept_id": "37018196", "condition_count": 2627}, {"condition_concept_id": "81902", "condition_count": 2458}], "text": "The distribution analysis of conditions, based on the top 10 most frequent condition concept IDs, shows that '316866' is the most commonly recorded condition, with 5,635 occurrences. The top four conditions (316866, 4159131, 2005200547, and 4291025) account for a significant portion of the recorded conditions, indicating these are the most prevalent diagnoses in the dataset."}]}
+ # _data['data'] = pd.DataFrame(_data['data']).to_dict(orient='split',index=False)
+ # for _row in _data['analysis'] :
+ # if 'data' in _row :
+ # _row['data'] = pd.DataFrame(_row['data']).to_dict(orient='records')
+ # return _data
+
@cms.Plugin(mimetype="text/plain")
def enabled(**_args):
_config = _args['config']
- return str(int('llm' in _config['system']['source'] ))
+ return str(int('ai' in _config['system']['source'] ))
diff --git a/www/html/_plugins/readme.md b/www/html/_plugins/readme.md
new file mode 100644
index 0000000..e6a3f4c
--- /dev/null
+++ b/www/html/_plugins/readme.md
@@ -0,0 +1,42 @@
+# Transport Agent Notebook
+
+This notebook provides a way to interact with the **data-transport MCP server** using LangChain and Marimo.
+
+## Overview
+
+The `notebook.py` file implements a `TransportAgent` class that bridges the gap between Large Language Models (LLMs) and the capabilities of the data-transport system via the **Model Context Protocol (MCP)**. It also demonstrates how to perform direct data access using the `transport` library.
+
+## How it Works
+
+The `notebook.py` file demonstrates how to integrate MCP with LangChain through the following steps:
+
+1. **Dynamic Tool Discovery**: Instead of hard-coding tool definitions, the `TransportAgent` fetches the OpenAPI specification from the MCP server (at `http://localhost:8090/openapi.json`).
+2. **Schema Mapping**: It automatically parses the OpenAPI `components` and `paths` to generate LangChain `StructuredTool` instances. This ensures that the agent always has the latest tools available as defined by the MCP server.
+3. **Agent Construction**: These tools are then passed into a LangChain agent. The agent uses these tools to fulfill user requests, such as listing databases or performing data analysis.
+4. **Interactive Interface**: The Marimo framework is used to provide a clean, interactive UI where users can type questions and see the agent's thought process and results.
+
+## Code Breakdown
+
+### `TransportAgent` Class
+The heart of the notebook. It manages:
+- **LLM Configuration**: Supports `ollama`, `openai`, and `azure_openai` backends.
+- **Tool Initialization (`_init_mcp_tools`)**:
+ - Fetches the `openapi.json` from the local MCP server.
+ - Maps API types (string, integer, etc.) to Python types.
+ - Dynamically creates `StructuredTool` objects with proper descriptions and argument schemas.
+- **Agent Initialization (`_init_agent`)**: Combines the chosen LLM with the discovered tools into a functional agent.
+- **Query Handling (`get`)**: Takes a natural language prompt and returns the final response from the agent's execution chain.
+
+## How to leverage MCP with LangChain (Summary)
+
+To integrate MCP with LangChain in this project:
+1. **Expose via OpenAPI**: The MCP server must provide an `openapi.json` endpoint.
+2. **Automate Tool Creation**: Use a helper function (like `_init_mcp_tools` in this notebook) to iterate over the OpenAPI paths and generate `StructuredTool` objects.
+3. **Bind to Agent**: Pass the list of generated tools to a LangChain agent (e.g., `create_agent`).
+4. **Dynamic Updates**: Because the tools are loaded at runtime from the OpenAPI spec, any update to the MCP server's capabilities is automatically reflected in the agent.
+
+## Usage
+To run this notebook, ensure the MCP server is running on `localhost:8090` and then execute the marimo notebook:
+```bash
+marimo edit www/html/_plugins/notebook.py
+```
diff --git a/www/html/_plugins/register.py b/www/html/_plugins/register.py
index ed4d83b..585b96e 100644
--- a/www/html/_plugins/register.py
+++ b/www/html/_plugins/register.py
@@ -14,10 +14,10 @@ def get (**_args) :
transport.registry.load()
_data = copy.copy(transport.registry.DATA)
_context = _args['config']['system']['context']
- print ()
- _CATALOGS = _args['config']['system']['source'].get('catalogs',{})
- _HAS_LLM = 'llm' in _args['config']['system']['source']
+ # _CATALOGS = _args['config']['system']['source'].get('catalogs',[])
+ _HAS_LLM = 'ai' in _args['config']['system']['source']
+ _ALLOWED = [] if not _HAS_LLM else _args['config']['system']['source']['ai']['allowed']
_labels = []
for _key in _data :
@@ -32,7 +32,7 @@ def get (**_args) :
_plugins = [] if 'plugins' not in _data[_key]else _data[_key]['plugins']
_icon = f'{_context}/api/disk/read?uri=www/html/_assets/images/{_provider}.png'
_icon = f'_assets/images/{_provider}.png'
- _labels.append({"label":_key,"provider":_provider,'table':_table,'icon':_icon,'AI': (_provider in _CATALOGS and _HAS_LLM)})
+ _labels.append({"label":_key,"provider":_provider,'table':_table,'icon':_icon,'AI': _key in _ALLOWED })
else:
continue
return _labels
diff --git a/www/html/_plugins/transport.py b/www/html/_plugins/transport.py
index 3b3feb5..d6c8f0d 100644
--- a/www/html/_plugins/transport.py
+++ b/www/html/_plugins/transport.py
@@ -49,18 +49,15 @@ def apply (**_args):
try:
if _query.strip()[0] not in ['{','[']:
_query = _query.replace('%%','%').replace('%','%%')
- _data = _qreader.apply(_query) #.astype(str).to_dict(orient='split')
+ _data = _qreader.read(sql=_query) #.astype(str).to_dict(orient='split')
else:
_data = _qreader.read( **json.loads(_query))
if _data.shape[0] :
- _data = _data.astype(str).to_dict(orient='split')
- if 'index' in _data:
- del _data['index']
-
+ _data = _data.astype(str).to_dict(orient='records') #',index=False)
else:
_data = []
_qreader.close()
- return _data
+ return {"data":_data}
except Exception as e:
raise Exception( str(e))
diff --git a/www/html/about.html b/www/html/about.html
index f6dc5e0..e9b0837 100644
--- a/www/html/about.html
+++ b/www/html/about.html
@@ -79,6 +79,9 @@ $(document).ready(()=>{
Credit
+
+ All-of-Us Program @ National Insitute of Health
+
- Setup system wide AI capability to translate a prompt to an SQL query. This feature is supported for the following technologies :
+ Setup system wide AI capability to translate a prompt to an SQL query. This feature is supported for the following databases :
-
Ollama & AzureOpenAI
-
+
Ollama & AzureOpenAI (for now)
+
+
+ Databases are whitelisted to enable the AI agent to interact with them
+