bug fixe: add to registry, wizard form rewrite

main
Steve Nyemba 2 weeks ago
parent feec5aff22
commit 169077002d

@ -22,9 +22,8 @@
}
],
"header": {
"title": "Wrangle Workbench",
"subtitle": "Powered by data-transport"
},
"title": "Data Transport",
"subtitle": "Open Workbench: Read, write & wrangle data"},
"index": "index.html",
"on": {
"load": {
@ -46,9 +45,9 @@
"apply",
"version"
],
"register":["get","add"],
"io": [
"read", "open",
"write"
"read", "open", "write"
]
}
}

@ -0,0 +1,198 @@
var _config = {
"sql":{
"host":{"type":"text","default":"localhost"},"database":"text","table":"text","username":"text","password":"password"
},
"cloudant":{
"host":{"type":"text","default":"localhost"},
"port":{"type":"number","default":5984},
"dbname":{"type":"text","label":"database"},
"username":{"type":"text"},
"password":"password",
"doc":{"type":"text","label":"document"}
},
"couchdb":{"@ref":"cloudant"},
"bigquery":{
"database":"text","table":"text",
"auth_file":{"type":"file","label":"service account json file"}
},
"postgresql":{
"@ref":"sql",
"port":{"type":"number","default":5432}
},
"redshift":{"@ref":"postgresql"},
"mariadb":{
"@ref":"sql",
"port":{"type":"number","default":3306}
},
"mysql":{
"@ref":"sql",
"port":{"type":"number","default":3306}
},
"netezza":{
"@ref":"sql",
"port":{"type":"number","default":5480}},
"sqlserver":{
"@ref":"sql",
"port":{"type":"number","default":1433}
},
"mongodb":{
"host":{"type":"text","default":"localhost"},
"port":{"type":"number","default":2717},
"db":{"type":"text","label":"database"},
"collection":{"type":"text"},"mechanism":{"type":"text","default":"SCRAM-SHA-256","values":["SCRAM-SHA-256","MONGODB-X509","MONGODB-CR"]}
},
"nextcloud":{
"uid":{"type":"text","label":"user id"},
"url":{"type":"text","label":"url of server"},
"token":"text"
},
"databricks":{
"host":"text","token":"text","cluster_path":"text","schema":"text","catalog":"text"
},
"s3":{
"bucket":"text","file":{"type":"text","desc":"aws s3 path of the file"},
"region":"text"
},
"sqlite":{"database":{"type":"text","desc":"path on disk"},"table":"text"},
"sqlite3":{"@ref":"sqlite"},
"duckdb":{"database":{"type":"text","desc":"path on disk"},"table":"text"},
"drill":{
"host":"text","port":{"type":"number","default":8047},"ssl":{"type":"bool","values":[true,false],"default":false}
},
"iceberg":{
"catalog":"text","database":"text","table":"text"
}
}
var ProviderInputForm = function (_provider){
this._provider = _provider
/**
* This section will setup the configuration file as it is intended to be
* The configuration fully specified will be used to build the input form
*/
this._config = {}
_parent = {}
var _tmpconfig = _config[_provider]
this._config = _config[_provider]
if (_tmpconfig['@ref'] != null){
_pkey = _tmpconfig['@ref']
_parent = _config[_pkey]
}
//-- merge the objects ...
this._config = Object.assign({'label':{'type':'text',desc:'reference to this database'}},_parent, _config[_provider])
if (this._config['@ref']){
delete this._config['@ref']
}
this.getInput = function(_item){//_type,_text,_desc){
// if(_type.match(/text|number|password/ig)){
_item.label= (_item.label == null)?_item.context: _item.label
_item.desc = (_item.desc != null)?_item.desc:_item.label
_label = $('<label></label>').html(_item.label)
_input = $('<input />')
$(_input).attr('type',_item.type)
if(_item.type.match(/number/)) {
$(_input).attr("pattern","^[1-9]([0-9])*$")
}
$(_input).attr('placeholder',_item.desc)
$(_input).addClass(_item.context)
$(_input).attr('id',_item.context)
if (_item.default != null){
$(_input).val(_item.default)
}
$(_label).attr('for',_item.context)
return $('<div class="form-item"></div>').append(_label,_input)
// }
}
this.submit = function (_data){
var http = HttpClient.instance()
http.setData(JSON.stringify(_data))
http.setHeader('Content-Type','application/json')
var uri = 'api/register/add'
http.post(uri,(x)=>{
if(x.status == 200 && x.readyState == 4){
//
//
_init(_data.label)
$('.jxmodal').slideUp()
}
})
}
this.build = function(_id){
//
// This will build the form on a pane
var _form = $('<div class="provider-input-form"></div>')
Object.keys(this._config).forEach(_key=>{
var _item = this._config[_key]
var _type = (_item.constructor == String)?_key:_item.type
if(_item.constructor == String){
_item = {type:_type}
}
_item.type = _type
_item.context = _key
// var _values = (_item.values)?[]:_item.values
var _inputLine = this.getInput(_item) //_type,_key,_item.label)
$(_form).append(_inputLine)
// }
})
//
// adding all the items to where they need to be
//
_back = $('<div class="border-round border db-back"><div class="active"> <i class="fa-solid fa-chevron-left"></i> Go Back</div></div>')
$(_back).on('click',()=>{
$('.db-form').slideUp(()=>{
$('.db-provider').slideDown()
})
})
var _submit = this.submit
var _provider= this._provider
_save = $('<div class="border-round border db-save"><div class="active"> <i class="fa-solid fa-check" style="color:green"></i> Save Now</div></div>')
$(_save).attr({'_object':this})
$(_save).on('click',()=>{
var _nodes = $('.db-form input')
var _data = {}
_errorCount = 0
_nodes.each((_index)=>{
var _input = _nodes[_index]
_data[_input.id] = _input.value
if(_input.value.trim().length == 0){
_errorCount += 1
$(_input).addClass('input-error')
}else{
$(_input).removeClass('input-error')
}
})
//
// assuming no error ...
if (_errorCount == 0){
_data = Object.assign({},{'provider':_provider}, _data)
_submit(_data)
}
})
_pane = $('<div class="form-controls"></div>')
$(_pane).append(_back,$("<div></div>"),_save)
$(_id).empty()
$(_id).append(_form)
$('.db-form .form-controls').remove()
$('.db-form').append(_pane)
}
}

@ -178,8 +178,9 @@ studio.grid = function (){
$(_id)[0].spreadsheet = spreadsheet
// console.log([' **** ',(_id+' .e-input-group')])
// $(_id+' .e-input-group').remove()
spreadsheet.refresh()
spreadsheet.hideFileMenuItems(["File"], true);
spreadsheet.refresh()
}
}
@ -246,6 +247,21 @@ studio.frame = function (_args){
}
})
}
this.show = function (){
var _label = this._args.label
$('.studio').each(_index=>{
var _bench = $('.studio')[_index]
if (_bench.className.match(_label) == null){
$(_bench).css({display:'none'})
}else{
$(_bench).css({display:'grid'})
_id = '.'+_label+' .output'
console.log(_id)
$(_id)[0].spreadsheet.refresh()
}
})
}
this.export = function(_label,spreadsheet){
var uri = 'api/io/write'
var http = HttpClient.instance()
@ -312,7 +328,9 @@ studio.frame = function (_args){
})
}
}
this.render = function (){
var _args = this._args
var _importSheet = this.open
var _exportSheet = this.export

@ -35,7 +35,7 @@ def get (**_args) :
@cms.Plugin(mimetype="application/json",method="GET")
def providers (**_args):
technologies = []
transport.supported().apply(lambda row: [technologies.append({"group":row.name,"name":_name}) for _name in row if _name != '' and row.name in ['sql','nosql','warehouse']],axis=0).tolist()
transport.supported().apply(lambda row: [technologies.append({"group":row.name,"name":_name}) for _name in row if _name != '' and row.name in ['sql','nosql','warehouse','cloud']],axis=0).tolist()
return technologies
@cms.Plugin(mimetype="text/plain")
def version (**_args) :

@ -0,0 +1,51 @@
import transport
from io import StringIO
import cms
import copy
import json
@cms.Plugin(mimetype='application/json',method='GET')
def get (**_args) :
"""
This function will return the list of labels available
"""
transport.registry.load()
_data = copy.copy(transport.registry.DATA)
_context = _args['config']['system']['context']
_labels = []
for _key in _data :
if _key not in ['default','email','version'] :
print (_key)
_provider = _data[_key]['provider']
_name = None
if 'table' in _data[_key] :
_name = 'table'
elif 'collection' in _data[_key] :
_name = 'collection'
_table = 'NA' if not _name else _data[_key][_name]
_plugins = [] if 'plugins' not in _data[_key]else _data[_key]['plugins']
_icon = f'{_context}/api/disk/read?uri=www/html/_assets/images/{_provider}.png'
_labels.append({"label":_key,"provider":_provider,'table':_table,'icon':_icon})
else:
continue
return _labels
@cms.Plugin(mimetype="application/json",method="POST")
def add(**_args):
"""
adding a label to the registry
"""
_request = _args['request']
_entry = _request.json
_label = _entry['label']
del _entry['label']
f = StringIO(json.dumps(_entry))
#
# @TODO: We need to test the parameters and return the response to the client
#
transport.registry.set(_label,f)
#
# now we can/should get the rest of the list
return get(**_args)

@ -31,6 +31,7 @@
<script src="www/html/_assets/lib/spreadsheet/syncfusion/syncfusion-helper.js"></script>
<script src="www/html/_assets/js/forms.js"></script>
<!-- <script src="https://bossanova.uk/jspreadsheet/v5/jspreadsheet.js"></script>
<script src="https://jsuites.net/v5/jsuites.js"></script>
<link rel="stylesheet" href="https://jsuites.net/v5/jsuites.css" type="text/css" />

@ -35,9 +35,10 @@
</style>
<script>
var _init = function (){
var _init = function (_term){
var http = HttpClient.instance()
var uri = (['{{system.context}}','api/dbe/get']).join('/')
var uri = (['{{system.context}}','api/register/get']).join('/')
$('.search-results .labels').empty()
http.get(uri,function (x){
var labels = JSON.parse(x.responseText)
@ -48,21 +49,32 @@
_div = $('<div class="active button"></div>').append(_icon,_label).attr('label',_data.label).attr('provider',_data.provider)
_div[0]._data = _data
_div[0].work_bench = new studio.frame(_data)
_div[0].onclick = function (){
_id = '.studio-pane .'+this._data.label.trim()
$('.studio').slideUp('fast',()=>{
// $('.studio').slideUp('fast',()=>{
})
// })
if ($(_id).length == 0){
_object = new studio.frame(this._data)
_object.render()
// _object = new studio.frame(this._data)
// _object.render()
this.work_bench.render()
}else{
$(_id).slideDown()
// $(_id).slideDown()
}
// var _label = this._data.label
this.work_bench.show()
}
$('.search-results .labels').append(_div)
});
if(_term){
$('.search-box .search-label').val(_term)
_find()
}
})
}
var _find = function (){
@ -131,5 +143,22 @@
</div>
</div>
<p>
<div class="border" style="padding:16px;">
<div class="active" style="margin:4px;" onclick="dialog.show({uri:'www/html/wizard.html',title:'Create New Connection',context:'{{system.context}}' })">
<i class="fa-solid fa-link"></i> Create a new database connection
</div>
<div class="active" style="margin:4px" onclick='window.open("https://dev.the-phi.com/git/data-transport")'>
<i class="fa-solid fa-link"></i> Contribute to <i class="fa-brands fa-git-alt"></i> source code
</div>
<div style="margin:4px">
<i class="fa-solid fa-link"></i> Install from <i class="fa-brands fa-docker"></i> docker/podman container</div>
</div>
<br>
<b>Credits & Thanks</b>
<br><i class="fa-solid fa-minus"></i> Attributions
</p>
</div>

@ -1,4 +1,5 @@
<script>
if (!wizard){
var wizard = {}
}
@ -9,24 +10,36 @@
_text = $('<div class="small bold" align="center"></div>').html(_item.name)
_icon = $('<img>').attr('src','www/html/_assets/images/'+_item.name+'.png')
_pane = $('<div class="provider active"></div>').append(_icon,_text)
$(_frame).attr('icon',$(_icon).attr('src'))
_frame.append(_pane)
_frame.on('click',function (){
_group = $(this).attr('group').trim()
_provider = $(this).attr('provider').trim()
$('.db-provider').slideUp('fast',function(){
$('.db-form').slideDown()
_title = '.::. '+_provider
$('.db-form-title').html(_title)
$('.db-form-title').attr('provider',_provider)
if (_provider.match(/sqlite|duckdb|databricks|bigquery/)){
$('.db-form .database').attr('placeholder','Absolute to database')
if (_provider.match(/databricks|bigquery/)){
$('.db-form .database').attr('placeholder','Absolute to service account file')
}
$('.db-form .server').slideUp()
$('.db-form .auth').slideUp()
}
})
_title = $('<div></div>').html(_provider)
// _title = '.::. '+_provider
// $('.db-form-title').html(_title)
var _icon = $('<img>').attr('src',$(this).attr('icon'))
$('.db-form-title').empty()
$('.db-form-title').append(_icon,_title)
$('.db-provider').slideUp( ()=>{ $('.db-form').slideDown()})
var form = new ProviderInputForm(_provider)
form.build('.db-form-input')
// $('.db-provider').slideUp('fast',function(){
// $('.db-form').slideDown()
// _title = '.::. '+_provider
// $('.db-form-title').html(_title)
// $('.db-form-title').attr('provider',_provider)
// if (_provider.match(/sqlite|duckdb|databricks|bigquery/)){
// $('.db-form .database').attr('placeholder','Absolute to database')
// if (_provider.match(/databricks|bigquery/)){
// $('.db-form .database').attr('placeholder','Absolute to service account file')
// }
// $('.db-form .server').slideUp()
// $('.db-form .auth').slideUp()
// }
// })
})
@ -79,7 +92,7 @@
// $('.db-form').slideUp()
wizard.jumpTo('.db-provider')
window.wizard = wizard
$('.provider-search').focus()
$('.provider-s.earch').focus()
})
</script>
@ -94,8 +107,62 @@
.chose-provider .provider {justify-content: center; display:grid;}
.chose-provider img {height:70px; justify-self: center; margin:4px;}
.tech-frame {height: 305px; overflow: hidden; overflow-y: auto;}
.db-form {display:block; margin:4px; }
.db-form .server {display:grid; grid-template-columns: auto 100px; gap:4px; height:48px;}
.db-setup{width:700px;}
.db-form {
display:grid; gap:4px; padding:8px; font-family:sans-serif;
grid-template-rows: 48px calc(100% - 120px) 48px; gap:4px;;
height:100%;
}
.db-form-title { display:grid;
grid-template-columns: 40px auto; gap:8px;
align-items: center;
text-transform: capitalize;
}
.db-form-title img {width:32px; height:32px;}
.db-form .form-item {
display:grid;
grid-template-columns:35% 65%; gap:4px;
align-items: center;
align-content: center;
margin:2px;
}
.db-form .form-item label {font-weight: normal; text-transform: capitalize;}
.db-form .form-item input {
background-color: #f3f3f3;
padding:8px; outline: 0; border:2px solid transparent;
margin:0px;
}
.db-form .form-item input::file-selector-button {
font-weight: bold;
color:#000000;
/* padding: 0.5em; */
padding:4px;
width:115px;
border: thin solid #CAD5E0;
border-radius: 3px;
}
.db-form .form-item input:focus {border-left-color: #4682b4;;}
.db-form .form-controls {
bottom:0px;
display:grid;
grid-template-columns: 125px auto 125px;
padding:4px;
margin-top:8px;
font-size:14px; font-weight:lighter;
}
.db-form .form-item .input-error {
background-color: yellow;
}
/* For WebKit browsers (Chrome, Safari, Edge, Opera) */
/*.db-form {display:block; margin:4px; }
.db-form .server {display:grid; grid-template-columns: auto 100px; gap:4px; height:48px;}
.db-form .internal {display:grid; grid-template-rows: auto; gap:4px;}
input{padding:8px; outline: 0; border:4px solid transparent;
background-color: #f3f3f3;}
@ -103,9 +170,9 @@
border-left-color: #4682B4;
}
.db-setup {width:800px; font-family: sans-serif; font-weight:lighter;}
.db-form-input {height:300px; display:grid;grid-template-rows: repeat(3,1fr); gap:4px;}
.db-form-input {height:300px; display:grid;grid-template-rows: repeat(3,1fr); gap:4px;} */
</style>
<div class="db-setup" style="height:500px;">
<div class="db-setup">
<div class="db-provider page">
<div class="search-box" style="align-items:center">
@ -126,88 +193,10 @@
<div class="db-form page">
<div class="db-form-title bold"></div>
<br>
<div class="db-form-input">
<div class="server">
<input type="text" class="host" placeholder="Host"/>
<input type="text" class="port" placeholder="Port"/>
</div>
<div class="internal">
<input type="text" class="database" placeholder="Database"/>
<input type="text" class="table" placeholder="Table"/>
</div>
<div class="internal auth">
<div class="bold">Authentication</div>
<input type="text" class="username" placeholder="Username"/>
<input type="password" class="password" placeholder="Password"/>
</div>
</div>
<div style="display:grid; grid-template-columns: 125px auto 125px; align-items: center;">
<div style="display:grid; justify-items: left; margin-top:28px;">
<div class="border-round border" onclick="wizard.jumpTo('.db-provider')">
<div class="active" align="center"><i class="fa-solid fa-arrow-left"></i> Back</div>
</div>
</div>
<div>
</div>
<div style="display:grid; justify-items: right; margin-top:28px;">
<div class="border-round border" onclick="wizard.jumpTo('.db-final')">
<div class="active" align="center"><i class="fa-solid fa-arrow-right"></i> Next</div>
</div>
</div>
</div>
<ul class="small">
<br><i class="fa-solid fa-minus"></i> Missing ports and host will refer to their default values
<br><i class="fa-solid fa-minus"></i> Make sure you have configured your database properly
</ul>
</div>
<div class="db-form-title bold border-bottom"></div>
<div class="db-form-input"></div>
<div class="db-final page">
<div class="db-form-title bold"></div>
<br>
<div class="db-form-input">
<div style="display: grid; grid-template-columns: 50% 50%; gap:4px;">
<div class="source-code">
{}
</div>
<div class="">
<input type="text" class="label" placeholder="label your configuration" style="width:100%"/>
<ul class="">
<i class="fa-solid fa-minus"></i> Make sure the label is short and descriptive
<br> <i class="fa-solid fa-minus"></i> The label will be used in code to point at the configuration
<br><i class="fa-solid fa-minus"></i> No data connectivity/location will be shared with your notebooks.
</ul>
</div>
</div>
</div>
<div style="display:grid; grid-template-columns: 125px auto 125px; align-items: center;">
<div style="display:grid; justify-items: left; margin-top:28px;">
<div class="border-round border" onclick="wizard.jumpTo('.db-form')">
<div class="active" align="center"><i class="fa-solid fa-arrow-left"></i> Back</div>
</div>
</div>
<div>
</div>
<div style="display:grid; justify-items: right; margin-top:28px;">
<div class="border-round border" onclick="wizard.jumpTo('.db-final')">
<div class="active" align="center"><i class="fa-solid fa-save"></i> Save</div>
</div>
</div>
</div>
</div>
</div>
Loading…
Cancel
Save