From 169077002d5e79a812e36e5b27706f5379c28049 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 27 Jun 2025 16:08:19 -0500 Subject: [PATCH] bug fixe: add to registry, wizard form rewrite --- qcms-manifest.json | 9 +- www/html/_assets/js/forms.js | 198 ++++++++++++++++++++++++++++++++++ www/html/_assets/js/studio.js | 22 +++- www/html/_plugins/dbe.py | 2 +- www/html/_plugins/register.py | 51 +++++++++ www/html/index.html | 1 + www/html/menu.html | 43 ++++++-- www/html/wizard.html | 191 ++++++++++++++++---------------- 8 files changed, 401 insertions(+), 116 deletions(-) create mode 100644 www/html/_assets/js/forms.js create mode 100644 www/html/_plugins/register.py diff --git a/qcms-manifest.json b/qcms-manifest.json index 68734e6..ff389c5 100644 --- a/qcms-manifest.json +++ b/qcms-manifest.json @@ -22,9 +22,8 @@ } ], "header": { - "title": "Wrangle Workbench", - "subtitle": "Powered by data-transport" - }, + "title": "Data Transport", + "subtitle": "Open Workbench: Read, write & wrangle data"}, "index": "index.html", "on": { "load": { @@ -46,9 +45,9 @@ "apply", "version" ], + "register":["get","add"], "io": [ - "read", "open", - "write" + "read", "open", "write" ] } } diff --git a/www/html/_assets/js/forms.js b/www/html/_assets/js/forms.js new file mode 100644 index 0000000..f4f8239 --- /dev/null +++ b/www/html/_assets/js/forms.js @@ -0,0 +1,198 @@ + +var _config = { + "sql":{ + + "host":{"type":"text","default":"localhost"},"database":"text","table":"text","username":"text","password":"password" + }, + "cloudant":{ + "host":{"type":"text","default":"localhost"}, + "port":{"type":"number","default":5984}, + "dbname":{"type":"text","label":"database"}, + "username":{"type":"text"}, + "password":"password", + "doc":{"type":"text","label":"document"} + }, + "couchdb":{"@ref":"cloudant"}, + "bigquery":{ + "database":"text","table":"text", + "auth_file":{"type":"file","label":"service account json file"} + }, + "postgresql":{ + "@ref":"sql", + "port":{"type":"number","default":5432} + }, + "redshift":{"@ref":"postgresql"}, + "mariadb":{ + "@ref":"sql", + "port":{"type":"number","default":3306} + }, + "mysql":{ + "@ref":"sql", + "port":{"type":"number","default":3306} + }, + "netezza":{ + "@ref":"sql", + "port":{"type":"number","default":5480}}, + "sqlserver":{ + "@ref":"sql", + "port":{"type":"number","default":1433} + }, + + "mongodb":{ + + "host":{"type":"text","default":"localhost"}, + + "port":{"type":"number","default":2717}, + "db":{"type":"text","label":"database"}, + "collection":{"type":"text"},"mechanism":{"type":"text","default":"SCRAM-SHA-256","values":["SCRAM-SHA-256","MONGODB-X509","MONGODB-CR"]} + }, + "nextcloud":{ + "uid":{"type":"text","label":"user id"}, + "url":{"type":"text","label":"url of server"}, + "token":"text" + + }, + "databricks":{ + "host":"text","token":"text","cluster_path":"text","schema":"text","catalog":"text" + + }, + "s3":{ + "bucket":"text","file":{"type":"text","desc":"aws s3 path of the file"}, + "region":"text" + }, + "sqlite":{"database":{"type":"text","desc":"path on disk"},"table":"text"}, + "sqlite3":{"@ref":"sqlite"}, + "duckdb":{"database":{"type":"text","desc":"path on disk"},"table":"text"}, + "drill":{ + "host":"text","port":{"type":"number","default":8047},"ssl":{"type":"bool","values":[true,false],"default":false} + + }, + "iceberg":{ + "catalog":"text","database":"text","table":"text" + } +} + +var ProviderInputForm = function (_provider){ + this._provider = _provider + /** + * This section will setup the configuration file as it is intended to be + * The configuration fully specified will be used to build the input form + */ + this._config = {} + _parent = {} + var _tmpconfig = _config[_provider] + this._config = _config[_provider] + if (_tmpconfig['@ref'] != null){ + _pkey = _tmpconfig['@ref'] + _parent = _config[_pkey] + } + //-- merge the objects ... + this._config = Object.assign({'label':{'type':'text',desc:'reference to this database'}},_parent, _config[_provider]) + if (this._config['@ref']){ + delete this._config['@ref'] + } + + this.getInput = function(_item){//_type,_text,_desc){ + // if(_type.match(/text|number|password/ig)){ + + _item.label= (_item.label == null)?_item.context: _item.label + _item.desc = (_item.desc != null)?_item.desc:_item.label + _label = $('').html(_item.label) + _input = $('') + $(_input).attr('type',_item.type) + + if(_item.type.match(/number/)) { + $(_input).attr("pattern","^[1-9]([0-9])*$") + } + + $(_input).attr('placeholder',_item.desc) + $(_input).addClass(_item.context) + $(_input).attr('id',_item.context) + if (_item.default != null){ + $(_input).val(_item.default) + } + $(_label).attr('for',_item.context) + return $('
').append(_label,_input) + // } + + } + this.submit = function (_data){ + var http = HttpClient.instance() + http.setData(JSON.stringify(_data)) + http.setHeader('Content-Type','application/json') + var uri = 'api/register/add' + http.post(uri,(x)=>{ + if(x.status == 200 && x.readyState == 4){ + // + // + _init(_data.label) + $('.jxmodal').slideUp() + } + }) + } + this.build = function(_id){ + // + // This will build the form on a pane + + var _form = $('
') + Object.keys(this._config).forEach(_key=>{ + var _item = this._config[_key] + var _type = (_item.constructor == String)?_key:_item.type + if(_item.constructor == String){ + _item = {type:_type} + } + _item.type = _type + _item.context = _key + // var _values = (_item.values)?[]:_item.values + var _inputLine = this.getInput(_item) //_type,_key,_item.label) + $(_form).append(_inputLine) + // } + }) + + // + // adding all the items to where they need to be + // + _back = $('
Go Back
') + $(_back).on('click',()=>{ + $('.db-form').slideUp(()=>{ + $('.db-provider').slideDown() + + }) + }) + var _submit = this.submit + var _provider= this._provider + _save = $('
Save Now
') + $(_save).attr({'_object':this}) + $(_save).on('click',()=>{ + var _nodes = $('.db-form input') + var _data = {} + _errorCount = 0 + _nodes.each((_index)=>{ + var _input = _nodes[_index] + _data[_input.id] = _input.value + if(_input.value.trim().length == 0){ + _errorCount += 1 + $(_input).addClass('input-error') + }else{ + $(_input).removeClass('input-error') + } + + }) + // + // assuming no error ... + if (_errorCount == 0){ + _data = Object.assign({},{'provider':_provider}, _data) + _submit(_data) + } + }) + + _pane = $('
') + $(_pane).append(_back,$("
"),_save) + + $(_id).empty() + $(_id).append(_form) + $('.db-form .form-controls').remove() + $('.db-form').append(_pane) + } + +} diff --git a/www/html/_assets/js/studio.js b/www/html/_assets/js/studio.js index afc2a1d..b922c6d 100644 --- a/www/html/_assets/js/studio.js +++ b/www/html/_assets/js/studio.js @@ -178,8 +178,9 @@ studio.grid = function (){ $(_id)[0].spreadsheet = spreadsheet // console.log([' **** ',(_id+' .e-input-group')]) // $(_id+' .e-input-group').remove() - spreadsheet.refresh() spreadsheet.hideFileMenuItems(["File"], true); + spreadsheet.refresh() + } } @@ -245,7 +246,22 @@ studio.frame = function (_args){ // handle the error in a graceful way } }) - } + } + this.show = function (){ + var _label = this._args.label + $('.studio').each(_index=>{ + var _bench = $('.studio')[_index] + if (_bench.className.match(_label) == null){ + $(_bench).css({display:'none'}) + }else{ + $(_bench).css({display:'grid'}) + _id = '.'+_label+' .output' + console.log(_id) + $(_id)[0].spreadsheet.refresh() + } + + }) + } this.export = function(_label,spreadsheet){ var uri = 'api/io/write' var http = HttpClient.instance() @@ -312,7 +328,9 @@ studio.frame = function (_args){ }) } } + this.render = function (){ + var _args = this._args var _importSheet = this.open var _exportSheet = this.export diff --git a/www/html/_plugins/dbe.py b/www/html/_plugins/dbe.py index be48586..733ad40 100644 --- a/www/html/_plugins/dbe.py +++ b/www/html/_plugins/dbe.py @@ -35,7 +35,7 @@ def get (**_args) : @cms.Plugin(mimetype="application/json",method="GET") def providers (**_args): technologies = [] - transport.supported().apply(lambda row: [technologies.append({"group":row.name,"name":_name}) for _name in row if _name != '' and row.name in ['sql','nosql','warehouse']],axis=0).tolist() + transport.supported().apply(lambda row: [technologies.append({"group":row.name,"name":_name}) for _name in row if _name != '' and row.name in ['sql','nosql','warehouse','cloud']],axis=0).tolist() return technologies @cms.Plugin(mimetype="text/plain") def version (**_args) : diff --git a/www/html/_plugins/register.py b/www/html/_plugins/register.py new file mode 100644 index 0000000..f40bf11 --- /dev/null +++ b/www/html/_plugins/register.py @@ -0,0 +1,51 @@ +import transport +from io import StringIO +import cms +import copy +import json + +@cms.Plugin(mimetype='application/json',method='GET') +def get (**_args) : + """ + This function will return the list of labels available + """ + transport.registry.load() + _data = copy.copy(transport.registry.DATA) + _context = _args['config']['system']['context'] + _labels = [] + for _key in _data : + + if _key not in ['default','email','version'] : + print (_key) + _provider = _data[_key]['provider'] + _name = None + if 'table' in _data[_key] : + _name = 'table' + elif 'collection' in _data[_key] : + _name = 'collection' + _table = 'NA' if not _name else _data[_key][_name] + _plugins = [] if 'plugins' not in _data[_key]else _data[_key]['plugins'] + _icon = f'{_context}/api/disk/read?uri=www/html/_assets/images/{_provider}.png' + _labels.append({"label":_key,"provider":_provider,'table':_table,'icon':_icon}) + else: + continue + return _labels + +@cms.Plugin(mimetype="application/json",method="POST") +def add(**_args): + """ + adding a label to the registry + """ + _request = _args['request'] + _entry = _request.json + _label = _entry['label'] + del _entry['label'] + f = StringIO(json.dumps(_entry)) + # + # @TODO: We need to test the parameters and return the response to the client + # + transport.registry.set(_label,f) + + # + # now we can/should get the rest of the list + return get(**_args) diff --git a/www/html/index.html b/www/html/index.html index 9d2bb19..efd4fa9 100644 --- a/www/html/index.html +++ b/www/html/index.html @@ -31,6 +31,7 @@ +