diff --git a/qcms-manifest.json b/qcms-manifest.json index 6f77f19..68734e6 100644 --- a/qcms-manifest.json +++ b/qcms-manifest.json @@ -9,9 +9,9 @@ "context": "", "logo": "www/html/_assets/images/logo.png", "source": { - "id": "disk" + "id": "disk", + "key": "/home/steve/dev/data/qcms/data-transport.key" }, - "theme": "default", "version": "0.1" }, @@ -28,7 +28,9 @@ "index": "index.html", "on": { "load": { - "pane":["www/html/menu.html"] + "pane": [ + "www/html/menu.html" + ] } }, "order": { @@ -38,7 +40,15 @@ "root": "www/html" }, "plugins": { - "dbe":["get","providers","apply","version"], - "io":["read","write"] + "dbe": [ + "get", + "providers", + "apply", + "version" + ], + "io": [ + "read", "open", + "write" + ] } } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..75c4401 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +xlsxwriter +openpyxl \ No newline at end of file diff --git a/www/html/_assets/js/studio.js b/www/html/_assets/js/studio.js index c9cf15e..72eeda7 100644 --- a/www/html/_assets/js/studio.js +++ b/www/html/_assets/js/studio.js @@ -158,20 +158,25 @@ studio.grid = function (){ }) var spreadsheet = new ej.spreadsheet.Spreadsheet(); - // spreadsheet.allowSave = true - // spreadsheet.saveUrl = 'api/io/write' + + // spreadsheet.openUrl='/api/io/open' + // spreadsheet.openSettings = {chunkSize:1024} + // spreadsheet.saveUrl = 'https://services.syncfusion.com/js/production/api/spreadsheet/save' + // spreadsheet.created = function (){} spreadsheet.sheets = [ {name:_id.replace(/[#,.]/g,' '), ranges:[{dataSource:rows}], } ] + spreadsheet.appendTo(_id) $(_id)[0].spreadsheet = spreadsheet // console.log([' **** ',(_id+' .e-input-group')]) // $(_id+' .e-input-group').remove() spreadsheet.refresh() + spreadsheet.hideFileMenuItems(["File"], true); } } @@ -209,10 +214,105 @@ studio.frame = function (_args){ } }) + } + this.open = function (_id,file){ + var http = HttpClient.instance() + _uri = 'api/io/open' + var form = new FormData() + form.append('file',file) + http.setData(form) + var _index = $(_id)[0].spreadsheet.sheets.length + // _name = file.name + + // $(_id)[0].spreadsheet.insertSheet([{name:_name,_index:1}],0) + // console.log([_index, $(_id)[0].spreadsheet.sheets.length]) + // $(_id)[0].spreadsheet.sheets[_index].name = `sheet {_index}` + http.post(_uri,function(x){ + if(x.status == 200 && x.readyState == 4){ + var _data = JSON.parse(x.responseText) + _data.Workbook.sheets.forEach(sheet=>{ + sheet.name = file.name + sheet.index=_index + $(_id)[0].spreadsheet.insertSheet([sheet],0) + }) + + + }else{ + //@TODO: + // handle the error in a graceful way + } + }) } + this.export = function(_label,spreadsheet){ + var uri = 'api/io/write' + var http = HttpClient.instance() + var _data = {} + _rows = {} + spreadsheet.sheets.forEach((sheet)=>{ + _rows[sheet.name] = {columns:[],values:[]} + + sheet.rows.forEach((_item,_index)=>{ + // cells = _item.cells + if(_index == 0){ + _item.cells.forEach(_x =>{ + _rows[sheet.name].columns.push(_x.value) + }) + }else{ + rec = [] + _item.cells.forEach(_x=>{ + rec.push(_x.value) + }) + _rows[sheet.name].values.push(rec) + } + }) + console.log(_rows[sheet.name].values) + // if (sheet.ranges.length > 0){ + // if(sheet.ranges[0].dataSource.length > 0){ + + // _data.push(sheet) //.ranges[0].dataSource) + // } + // } + }) + if (_rows ){ + http.setHeader('Content-Type','application/json') + http.setData (JSON.stringify({rows:_rows,'label':_label})) + http.post(uri,(x)=>{ + if(x.status == 200 && x.readyState == 4){ + // + // + // console.log(x) + // var blob = new Blob([x.response], { type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' }); + const byteCharacters = atob(x.responseText); + const byteNumbers = new Array(byteCharacters.length); + for (let i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + const blob = new Blob([byteArray], { + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + }); + + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); + link.download = _label+'-export.xlsx'; + document.body.appendChild(link); + link.click(); + + // Cleanup + URL.revokeObjectURL(link.href); + document.body.removeChild(link); + + }else{ + // + // something went wront + } + }) + } + } this.render = function (){ var _args = this._args - + var _importSheet = this.open + var _exportSheet = this.export var _icon = $('').attr('src',this._args.icon) var _label = $('
').html(this._args.label+'
'+this._args.provider+'
') read = this.read @@ -229,9 +329,23 @@ studio.frame = function (_args){ } }) - var _imgSrc = "api/disk/read?uri=www/html/_assets/images/python.png" - _imgSrc = studio._context + _imgSrc - _codeButton = ('
') + + _openFile = $('
') + $(_openFile).on('click',()=>{ + var _id = '.'+_args.label+' .open-file' + $(_id).on('change',(event)=>{ + var file = event.target.files[0] + _id = '.'+_args.label + ' .output' + _importSheet(_id,file) + }) + $(_id).click() + }) + _saveFile = $('
') + $(_saveFile).on('click',()=>{ + var _id = '.'+_args.label +' .output' + var spreadsheet = $(_id)[0].spreadsheet + _exportSheet(_args.label,spreadsheet) + }) _wizButton = $('
') _wizButton.on('click',function(){ dialog.show({uri:'www/html/wizard.html',title:'Create New Connection',context:studio._context}) @@ -253,13 +367,11 @@ studio.frame = function (_args){ } $(_id+' .expand').empty().html(_icon).attr('title',_title) var spreadsheet = $(_id+' .output')[0].spreadsheet - if (spreadsheet != null){ - spreadsheet.refresh() - } + spreadsheet.refresh() }) - var _buttons = $('
 
') - _buttons.append(_codeButton,_wizButton,_xbutton,_expandButton) + var _buttons = $('
 
') + _buttons.append(_openFile,_saveFile,_wizButton,_xbutton,_expandButton) // _frame = $('
'.replace(/:label/,this._args.label)) var _frame = $('
') .addClass(this._args.label) @@ -280,11 +392,7 @@ studio.frame = function (_args){ $(_id).click() } }) - // _inputframe = $('
.::
') - // _inputframe = $('
') - // _inputframe.append(_ctrl,_textarea) - // _outputframe = $('
') - _outputframe = $('
') + _outputframe = $('
') // $(_frame).append(_inputframe, _outputframe) @@ -308,25 +416,7 @@ studio.frame = function (_args){ _gHandler.syncfusion(_id,_data) _rows = (_data.data)?_data.data.length : 0 $('.'+_label + ' .status').html(_rows + ' rows') - // $(_id).html('
') - // _id = _id + ' table' - // var _columns = [] - // _args = {} - // if (_data.columns ){ - // _data.columns.forEach(_name=>{ - // _columns.push({title:_name}) - // }) - // var _args = {'data':_data.data,'columns':_columns} - // // _args.layout = {topStart:{}} - // _args.buttons = ['excel','pdf'] - - // }else{ - // _data = {'data':[],columns:[]} - // } - // _args.dom = 'rtip' - // _args.scrollCollapse = true - // _args.scrollY = '300px' - // new DataTable(_id,_args) + } } \ No newline at end of file diff --git a/www/html/_plugins/dbe.py b/www/html/_plugins/dbe.py index 7a86365..7e93603 100644 --- a/www/html/_plugins/dbe.py +++ b/www/html/_plugins/dbe.py @@ -39,7 +39,7 @@ def providers (**_args): return technologies @cms.Plugin(mimetype="text/plain") def version (**_args) : - return transport.__edition__+ ' '+transport.__version__ + return transport.__edition__+ ' Edition '+transport.__version__ @cms.Plugin(mimetype="application/json",method="POST") def apply (**_args): _request = _args['request'] diff --git a/www/html/_plugins/io.py b/www/html/_plugins/io.py index b85ea71..b6d4c87 100644 --- a/www/html/_plugins/io.py +++ b/www/html/_plugins/io.py @@ -4,6 +4,12 @@ This interfaces the reader for a data-transport import cms import transport +from io import BytesIO, StringIO +import pandas as pd +import numpy as np +import json +import base64 +from flask import make_response, send_file @cms.Plugin(mimetype="application/json",method="POST") def read(**_args) : @@ -23,9 +29,44 @@ def read(**_args) : except Exception as e: print (e) return _data +def format (file,_mimetype) : + if 'csv' in _mimetype : + _df = pd.read_csv(file).fillna('n/a') + + rows = _df.apply(lambda row:row.values ,axis=1).apply(lambda row: {'cells':[{'value':x}for x in row]} ).tolist() + rows = [{"cells":[ {'value':_name} for _name in _df.columns]}] + rows + sheets = [{'usedRange':{'colIndex':1,'rowIndex':_df.shape[0]},'name':'sheet1','rows':rows,'standardHeight':20}] + return {'Workbook':{'sheets':sheets}} + -@cms.Plugin(mimetype="application/json") + +@cms.Plugin(mimetype="application/json",method="POST") +def open(**_args): + _request = _args['request'] + file = _request.files['file'] + spreadsheet = format(file,file.mimetype) + return spreadsheet #{"Workbook":{"sheets":[{"usedRange":{"colIndex":1,"rowIndex":3},"name":"Sheet1","rows":[{"cells":[{"value":"name"},{"value":"age"}]},{"cells":[{"value":"steve"},{"value":"44"}]},{"cells":[{"value":"elon"},{"value":"9"}]},{"cells":[{"value":"nico"},{"value":"33"}]}],"standardHeight":20}]}}) +@cms.Plugin(mimetype="application/octet-stream",method="POST") def write(**_args) : _request = _args['request'] - print (_request.json) - return {} \ No newline at end of file + _object = _request.json + _sheets = None + if _object : + _rows = _object['rows'] + _label = _object['label'] + # + # convert this to a spreadsheet + _excelFile = BytesIO() + with pd.ExcelWriter(_excelFile, engine='openpyxl') as writer: + _index = 1 + for _name in _rows : + _sheet = _rows[_name] + _df = pd.DataFrame(_sheet['values'],columns=_sheet['columns']) + + _df.to_excel(writer,sheet_name=_name,index=False) + _index += 1 + + # Write each DataFrame to a different sheet + _excelFile.seek(0) + _stream= _excelFile.read() #send_file(_excelFile,as_attachment=True,download_name=f'{_label}-export.xlsx', mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') + return base64.b64encode(_stream).decode('utf-8') \ No newline at end of file diff --git a/www/html/index.html b/www/html/index.html index 82cf6cf..9d2bb19 100644 --- a/www/html/index.html +++ b/www/html/index.html @@ -40,8 +40,19 @@
\ No newline at end of file diff --git a/www/html/menu.html b/www/html/menu.html index 61f520e..0cff5e9 100644 --- a/www/html/menu.html +++ b/www/html/menu.html @@ -6,14 +6,18 @@ grid-template-columns: auto 50px ; gap:4px; align-content:center; + align-items: center; + padding:0px; } .search-box input[type=text]{ - - padding:4px; + height:26px; + padding:6px; outline:0; - border: 4px solid transparent; - background-color: #d3d3d3; + border: 0px solid transparent; + border-left:4px solid transparent; + background-color: #D3D3D3; + margin:4px; } .search-box input[type=text]:focus { border-left-color: #4682b4;; @@ -46,7 +50,9 @@ _div[0]._data = _data _div[0].onclick = function (){ _id = '.studio-pane .'+this._data.label.trim() - $('.studio').slideUp() + $('.studio').slideUp('fast',()=>{ + + }) if ($(_id).length == 0){ _object = new studio.frame(this._data) _object.render() @@ -66,8 +72,6 @@ var _nodes = $('.search-results .labels .button') _nodes.each(_index => { _item = _nodes[_index] - console.log($(_item).attr('')) - console.log([' **** ']) p = $(_item).attr('provider') q = $(_item).attr('label') rgx = new RegExp(_val) @@ -107,19 +111,7 @@
- +