file save, open and minor bug fixes

qcms-manifest.json

@@ -9,9 +9,9 @@
     "context": "",
     "logo": "www/html/_assets/images/logo.png",
     "source": {
-      "id": "disk"
+      "id": "disk",
+      "key": "/home/steve/dev/data/qcms/data-transport.key"
     },
-    
     "theme": "default",
     "version": "0.1"
   },
@@ -28,7 +28,9 @@
     "index": "index.html",
     "on": {
       "load": {
-        "pane":["www/html/menu.html"]
+        "pane": [
+          "www/html/menu.html"
+        ]
       }
     },
     "order": {
@@ -38,7 +40,15 @@
     "root": "www/html"
   },
   "plugins": {
-    "dbe":["get","providers","apply","version"],
+    "dbe": [
-    "io":["read","write"]
+      "get",
+      "providers",
+      "apply",
+      "version"
+    ],
+    "io": [
+      "read", "open",
+      "write"
+    ]
   }
 }

requirements.txt

@@ -0,0 +1,2 @@
+xlsxwriter
+openpyxl

www/html/_assets/js/studio.js

@@ -158,8 +158,11 @@ studio.grid = function (){
         })
 
         var spreadsheet = new ej.spreadsheet.Spreadsheet();
-        // spreadsheet.allowSave = true
+        
-        // spreadsheet.saveUrl = 'api/io/write'
+        // spreadsheet.openUrl='/api/io/open'
+        // spreadsheet.openSettings = {chunkSize:1024}
+        // spreadsheet.saveUrl = 'https://services.syncfusion.com/js/production/api/spreadsheet/save'
+        // spreadsheet.created = function (){}
         spreadsheet.sheets = [
             {name:_id.replace(/[#,.]/g,' '),
                 ranges:[{dataSource:rows}],
@@ -167,11 +170,13 @@ studio.grid = function (){
             }
         ]
           
+        
         spreadsheet.appendTo(_id)
         $(_id)[0].spreadsheet = spreadsheet
         // console.log([' **** ',(_id+' .e-input-group')])
         // $(_id+' .e-input-group').remove()
         spreadsheet.refresh()
+        spreadsheet.hideFileMenuItems(["File"], true);
     }
     
 }
@@ -210,9 +215,104 @@ studio.frame = function (_args){
         })
 
     }
+    this.open = function (_id,file){
+        var http = HttpClient.instance()
+        _uri = 'api/io/open'
+        var form = new FormData()
+        form.append('file',file)
+        http.setData(form)
+        var _index = $(_id)[0].spreadsheet.sheets.length 
+        // _name = file.name
+        
+        // $(_id)[0].spreadsheet.insertSheet([{name:_name,_index:1}],0)
+        // console.log([_index, $(_id)[0].spreadsheet.sheets.length])
+        // $(_id)[0].spreadsheet.sheets[_index].name  = `sheet {_index}`
+        http.post(_uri,function(x){
+            if(x.status == 200 && x.readyState == 4){
+                var _data = JSON.parse(x.responseText)
+                _data.Workbook.sheets.forEach(sheet=>{
+                    sheet.name = file.name
+                    sheet.index=_index
+                    $(_id)[0].spreadsheet.insertSheet([sheet],0)
+                })
+                
+               
+            }else{
+                //@TODO:
+                // handle the error in a graceful way
+            }
+        })
+    }    
+    this.export = function(_label,spreadsheet){
+        var uri = 'api/io/write'
+        var http = HttpClient.instance()
+        var _data = {}
+        _rows = {}
+        spreadsheet.sheets.forEach((sheet)=>{
+            _rows[sheet.name] = {columns:[],values:[]}
+
+            sheet.rows.forEach((_item,_index)=>{ 
+                // cells = _item.cells
+                if(_index == 0){
+                    _item.cells.forEach(_x =>{
+                        _rows[sheet.name].columns.push(_x.value)
+                    })
+                }else{
+                    rec = []
+                    _item.cells.forEach(_x=>{
+                        rec.push(_x.value)
+                    })
+                    _rows[sheet.name].values.push(rec)
+                }
+            })
+            console.log(_rows[sheet.name].values)
+            // if (sheet.ranges.length > 0){
+            //     if(sheet.ranges[0].dataSource.length > 0){
+
+            //         _data.push(sheet) //.ranges[0].dataSource)
+            //     }
+            // }
+        })
+        if (_rows ){
+            http.setHeader('Content-Type','application/json')
+            http.setData (JSON.stringify({rows:_rows,'label':_label}))
+            http.post(uri,(x)=>{
+                if(x.status == 200 && x.readyState == 4){
+                    //
+                    //
+                    // console.log(x)
+                    // var blob = new Blob([x.response], { type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' });
+                    const byteCharacters = atob(x.responseText);
+                    const byteNumbers = new Array(byteCharacters.length);
+                    for (let i = 0; i < byteCharacters.length; i++) {
+                        byteNumbers[i] = byteCharacters.charCodeAt(i);
+                    }
+                    const byteArray = new Uint8Array(byteNumbers);
+                    const blob = new Blob([byteArray], {
+                        type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+                    });
+                                        
+                    const link = document.createElement("a");
+                    link.href = URL.createObjectURL(blob);
+                    link.download = _label+'-export.xlsx';
+                    document.body.appendChild(link);
+                    link.click();
+
+                    // Cleanup
+                    URL.revokeObjectURL(link.href);
+                    document.body.removeChild(link);
+                    
+                }else{
+                    //
+                    // something went wront
+                }
+            })
+        }
+    }
     this.render = function (){
         var _args = this._args
-        
+        var _importSheet = this.open
+        var _exportSheet = this.export
         var _icon = $('<img>').attr('src',this._args.icon)
         var _label = $('<div class="bold"></div>').html(this._args.label+'<div style="font-weight:lighter; font-size:11px;">'+this._args.provider+'</div>')
         read = this.read
@@ -229,9 +329,23 @@ studio.frame = function (_args){
 
             }
         })
-        var _imgSrc = "api/disk/read?uri=www/html/_assets/images/python.png"
+        
-        _imgSrc = studio._context + _imgSrc
+        _openFile = $('<div class="border-left"><div class="active" title="Open File"><i class="fa-regular fa-folder-open"></i></div></div>')
-        _codeButton = ('<div class="border-left"><div class="active" title="show in code"><i class="fa-solid fa-code"></i></div></div>')
+        $(_openFile).on('click',()=>{
+            var _id = '.'+_args.label+' .open-file'
+            $(_id).on('change',(event)=>{
+                var file = event.target.files[0]
+                _id = '.'+_args.label + ' .output'
+                _importSheet(_id,file)
+            })
+            $(_id).click()
+        })
+        _saveFile = $('<div class="border-left"><div class="active" title="Save File"><i class="fa-solid fa-floppy-disk"></i></div></div>')
+        $(_saveFile).on('click',()=>{
+            var _id = '.'+_args.label +' .output'
+            var spreadsheet = $(_id)[0].spreadsheet
+            _exportSheet(_args.label,spreadsheet)
+        })
         _wizButton = $('<div class="border-left"><div class="active" title="New Connection"><i class="fa-solid fa-plug-circle-plus" style="color:#4682B4;"></i></div></div>')
         _wizButton.on('click',function(){
             dialog.show({uri:'www/html/wizard.html',title:'Create New Connection',context:studio._context})
@@ -253,13 +367,11 @@ studio.frame = function (_args){
             }
             $(_id+' .expand').empty().html(_icon).attr('title',_title)
             var spreadsheet = $(_id+' .output')[0].spreadsheet
-            if (spreadsheet != null){
+            spreadsheet.refresh()
-                spreadsheet.refresh()
-            }
         })
         
-        var _buttons = $('<div style="display:grid; grid-template-columns: auto repeat(4,64px); gap:4px;" align="center"><div>&nbsp;</div></div>')
+        var _buttons = $('<div style="display:grid; grid-template-columns: auto repeat(5,64px); gap:4px;" align="center"><div>&nbsp;</div></div>')
-        _buttons.append(_codeButton,_wizButton,_xbutton,_expandButton)
+        _buttons.append(_openFile,_saveFile,_wizButton,_xbutton,_expandButton)
 
         // _frame = $('<div class="studio :label"></div>'.replace(/:label/,this._args.label))
         var _frame = $('<div class="studio"></div>') .addClass(this._args.label)
@@ -280,11 +392,7 @@ studio.frame = function (_args){
                 $(_id).click()
             }
         })
-        // _inputframe = $('<div class="input-frame"><div class="ctrl border-round border">.::</div> <textarea class="code"></textarea></div>')
+        _outputframe = $('<div class="output" ></div><div class="status"></div><input type="file" class="open-file" style="display:none" accept=".csv, .xls, .xlsx"/>')
-        // _inputframe = $('<div class="input-frame"></div>')
-        // _inputframe.append(_ctrl,_textarea)
-        // _outputframe = $('<div class="" style="display:grid; grid-template-rows:auto 32px; gap:4px;"><div class="output border-round" ></div><div class="status border"></div></div>')
-        _outputframe = $('<div class="output" ></div><div class="status"></div>')
         
 
         // $(_frame).append(_inputframe, _outputframe)
@@ -308,25 +416,7 @@ studio.frame = function (_args){
         _gHandler.syncfusion(_id,_data)
         _rows = (_data.data)?_data.data.length : 0
         $('.'+_label + ' .status').html(_rows + ' rows')
-        // $(_id).html('<table class="display"></table>')
+      
-        // _id = _id + ' table'
-        // var _columns = []
-        // _args = {}
-        // if (_data.columns ){
-        //     _data.columns.forEach(_name=>{
-        //         _columns.push({title:_name})
-        //     })
-        //     var _args = {'data':_data.data,'columns':_columns}
-        //     // _args.layout = {topStart:{}}
-        //     _args.buttons = ['excel','pdf']
-            
-        // }else{
-        //     _data = {'data':[],columns:[]}
-        // }
-        // _args.dom = 'rtip'
-        // _args.scrollCollapse = true
-        // _args.scrollY = '300px'
-        // new DataTable(_id,_args)
        
     }
 }

www/html/_plugins/dbe.py

@@ -39,7 +39,7 @@ def providers (**_args):
     return technologies
 @cms.Plugin(mimetype="text/plain")
 def version (**_args) :
-    return transport.__edition__+ ' '+transport.__version__
+    return transport.__edition__+ ' Edition '+transport.__version__
 @cms.Plugin(mimetype="application/json",method="POST")
 def apply (**_args):
     _request = _args['request']

www/html/_plugins/io.py

@@ -4,6 +4,12 @@ This interfaces the reader for a data-transport
 
 import cms
 import transport
+from io import BytesIO, StringIO
+import pandas as pd
+import numpy as np
+import json
+import base64
+from flask import make_response, send_file
 
 @cms.Plugin(mimetype="application/json",method="POST")
 def read(**_args) :
@@ -23,9 +29,44 @@ def read(**_args) :
     except Exception as e:
         print (e)
     return _data
+def format (file,_mimetype) :
+    if 'csv' in _mimetype :
+        _df = pd.read_csv(file).fillna('n/a')
         
-@cms.Plugin(mimetype="application/json")
+    rows = _df.apply(lambda row:row.values ,axis=1).apply(lambda row: {'cells':[{'value':x}for x in row]} ).tolist()
+    rows = [{"cells":[ {'value':_name} for _name in _df.columns]}] + rows
+    sheets = [{'usedRange':{'colIndex':1,'rowIndex':_df.shape[0]},'name':'sheet1','rows':rows,'standardHeight':20}]
+    return {'Workbook':{'sheets':sheets}}
+    
+
+
+@cms.Plugin(mimetype="application/json",method="POST")
+def open(**_args):
+    _request = _args['request']
+    file = _request.files['file']
+    spreadsheet = format(file,file.mimetype)
+    return spreadsheet #{"Workbook":{"sheets":[{"usedRange":{"colIndex":1,"rowIndex":3},"name":"Sheet1","rows":[{"cells":[{"value":"name"},{"value":"age"}]},{"cells":[{"value":"steve"},{"value":"44"}]},{"cells":[{"value":"elon"},{"value":"9"}]},{"cells":[{"value":"nico"},{"value":"33"}]}],"standardHeight":20}]}})
+@cms.Plugin(mimetype="application/octet-stream",method="POST")
 def write(**_args) :
     _request = _args['request']
-    print (_request.json)
+    _object = _request.json
-    return {}
+    _sheets = None
+    if _object :
+        _rows = _object['rows']
+        _label = _object['label']
+    #
+    # convert this to a spreadsheet
+    _excelFile = BytesIO()
+    with pd.ExcelWriter(_excelFile, engine='openpyxl') as writer:
+        _index = 1
+        for _name in _rows :
+            _sheet = _rows[_name]
+            _df  = pd.DataFrame(_sheet['values'],columns=_sheet['columns'])
+
+            _df.to_excel(writer,sheet_name=_name,index=False)
+            _index += 1
+        
+    # Write each DataFrame to a different sheet
+    _excelFile.seek(0)
+    _stream= _excelFile.read() #send_file(_excelFile,as_attachment=True,download_name=f'{_label}-export.xlsx', mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
+    return base64.b64encode(_stream).decode('utf-8')

www/html/index.html

@@ -40,8 +40,19 @@
 <script src="www/html/_assets/js/studio.js"></script> 
 
 <script>
+    var Initialize =function(){
+        var http = HttpClient.instance()
+        uri = 'api/dbe/version'
+        http.get(uri,(x)=>{
+            var _html = ('<div><span style="text-transform:lowercase; color:#4682B4;">data-transport</span> '+x.responseText+'</div>')
+            $
+            $('.footer').html(_html)
+
+        })
+    }
     $(document).ready(function (){
-        
+        Initialize()
+        $('.footer').css({'grid-template-columns':'100%','font-weight':'bold'})
     })
 </script>
 <div class="studio-pane"></div>

www/html/menu.html

@@ -6,14 +6,18 @@
     grid-template-columns: auto 50px ;
     gap:4px;
     align-content:center;
+    align-items: center;
+    padding:0px;
     
 }    
 .search-box input[type=text]{
-    
+    height:26px;
-    padding:4px;
+    padding:6px;
     outline:0;
-    border: 4px solid transparent;
+    border: 0px solid transparent;
-    background-color: #d3d3d3;
+    border-left:4px solid transparent;
+    background-color: #D3D3D3;
+    margin:4px;
 }
 .search-box input[type=text]:focus {
     border-left-color: #4682b4;;
@@ -46,7 +50,9 @@
                 _div[0]._data = _data
                 _div[0].onclick = function (){
                     _id = '.studio-pane .'+this._data.label.trim()
-                    $('.studio').slideUp()
+                    $('.studio').slideUp('fast',()=>{
+                        
+                    })
                     if ($(_id).length == 0){
                         _object = new studio.frame(this._data)
                         _object.render()
@@ -66,8 +72,6 @@
             var _nodes = $('.search-results .labels .button')
             _nodes.each(_index => {
                 _item = _nodes[_index]
-                console.log($(_item).attr(''))
-                console.log([' **** '])
                 p = $(_item).attr('provider')
                 q = $(_item).attr('label')
                 rgx = new RegExp(_val)
@@ -107,19 +111,7 @@
   
    
     <div class="settings">
-        <div class="setting-frame new-connection" style="display:none">
+      
-           <ul class="">
-            <i class="fa-solid fa-minus"></i> Create a new connection to a database, and label it.
-            <br><i class="fa-solid fa-minus"></i> The database will be referenced by its label
-
-           <br>
-           <div class="border-round border" style="width:150px;" onclick="dialog.show({uri:'www/html/wizard.html',title:'Create New Connection',context:'{{system.context}}'})">
-                <div class="active bold">
-                   <i class="fa-solid fa-power-off" style="color:#4682B4"></i> Start Now
-                </div>
-           </div>
-           </ul>
-        </div>
         <div class="setting-frame lookup">
 
             <div class="search-box border" style="margin-bottom:4px;">