Steve Nyemba
3 weeks ago
parent
773a7faa6f
commit
a8756b9947
@ -1 +1,80 @@
|
|||||||
These are basic <b>python functions</b> with a single argument (data:pd.DataFrame). The functions can be used as a pipeline to be called in the context of pre/post processing.
|
<style>
|
||||||
|
.double-page {
|
||||||
|
display:grid;
|
||||||
|
grid-template-columns: 49% 1% 49%;
|
||||||
|
gap:8px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
<script>
|
||||||
|
$(document).ready(()=>{
|
||||||
|
_layout = {on:{load:{'visit-us':['www/html/visit-us.html']}}}
|
||||||
|
bootup.init('',_layout)
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
<div>
|
||||||
|
<div class="large-text">Plugins</div>
|
||||||
|
Plugins are native python functions, that are integrated into <b>{{layout.header.title}}</b> and called as pre/post processing.
|
||||||
|
|
||||||
|
<div class="source-code">
|
||||||
|
$ transport plugins --help
|
||||||
|
</div>
|
||||||
|
<div class="double-page">
|
||||||
|
<div>
|
||||||
|
<b>0. Write a plugin function with a decorator</b>
|
||||||
|
<p>
|
||||||
|
Plugins are native python functions, that take in a single parameter. The following example should be save in a file <b>my-plugin.py</b>
|
||||||
|
</p>
|
||||||
|
<div class="source-code">
|
||||||
|
<b>import</b> transport
|
||||||
|
<br><b>import</b> numpy <b>as</b> np
|
||||||
|
|
||||||
|
<br>_index = 0
|
||||||
|
<br><b>@</b>transport.Plugin(name='autoincrement')
|
||||||
|
<br><b>def</b> _incr (_data):
|
||||||
|
<div style="margin-left:16px">
|
||||||
|
global _index
|
||||||
|
<br>_data['_id'] = _index + np.arange(_data.shape[0])
|
||||||
|
<br>_index = _data.shape[0]
|
||||||
|
<br><b>return</b> _data
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="border-top:4px dotted #CAD5E0; margin:8px;"></div>
|
||||||
|
<b>1. Register & test the plugin</b>
|
||||||
|
<p>
|
||||||
|
The plugin utility will make a copy of the file and allow it to be reused against any supported database techology.
|
||||||
|
More information is available when running
|
||||||
|
|
||||||
|
</p>
|
||||||
|
<div class="source-code">
|
||||||
|
$ transport plugins add myplugin my-plugin.py
|
||||||
|
</div>
|
||||||
|
<p>
|
||||||
|
Once registered it is important to see if the function can be tested
|
||||||
|
</p>
|
||||||
|
<div class="source-code">$ plugin-ix registry list --folder ~/.data-transport</div>
|
||||||
|
</div>
|
||||||
|
<div style="border-left:4px dotted #cad4e0"></div>
|
||||||
|
<div>
|
||||||
|
<b>Using our first plugin</b>
|
||||||
|
<p>
|
||||||
|
Plugins are used as pipelines i.e you can add more than one and they will execute accordingly in the order in which they are expressed.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div class="source-code">
|
||||||
|
<b>import</b> transport
|
||||||
|
<br>
|
||||||
|
<br>dbreader = transport.get.reader(label="address-db",plugins=["_incr@myplugin"])
|
||||||
|
<br>_df = dbreader.read()
|
||||||
|
</div>
|
||||||
|
<br>
|
||||||
|
The code above shows how a simple plugin function can be applied to a data
|
||||||
|
|
||||||
|
<br><i class="fa-solid fa-minus"></i> address-db, is the database label that points to the url with data
|
||||||
|
<br><i class="fa-solid fa-minus"></i> myplugin, points to a copy of "incr" in "my-plugin.py"
|
||||||
|
<p>
|
||||||
|
<div id="visit-us"></div>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@ -0,0 +1,83 @@
|
|||||||
|
<style>
|
||||||
|
.configurator {
|
||||||
|
width:70%; margin-left:15%;
|
||||||
|
}
|
||||||
|
.configurator input[type=text] {
|
||||||
|
padding:8px;
|
||||||
|
outline:0px;
|
||||||
|
border:0px;
|
||||||
|
font-family:sans-serif; font-size:18px;
|
||||||
|
background-color: #f3f3f3f3;;
|
||||||
|
font-weight: normal; color:#000000; line-height: 1.5;
|
||||||
|
}
|
||||||
|
.configurator .search-box {
|
||||||
|
display:grid;
|
||||||
|
grid-template-columns: auto 64px; gap:8px;
|
||||||
|
background-color: #f3f3f3f3;;
|
||||||
|
align-items: center;
|
||||||
|
align-content:center;
|
||||||
|
}
|
||||||
|
.configurator .source .item-select {display:grid;
|
||||||
|
grid-template-columns: 50% 25% 25%;
|
||||||
|
padding:4px; gap:8px;
|
||||||
|
margin:4px;
|
||||||
|
/* text-transform: capitalize; */
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
<script>
|
||||||
|
var _setup = {}
|
||||||
|
_setup.render = {}
|
||||||
|
_setup.render.source = function (_data) {
|
||||||
|
var _select = $('.configurator .source')[0]
|
||||||
|
_data.forEach(_item=>{
|
||||||
|
var _label = jx.dom.get.instance('DIV')
|
||||||
|
var _provider = jx.dom.get.instance('DIV')
|
||||||
|
var _plugins = jx.dom.get.instance('DIV')
|
||||||
|
_label.innerHTML = _item.label
|
||||||
|
_provider.innerHTML = _item.provider
|
||||||
|
_plugins.innerHTML = JSON.stringify(_item.plugins)
|
||||||
|
_provider.align='left'
|
||||||
|
_provider.className = ''
|
||||||
|
var _pane = jx.dom.get.instance('DIV')
|
||||||
|
_pane.appendChild(_label)
|
||||||
|
_pane.appendChild(_provider)
|
||||||
|
_pane.appendChild(_plugins)
|
||||||
|
_pane.data = _item
|
||||||
|
_pane.className = 'item-select active'
|
||||||
|
_select.appendChild(_pane)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
_setup.init = function (){
|
||||||
|
var uri='{{system.context}}/api/info/registry'
|
||||||
|
var http = HttpClient.instance()
|
||||||
|
http.get(uri,function (x){
|
||||||
|
_setup.render.source ( JSON.parse(x.responseText))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
$(document).ready(function(){
|
||||||
|
_setup.init()
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
<div class="configurator">
|
||||||
|
<div class="tabs">
|
||||||
|
<input type="radio" id="etl-tab" name="setup-tabs"/>
|
||||||
|
<label for="etl-tab">ETL : CLI</label>
|
||||||
|
<input type="radio" id="store-tab" name="setup-tabs"/>
|
||||||
|
<label for="store-tab">Read/Write</label>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="tab-content">
|
||||||
|
<div class="tab-content-container">
|
||||||
|
<div>
|
||||||
|
<h3>Source: Database Technology</h3>
|
||||||
|
<div class="small">Select a database technology as a <b>source</b></div>
|
||||||
|
<div class="search-box">
|
||||||
|
<input type="text" class="source-label" placeholder="Name of the label"/>
|
||||||
|
<div class="border-left"><div class="active" align="center"><i class="fa-solid fa-times"></i></div></div>
|
||||||
|
</div>
|
||||||
|
<div class="source border-top"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@ -0,0 +1,199 @@
|
|||||||
|
<script>
|
||||||
|
var _meta ={"postgresql":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0},"redshift":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0}, "mysql":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0},"mariadb":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0}, "bigquery":{"private_key":1,"dataset":1,"table":1},"mongodb":{"db":1,"collection":1,"host":0,"port":0,"username":0,"password":0}, "netezza":{"host":1,"port":1,"username":1,"password":1, "database":1,"table":1}, "sqlite":{"path":1,"table":1}, "sqlserver":{"host":0,"port":0,"username":1,"password":1,"database":1, "table":1}
|
||||||
|
,"databricks":{"host":1,"token":1,"schema":1,"cluster_path":1,"table":1},"couchdb":{"url":0,"dbname":1,"doc":1,"username":0,"password":0},"cloudant":{"url":0,"dbname":1,"doc":1,"username":0,"password":0}
|
||||||
|
}
|
||||||
|
|
||||||
|
var wizard = {}
|
||||||
|
wizard.init = function (){
|
||||||
|
|
||||||
|
Object.keys(_meta).forEach( function (_name){
|
||||||
|
var _div = jx.dom.get.instance('DIV')
|
||||||
|
_div.innerHTML = _name
|
||||||
|
_div.data = _meta[_name]
|
||||||
|
_div.data.provider = _name
|
||||||
|
_div.data.context = 'write'
|
||||||
|
_div.className = 'search-item active'
|
||||||
|
_div.placeholder = 'use '+_name+' provider'
|
||||||
|
_div.onclick = function (){
|
||||||
|
wizard.write_code(this.data)
|
||||||
|
$('.wizard-code').slideDown()
|
||||||
|
|
||||||
|
}
|
||||||
|
jx.dom.append('search-meta-items',_div)
|
||||||
|
})
|
||||||
|
|
||||||
|
jx.dom.set.value('search-provider','sqlite')
|
||||||
|
wizard.search()
|
||||||
|
|
||||||
|
}
|
||||||
|
wizard.write_code = function (_info){
|
||||||
|
$('.provider').html(_info.provider)
|
||||||
|
$('#auth-file').html( JSON.stringify(_info))
|
||||||
|
}
|
||||||
|
wizard.search = function (){
|
||||||
|
var term = jx.dom.get.value('search-provider').trim()
|
||||||
|
|
||||||
|
wizard.show(term)
|
||||||
|
|
||||||
|
}
|
||||||
|
wizard.clear = function (){
|
||||||
|
jx.dom.set.value('search-provider','')
|
||||||
|
wizard.show('')
|
||||||
|
jx.dom.set.focus('search-provider')
|
||||||
|
}
|
||||||
|
wizard.show = function (term){
|
||||||
|
_count = 0
|
||||||
|
$('.wizard-code').slideUp()
|
||||||
|
if (term == ''){
|
||||||
|
$('.search-item').slideDown()
|
||||||
|
}else{
|
||||||
|
var nodes = jx.dom.get.children('search-meta-items')
|
||||||
|
var _regx = new RegExp(term,"ig")
|
||||||
|
jx.utils.patterns.visitor(nodes,function (node){
|
||||||
|
if (node.data.provider.match(_regx)){
|
||||||
|
$(node).slideDown()
|
||||||
|
_count += 1
|
||||||
|
}else{
|
||||||
|
$(node).slideUp()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
$('.found-items').html(_count)
|
||||||
|
}
|
||||||
|
wizard.supported_providers = function (){
|
||||||
|
var uri = '{{system.context}}/api/info/about' ;
|
||||||
|
|
||||||
|
var http = HttpClient.instance()
|
||||||
|
http.get(uri,function(x){
|
||||||
|
var _object = JSON.parse(x.responseText)
|
||||||
|
jx.dom.set.value('wiz-vendors',_object.supported)
|
||||||
|
|
||||||
|
var layout = {on:{load:{'contact-wizard':['www/html/contact.html']}}}
|
||||||
|
bootup.init('{{system.context}}',layout)
|
||||||
|
|
||||||
|
|
||||||
|
})
|
||||||
|
}
|
||||||
|
$(document).ready(function(){
|
||||||
|
_node = jx.dom.get.instance('wz-clear-search')
|
||||||
|
_node.onclick = wizard.clear
|
||||||
|
|
||||||
|
_node = jx.dom.get.instance('search-provider')
|
||||||
|
_node.onkeyup=wizard.search
|
||||||
|
|
||||||
|
$('.wizard-code').slideUp()
|
||||||
|
wizard.supported_providers()
|
||||||
|
wizard.init()
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
<style>
|
||||||
|
.wizard-frame input {
|
||||||
|
background-color:#f3f3f3; width:96%;
|
||||||
|
padding:8px; border:4px solid transparent; outline: 0;
|
||||||
|
color:#000000;
|
||||||
|
}
|
||||||
|
.wizard-frame input:focus {
|
||||||
|
border-left-color: #4682b4;
|
||||||
|
}
|
||||||
|
.search-meta-frame {
|
||||||
|
width:calc(100% - 64px) ;
|
||||||
|
margin-top:4px;
|
||||||
|
}
|
||||||
|
#search-meta{
|
||||||
|
|
||||||
|
height:100px; overflow: hidden; overflow-y: auto;
|
||||||
|
|
||||||
|
padding:8px;
|
||||||
|
}
|
||||||
|
#search-meta .search-item {padding:4px; text-transform: capitalize;}
|
||||||
|
|
||||||
|
.wizard-code .provider {text-transform: uppercase;}
|
||||||
|
|
||||||
|
|
||||||
|
.supported-vendors table {width:100%; padding:4px; border:1px solid #CAD5E0;}
|
||||||
|
.supported-vendors thead tr {
|
||||||
|
background-color: #4682B4;
|
||||||
|
color: #ffffff;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
.supported-vendors tbody tr {
|
||||||
|
border-bottom: 1px solid #dddddd;
|
||||||
|
}
|
||||||
|
|
||||||
|
.supported-vendors tbody tr:nth-of-type(even) {
|
||||||
|
background-color: #f3f3f3;
|
||||||
|
}
|
||||||
|
|
||||||
|
</style>
|
||||||
|
|
||||||
|
<div class="wizard-frame" style="display:grid; grid-template-columns: 650px auto; gap:8px;">
|
||||||
|
<div>
|
||||||
|
<div class="large-text">Wizard: auth file generator</div>
|
||||||
|
<p>
|
||||||
|
<div>This wizard generates an <b>auth-file</b>. It is a template file to be used to setup a <b>data-transport</b> database connectivity to help with best practice when it comes to sensitive information in code.
|
||||||
|
<ul>
|
||||||
|
<i class="fa-solid fa-minus"></i> search for the database provider / vendors
|
||||||
|
<br><i class="fa-solid fa-minus"></i> click on the vendor and copy the generated code to a file
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<div style="display:grid; grid-template-columns: auto 48px; gap:4px">
|
||||||
|
<div>
|
||||||
|
<input id="search-provider" type="text" placeholder="[database provider]" onkeyup="wizard.search()">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div id="wz-clear-search" class="active" align="center"><i class="fa-solid fa-trash"></i></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="search-meta-frame border-round border">
|
||||||
|
<div align="right">
|
||||||
|
<span class="found-items">0</span><span> found</span>
|
||||||
|
</div>
|
||||||
|
<div id="search-meta"><div id="search-meta-items"></div></div>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<div style="height:150px;">
|
||||||
|
|
||||||
|
<div class="wizard-code">
|
||||||
|
<div class="provider bold"></div>
|
||||||
|
<div id="auth-file" class="source-code" style="text-wrap: wrap; overflow: hidden; overflow-x: auto;"> </div>
|
||||||
|
<br><b>Note :</b>
|
||||||
|
<br><i class="fa-solid fa-minus"> </i> Copy the code above to the <b>auth-file</b> and fill with appropriate values
|
||||||
|
<br><i class="fa-solid fa-minus"> </i> Attributes with <b>zero</b> i.e <b>0</b> are optional
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<div id="contact-wizard"></div>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div class="large-text">Prerequisites</div>
|
||||||
|
<ul class="border-left">
|
||||||
|
<li type="square">Familiarity with JSON format</li>
|
||||||
|
<li type="square">Understand your current database security access policy
|
||||||
|
<br><i class="fa-solid fa-minus"> </i> Insure your policy (permissions) match your use case
|
||||||
|
<br>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<div class="large-text">Thing to know</div>
|
||||||
|
<ul class="border-left">
|
||||||
|
<li type="square">Values assigned to attributes
|
||||||
|
<br><i class="fa-solid fa-minus"> </i> value of <b>one</b> i.e <b>1</b> suggests a value must be provided
|
||||||
|
<br><i class="fa-solid fa-minus"> </i> value of <b>zero</b> i.e <b>0</b> suggests the attribute is optional and can be removed
|
||||||
|
</li>
|
||||||
|
<li type="square">
|
||||||
|
Supported databases (or database providers) to use in search
|
||||||
|
<div id="wiz-vendors" class="supported-vendors"></div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@ -0,0 +1,199 @@
|
|||||||
|
<style>
|
||||||
|
.notebook-pane {
|
||||||
|
display:grid; grid-template-columns: 40% 60%; gap:8px;;
|
||||||
|
font-family: sans-serif; line-height: 1.5; padding:8px; font-size:14px;
|
||||||
|
}
|
||||||
|
.search-box {display:grid; grid-template-columns: auto 64px; gap:8px; }
|
||||||
|
.search-box input {padding:8px; border:4px solid transparent; outline: 0; background-color: #f3f3f3; width:99%;}
|
||||||
|
.search-box input:focus {border-left-color: #4682b4;}
|
||||||
|
.notebook-search-results-pane {
|
||||||
|
height:105x;
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
.notebook-search-results {
|
||||||
|
height:105px; overflow: hidden; overflow-y: auto;
|
||||||
|
}
|
||||||
|
.notebook-pane .right-pane .notebook{
|
||||||
|
height:65vh;
|
||||||
|
padding:8px;
|
||||||
|
overflow: hidden ;
|
||||||
|
}
|
||||||
|
.notebook-pane .right-pane .notebook iframe{
|
||||||
|
height:98%;
|
||||||
|
width:100%;
|
||||||
|
}
|
||||||
|
.notebook-pane .right-pane{
|
||||||
|
display:grid;
|
||||||
|
grid-template-rows: 32px auto; gap:4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
</style>
|
||||||
|
<script>
|
||||||
|
|
||||||
|
var notebook = {}
|
||||||
|
notebook.meta ={"postgresql":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0},"redshift":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0}, "mysql":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0},"mariadb":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0}, "bigquery":{"private_key":1,"dataset":1,"table":1},"mongodb":{"db":1,"collection":1,"host":0,"port":0,"username":0,"password":0}, "netezza":{"host":1,"port":1,"username":1,"password":1, "database":1,"table":1}, "sqlite":{"path":1,"table":1}, "sqlserver":{"host":0,"port":0,"username":1,"password":1,"database":1, "table":1}
|
||||||
|
,"databricks":{"host":1,"token":1,"schema":1,"cluster_path":1,"table":1},"couchdb":{"url":0,"dbname":1,"doc":1,"username":0,"password":0},"cloudant":{"url":0,"dbname":1,"doc":1,"username":0,"password":0}
|
||||||
|
|
||||||
|
}
|
||||||
|
notebook.meta.etl = "transport generate ./my-etl-config.json<br>transport apply ./my-etl-config.json"
|
||||||
|
notebook.meta.s3 = {"access_key":1,"secret_key":1, "file":1, "bucket":1, "region":1}
|
||||||
|
notebook.meta['mssqlserver'] = notebook.meta.sqlserver
|
||||||
|
notebook.init = function (){
|
||||||
|
var uri = '{{system.context}}/api/info/_data'
|
||||||
|
var http = HttpClient.instance()
|
||||||
|
http.post(uri,function(x){
|
||||||
|
if(x.status == 200 && x.readyState == 4){
|
||||||
|
var _data = JSON.parse(x.responseText)
|
||||||
|
|
||||||
|
notebook.render(_data)
|
||||||
|
notebook.search('sqlite')
|
||||||
|
$('.notebook-pane input').val('sqlite')
|
||||||
|
$('.notebook-pane .left-pane .sqlite').click()
|
||||||
|
}else{
|
||||||
|
//
|
||||||
|
// Find a way to alert of the error
|
||||||
|
//
|
||||||
|
;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
notebook.render = function (data){
|
||||||
|
data.forEach(_item =>{
|
||||||
|
var _div = jx.dom.get.instance('DIV')
|
||||||
|
_div.innerHTML = _item.label
|
||||||
|
_div.data = _item
|
||||||
|
_div.onclick = function (){
|
||||||
|
$('.notebook-pane .label').html(this.data.label)
|
||||||
|
$('.notebook-pane .left-pane .vendor-info').html(this.data.doc)
|
||||||
|
$('.notebook-pane .left-pane .vendor-url').html(this.data.url)
|
||||||
|
_uri = 'www/html/docs/html/'+this.data.provider.trim()+'.html'
|
||||||
|
// _context = '{{system.context}}'
|
||||||
|
// if (_context != '') {
|
||||||
|
// _uri = _context + '/'+_uri
|
||||||
|
// }
|
||||||
|
frame = jx.dom.get.instance('IFRAME')
|
||||||
|
frame.frameBorder = 0
|
||||||
|
frame.className = ''
|
||||||
|
frame.src = _uri //'{{system.context}}'+'/www/html/docs/html/'+this.data.provider.trim()+'.html'
|
||||||
|
$('.notebook-pane .right-pane .notebook').empty()
|
||||||
|
$('.notebook-pane .right-pane .notebook').append(frame)
|
||||||
|
$('.notebook-pane .search-box input').focus()
|
||||||
|
_info = notebook.meta[this.data.provider]
|
||||||
|
$('.notebook-pane .left-pane .source-code').html(JSON.stringify(_info))
|
||||||
|
} //-- binding
|
||||||
|
_div.className = 'active notebook-item '+_item.provider
|
||||||
|
$('.notebook-search-results').append(_div)
|
||||||
|
|
||||||
|
|
||||||
|
})
|
||||||
|
// $('.notebook-search-results .notebook-item').slideUp()
|
||||||
|
}
|
||||||
|
notebook.search = function (term) {
|
||||||
|
_count = 0
|
||||||
|
if (term == ''){
|
||||||
|
$('.notebook-search-results .notebook-item').slideDown()
|
||||||
|
}else{
|
||||||
|
var nodes = jx.dom.get.children('notebook-search-results')
|
||||||
|
var _regx = new RegExp(term,"ig")
|
||||||
|
|
||||||
|
jx.utils.patterns.visitor(nodes,function (node){
|
||||||
|
if (node.data.provider.match(_regx) || node.data.label.match(_regx)) {
|
||||||
|
$(node).slideDown()
|
||||||
|
_count += 1
|
||||||
|
}else{
|
||||||
|
$(node).slideUp()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
$('.notebook-pane .found').html(_count)
|
||||||
|
}
|
||||||
|
notebook.find = function (){
|
||||||
|
var term = $('.search-box input').val().trim()
|
||||||
|
notebook.search(term)
|
||||||
|
}
|
||||||
|
notebook.clear = function (){
|
||||||
|
$('.notebook-pane .search-box input').val('')
|
||||||
|
notebook.search('')
|
||||||
|
$('.notebook-pane .search-box input').focus()
|
||||||
|
$('.notebook-pane .right-pane .notebook').empty()
|
||||||
|
$('.notebook-pane .label').html('')
|
||||||
|
$('.notebook-pane .left-pane .vendor-info').html('')
|
||||||
|
$('.notebook-pane .left-pane .vendor-url').html('')
|
||||||
|
$('.notebook-pane .source-code').html(' ')
|
||||||
|
|
||||||
|
// $('.notebook-pane .search-box input').focus()
|
||||||
|
|
||||||
|
}
|
||||||
|
notebook.open_wizard =function(){
|
||||||
|
try{
|
||||||
|
_args = {{layout.overwrite.wizard}}
|
||||||
|
menu.events._dialog(_args,'{{context}}')
|
||||||
|
}catch(e){}
|
||||||
|
}
|
||||||
|
$(document).ready(function (){
|
||||||
|
|
||||||
|
notebook.init()
|
||||||
|
$('.search-box .active').on('click',notebook.clear)
|
||||||
|
$('.search-box input').on('keypress',notebook.find)
|
||||||
|
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
<div class="notebook-pane">
|
||||||
|
|
||||||
|
<div class="left-pane">
|
||||||
|
<div class="search-box border-round border">
|
||||||
|
<div class="border-right"><input type="search" placeholder="[provider or vendor]"/></div>
|
||||||
|
<div class="active" align="center"><i class="fa-solid fa-trash"></i></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p><div class="border-round border notebook-search-results-pane">
|
||||||
|
<div align="right" class="">
|
||||||
|
<span class="found"></span><span> Found </span>
|
||||||
|
</div>
|
||||||
|
<div id = "notebook-search-results" class="notebook-search-results"></div>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<div style="height:125px">
|
||||||
|
<ul>
|
||||||
|
<i class="fa-solid fa-minus "></i> Preview notebooks with database providers/vendors
|
||||||
|
<br><i class="fa-solid fa-minus "></i> The notebooks show how to use <b>data-transport</b> as a library
|
||||||
|
<br><i class="fa-solid fa-minus "></i> <span class="vendor-info"></span>
|
||||||
|
<br><i class="fa-solid fa-minus "></i> <span class="vendor-url"></span>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="border-top"> </div>
|
||||||
|
<p>
|
||||||
|
<h3>Generated: auth-file <span class="label" style="text-transform: uppercase;"></span></h3>
|
||||||
|
<ul>
|
||||||
|
<i class="fa-solid fa-minus "></i> An <b>auth-file</b> is a file used to store database parameters.
|
||||||
|
<br><i class="fa-solid fa-minus "></i> Copy the code above to the auth-file and fill with appropriate values
|
||||||
|
<br><i class="fa-solid fa-minus "></i> Attributes with zero i.e 0 are optional
|
||||||
|
</ul>
|
||||||
|
<div class="source-code"></div>
|
||||||
|
|
||||||
|
<ul>
|
||||||
|
<b>Note:</b>
|
||||||
|
<br><i class="fa-solid fa-minus "></i> The database providers/vendors above is exhaustive
|
||||||
|
<br><i class="fa-solid fa-minus "></i> Generate files for database provider/vendors <span class="bold active" onclick="notebook.open_wizard()">click here</span>
|
||||||
|
</ul>
|
||||||
|
</p>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div class="right-pane">
|
||||||
|
<div>
|
||||||
|
<div style="text-transform: capitalize;">
|
||||||
|
<div class="label bold" style="font-size:20px; text-transform: uppercase;"></div>
|
||||||
|
<div>Jupyter Notebook preview</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<p><div class="notebook border border-round"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@ -1,199 +1,58 @@
|
|||||||
<style>
|
<style>
|
||||||
.notebook-pane {
|
iframe {
|
||||||
display:grid; grid-template-columns: 40% 60%; gap:8px;;
|
border:1px solid transparent;
|
||||||
font-family: sans-serif; line-height: 1.5; padding:8px; font-size:14px;
|
width:100%; height:100%; margin:8px;}
|
||||||
}
|
.page-split {display:grid; grid-template-columns: 25% 4px auto; gap:8px; height:75vh;}
|
||||||
.search-box {display:grid; grid-template-columns: auto 64px; gap:8px; }
|
.page-split ul {margin:4px}
|
||||||
.search-box input {padding:8px; border:4px solid transparent; outline: 0; background-color: #f3f3f3; width:99%;}
|
|
||||||
.search-box input:focus {border-left-color: #4682b4;}
|
|
||||||
.notebook-search-results-pane {
|
|
||||||
height:105x;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
.notebook-search-results {
|
|
||||||
height:105px; overflow: hidden; overflow-y: auto;
|
|
||||||
}
|
|
||||||
.notebook-pane .right-pane .notebook{
|
|
||||||
height:65vh;
|
|
||||||
padding:8px;
|
|
||||||
overflow: hidden ;
|
|
||||||
}
|
|
||||||
.notebook-pane .right-pane .notebook iframe{
|
|
||||||
height:98%;
|
|
||||||
width:100%;
|
|
||||||
}
|
|
||||||
.notebook-pane .right-pane{
|
|
||||||
display:grid;
|
|
||||||
grid-template-rows: 32px auto; gap:4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
</style>
|
</style>
|
||||||
<script>
|
<script>
|
||||||
|
var _layout = {on:{load:{'visit-us':['www/html/visit-us.html','www/html/visit-us.html']}}}
|
||||||
var notebook = {}
|
bootup.init('',_layout)
|
||||||
notebook.meta ={"postgresql":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0},"redshift":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0}, "mysql":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0},"mariadb":{"host":0,"port":0,"database":1,"table":1,"username":0,"password":0}, "bigquery":{"private_key":1,"dataset":1,"table":1},"mongodb":{"db":1,"collection":1,"host":0,"port":0,"username":0,"password":0}, "netezza":{"host":1,"port":1,"username":1,"password":1, "database":1,"table":1}, "sqlite":{"path":1,"table":1}, "sqlserver":{"host":0,"port":0,"username":1,"password":1,"database":1, "table":1}
|
var notebook = {}
|
||||||
,"databricks":{"host":1,"token":1,"schema":1,"cluster_path":1,"table":1},"couchdb":{"url":0,"dbname":1,"doc":1,"username":0,"password":0},"cloudant":{"url":0,"dbname":1,"doc":1,"username":0,"password":0}
|
notebook.open = function (_id){
|
||||||
|
$('.notebook iframe').attr('src',`www/html/docs/html/${_id}.html`)
|
||||||
}
|
|
||||||
notebook.meta.etl = "transport generate ./my-etl-config.json<br>transport apply ./my-etl-config.json"
|
|
||||||
notebook.meta.s3 = {"access_key":1,"secret_key":1, "file":1, "bucket":1, "region":1}
|
|
||||||
notebook.meta['mssqlserver'] = notebook.meta.sqlserver
|
|
||||||
notebook.init = function (){
|
|
||||||
var uri = '{{system.context}}/api/info/_data'
|
|
||||||
var http = HttpClient.instance()
|
|
||||||
http.post(uri,function(x){
|
|
||||||
if(x.status == 200 && x.readyState == 4){
|
|
||||||
var _data = JSON.parse(x.responseText)
|
|
||||||
|
|
||||||
notebook.render(_data)
|
|
||||||
notebook.search('sqlite')
|
|
||||||
$('.notebook-pane input').val('sqlite')
|
|
||||||
$('.notebook-pane .left-pane .sqlite').click()
|
|
||||||
}else{
|
|
||||||
//
|
|
||||||
// Find a way to alert of the error
|
|
||||||
//
|
|
||||||
;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
notebook.render = function (data){
|
|
||||||
data.forEach(_item =>{
|
|
||||||
var _div = jx.dom.get.instance('DIV')
|
|
||||||
_div.innerHTML = _item.label
|
|
||||||
_div.data = _item
|
|
||||||
_div.onclick = function (){
|
|
||||||
$('.notebook-pane .label').html(this.data.label)
|
|
||||||
$('.notebook-pane .left-pane .vendor-info').html(this.data.doc)
|
|
||||||
$('.notebook-pane .left-pane .vendor-url').html(this.data.url)
|
|
||||||
_uri = 'www/html/docs/html/'+this.data.provider.trim()+'.html'
|
|
||||||
// _context = '{{system.context}}'
|
|
||||||
// if (_context != '') {
|
|
||||||
// _uri = _context + '/'+_uri
|
|
||||||
// }
|
|
||||||
frame = jx.dom.get.instance('IFRAME')
|
|
||||||
frame.frameBorder = 0
|
|
||||||
frame.className = ''
|
|
||||||
frame.src = _uri //'{{system.context}}'+'/www/html/docs/html/'+this.data.provider.trim()+'.html'
|
|
||||||
$('.notebook-pane .right-pane .notebook').empty()
|
|
||||||
$('.notebook-pane .right-pane .notebook').append(frame)
|
|
||||||
$('.notebook-pane .search-box input').focus()
|
|
||||||
_info = notebook.meta[this.data.provider]
|
|
||||||
$('.notebook-pane .left-pane .source-code').html(JSON.stringify(_info))
|
|
||||||
} //-- binding
|
|
||||||
_div.className = 'active notebook-item '+_item.provider
|
|
||||||
$('.notebook-search-results').append(_div)
|
|
||||||
|
|
||||||
|
|
||||||
})
|
|
||||||
// $('.notebook-search-results .notebook-item').slideUp()
|
|
||||||
}
|
|
||||||
notebook.search = function (term) {
|
|
||||||
_count = 0
|
|
||||||
if (term == ''){
|
|
||||||
$('.notebook-search-results .notebook-item').slideDown()
|
|
||||||
}else{
|
|
||||||
var nodes = jx.dom.get.children('notebook-search-results')
|
|
||||||
var _regx = new RegExp(term,"ig")
|
|
||||||
|
|
||||||
jx.utils.patterns.visitor(nodes,function (node){
|
|
||||||
if (node.data.provider.match(_regx) || node.data.label.match(_regx)) {
|
|
||||||
$(node).slideDown()
|
|
||||||
_count += 1
|
|
||||||
}else{
|
|
||||||
$(node).slideUp()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
$('.notebook-pane .found').html(_count)
|
|
||||||
}
|
|
||||||
notebook.find = function (){
|
|
||||||
var term = $('.search-box input').val().trim()
|
|
||||||
notebook.search(term)
|
|
||||||
}
|
|
||||||
notebook.clear = function (){
|
|
||||||
$('.notebook-pane .search-box input').val('')
|
|
||||||
notebook.search('')
|
|
||||||
$('.notebook-pane .search-box input').focus()
|
|
||||||
$('.notebook-pane .right-pane .notebook').empty()
|
|
||||||
$('.notebook-pane .label').html('')
|
|
||||||
$('.notebook-pane .left-pane .vendor-info').html('')
|
|
||||||
$('.notebook-pane .left-pane .vendor-url').html('')
|
|
||||||
$('.notebook-pane .source-code').html(' ')
|
|
||||||
|
|
||||||
// $('.notebook-pane .search-box input').focus()
|
|
||||||
|
|
||||||
}
|
|
||||||
notebook.open_wizard =function(){
|
|
||||||
try{
|
|
||||||
_args = {{layout.overwrite.wizard}}
|
|
||||||
menu.events._dialog(_args,'{{context}}')
|
|
||||||
}catch(e){}
|
|
||||||
}
|
|
||||||
$(document).ready(function (){
|
|
||||||
|
|
||||||
notebook.init()
|
|
||||||
$('.search-box .active').on('click',notebook.clear)
|
|
||||||
$('.search-box input').on('keypress',notebook.find)
|
|
||||||
|
|
||||||
})
|
|
||||||
</script>
|
</script>
|
||||||
<div class="notebook-pane">
|
<div class="notebook page-split">
|
||||||
|
<div>
|
||||||
<div class="left-pane">
|
<div class="large-text">Notebooks</div>
|
||||||
<div class="search-box border-round border">
|
<p class="small">
|
||||||
<div class="border-right"><input type="search" placeholder="[provider or vendor]"/></div>
|
Click on a link below to preview a notebook that illustrates how {{layout.header.title}} can be used.
|
||||||
<div class="active" align="center"><i class="fa-solid fa-trash"></i></div>
|
</p>
|
||||||
</div>
|
|
||||||
|
|
||||||
<p><div class="border-round border notebook-search-results-pane">
|
|
||||||
<div align="right" class="">
|
|
||||||
<span class="found"></span><span> Found </span>
|
|
||||||
</div>
|
|
||||||
<div id = "notebook-search-results" class="notebook-search-results"></div>
|
|
||||||
|
|
||||||
|
|
||||||
</div>
|
|
||||||
<div style="height:125px">
|
|
||||||
<ul>
|
|
||||||
<i class="fa-solid fa-minus "></i> Preview notebooks with database providers/vendors
|
|
||||||
<br><i class="fa-solid fa-minus "></i> The notebooks show how to use <b>data-transport</b> as a library
|
|
||||||
<br><i class="fa-solid fa-minus "></i> <span class="vendor-info"></span>
|
|
||||||
<br><i class="fa-solid fa-minus "></i> <span class="vendor-url"></span>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
<div class="border-top"> </div>
|
|
||||||
<p>
|
|
||||||
<h3>Generated: auth-file <span class="label" style="text-transform: uppercase;"></span></h3>
|
|
||||||
<ul>
|
|
||||||
<i class="fa-solid fa-minus "></i> An <b>auth-file</b> is a file used to store database parameters.
|
|
||||||
<br><i class="fa-solid fa-minus "></i> Copy the code above to the auth-file and fill with appropriate values
|
|
||||||
<br><i class="fa-solid fa-minus "></i> Attributes with zero i.e 0 are optional
|
|
||||||
</ul>
|
|
||||||
<div class="source-code"></div>
|
|
||||||
|
|
||||||
<ul>
|
<ul>
|
||||||
<b>Note:</b>
|
<div class="active" onclick="notebook.open('sqlite')">
|
||||||
<br><i class="fa-solid fa-minus "></i> The database providers/vendors above is exhaustive
|
<i class="fa-solid fa-minus"></i> sqlite
|
||||||
<br><i class="fa-solid fa-minus "></i> Generate files for database provider/vendors <span class="bold active" onclick="notebook.open_wizard()">click here</span>
|
</div>
|
||||||
|
<div class="active" onclick="notebook.open('bigquery')">
|
||||||
|
<i class="fa-solid fa-minus"></i> bigquery
|
||||||
|
</div>
|
||||||
|
<div class="active" onclick="notebook.open('mssqlserver')">
|
||||||
|
<i class="fa-solid fa-minus"></i> mssqlserver
|
||||||
|
</div>
|
||||||
|
<div class="active" onclick="notebook.open('mysql')">
|
||||||
|
<i class="fa-solid fa-minus"></i> mysql
|
||||||
|
</div>
|
||||||
|
<div class="active" onclick="notebook.open('postgresql')">
|
||||||
|
<i class="fa-solid fa-minus"></i> postgresql
|
||||||
|
</div>
|
||||||
|
<div class="active" onclick="notebook.open('s3')">
|
||||||
|
<i class="fa-solid fa-minus"></i> AWS S3
|
||||||
|
</div>
|
||||||
|
<div class="active" onclick="notebook.open('mongodb')">
|
||||||
|
<div ><i class="fa-solid fa-minus"></i> mongodb</div>
|
||||||
|
</div>
|
||||||
</ul>
|
</ul>
|
||||||
</p>
|
<p>
|
||||||
</p>
|
<br>
|
||||||
</div>
|
<div style="border-top:4px dotted #CAD5E0"></div>
|
||||||
<div class="right-pane">
|
<br>
|
||||||
<div>
|
<div id="visit-us"></div>
|
||||||
<div style="text-transform: capitalize;">
|
</p>
|
||||||
<div class="label bold" style="font-size:20px; text-transform: uppercase;"></div>
|
|
||||||
<div>Jupyter Notebook preview</div>
|
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
||||||
</div>
|
|
||||||
<p><div class="notebook border border-round"></div>
|
<div style="border-left:4px dotted #CAD5E0;"></div>
|
||||||
</div>
|
<iframe src="www/html/docs/html/mongodb.html"></iframe>
|
||||||
</div>
|
</div>
|
||||||
@ -1,166 +0,0 @@
|
|||||||
<script>
|
|
||||||
_layout = {on:{load:{'plugin-intro':['www/html/_notes/plugins-intro.html']}}}
|
|
||||||
var _plugins = {}
|
|
||||||
_plugins.open = function (_id){
|
|
||||||
$('.tab-content').slideUp(function (){
|
|
||||||
})
|
|
||||||
_id = '.'+_id
|
|
||||||
$(_id).slideDown()
|
|
||||||
|
|
||||||
}
|
|
||||||
_plugins.copy = function(_node) {
|
|
||||||
var _code = $(_node.parentNode).text().trim().replace(/ {8}/g,'').replace(/ {4}/g,'\t').replace(/\r/g,'\n')
|
|
||||||
navigator.clipboard.writeText(_code);
|
|
||||||
$(_node).empty()
|
|
||||||
$(_node).html('<i class="fa-solid fa-check"></i>')
|
|
||||||
setTimeout(function(){
|
|
||||||
$(_node).empty()
|
|
||||||
$(_node).html('<i class="fa-regular fa-copy"></i>')
|
|
||||||
},750)
|
|
||||||
}
|
|
||||||
$(document).ready(function(){
|
|
||||||
bootup.init('{{system.context}}',_layout)
|
|
||||||
// _plugins.open('makeplug-src')
|
|
||||||
var label = $('.process-steps .tabs label')[0]
|
|
||||||
label.click()
|
|
||||||
})
|
|
||||||
|
|
||||||
</script>
|
|
||||||
<style>
|
|
||||||
/* .source-code {background-color:#f3f3f3; color:black;} */
|
|
||||||
.source-code b {color: yellow;}
|
|
||||||
.split-1 {display:grid; grid-template-columns: 45% auto; gap:4px}
|
|
||||||
.split-2 {display:grid; grid-template-columns: 55% auto; gap:4px}
|
|
||||||
|
|
||||||
.process-steps .list { font-size:16px; padding:4px; margin-left:8px; background-color: #f3f3f3;}
|
|
||||||
.process-steps .list div {margin:8px;}
|
|
||||||
/* .fa-copy {float:right; cursor:pointer; padding:4px; margin:4px; background-color: transparent;} */
|
|
||||||
/* .fa-copy:hover {color:#4682b4;} */
|
|
||||||
</style>
|
|
||||||
<div style="min-height:650px;">
|
|
||||||
<div class="split-2">
|
|
||||||
<div>
|
|
||||||
<h3>Plugins: Usage & Development</h3>
|
|
||||||
<p>
|
|
||||||
<div id="plugin-intro" >
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
<h3>Plugins: Registry</h3>
|
|
||||||
|
|
||||||
The plugins registry is a registry of plugins intended to be used in pre/post processing. This feature comes in handy :
|
|
||||||
<ul>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> During ETL: Cleanup data, adding columns enforcing data-typing, removing/encrypting PHI ... </div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> In a collaborative environment (Jupyter-x; Zeppelin; AWS Service Workbench)</div>
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
<h3>Plugins: Architecture & Design</h3>
|
|
||||||
Plugins are designed around <b class="active" onclick="window.open('https://en.wikipedia.org/wiki/Plug-in_(computing)','_doc')">plugin architecture</b> using <b class="active" onclick="window.open('https://en.wikipedia.org/wiki/Iterator_pattern','doc')">Iterator design-pattern</b>. In that respect and function as a <b>pipeline</b> i.e executed sequentially in the order in which they are expressed in the parameter. Effectively the output of one function will be the input to the next.
|
|
||||||
<p>
|
|
||||||
<div class="border figure" align="center" >
|
|
||||||
<img src="www/html/_assets/images/plugins-components.png">
|
|
||||||
<div class="small border-top" align="center" style="margin-top:4px; padding-top:4px;">Data Transport UML Plugin Component View</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<div class="process-steps" >
|
|
||||||
<h3>Quick Start</h3>
|
|
||||||
<div class="tabs">
|
|
||||||
<input type="radio" name="step" id="makeplug">
|
|
||||||
<label for="makeplug" onclick="_plugins.open('makeplug-src')">1. Make Plugin</label>
|
|
||||||
|
|
||||||
<input type="radio" name="step" id="regplug">
|
|
||||||
<label for="regplug" onclick="_plugins.open('regplug-src')">2. Register Plugin</label>
|
|
||||||
|
|
||||||
<input type="radio" name="step" id="useplug">
|
|
||||||
<label for="useplug" onclick="_plugins.open('useplug-src')">3. Use The Plugin</label>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="tab-content makeplug-src">
|
|
||||||
<p>
|
|
||||||
|
|
||||||
<div class="split-x">
|
|
||||||
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<ul class="list">
|
|
||||||
<div><i class="fa-solid fa-minus"></i> The code here shows a function that will be registered as <b><i>"autoincrement"</i></b>.</div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> The data, will always be a <b>pandas.DataFrame</b></b></div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> For the sake of this example the file will be <b>my-plugin.py</b></div>
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
<div class="source-code"><i class="fa-regular fa-copy" onclick="_plugins.copy(this)"></i>
|
|
||||||
<b>import</b> transport
|
|
||||||
<div><b>import</b> numpy <b>as</b> np</div>
|
|
||||||
<br>
|
|
||||||
<br>_index = 0
|
|
||||||
<br><div>@transport.Plugin(name='autoincrement')</div>
|
|
||||||
<div><b>def</b> _incr (_data):</div>
|
|
||||||
<ul>
|
|
||||||
<div><b>global</b> _index</div>
|
|
||||||
<div>_data['_id'] = _index + np.arange(_data.shape[0])</div>
|
|
||||||
<div>_index = _data.shape[0]</div>
|
|
||||||
<div><b>return</b> _data</div>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<!-- register plugin-->
|
|
||||||
<div class="tab-content regplug-src">
|
|
||||||
<p>
|
|
||||||
<div class="">
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<b>data-transport</b> comes with a built-in command line interface (CLI). It allows plugins to be registered and reused.
|
|
||||||
<ul class="list">
|
|
||||||
<div><i class="fa-solid fa-minus"></i> Registered functions are stored in <b><i>$HOME/.data-transport/plugins/code</i></b></div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> Any updates to <b>my-plugin.py</b> will require re-registering the file</div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> Additional plugin registry functions (list, test) are available </div>
|
|
||||||
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<div class="source-code"><i class="fa-regular fa-copy" onclick="_plugins.copy(this)"></i>
|
|
||||||
<b>$</b> transport plugin-add demo ./my-plugin.py
|
|
||||||
</div>
|
|
||||||
<p>
|
|
||||||
The following command allows <b>data-transport</b> to determine what is knows about the function i.e <b>real name</b> and <b>name</b> to be used in code.
|
|
||||||
<div class="source-code"><i class="fa-regular fa-copy" onclick="_plugins.copy(this)"></i>
|
|
||||||
<b>$</b> transport plugin-test demo.autoincrement
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<!-- useplug-src -->
|
|
||||||
<div class="tab-content useplug-src">
|
|
||||||
<p>
|
|
||||||
Once registered, the plugins are ready for use within code or configuration file (auth-file).
|
|
||||||
<div class="source-code"><i class="fa-regular fa-copy" onclick="_plugins.copy(this)"></i><b>import</b> transport
|
|
||||||
<div><b>from</b> transport <b>import</b> providers</div>
|
|
||||||
|
|
||||||
_args = {
|
|
||||||
<div>"provider":providers.HTTP,</div>
|
|
||||||
<div>"url":"https://raw.githubusercontent.com/codeforamerica/ohana-api/master/data/sample-csv/addresses.csv",</div>
|
|
||||||
<div>"plugins":["demo@autoincrement"]</div>
|
|
||||||
}
|
|
||||||
|
|
||||||
<div>reader = transport.get.reader(**_args)</div>
|
|
||||||
<div>_data = reader.read()</div>
|
|
||||||
<div>print (_data.head())</div>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
@ -1,186 +0,0 @@
|
|||||||
<style>
|
|
||||||
.terminal {
|
|
||||||
display:grid; grid-template-columns: 55% 45%; gap:8px; font-weight: lighter;
|
|
||||||
}
|
|
||||||
hr{border:0px; border-top:3px dotted #CAD5E0; width:50%; margin-left:25%}
|
|
||||||
.source-code pre {font-size:12px; font-weight:lighter; font-family: courier;}
|
|
||||||
</style>
|
|
||||||
<script>
|
|
||||||
layout = {on:{load:{'install':['www/html/_notes/install.html'],'documentation':['www/html/_notes/documentation.html']}}}
|
|
||||||
// bootup.init('{{system.context}}',layout)
|
|
||||||
var label = $('.terminal .tabs label')[0]
|
|
||||||
label.click()
|
|
||||||
// menu.events._openTabs('.terminal .tab-content','etl-conf-tab')
|
|
||||||
// menu.events._openTabs('.make-config','.manual')
|
|
||||||
label = $('.terminal .etl-conf-tab .tabs label')[0]
|
|
||||||
label.click()
|
|
||||||
</script>
|
|
||||||
<div class="terminal">
|
|
||||||
<div>
|
|
||||||
<p>
|
|
||||||
<h3>ETL: Introduction</h3>
|
|
||||||
Extract Load & Transform (ETL) consists in copying data from one database to one or many others. This can be done in two different ways:
|
|
||||||
<ul>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> Command Line Interface (CLI), driven by JSON configuration</div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> Or within custom python code</div>
|
|
||||||
</ul>
|
|
||||||
The ETL process will take advantage of registries for <b>plugins</b> and <b>labeled database connectivity</b> to perform <b>pre</b>/<b>post</b> processing tasks.
|
|
||||||
|
|
||||||
</p>
|
|
||||||
<hr>
|
|
||||||
<p>
|
|
||||||
<h3>ETL: Command Line Interface</h3>
|
|
||||||
<p>
|
|
||||||
The configuration file needed to run the ETL is a JSON formatted file where each entry contains:
|
|
||||||
<ul>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> <b>source</b> with the content of an <b>auth-file</b></div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> <b>target</b> with <b>list</b> of elements of an <b>auth-file</b></div>
|
|
||||||
</ul>
|
|
||||||
<!-- The auth-file<span class="active bold" onclick="menu.events._dialog({type:'dialog',uri:'www/html/wizard/wizard.html',title:'Auth-file Generator'},'{{system.context}}')">generator</span>, shows how the auth-file entry is structured. -->
|
|
||||||
<div>
|
|
||||||
The <b>CLI</b> (transport), is capable of generating a demo ETL :
|
|
||||||
<ul>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> with <b>source</b>: reads CSV data from github</b></div>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> and <b>target</b>: writes the data to CSV & SQLite3 database</div>
|
|
||||||
</ul>
|
|
||||||
<div class="source-code"><i class="fa-solid fa-copy" onclick="_plugins.copy(this)"></i>
|
|
||||||
$ transport generate ./demo-etl.json
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
</p>
|
|
||||||
<br>
|
|
||||||
<hr>
|
|
||||||
<br><div align="center" class="border figure"><img src="www/html/_images/uml-activity.png">
|
|
||||||
<div class="small border-top" style="margin-top:4px; padding-top:4px">
|
|
||||||
Data-transport UML Extract-Load-Transform (ETL) Workflow
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<div>
|
|
||||||
<div class="tabs" >
|
|
||||||
<input type="radio" name="etl" id="etl-conf"/>
|
|
||||||
<label for="etl-conf" onclick="menu.events._openTabs('.terminal .tab-content','etl-conf-tab')">1. Configuration</label>
|
|
||||||
|
|
||||||
<input type="radio" name="etl" id="etl-exe"/>
|
|
||||||
<label for="etl-exe" onclick="menu.events._openTabs('.terminal .tab-content','etl-exe-tab')">2. Run ETL CLI</label>
|
|
||||||
|
|
||||||
<input type="radio" name="etl" id="etl-code"/>
|
|
||||||
<label for="etl-code" onclick="menu.events._openTabs('.terminal .tab-content','etl-code-tab')">ETL: Custom Code</label>
|
|
||||||
<input type="radio" name="etl" id="none" disabled>
|
|
||||||
<label for="none" style="grid-column:4"> </label>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<p>
|
|
||||||
|
|
||||||
<div class="tab-content">
|
|
||||||
<div class="etl-exe-tab">
|
|
||||||
<p>
|
|
||||||
The command-line interface should be instructed to run the ETL by calling the <b>apply</b> function.
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
<div class="source-code">
|
|
||||||
$ transport apply ./demo-etl.json
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
Additional parameters can be invoked by providing the <b>--help</b> switch
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<p>
|
|
||||||
<div class="source-code">
|
|
||||||
$ transport apply --help
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
<div class="etl-code-tab"></div>
|
|
||||||
<div class="etl-conf-tab">
|
|
||||||
The following examples shows simple configuration files that do NOT require any database to be installed. Feel free to change and edit at your own discression.
|
|
||||||
<br>
|
|
||||||
<p>
|
|
||||||
<h3>Example # 1: Basic ETL</h3>
|
|
||||||
<div class="tabs" style="margin:0px; padding:0px; background-color:#ffffff;grid-template-columns: 50% 50%; display:grid;" align="center">
|
|
||||||
<input type="radio" name="mk-conf" id="mk-man">
|
|
||||||
<label for="mk-man" onclick="menu.events._openTabs('.make-config','.manual')" style="background-color:#ffffff; border-radius:0px;">Manual</label>
|
|
||||||
<input type="radio" name="mk-conf" id="mk-gen">
|
|
||||||
<label for="mk-gen" onclick="menu.events._openTabs('.make-config','.generated')" style="background-color:#ffffff; border-radius:0px;">Generate</label>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div style="border-top:0px; min-height:400px;">
|
|
||||||
<div class="make-config">
|
|
||||||
<div class="generated">
|
|
||||||
<p>
|
|
||||||
<b>data-transport</b> comes with a CLI integrated that will
|
|
||||||
<ul>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> <b>generate</b> an EL configuration file</div>
|
|
||||||
<div class=" source-code"><i class="fa-solid fa-copy" onclick="_plugins.copy(this)"></i>
|
|
||||||
<span>$ transport generate ./demo-etl.json</span>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</ul>
|
|
||||||
<div><i class="fa-solid fa-minus"></i> <b>NOTE:</b>The configuration file supports <b>labels</b> and/or <b>plugins</b>, these would have to be done manually</div>
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<div class="manual">
|
|
||||||
<p>Copy the content and save it to a file <b>"demo-etl.json"</b></p>
|
|
||||||
<div class="source-code" style="text-overflow: ellipsis;">
|
|
||||||
<i class="fa-solid fa-copy" onclick="code.copy(this)"></i>
|
|
||||||
<pre>[{
|
|
||||||
"source": {
|
|
||||||
"provider": "http",
|
|
||||||
"url": "https://github.com/codeforamerica/ohana-api/blob/master/data/sample-csv/addresses.csv"
|
|
||||||
},
|
|
||||||
"target": [
|
|
||||||
{"provider": "files", "path": "addresses.csv", "delimiter": ","},
|
|
||||||
{"provider": "sqlite3", "database": "sample.db3", "table": "addresses"}
|
|
||||||
]}]</pre>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
<hr>
|
|
||||||
|
|
||||||
<p>
|
|
||||||
<h3>Example # 2: ETL With Plugins</h3>
|
|
||||||
<p>Copy the content and save it to a file <b>"demo-etl.json"</b></p>
|
|
||||||
<div class="source-code" style="text-overflow: ellipsis;">
|
|
||||||
<i class="fa-solid fa-copy" onclick="code.copy(this)"></i>
|
|
||||||
<pre >[{
|
|
||||||
"source": {
|
|
||||||
"provider": "http",
|
|
||||||
"plugins":["demo@autoincrement"],
|
|
||||||
"url": "https://github.com/codeforamerica/ohana-api/blob/master/data/sample-csv/addresses.csv"
|
|
||||||
},
|
|
||||||
"target": [
|
|
||||||
{"provider": "files", "path": "addresses.csv", "delimiter": ","},
|
|
||||||
{"provider": "sqlite3", "database": "sample.db3", "table": "addresses"}
|
|
||||||
]}]</pre>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</p>
|
|
||||||
<!-- <div class="border-round border">
|
|
||||||
<h3 style="border-color: transparent;">UML Activity Diagram - ETL</h3>
|
|
||||||
|
|
||||||
<br><div class="border-top">
|
|
||||||
<ul>
|
|
||||||
<i class="fa-solid fa-minus"> </i> The diagram shows <b>1-to-many</b> database support
|
|
||||||
<br><i class="fa-solid fa-minus"> </i> The ETL job is specified by JSON configuration file
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<br><div id="documentation" class="border-round border" style="min-height:250px"></div> -->
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
<style>
|
||||||
|
.visit-us-action-frame {display:grid; grid-template-columns: auto 250px auto;}
|
||||||
|
</style>
|
||||||
|
<div>
|
||||||
|
We encourage you to visit the github repository for examples and various ways to use <b>{{layout.header.title}}</b>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
|
||||||
|
<div class="visit-us-action-frame">
|
||||||
|
<div></div>
|
||||||
|
<div onclick="window.open('https://github.com/lnyemba/data-transport')">
|
||||||
|
<div align="center" class="border-round border"><div class="active bold">Github</div></div>
|
||||||
|
</div>
|
||||||
|
<div></div>
|
||||||
|
</div>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
Loading…
Reference in new issue