diff --git a/healthcareio/Dockerfile b/healthcareio/Dockerfile index 5cf8fd8..23464ef 100644 --- a/healthcareio/Dockerfile +++ b/healthcareio/Dockerfile @@ -1,4 +1,8 @@ -FROM ubuntu:bionic-20200403 +# +# Let us create an image for healthcareio +# The image will contain the {X12} Parser and the +# FROM ubuntu:bionic-20200403 +FROM ubuntu:focal RUN ["apt","update","--fix-missing"] RUN ["apt-get","upgrade","-y"] @@ -6,9 +10,22 @@ RUN ["apt-get","-y","install","apt-utils"] RUN ["apt","update","--fix-missing"] RUN ["apt-get","upgrade","-y"] -RUN ["apt-get","install","-y","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv"] +RUN ["apt-get","install","-y","mongo","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv","wget"] # # +RUN ["pip3","install","--upgrade","pip"] +# RUN ["pip3","install","git+https://healthcare.the-phi.com/git/code/parser.git","botocore"] USER health-user +# +# This volume is where the data will be loaded from (otherwise it is assumed the user will have it in the container somehow) +# +VOLUME ["/data"] +# +# This is the port from which some degree of monitoring can/will happen +EXPOSE 80 +# wget https://healthcareio.the-phi.com/git/code/parser.git/bootup.sh +COPY bootup.sh bootup.sh +ENTRYPOINT ["bash","-C"] +CMD ["bootup.sh"] # VOLUME ["/home/health-user/healthcare-io/","/home-healthuser/.healthcareio"] -# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"] \ No newline at end of file +# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"] diff --git a/healthcareio/server/__init__.py b/healthcareio/server/__init__.py index 3910930..d723bca 100644 --- a/healthcareio/server/__init__.py +++ b/healthcareio/server/__init__.py @@ -3,7 +3,78 @@ from healthcareio.params import SYS_ARGS import healthcareio.analytics import os import json +import time +import smart +import transport +import pandas as pd +import numpy as np +import x12 + +from multiprocessing import Process +from flask_socketio import SocketIO, emit, disconnect,send +from healthcareio.server import proxy +PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) app = Flask(__name__) +socket_ = SocketIO(app) + +def resume (files): + _args = SYS_ARGS['config']['store'].copy() + if 'mongo' in SYS_ARGS['config']['store']['type'] : + _args['type'] = 'mongo.MongoReader' + reader = transport.factory.instance(**_args) + _files = [] + try: + pipeline = [{"$match":{"completed":{"$eq":True}}},{"$group":{"_id":"$name"}},{"$project":{"name":"$_id","_id":0}}] + _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline} + _files = reader.read(mongo = _args) + _files = [item['name'] for item in _files] + except Exception as e : + pass + print (["found ",len(files),"\tProcessed ",len(_files)]) + return list(set(files) - set(_files)) + + + +def run (): + # + # let's get the files in the folder (perhaps recursively traverse them) + # + FILES = [] + BATCH = int(SYS_ARGS['config']['args']['batch']) #-- number of processes (poorly named variable) + + for root,_dir,f in os.walk(SYS_ARGS['config']['args']['folder']) : + if f : + FILES += [os.sep.join([root,name]) for name in f] + FILES = resume(FILES) + FILES = np.array_split(FILES,BATCH) + procs = [] + for FILE_GROUP in FILES : + + FILE_GROUP = FILE_GROUP.tolist() + # logger.write({"process":index,"parse":SYS_ARGS['parse'],"file_count":len(row)}) + # proc = Process(target=apply,args=(row,info['store'],_info,)) + parser = x12.Parser(PATH) #os.sep.join([PATH,'config.json'])) + parser.set.files(FILE_GROUP) + parser.start() + procs.append(parser) + SYS_ARGS['procs'] = procs +# @socket_.on('data',namespace='/stream') +def push() : + _args = dict(SYS_ARGS['config']['store'].copy(),**{"type":"mongo.MongoReader"}) + reader = transport.factory.instance(**_args) + pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}] + _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline} + r = pd.DataFrame(reader.read(mongo=_args)) + r = healthcareio.analytics.Apex.apply({"chart":{"type":"donut","axis":{"x":"count","y":"type"}},"data":r}) + emit("update",r,json=True) + return r +@socket_.on('connect') +def client_connect(**r): + print ('Connection received') + print (r) + push() + pass + @app.route("/favicon.ico") def _icon(): return send_from_directory(os.path.join([app.root_path, 'static','img','logo.svg']), @@ -12,7 +83,7 @@ def _icon(): def init(): e = SYS_ARGS['engine'] sections = {"remits":e.info['835'],"claims":e.info['837']} - _args = {"sections":sections} + _args = {"sections":sections,"store":SYS_ARGS["config"]["store"],"owner":SYS_ARGS['config']['owner'],"args":SYS_ARGS["config"]["args"]} return render_template("index.html",**_args) @app.route("/format//",methods=['POST']) def _format(id,index): @@ -21,43 +92,117 @@ def _format(id,index): key = '837' if id == 'claims' else '835' index = int(index) # p = e.info[key][index] - p = e.apply(type=id,index=index) - - # + p = e.filter(type=id,index=index) + r = [] - for item in p[0]['pipeline'] : - _item= dict(item) - del _item['sql'] - del _item ['data'] - + for item in p['pipeline'] : + _item= dict(item) _item = dict(_item,**healthcareio.analytics.Apex.apply(item)) + del _item['data'] if 'apex' in _item or 'html' in _item: r.append(_item) - r = {"id":p[0]['id'],"pipeline":r} + + r = {"id":p['id'],"pipeline":r} return json.dumps(r),200 + @app.route("/get//",methods=['GET']) def get(id,index): e = SYS_ARGS['engine'] key = '837' if id == 'claims' else '835' index = int(index) # p = e.info[key][index] - p = e.apply(type=id,index=index) + p = e.filter(type=id,index=index) r = {} for item in p[0]['pipeline'] : _item= [dict(item)] - r[item['label']] = item['data'].to_dict(orient='record') - # del _item['sql'] - # del _item ['data'] - # print (item['label']) - # _item['apex'] = healthcareio.analytics.Apex.apply(item) - # if _item['apex']: - # r.append(_item) - - # r = {"id":p[0]['id'],"pipeline":r} + # r[item['label']] = item['data'].to_dict(orient='record') + r[item['label']] = item['data'].to_dict('record') return json.dumps(r),200 + +@app.route("/reset",methods=["POST"]) +def reset(): + return "1",200 +@app.route("/data",methods=['GET']) +def get_data (): + """ + This function will return statistical data about the services i.e general statistics about what has/been processed + """ + HEADER = {"Content-type":"application/json"} + _args = SYS_ARGS['config'] + options = dict(proxy.get.files(_args),**proxy.get.processes(_args)) + return json.dumps(options),HEADER +@app.route("/log/",methods=["POST","PUT","GET"]) +def log(id) : + HEADER = {"Content-Type":"application/json; charset=utf8"} + if id == 'params' and request.method in ['PUT', 'POST']: + info = request.json + _args = {"batch":info['batch'] if 'batch' in info else 1,"resume":True} + # + # We should update the configuration + SYS_ARGS['config']['args'] = _args + PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) + write = lambda content: (open(PATH,'w')).write(json.dumps(content)) + proc = Process(target=write,args=(SYS_ARGS['config'],)) + proc.start() + return "1",HEADER + pass +@app.route("/io/",methods=['POST']) +def io_data(id): + if id == 'params' : + _args = request.json + # + # Expecting batch,folder as parameters + _args = request.json + _args['resume'] = True + print (_args) + # + # We should update the configuration + SYS_ARGS['config']['args'] = _args + # PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) + try: + write = lambda content: (open(PATH,'w')).write(json.dumps(content)) + proc = Process(target=write,args=(SYS_ARGS['config'],)) + proc.start() + # proc.join() + return "1",200 + except Exception as e : + return "0",403 + pass + elif id == 'stop' : + stop() + pass + elif id == 'run' : + # run() + _args = {"args":SYS_ARGS['config']['args'],"store":SYS_ARGS["config"]["store"]} + proxy.run(_args) + return "1",200 + pass + +@app.route("/export") +def export_form(): + _args = {"context":SYS_ARGS['context']} + return render_template("store.html",**_args) +@app.route("/export",methods=['POST','PUT']) +def apply_etl(): + _info = request.json + m = {'s3':'s3.s3Writer','mongo':'mongo.MongoWriter'} + if _info : + dest_args = {'type':m[_info['type']],"args": _info['content'] } + src_args = SYS_ARGS['config']['store'] + # print (_args) + # writer = transport.factory.instance(**_args) + proxy.publish(src_args,dest_args) + return "1",405 + + else: + return "0",404 +@app.route("/update") +def update(): + pass + return "0",405 @app.route("/reload",methods=['POST']) def reload(): # e = SYS_ARGS['engine'] @@ -74,11 +219,20 @@ if __name__ == '__main__' : PORT = int(SYS_ARGS['port']) if 'port' in SYS_ARGS else 5500 DEBUG= int(SYS_ARGS['debug']) if 'debug' in SYS_ARGS else 0 SYS_ARGS['context'] = SYS_ARGS['context'] if 'context' in SYS_ARGS else '' + # # PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) + # + # Adjusting configuration with parameters (batch,folder,resume) + if 'args' not in SYS_ARGS['config'] : + SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"} + + SYS_ARGS['procs'] = [] + + # SYS_ARGS['path'] = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) e = healthcareio.analytics.engine(PATH) - # e.apply(type='claims',serialize=True) + e.apply(type='claims',serialize=False) SYS_ARGS['engine'] = e app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=True) \ No newline at end of file diff --git a/healthcareio/server/index.py b/healthcareio/server/index.py index c61f130..99f3709 100644 --- a/healthcareio/server/index.py +++ b/healthcareio/server/index.py @@ -12,8 +12,9 @@ def _icon(): def init(): e = SYS_ARGS['engine'] sections = {"remits":e.info['835'],"claims":e.info['837']} - _args = {"sections":sections} - return render_template("index.html",**_args) + _args = {"sections":sections,"store":SYS_ARGS['config']['store']} + print (SYS_ARGS['config']['store']) + return render_template("setup.html",**_args) @app.route("/format//",methods=['POST']) def _format(id,index): diff --git a/healthcareio/server/static/css/default.css b/healthcareio/server/static/css/default.css index 0bded6c..5ea27b2 100644 --- a/healthcareio/server/static/css/default.css +++ b/healthcareio/server/static/css/default.css @@ -1,3 +1,4 @@ + .active { padding:4px; cursor:pointer; @@ -6,3 +7,51 @@ .active:hover{ border-bottom:2px solid #ff6500; } +input[type=text]{ + border:1px solid transparent; + background-color:#f3f3f3; + outline: 0px; + padding:8px; + font-weight:normal; + font-family:sans-serif; + color:black; +} +.active-button { + display:grid; + grid-template-columns: 32px auto; + gap:2px; + align-items:center; + border:2px solid #CAD5E0; + cursor:pointer; + +} +.active-button i {padding:4px;;} +.active-button:hover { border-color:#ff6500} + +.system {display:grid; grid-template-columns: 45% 1px auto; gap:20px; margin-left:5%; width:90%;} +.system .status .item {display:grid; grid-template-columns: 75px 8px auto; gap:2px;} +.input-form {display:grid; gap:2px;} +.input-form .item {display:grid; grid-template-columns: 125px auto; gap:2px; align-items:center;} +.input-form .item .label { font-weight:bold; padding-left:10px} +.fa-cog {color:#4682B4} +.fa-check {color:#00c6b3} +.fa-times {color:maroon} + +.code { + margin:4px; + background:#000000 ; + padding:8px; + font-family: 'Courier New', Courier, monospace; + color:#d3d3d3; + font-size:12px; + line-height: 2; +} + +.tabs {display:grid; grid-template-columns: repeat(3,1fr) auto; gap:0px; align-items:center; text-align: center;} +.tab {border:1px solid transparent; border-bottom-color:#D3D3D3; font-weight:bold; padding:4px} + +.tabs .selected {border-color:#CAD5E0; border-bottom-color:transparent; } +.system iframe {width:100%; height:100%; border:1px solid transparent;} +.data-info {height:90%; padding:8px;} +.fa-cloud {color:#4682B4} +.fa-database{color:#cc8c91} diff --git a/healthcareio/server/static/js/jx/rpc.js b/healthcareio/server/static/js/jx/rpc.js index 8dd08fe..22033f9 100755 --- a/healthcareio/server/static/js/jx/rpc.js +++ b/healthcareio/server/static/js/jx/rpc.js @@ -12,119 +12,128 @@ * Improve on how returned data is handled (if necessary). */ if(!jx){ - var jx = {} -} -/** - * These are a few parsers that can come in handy: - * urlparser: This parser is intended to break down a url parameter string in key,value pairs - */ - -function urlparser(url){ - if(url.toString().match(/\x3F/) != null){ - url = url.split('\x3F')[1] - - } - var p = url.split('&') ; - var r = {} ; - r.meta = [] ; - r.data = {} ; - var entry; - for(var i=0; i < p.length; i++){ - entry = p[i] ; - key = (entry.match('(.*)=') !=null)? entry.match('(.*)=')[1]:null ; - value = (entry.match('=(.*)$') != null)? entry.match('=(.*)$')[1]:null - if(key != null){ - key = key.replace('\x3F','') - r.meta.push(key) ; - r.data[key] = value ; - } - } - - return r.data; -} - -/** -* The following are corrections related to consistency in style & cohesion -*/ -jx.ajax = {} -jx.ajax.get = {} ; -jx.ajax.debug = null; -jx.ajax.get.instance = function(){ - var factory = function(){ - this.obj = {} - this.obj.headers = {} - this.obj.async = true; - this.setHeader = function(key,value){ - if(key.constructor != String && value == null){ - this.obj.headers = key ; - }else{ - this.obj.headers[key] = value; - } - } - this.setData = function(data){ - this.obj.data = data; - } - this.setAsync = function(flag){ - this.obj.async = (flag == true) ; - } - this.send = function(url,callback,method){ - - if(method == null){ - method = 'GET' - } - - p = jx.ajax.debug != null; - q = false; - if(p){ - q = jx.ajax.debug[url] != null; - } - - is_debuggable = p && q - - if(is_debuggable){ - x = {} ; - x.responseText = jx.ajax.debug [url] ; - callback(x) - }else{ - var http = new XMLHttpRequest() ; - http.onreadystatechange = function(){ - if(http.readyState == 4){ - - callback(http) - } - } - // - // In order to insure backward compatibility - // Previous versions allowed the user to set the variable on the wrapper (poor design) - if(this.async != null){ - this.setAsync(this.async) ; - } - http.open(method,url,this.obj.async) ; - for(key in this.obj.headers){ - value = this.obj.headers[key] ; - - http.setRequestHeader(key,value) - } - - http.send(this.obj.data) - } - - - } - this.put = function(url,callback){ - this.send(url,callback,'PUT') ; - } - this.get = function(url,callback){ - this.send(url,callback,'GET') ; - } - this.post = function(url,callback){ - this.send(url,callback,'POST') ; - } - }//-- end of the factory method - return new factory() ; -} - -// -// backward compatibility -jx.ajax.getInstance = jx.ajax.get.instance ; -var HttpClient = jx.ajax.get ; + var jx = {} + } + /** + * These are a few parsers that can come in handy: + * urlparser: This parser is intended to break down a url parameter string in key,value pairs + */ + + function urlparser(url){ + if(url.toString().match(/\x3F/) != null){ + url = url.split('\x3F')[1] + + } + var p = url.split('&') ; + var r = {} ; + r.meta = [] ; + r.data = {} ; + var entry; + for(var i=0; i < p.length; i++){ + entry = p[i] ; + key = (entry.match('(.*)=') !=null)? entry.match('(.*)=')[1]:null ; + value = (entry.match('=(.*)$') != null)? entry.match('=(.*)$')[1]:null + if(key != null){ + key = key.replace('\x3F','') + r.meta.push(key) ; + r.data[key] = value ; + } + } + + return r.data; + } + + /** + * The following are corrections related to consistency in style & cohesion + */ + jx.ajax = {} + jx.ajax.get = {} ; + jx.ajax.debug = null; + jx.ajax.get.instance = function(){ + var factory = function(){ + this.obj = {} + this.obj.headers = {} + this.obj.async = true; + this.setHeader = function(key,value){ + if(key.constructor != String && value == null){ + this.obj.headers = key ; + }else{ + this.obj.headers[key] = value; + } + } + this.setData = function(data,mimetype){ + if(mimetype == null) + this.obj.data = data; + else { + this.obj.headers['Content-Type'] = mimetype + if(mimetype.match(/application\/json/i)){ + this.obj.data = JSON.stringify(data) + } + } + + } + this.setAsync = function(flag){ + this.obj.async = (flag == true) ; + } + this.send = function(url,callback,method){ + + if(method == null){ + method = 'GET' + } + + p = jx.ajax.debug != null; + q = false; + if(p){ + q = jx.ajax.debug[url] != null; + } + + is_debuggable = p && q + + if(is_debuggable){ + x = {} ; + x.responseText = jx.ajax.debug [url] ; + callback(x) + }else{ + var http = new XMLHttpRequest() ; + http.onreadystatechange = function(){ + if(http.readyState == 4){ + + callback(http) + } + } + // + // In order to insure backward compatibility + // Previous versions allowed the user to set the variable on the wrapper (poor design) + if(this.async != null){ + this.setAsync(this.async) ; + } + http.open(method,url,this.obj.async) ; + for(key in this.obj.headers){ + value = this.obj.headers[key] ; + + http.setRequestHeader(key,value) + } + + http.send(this.obj.data) + } + + + } + this.put = function(url,callback){ + this.send(url,callback,'PUT') ; + } + this.get = function(url,callback){ + this.send(url,callback,'GET') ; + } + this.post = function(url,callback){ + this.send(url,callback,'POST') ; + } + }//-- end of the factory method + return new factory() ; + } + + // + // backward compatibility + jx.ajax.getInstance = jx.ajax.get.instance ; + var HttpClient = jx.ajax.get ; + \ No newline at end of file diff --git a/healthcareio/server/templates/index.html b/healthcareio/server/templates/index.html index 915cddc..768618e 100644 --- a/healthcareio/server/templates/index.html +++ b/healthcareio/server/templates/index.html @@ -8,6 +8,7 @@ + @@ -15,6 +16,9 @@ + + + Healthcare/IO Analytics -
+
-
Healthcare/IO
-
Analytics Dashboard
+
Healthcare/IO :: Parser
+
Dashboard
-
+
+
+
+
+
Setup
+
+ + +
+
+
+
Configure Parser
+ +
Reset Parser
+ +
+
{% for key in sections %} -
-
-
{{key|safe}}
-
+
+
+
{{key|safe}}
+
+ + +
+
+
- + {% for item in sections[key] %} +
{{item.id}}
+ {% endfor %}
-
-
- - {% for item in sections[key] %} -
{{item.id}}
- {% endfor %}
-
- {% endfor %} + {% endfor %}
-
-
+
+
-
+
+ -
+ {%include 'setup.html' %} + +
- © Vanderbilt University Medical Center + Healthcare/IO :: Parser ©
\ No newline at end of file