Merge branch 'dev' of lab/parse-edi into master

master
steve 4 years ago committed by Gogs
commit ad56951807

@ -1,4 +1,8 @@
FROM ubuntu:bionic-20200403
#
# Let us create an image for healthcareio
# The image will contain the {X12} Parser and the
# FROM ubuntu:bionic-20200403
FROM ubuntu:focal
RUN ["apt","update","--fix-missing"]
RUN ["apt-get","upgrade","-y"]
@ -6,9 +10,22 @@ RUN ["apt-get","-y","install","apt-utils"]
RUN ["apt","update","--fix-missing"]
RUN ["apt-get","upgrade","-y"]
RUN ["apt-get","install","-y","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv"]
RUN ["apt-get","install","-y","mongo","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv","wget"]
#
#
RUN ["pip3","install","--upgrade","pip"]
# RUN ["pip3","install","git+https://healthcare.the-phi.com/git/code/parser.git","botocore"]
USER health-user
#
# This volume is where the data will be loaded from (otherwise it is assumed the user will have it in the container somehow)
#
VOLUME ["/data"]
#
# This is the port from which some degree of monitoring can/will happen
EXPOSE 80
# wget https://healthcareio.the-phi.com/git/code/parser.git/bootup.sh
COPY bootup.sh bootup.sh
ENTRYPOINT ["bash","-C"]
CMD ["bootup.sh"]
# VOLUME ["/home/health-user/healthcare-io/","/home-healthuser/.healthcareio"]
# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"]
# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"]

@ -11,7 +11,7 @@ import transport
import matplotlib.pyplot as plt
import re, base64
# from weasyprint import HTML, CSS
COLORS = ["#f79256","#7dcfb6","#fbd1a2","#00b2ca","#1d4e89","#4682B4","#c5c3c6","#4c5c68","#1985a1","#f72585","#7209b7","#3a0ca3","#4361ee","#4cc9f0","#ff595e","#ffca3a","#8ac926","#1982c4","#6a4c93"]
COLORS = ["#fbd1a2","#00b2ca","#1d4e89","#4682B4","#c5c3c6","#4c5c68","#1985a1","#f72585","#7209b7","#3a0ca3","#4361ee","#4cc9f0","#ff595e","#ffca3a","#8ac926","#1982c4","#6a4c93"]
class stdev :
def __init__(self) :
self.values = []
@ -149,11 +149,16 @@ class Apex :
This class will format a data-frame to work with Apex charting engine
"""
@staticmethod
def apply(item):
def apply(item,theme={'mode':'light','palette':'palette6'}):
pointer = item['chart']['type']
if hasattr(Apex,pointer) :
pointer = getattr(Apex,pointer)
options = pointer(item)
if 'apex' in options and 'colors' in options['apex'] :
del options['apex']['colors']
if 'apex' in options :
options['apex']['theme'] = theme
options['responsive']= [
{
'breakpoint': 1,
@ -168,6 +173,18 @@ class Apex :
print ("Oops")
pass
@staticmethod
def radial(item):
df = item['data']
x = item['chart']['axis']['x']
y = item['chart']['axis']['y']
labels = df[y].tolist()
values = [float(np.round(value,2)) for value in df[x].tolist()]
chart = {"type":"radialBar","height":200}
option = {"chart":chart,"series":values,"labels":labels,"plotOptions":{"radialBar":{"hollow":{"size":"70%"}}}}
return {'apex':option}
@staticmethod
def scatter(item):
options = Apex.spline(item)
options['apex']['chart']['type'] = 'scatter'
@ -175,7 +192,7 @@ class Apex :
@staticmethod
def scalar(item):
_df = item['data']
print (_df)
name = _df.columns.tolist()[0]
value = _df[name].values.round(2)[0]
html = '<div class="scalar"><div class="value">:value</div><div class="label">:label</div></div>'
@ -235,16 +252,17 @@ class Apex :
@TODO: alias this with bar (!= column)
"""
df = item['data']
N = df.shape[0] if df.shape[0] < 10 else 10
axis = item['chart']['axis']
y = axis['y']
if type(y) == list :
y = y[0]
axis['x'] = [axis['x']] if type(axis['x']) != list else axis['x']
if not set(axis['x']) & set(df.columns.tolist()) :
print (set(axis['x']) & set(df.columns.tolist()))
print (axis['x'])
print (df.columns)
# if not set(axis['x']) & set(df.columns.tolist()) :
# print (set(axis['x']) & set(df.columns.tolist()))
# print (axis['x'])
# print (df.columns)
# df.columns = axis['x']
series = []
_min=_max = 0
@ -294,7 +312,6 @@ class Apex :
values are x-axis
"""
df = item['data']
if df.shape [0]> 1 :
y_cols,x_cols = item['chart']['axis']['y'],item['chart']['axis']['x']
labels = df[y_cols].values.tolist()
@ -302,10 +319,11 @@ class Apex :
values = df[x_cols].values.round(2).tolist()
else:
labels = [name.upper().replace('_',' ') for name in df.columns.tolist()]
df = df.astype(float)
values = df.values.round(2).tolist()[0] if df.shape[1] > 1 else df.values.round(2).tolist()
colors = COLORS[:len(values)]
options = {"series":values,"colors":colors,"labels":labels,"chart":{"type":"donut"},"plotOptions":{"pie":{"customScale":.8}},"legend":{"position":"right"}}
options = {"series":values,"colors":colors,"labels":labels,"dataLabels":{"enabled":True,"style":{"colors":["#000000"]},"dropShadow":{"enabled":False}},"chart":{"type":"donut","width":200},"plotOptions":{"pie":{"customScale":.9}},"legend":{"position":"right"}}
return {"apex":options}
pass
@ -329,43 +347,117 @@ class engine :
_args['type'] = 'mongo.MongoReader'
else:
_args['type'] = 'disk.SQLiteReader'
self.reader = transport.factory.instance(**_args)
self.store_config = _args ;
def filter (self,**args):
"""
type: claims or remits
filter optional identifier claims, procedures, taxonomy, ...
"""
_m = {'claim':'837','claims':'837','remits':'835','remit':'835'}
table = _m[ args['type']]
_analytics = self.info[table]
if 'index' in args :
index = int(args['index'])
_analytics = [_analytics[index]]
_info = list(_analytics) #if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']]
# conn = lite.connect(self.store_config['args']['path'],isolation_level=None)
# conn.create_aggregate("stdev",1,stdev)
DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql'
if DB_TYPE == 'mongo' :
self.store_config['args']['doc'] = args['type']
self.reader = transport.factory.instance(**self.store_config)
r = []
for row in _info :
pipeline = row['pipeline']
index = 0
for item in pipeline:
if not item[DB_TYPE] :
continue
query = {DB_TYPE:item[DB_TYPE]}
df = pd.DataFrame(self.reader.read(**query)) #item)
df = df.fillna('N/A')
# item['data'] = df
chart = item['chart']
pipe = {"data":df,"chart":chart}
for key in list(item.keys()) :
if key not in ["chart","data","mongo","sql","couch"] :
pipe[key] = item[key]
r.append(pipe)
self.reader.close()
return {"id":_info[0]['id'],'pipeline':r}
def apply (self,**args) :
"""
type: claims or remits
filter optional identifier claims, procedures, taxonomy, ...
"""
_m = {'claim':'837','claims':'837','remits':'835','remit':'835'}
# key = '837' if args['type'] == 'claims' else '835'
table = _m[ args['type']]
analytics = self.info[table]
_analytics = self.info[table]
if 'index' in args :
index = int(args['index'])
analytics = [analytics[index]]
_analytics = [_analytics[index]]
_info = list(analytics) if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']]
_info = list(_analytics) if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']]
# conn = lite.connect(self.store_config['args']['path'],isolation_level=None)
# conn.create_aggregate("stdev",1,stdev)
DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql'
#
# @TODO: Find a better way to handle database variance
#
# DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql'
if 'mongo' in self.store_config['type'] :
DB_TYPE='mongo'
else:
DB_TYPE='sql'
self.store_config['args']['table'] = args['type']
self.reader = transport.factory.instance(**self.store_config)
r = []
for row in _info :
for item in row['pipeline'] :
pipeline = row['pipeline']
index = 0
for item in pipeline:
# item['data'] = pd.read_sql(item['sql'],conn)
query = {DB_TYPE:item[DB_TYPE]}
item['data'] = self.reader.read(**item)
# query = {DB_TYPE:item[DB_TYPE]}
query = item[DB_TYPE]
if not query :
continue
if DB_TYPE == 'sql' :
query = {"sql":query}
item['data'] = self.reader.read(**query) #item)
if 'serialize' in args :
item['data'] = json.dumps(item['data'].to_dict(orient='record')) if type(item['data']) == pd.DataFrame else item['data']
# item['data'] = json.dumps(item['data'].to_dict(orient='record')) if type(item['data']) == pd.DataFrame else item['data']
item['data'] = json.dumps(item['data'].to_dict('record')) if type(item['data']) == pd.DataFrame else item['data']
else:
item['data'] = (pd.DataFrame(item['data']))
pipeline[index] = item
index += 1
#
#
row['pipeline']= pipeline
# if 'info' in item:
# item['info'] = item['info'].replace(":rows",str(item["data"].shape[0]))
# conn.close()
self.reader.close()
return _info
def _html(self,item) :

@ -0,0 +1,37 @@
#
# Let us create an image for healthcareio
# The image will contain the {X12} Parser and the
# FROM ubuntu:bionic-20200403
FROM ubuntu:focal
RUN ["apt-get","update","--fix-missing"]
RUN ["apt-get","upgrade","-y"]
RUN ["apt-get","-y","install","apt-utils"]
RUN ["apt-get","update","--fix-missing"]
RUN ["apt-get","upgrade","-y"]
RUN ["apt-get","install","-y","mongodb","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv","wget"]
#
#
RUN ["pip3","install","--upgrade","pip"]
RUN ["pip3","install","numpy","pandas","git+https://dev.the-phi.com/git/steve/data-transport","botocore","matplotlib"]
# RUN ["pip3","install","git+https://healthcare.the-phi.com/git/code/parser.git","botocore"]
# RUN ["useradd", "-ms", "/bin/bash", "health-user"]
# USER health-user
#
# This volume is where the data will be loaded from (otherwise it is assumed the user will have it in the container somehow)
#
VOLUME ["/data","/app/healthcareio"]
WORKDIR /app
ENV PYTHONPATH="/app"
#
# This is the port from which some degree of monitoring can/will happen
EXPOSE 80
EXPOSE 27017
# wget https://healthcareio.the-phi.com/git/code/parser.git/bootup.sh
COPY bootup.sh bootup.sh
ENTRYPOINT ["bash","-C"]
CMD ["bootup.sh"]
# VOLUME ["/home/health-user/healthcare-io/","/home-healthuser/.healthcareio"]
# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"]

@ -0,0 +1,10 @@
set -e
/etc/init.d/mongodb start
cd /app
export
export PYTHONPATH=$PWD
ls
# python3 healthcareio/healthcare-io.py --signup $EMAIL --store mongo
# python3 healthcareio/healthcare-io.py --analytics --port 80 --debug
bash

@ -43,6 +43,8 @@ import numpy as np
from multiprocessing import Process
import time
from healthcareio import x12
import smart
import pandas as pd
PATH = os.sep.join([os.environ['HOME'],'.healthcareio'])
OUTPUT_FOLDER = os.sep.join([os.environ['HOME'],'healthcare-io'])
@ -337,10 +339,20 @@ if __name__ == '__main__' :
# PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
e = analytics.engine(os.sep.join([PATH,'config.json'])) #--@TODO: make the configuration file globally accessible
e.apply(type='claims',serialize=True)
SYS_ARGS['engine'] = e
if os.path.exists(os.sep.join([PATH,'config.json'])) :
e = analytics.engine(os.sep.join([PATH,'config.json'])) #--@TODO: make the configuration file globally accessible
e.apply(type='claims',serialize=True)
SYS_ARGS['engine'] = e
SYS_ARGS['config'] = json.loads(open(os.sep.join([PATH,'config.json'])).read())
else:
SYS_ARGS['config'] = {"owner":None,"store":None}
if 'args' not in SYS_ARGS['config'] :
SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"}
me = pd.DataFrame(smart.top.read(name='healthcare-io.py')).args.unique().tolist()
SYS_ARGS['me'] = me[0] #-- This key will identify the current process
pointer = lambda : server.app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=False)
pthread = Process(target=pointer,args=())
pthread.start()

@ -3,7 +3,78 @@ from healthcareio.params import SYS_ARGS
import healthcareio.analytics
import os
import json
import time
import smart
import transport
import pandas as pd
import numpy as np
import x12
from multiprocessing import Process
from flask_socketio import SocketIO, emit, disconnect,send
from healthcareio.server import proxy
PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
app = Flask(__name__)
socket_ = SocketIO(app)
def resume (files):
_args = SYS_ARGS['config']['store'].copy()
if 'mongo' in SYS_ARGS['config']['store']['type'] :
_args['type'] = 'mongo.MongoReader'
reader = transport.factory.instance(**_args)
_files = []
try:
pipeline = [{"$match":{"completed":{"$eq":True}}},{"$group":{"_id":"$name"}},{"$project":{"name":"$_id","_id":0}}]
_args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
_files = reader.read(mongo = _args)
_files = [item['name'] for item in _files]
except Exception as e :
pass
print (["found ",len(files),"\tProcessed ",len(_files)])
return list(set(files) - set(_files))
def run ():
#
# let's get the files in the folder (perhaps recursively traverse them)
#
FILES = []
BATCH = int(SYS_ARGS['config']['args']['batch']) #-- number of processes (poorly named variable)
for root,_dir,f in os.walk(SYS_ARGS['config']['args']['folder']) :
if f :
FILES += [os.sep.join([root,name]) for name in f]
FILES = resume(FILES)
FILES = np.array_split(FILES,BATCH)
procs = []
for FILE_GROUP in FILES :
FILE_GROUP = FILE_GROUP.tolist()
# logger.write({"process":index,"parse":SYS_ARGS['parse'],"file_count":len(row)})
# proc = Process(target=apply,args=(row,info['store'],_info,))
parser = x12.Parser(PATH) #os.sep.join([PATH,'config.json']))
parser.set.files(FILE_GROUP)
parser.start()
procs.append(parser)
SYS_ARGS['procs'] = procs
# @socket_.on('data',namespace='/stream')
def push() :
_args = dict(SYS_ARGS['config']['store'].copy(),**{"type":"mongo.MongoReader"})
reader = transport.factory.instance(**_args)
pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}]
_args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
r = pd.DataFrame(reader.read(mongo=_args))
r = healthcareio.analytics.Apex.apply({"chart":{"type":"donut","axis":{"x":"count","y":"type"}},"data":r})
emit("update",r,json=True)
return r
@socket_.on('connect')
def client_connect(**r):
print ('Connection received')
print (r)
push()
pass
@app.route("/favicon.ico")
def _icon():
return send_from_directory(os.path.join([app.root_path, 'static','img','logo.svg']),
@ -12,7 +83,7 @@ def _icon():
def init():
e = SYS_ARGS['engine']
sections = {"remits":e.info['835'],"claims":e.info['837']}
_args = {"sections":sections}
_args = {"sections":sections,"store":SYS_ARGS["config"]["store"],"owner":SYS_ARGS['config']['owner'],"args":SYS_ARGS["config"]["args"]}
return render_template("index.html",**_args)
@app.route("/format/<id>/<index>",methods=['POST'])
def _format(id,index):
@ -21,43 +92,117 @@ def _format(id,index):
key = '837' if id == 'claims' else '835'
index = int(index)
# p = e.info[key][index]
p = e.apply(type=id,index=index)
#
p = e.filter(type=id,index=index)
r = []
for item in p[0]['pipeline'] :
_item= dict(item)
del _item['sql']
del _item ['data']
for item in p['pipeline'] :
_item= dict(item)
_item = dict(_item,**healthcareio.analytics.Apex.apply(item))
del _item['data']
if 'apex' in _item or 'html' in _item:
r.append(_item)
r = {"id":p[0]['id'],"pipeline":r}
r = {"id":p['id'],"pipeline":r}
return json.dumps(r),200
@app.route("/get/<id>/<index>",methods=['GET'])
def get(id,index):
e = SYS_ARGS['engine']
key = '837' if id == 'claims' else '835'
index = int(index)
# p = e.info[key][index]
p = e.apply(type=id,index=index)
p = e.filter(type=id,index=index)
r = {}
for item in p[0]['pipeline'] :
_item= [dict(item)]
r[item['label']] = item['data'].to_dict(orient='record')
# del _item['sql']
# del _item ['data']
# print (item['label'])
# _item['apex'] = healthcareio.analytics.Apex.apply(item)
# if _item['apex']:
# r.append(_item)
# r = {"id":p[0]['id'],"pipeline":r}
# r[item['label']] = item['data'].to_dict(orient='record')
r[item['label']] = item['data'].to_dict('record')
return json.dumps(r),200
@app.route("/reset",methods=["POST"])
def reset():
return "1",200
@app.route("/data",methods=['GET'])
def get_data ():
"""
This function will return statistical data about the services i.e general statistics about what has/been processed
"""
HEADER = {"Content-type":"application/json"}
_args = SYS_ARGS['config']
options = dict(proxy.get.files(_args),**proxy.get.processes(_args))
return json.dumps(options),HEADER
@app.route("/log/<id>",methods=["POST","PUT","GET"])
def log(id) :
HEADER = {"Content-Type":"application/json; charset=utf8"}
if id == 'params' and request.method in ['PUT', 'POST']:
info = request.json
_args = {"batch":info['batch'] if 'batch' in info else 1,"resume":True}
#
# We should update the configuration
SYS_ARGS['config']['args'] = _args
PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
write = lambda content: (open(PATH,'w')).write(json.dumps(content))
proc = Process(target=write,args=(SYS_ARGS['config'],))
proc.start()
return "1",HEADER
pass
@app.route("/io/<id>",methods=['POST'])
def io_data(id):
if id == 'params' :
_args = request.json
#
# Expecting batch,folder as parameters
_args = request.json
_args['resume'] = True
print (_args)
#
# We should update the configuration
SYS_ARGS['config']['args'] = _args
# PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
try:
write = lambda content: (open(PATH,'w')).write(json.dumps(content))
proc = Process(target=write,args=(SYS_ARGS['config'],))
proc.start()
# proc.join()
return "1",200
except Exception as e :
return "0",403
pass
elif id == 'stop' :
stop()
pass
elif id == 'run' :
# run()
_args = {"args":SYS_ARGS['config']['args'],"store":SYS_ARGS["config"]["store"]}
proxy.run(_args)
return "1",200
pass
@app.route("/export")
def export_form():
_args = {"context":SYS_ARGS['context']}
return render_template("store.html",**_args)
@app.route("/export",methods=['POST','PUT'])
def apply_etl():
_info = request.json
m = {'s3':'s3.s3Writer','mongo':'mongo.MongoWriter'}
if _info :
dest_args = {'type':m[_info['type']],"args": _info['content'] }
src_args = SYS_ARGS['config']['store']
# print (_args)
# writer = transport.factory.instance(**_args)
proxy.publish(src_args,dest_args)
return "1",405
else:
return "0",404
@app.route("/update")
def update():
pass
return "0",405
@app.route("/reload",methods=['POST'])
def reload():
# e = SYS_ARGS['engine']
@ -74,11 +219,20 @@ if __name__ == '__main__' :
PORT = int(SYS_ARGS['port']) if 'port' in SYS_ARGS else 5500
DEBUG= int(SYS_ARGS['debug']) if 'debug' in SYS_ARGS else 0
SYS_ARGS['context'] = SYS_ARGS['context'] if 'context' in SYS_ARGS else ''
#
#
PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
#
# Adjusting configuration with parameters (batch,folder,resume)
if 'args' not in SYS_ARGS['config'] :
SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"}
SYS_ARGS['procs'] = []
# SYS_ARGS['path'] = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
e = healthcareio.analytics.engine(PATH)
# e.apply(type='claims',serialize=True)
e.apply(type='claims',serialize=False)
SYS_ARGS['engine'] = e
app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=True)

@ -12,8 +12,9 @@ def _icon():
def init():
e = SYS_ARGS['engine']
sections = {"remits":e.info['835'],"claims":e.info['837']}
_args = {"sections":sections}
return render_template("index.html",**_args)
_args = {"sections":sections,"store":SYS_ARGS['config']['store']}
print (SYS_ARGS['config']['store'])
return render_template("setup.html",**_args)
@app.route("/format/<id>/<index>",methods=['POST'])
def _format(id,index):

@ -0,0 +1,170 @@
"""
This file serves as proxy to healthcare-io, it will be embedded into the API
"""
import os
import transport
import numpy as np
import x12
import pandas as pd
import smart
from analytics import Apex
import time
class get :
PROCS = []
PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
@staticmethod
def resume (files,args):
"""
This function will determine the appropriate files to be processed by performing a simple complementary set operation against the logs
@TODO: Support data-stores other than mongodb
:param files list of files within a folder
:param _args configuration
"""
_args = args['store'].copy()
if 'mongo' in _args['type'] :
_args['type'] = 'mongo.MongoReader'
reader = transport.factory.instance(**_args)
_files = []
try:
pipeline = [{"$match":{"completed":{"$eq":True}}},{"$group":{"_id":"$name"}},{"$project":{"name":"$_id","_id":0}}]
_args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
_files = reader.read(mongo = _args)
_files = [item['name'] for item in _files]
except Exception as e :
pass
print (["found ",len(files),"\tProcessed ",len(_files)])
return list(set(files) - set(_files))
@staticmethod
def processes(_args):
_info = pd.DataFrame(smart.top.read(name='healthcare-io.py'))[['name','cpu','mem']]
if _info.shape[0] == 0 :
_info = pd.DataFrame({"name":["healthcare-io.py"],"cpu":[0],"mem":[0]})
# _info = pd.DataFrame(_info.groupby(['name']).sum())
# _info['name'] = ['healthcare-io.py']
m = {'cpu':'CPU','mem':'RAM','name':'name'}
_info.columns = [m[name] for name in _info.columns.tolist()]
_info.index = np.arange(_info.shape[0])
charts = []
for label in ['CPU','RAM'] :
value = _info[label].sum()
df = pd.DataFrame({"name":[label],label:[value]})
charts.append (
Apex.apply(
{"data":df, "chart":{"type":"radial","axis":{"x":label,"y":"name"}}}
)['apex']
)
#
# This will update the counts for the processes, upon subsequent requests so as to show the change
#
N = 0
lprocs = []
for proc in get.PROCS :
if proc.is_alive() :
lprocs.append(proc)
N = len(lprocs)
get.PROCS = lprocs
return {"process":{"chart":charts,"counts":N}}
@staticmethod
def files (_args):
_info = smart.folder.read(path='/data')
N = _info.files.tolist()[0]
if 'mongo' in _args['store']['type'] :
store_args = dict(_args['store'].copy(),**{"type":"mongo.MongoReader"})
# reader = transport.factory.instance(**_args)
pipeline = [{"$group":{"_id":"$name","count":{"$sum":{"$cond":[{"$eq":["$completed",True]},1,0]}} }},{"$group":{"_id":None,"count":{"$sum":"$count"}}},{"$project":{"_id":0,"status":"completed","count":1}}]
query = {"mongo":{"aggregate":"logs","allowDiskUse":True,"cursor":{},"pipeline":pipeline}}
# _info = pd.DataFrame(reader.read(mongo={"aggregate":"logs","allowDiskUse":True,"cursor":{},"pipeline":pipeline}))
pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}]
_query = {"mongo":{"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}} #-- distribution claims/remits
else:
store_args = dict(_args['store'].copy(),**{"type":"disk.SQLiteReader"})
store_args['args']['table'] = 'logs'
query= {"sql":"select count(distinct json_extract(data,'$.name')) as count, 'completed' status from logs where json_extract(data,'$.completed') = true"}
_query={"sql":"select json_extract(data,'$.parse') as type,count(distinct json_extract(data,'$.name')) as count from logs group by type"} #-- distribution claim/remits
reader = transport.factory.instance(**store_args)
_info = pd.DataFrame(reader.read(**query))
if not _info.shape[0] :
_info = pd.DataFrame({"status":["completed"],"count":[0]})
_info['count'] = np.round( (_info['count'] * 100 )/N,2)
charts = [Apex.apply({"data":_info,"chart":{"type":"radial","axis":{"y":"status","x":"count"}}})['apex']]
#
# Let us classify the files now i.e claims / remits
#
# pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}]
# _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
# r = pd.DataFrame(reader.read(mongo=_args))
r = pd.DataFrame(reader.read(**_query)) #-- distribution claims/remits
r = Apex.apply({"chart":{"type":"donut","axis":{"x":"count","y":"type"}},"data":r})['apex']
r['chart']['height'] = '100%'
r['legend']['position'] = 'bottom'
charts += [r]
return {"files":{"counts":N,"chart":charts}}
pass
#
# Process handling ....
def run (_args) :
"""
This function will run the jobs and insure as processes (as daemons).
:param _args system configuration
"""
FILES = []
BATCH = int(_args['args']['batch']) #-- number of processes (poorly named variable)
for root,_dir,f in os.walk(_args['args']['folder']) :
if f :
FILES += [os.sep.join([root,name]) for name in f]
FILES = get.resume(FILES,_args)
FILES = np.array_split(FILES,BATCH)
for FILE_GROUP in FILES :
FILE_GROUP = FILE_GROUP.tolist()
# logger.write({"process":index,"parse":_args['parse'],"file_count":len(row)})
# proc = Process(target=apply,args=(row,info['store'],_info,))
parser = x12.Parser(get.PATH) #os.sep.join([PATH,'config.json']))
parser.set.files(FILE_GROUP)
parser.daemon = True
parser.start()
get.PROCS.append(parser)
time.sleep(3)
#
# @TODO:consider submitting an update to clients via publish/subscribe framework
#
return get.PROCS
def stop(_args):
for job in get.PROCS :
if job.is_alive() :
job.terminate()
get.PROCS = []
#
# @TODO: consider submitting an update to clients via publish/subscribe framework
pass
def write(src_args,dest_args,files) :
#
# @TODO: Support for SQLite
pass
def publish (src_args,dest_args,folder="/data"):
FILES = []
for root,_dir,f in os.walk(folder) :
if f :
FILES += [os.sep.join([root,name]) for name in f]
#
# @TODO: Add support for SQLite ....
FILES = np.array_split(FILES,4)

@ -1,3 +1,4 @@
.active {
padding:4px;
cursor:pointer;
@ -6,3 +7,51 @@
.active:hover{
border-bottom:2px solid #ff6500;
}
input[type=text]{
border:1px solid transparent;
background-color:#f3f3f3;
outline: 0px;
padding:8px;
font-weight:normal;
font-family:sans-serif;
color:black;
}
.active-button {
display:grid;
grid-template-columns: 32px auto;
gap:2px;
align-items:center;
border:2px solid #CAD5E0;
cursor:pointer;
}
.active-button i {padding:4px;;}
.active-button:hover { border-color:#ff6500}
.system {display:grid; grid-template-columns: 45% 1px auto; gap:20px; margin-left:5%; width:90%;}
.system .status .item {display:grid; grid-template-columns: 75px 8px auto; gap:2px;}
.input-form {display:grid; gap:2px;}
.input-form .item {display:grid; grid-template-columns: 125px auto; gap:2px; align-items:center;}
.input-form .item .label { font-weight:bold; padding-left:10px}
.fa-cog {color:#4682B4}
.fa-check {color:#00c6b3}
.fa-times {color:maroon}
.code {
margin:4px;
background:#000000 ;
padding:8px;
font-family: 'Courier New', Courier, monospace;
color:#d3d3d3;
font-size:12px;
line-height: 2;
}
.tabs {display:grid; grid-template-columns: repeat(3,1fr) auto; gap:0px; align-items:center; text-align: center;}
.tab {border:1px solid transparent; border-bottom-color:#D3D3D3; font-weight:bold; padding:4px}
.tabs .selected {border-color:#CAD5E0; border-bottom-color:transparent; }
.system iframe {width:100%; height:100%; border:1px solid transparent;}
.data-info {height:90%; padding:8px;}
.fa-cloud {color:#4682B4}
.fa-database{color:#cc8c91}

@ -0,0 +1,21 @@
<div class="dialog">
<div class="title-bar">
<div class="title bold" ></div>
<div class="active close" align="center"><i class="fas fa-times"></i></div>
</div>
<div class="message">
<div class="icon" align="center">
<i id="msg-icon"></i>
</div>
<div class="text"></div>
</div>
<div class="action">
<div class="active-button border-round" align="center">
<div align="center">
<i class="fas fa-check"></i>
</div>
<div class="bold">Ok</div>
</div>
</div>
</div>

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

@ -0,0 +1,75 @@
/***
* This file will handle the dialog boxes as they and their associated configurations and function binding
*/
if (!dialog){
var dialog = {}
}
dialog.open = function(title,msg,pointer){
if (sessionStorage.dialog == null){
var http = HttpClient.instance()
http.get(sessionStorage.io_context+'/static/dialog.html',function(x){
var html = x.responseText
jx.modal.show({html:html,id:'dialog'})
$('.dialog .title').text(title)
$('.dialog .message .text').text(msg)
dialog.status.ask()
$('.dialog .action .active-button').on('click',pointer)
$('.dialog .title-bar .close').on('click',function(){dialog.close(0)})
})
}else{
var html = sessionStorage.dialog
jx.modal.show({html:html,id:'dialog'})
dialog.status.ask()
$('.dialog .action .active-button').on('click',pointer)
$('.dialog .title-bar .close').on('click',function(){dialog.close(0)})
}
}
dialog.bind = function(pointer){
if (pointer == null){
pointer = dialog.close
}
$('.dialog .action .active-button').off()
$('.dialog .action .active-button').on('click',pointer)
}
dialog.close = function(delay){
delay = (delay == null)?1750:delay
setTimeout(function(){
if ( $('.dialog').length > 0){
jx.modal.close()
}
},delay)
}
dialog.status = {}
dialog.status.wait = function(){
$('.dialog .action .active-button').hide()
}
dialog.status.confirm = function(){
$('.dialog .action .active-button').show()
}
dialog.status.busy = function(){
$('.dialog .message #msg-icon').removeClass()
$('.dialog .message #msg-icon').addClass('fas fa-cog fa-4x fa-spin')
}
dialog.status.fail = function(){
$('.dialog .message #msg-icon').removeClass()
$('.dialog .message #msg-icon').addClass('fas fa-times fa-4x')
}
dialog.status.ask = function(){
$('.dialog .message #msg-icon').removeClass()
$('.dialog .message #msg-icon').addClass('far fa-question-circle fa-4x')
}
dialog.status.warn = function(){
$('.dialog .message #msg-icon').removeClass()
$('.dialog .message #msg-icon').addClass('fas fa-exclamation-triangle fa-4x')
}
dialog.status.success = function(){
$('.dialog .message #msg-icon').removeClass()
$('.dialog .message #msg-icon').addClass('fas fa-check fa-4x')
}

@ -0,0 +1,15 @@
if (!healthcare) {
var healthcare = {io:{}}
}
healthcare.io = {'dialog':dialog,'confirmed':confirmed,'reset':reset,'update':update,'run':run,'publish':publish}
healthcare.io.apply = function(){
var value = $('.input-form .item .procs').val()
var folder= $('.input-form .item .folder').val()
$('.code .batch').html(value)
var http = HttpClient.instance()
http.setData({"batch":value,"resume":true,"folder":folder},"application/json")
http.post(sessionStorage.io_context+'/io/params',function(x){})
}

@ -0,0 +1,182 @@
/**
* This file will depend on dialog.js (soft dependency). Some functions here will make calls to resources in dialog.js
*/
var reset = function(){
dialog.open('Healthcare/IO::Parser', 'Are you sure you would like to delete all data parsed? Click Ok to confirm',confirmed.reset)
}
var update= function(){
dialog.open('Healthcare/IO::Parser','Update will change parsing configuration. Would you like to continue ?',confirmed.update)
}
var run = function(){
dialog.open('Healthcare/IO::Parser','Preparing parser, confirm to continue',confirmed.run)
}
var _queue = {socket:null}
var confirmed = {}
confirmed.run = function(){
dialog.status.busy()
dialog.status.wait()
$('.dialog .message .text').html('Initiating Parsing ...')
setTimeout(function(){
var http = HttpClient.instance()
http.post(sessionStorage.io_context+'/io/run',function(x){
// dialog.handler = setInterval(function(){monitor.data()},750)
monitor.data()
//dialog.close()
})
},1000)
}
confirmed.reset = function(){
var uri = sessionStorage.io_context+'/reset'
var http= HttpClient.instance()
dialog.status.busy()
dialog.status.wait()
http.post(uri,function(x){
setTimeout(function(){
if (x.status == 200 && x.responseText == "1"){
dialog.status.success()
$('.dialog .message .text').html('Reset Healthcare/IO::Parser was successful!<br><div align="center">Dialog will be closing</div>')
dialog.close()
}else{
dialog.status.fail()
}
},2000)
})
}
confirmed.update = function(){
var uri = sessionStorage.io_context+'/update'
var email = $('#email').val()
//
//-- validate the email
if (email.match(/^([^\s]+)@([^\s@]+)\.(org|com|edu|io)$/i)){
dialog.status.wait()
dialog.status.busy()
var http = HttpClient.instance()
http.setData({"email":email},"application/son")
setTimeout(function(){
http.post(uri,function(x){
if(x.status == 200 && x.responseText == "1"){
dialog.status.success()
}else{
dialog.status.fail()
$('.dialog .message .text').html('Error code '+x.status)
dialog.bind()
dialog.status.confirm()
$('.dialog .title-bar .title').html("Error found")
}
})
},1000)
}else{
dialog.status.fail()
dialog.bind()
$('.dialog .title-bar .title').text("Error found")
$('.dialog .message .text').html('Invvalid Email entered')
dialog.status.confirm()
}
}
/**
* This namespace is designed to export data to either the cloud or to a database
*/
var publish={set:{}}
publish.post = function(){
if($('.jxmodal').length > 0){
jx.modal.close()
}
dialog.open('Export/ETL','Please wait')
dialog.status.busy()
var http = HttpClient.instance()
http.setData(JSON.parse(sessionStorage.export),"application/json")
http.post(sessionStorage.io_context+'/export',function(x){
if (x.status != 200){
setTimeout(function(){
$('.dialog .message .text').html('An error occurred with code '+x.status)
dialog.status.fail()
dialog.status.wait()
},1500)
}
//
// @TODO: Have progress be monitored for this bad boy i.e open the connection to socket and read in ...
//
})
}
publish.set.file = function(){
var file = $('#file')[0].files[0]
$('.file .name').html(file.name)
var button = $('.cloud input').prop('disabled',true)
var div = $('.cloud .file .fa-file-upload')[0]
$(div).empty()
$(div).addClass('fas fa-cog fa-spin')
var reader = new FileReader()
reader.readAsText(file)
reader.onload = function(){
_args = {"type":$('.cloud .id').html().trim(),"content":reader.result}
// _args = JSON.stringify(_args)
if (_args.content.match(/^\{.+/i) == null){
content = _args.content.split('\n')[1].split(',')
_args.content = {'bucket':'healthcareio','access_key':content[0].trim(),'secret_key':content[1].trim()}
}
sessionStorage.export = JSON.stringify(_args)
}
reader.onloadend = function(){
setTimeout(function(){
var div = $('.cloud .file .fa-cog')[0]
$(div).empty()
$(div).addClass('fas fa-check')
$(div).removeClass('fa-spin')
// jx.modal.close()
//setTimeout(jx.modal.close,1500)
publish.post()
},2000)
}
}
publish.database = {}
publish.database.init = function(id){
//
// we are expecting id in {mongo,couch,postgresql,mysql,sqlite}
// @TODO: Account for cloud service brokers like dropbox, box, one-drive and google-drive
sessionStorage.export = "{}"
p = {'id':id}
if (id.match(/(mongodb|postgresql|mysql|sqlite|couchdb)/i)){
var hide_id = '.store .cloud'
var show_id = '.store .database'
}else{
//
// @TODO: generate an error message
var show_id = '.store .cloud'
var hide_id = '.store .database'
}
var http = HttpClient.instance()
http.get(sessionStorage.io_context+'/export',function(x){
var html = x.responseText
jx.modal.show({'html':html,'id':'dialog'})
$(hide_id).hide(function(){
$(show_id).show()
})
$('.store .id').text(id)
})
}

@ -12,119 +12,128 @@
* Improve on how returned data is handled (if necessary).
*/
if(!jx){
var jx = {}
}
/**
* These are a few parsers that can come in handy:
* urlparser: This parser is intended to break down a url parameter string in key,value pairs
*/
function urlparser(url){
if(url.toString().match(/\x3F/) != null){
url = url.split('\x3F')[1]
}
var p = url.split('&') ;
var r = {} ;
r.meta = [] ;
r.data = {} ;
var entry;
for(var i=0; i < p.length; i++){
entry = p[i] ;
key = (entry.match('(.*)=') !=null)? entry.match('(.*)=')[1]:null ;
value = (entry.match('=(.*)$') != null)? entry.match('=(.*)$')[1]:null
if(key != null){
key = key.replace('\x3F','')
r.meta.push(key) ;
r.data[key] = value ;
}
}
return r.data;
}
/**
* The following are corrections related to consistency in style & cohesion
*/
jx.ajax = {}
jx.ajax.get = {} ;
jx.ajax.debug = null;
jx.ajax.get.instance = function(){
var factory = function(){
this.obj = {}
this.obj.headers = {}
this.obj.async = true;
this.setHeader = function(key,value){
if(key.constructor != String && value == null){
this.obj.headers = key ;
}else{
this.obj.headers[key] = value;
}
}
this.setData = function(data){
this.obj.data = data;
}
this.setAsync = function(flag){
this.obj.async = (flag == true) ;
}
this.send = function(url,callback,method){
if(method == null){
method = 'GET'
}
p = jx.ajax.debug != null;
q = false;
if(p){
q = jx.ajax.debug[url] != null;
}
is_debuggable = p && q
if(is_debuggable){
x = {} ;
x.responseText = jx.ajax.debug [url] ;
callback(x)
}else{
var http = new XMLHttpRequest() ;
http.onreadystatechange = function(){
if(http.readyState == 4){
callback(http)
}
}
//
// In order to insure backward compatibility
// Previous versions allowed the user to set the variable on the wrapper (poor design)
if(this.async != null){
this.setAsync(this.async) ;
}
http.open(method,url,this.obj.async) ;
for(key in this.obj.headers){
value = this.obj.headers[key] ;
http.setRequestHeader(key,value)
}
http.send(this.obj.data)
}
}
this.put = function(url,callback){
this.send(url,callback,'PUT') ;
}
this.get = function(url,callback){
this.send(url,callback,'GET') ;
}
this.post = function(url,callback){
this.send(url,callback,'POST') ;
}
}//-- end of the factory method
return new factory() ;
}
//
// backward compatibility
jx.ajax.getInstance = jx.ajax.get.instance ;
var HttpClient = jx.ajax.get ;
var jx = {}
}
/**
* These are a few parsers that can come in handy:
* urlparser: This parser is intended to break down a url parameter string in key,value pairs
*/
function urlparser(url){
if(url.toString().match(/\x3F/) != null){
url = url.split('\x3F')[1]
}
var p = url.split('&') ;
var r = {} ;
r.meta = [] ;
r.data = {} ;
var entry;
for(var i=0; i < p.length; i++){
entry = p[i] ;
key = (entry.match('(.*)=') !=null)? entry.match('(.*)=')[1]:null ;
value = (entry.match('=(.*)$') != null)? entry.match('=(.*)$')[1]:null
if(key != null){
key = key.replace('\x3F','')
r.meta.push(key) ;
r.data[key] = value ;
}
}
return r.data;
}
/**
* The following are corrections related to consistency in style & cohesion
*/
jx.ajax = {}
jx.ajax.get = {} ;
jx.ajax.debug = null;
jx.ajax.get.instance = function(){
var factory = function(){
this.obj = {}
this.obj.headers = {}
this.obj.async = true;
this.setHeader = function(key,value){
if(key.constructor != String && value == null){
this.obj.headers = key ;
}else{
this.obj.headers[key] = value;
}
}
this.setData = function(data,mimetype){
if(mimetype == null)
this.obj.data = data;
else {
this.obj.headers['Content-Type'] = mimetype
if(mimetype.match(/application\/json/i)){
this.obj.data = JSON.stringify(data)
}
}
}
this.setAsync = function(flag){
this.obj.async = (flag == true) ;
}
this.send = function(url,callback,method){
if(method == null){
method = 'GET'
}
p = jx.ajax.debug != null;
q = false;
if(p){
q = jx.ajax.debug[url] != null;
}
is_debuggable = p && q
if(is_debuggable){
x = {} ;
x.responseText = jx.ajax.debug [url] ;
callback(x)
}else{
var http = new XMLHttpRequest() ;
http.onreadystatechange = function(){
if(http.readyState == 4){
callback(http)
}
}
//
// In order to insure backward compatibility
// Previous versions allowed the user to set the variable on the wrapper (poor design)
if(this.async != null){
this.setAsync(this.async) ;
}
http.open(method,url,this.obj.async) ;
for(key in this.obj.headers){
value = this.obj.headers[key] ;
http.setRequestHeader(key,value)
}
http.send(this.obj.data)
}
}
this.put = function(url,callback){
this.send(url,callback,'PUT') ;
}
this.get = function(url,callback){
this.send(url,callback,'GET') ;
}
this.post = function(url,callback){
this.send(url,callback,'POST') ;
}
}//-- end of the factory method
return new factory() ;
}
//
// backward compatibility
jx.ajax.getInstance = jx.ajax.get.instance ;
var HttpClient = jx.ajax.get ;

@ -0,0 +1,29 @@
<meta charset="utf8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" />
<meta http-equiv="Pragma" content="no-cache" />
<meta http-equiv="Expires" content="0" />
<link rel="shortcut icon" href="{{context}}/static/img/logo.svg" type="image/icon type">
<link rel="stylesheet" href="{{context}}/static/css/default.css" type="text/css">
<script src="{{context}}/static/js/jx/rpc.js"></script>
<script src="{{context}}/static/js/jx/dom.js"></script>
<script src="{{context}}/static/js/jx/utils.js"></script>
<script src="{{context}}/static/js/jquery.js"></script>
<script src="https://cdn.jsdelivr.net/npm/apexcharts"></script>
<link href="{{context}}/static/css/borders.css" type="text/css" rel="stylesheet">
<link href="{{context}}/static/css/fa/css/all.css" type="text/css" rel="stylesheet">
<script src="{{context}}/static/css/fa/js/all.js"></script>
<div class="border-round border menu-bar">
<div class="menu">
Admin
<div class="menu-items border">
<div class="item">Setup</div>
</div>
</div>
<div class="menu">
Claims & Remits
</div>
</div>

@ -8,6 +8,7 @@
<script src="{{context}}/static/js/jx/rpc.js"></script>
<script src="{{context}}/static/js/jx/dom.js"></script>
<script src="{{context}}/static/js/jx/utils.js"></script>
<script src="{{context}}/static/js/jx/ext/modal.js"></script>
<script src="{{context}}/static/js/jquery.js"></script>
@ -15,6 +16,9 @@
<link href="{{context}}/static/css/borders.css" type="text/css" rel="stylesheet">
<link href="{{context}}/static/css/fa/css/all.css" type="text/css" rel="stylesheet">
<script src="{{context}}/static/css/fa/js/all.js"></script>
<script src="{{context}}/static/js/io/dialog.js"></script>
<script src="{{context}}/static/js/io/io.js"></script>
<script src="{{context}}/static/js/io/healthcare.js"></script>
<style>
body {
font-size:16px;
@ -64,6 +68,7 @@
scroll-behavior: smooth;
gap:2px;
padding:4px;
height:95%;
}
@ -81,12 +86,12 @@
}
.dashboard .chart-pane .chart {
.dashboard .chart-pane .chart2 {
max-height:99%;
min-height:99%;
height:99%;
}
.dashboard .chart-pane .chart .graph {
.dashboard .chart-pane .chart2 .graph {
max-height:100%;
@ -95,7 +100,7 @@
}
.dashboard .chart-pane .chart .graph .apexcharts-svg {
.dashboard .chart-pane .chart2 .graph .apexcharts-svg {
/*border-radius:8px;*/
background-color:#f3f3f3;
max-height:100%;
@ -161,7 +166,7 @@
}
.gradient { background-image: linear-gradient(to top,#F3F3F3, #D3D3D3, #F3F3F3);}
.gradient { background-image: linear-gradient(to top,#F3F3F3, #FFFFFF, #FFFFFF);}
.white {color:#ffffff}
.scalar {
display:grid;
@ -172,23 +177,39 @@
}
.shadow {
box-shadow: 0px 4px 4px #d3d3d3;
}
.scalar-title { padding:8px; text-transform: capitalize; }
.scalar .value {font-size:32px; font-weight:bold; margin:4%;}
.scalar .value {font-size:32px; margin:4%;}
.scalar .label {font-size:12px; text-transform:capitalize; text-overflow: ellipsis; display:grid; align-items: flex-end; text-align:center ;}
.monthly_patient_count, .taxonomy_code_distribution, .top_adjustment_codes, .adjustment_reasons {
grid-row:1 / span 3 ;
}
</style>
<script>
sessionStorage.io_context = "{{context}}"
var plot = function(id,index){
$('.system').slideUp(function(){
$('.dashboard').slideDown()
})
var uri = ([sessionStorage.io_context,'format',id,index]).join('/')
var httpclient = HttpClient.instance()
//
// @TODO: Let the user know something is going on .. spinner
httpclient.post(uri,function(x){
var r = JSON.parse(x.responseText)
var pane = jx.dom.get.instance('DIV')
var scalar_pane = jx.dom.get.instance('DIV')
pane.id = r.id
@ -202,9 +223,9 @@
var p = jx.utils.patterns.visitor(r.pipeline,function(item){
var div = jx.dom.get.instance('DIV')
var frame = jx.dom.get.instance('DIV')
//div.className = 'chart border-round border'
frame.className = 'chart border'
div.className = 'graph'
//div.className = 'chart2 border-round border'
frame.className = 'chart2 border ' + item.label.toLowerCase().replace(/ /g,'_')
div.className = 'graph '
//frame.append(div)
//pane.append(frame)
@ -212,8 +233,11 @@
if(item.apex != null){ item.apex.title = {text:item.label}
frame.append(div)
pane.append(frame)
delete item.apex.colors
item.apex.theme= {
if (item.apex.colors ){
delete item.apex.colors
}
/*item.apex.theme= {
mode: 'material',
palette: 'palette6',
monochrome: {
@ -222,8 +246,11 @@
shadeTo: 'light',
shadeIntensity: 0.65
},
}
}*/
item.apex.chart.height = '100%'
delete item.apex.chart.width
return new ApexCharts(div,item.apex)
}else{
//frame.className = ''
@ -283,50 +310,80 @@
}
}
$(document).ready(function(){
$('.dashabord').hide()
$('.item-group').slideUp()
var index = 0;
jx.utils.patterns.visitor($('.menu .items'), function(_item){
var node = $(_item).children()[0]
$(node).attr('index',index)
node.onclick = function(){ toggle($(this).attr('index')) }
index += 1;
})
var year = (new Date()).getFullYear()
$('.year').text(year)
})
</script>
<title>Healthcare/IO Analytics</title>
<body>
<div class="pane border">
<div class="pane">
<div class="header border-bottom">
<div class="caption">Healthcare/IO</div>
<div class="small">Analytics Dashboard</div>
<div class="caption">Healthcare/IO :: Parser</div>
<div class="small">Dashboard</div>
</div>
<div class="menu border-right">
<div class="menu">
<div>
<div class="items">
<div class="bold active" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;">
<div>Setup</div>
<div align="center" class="glyph" >
<i class="fas fa-angle-down"></i>
</div>
</div>
<div class="item-group border border-round">
<div class="item small active" onclick="setup.open()">Configure Parser</div>
<div class="item small active" onclick="healthcare.io.reset()">Reset Parser</div>
</div>
</div>
{% for key in sections %}
<div class="items">
<div class="bold active" onclick="toggle({{loop.index -1}})" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;">
<div>{{key|safe}}</div>
<div align="center" class="glyph" >
<div class="items">
<div class="bold active" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;">
<div>{{key|safe}}</div>
<div align="center" class="glyph" >
<i class="fas fa-angle-down"></i>
</div>
</div>
<div class="item-group border border-round">
<i class="fas fa-angle-down"></i>
{% for item in sections[key] %}
<div class="item small active" onclick="plot('{{key}}',{{loop.index-1}})">{{item.id}}</div>
{% endfor %}
</div>
</div>
<div class="item-group border">
{% for item in sections[key] %}
<div class="item small active" onclick="plot('{{key}}',{{loop.index-1}})">{{item.id}}</div>
{% endfor %}
</div>
</div>
{% endfor %}
{% endfor %}
</div>
<div style="display:grid; align-items:flex-end">
<div class="logs border" style="height:250px"></div>
<div style="display:none; align-items:flex-end">
<div class="logs chart" style=" align-items:center; display:grid" align="center"></div>
</div>
</div>
<div class="dashboard">
<div>
<div class="dashboard" style="display:none"></div>
</div>
{%include 'setup.html' %}
</div>
</div>
<div class="footer small">
&copy; Vanderbilt University Medical Center
Healthcare/IO :: Parser &copy; <span class="year"> </span>
</div>
</body>

@ -0,0 +1,391 @@
<style>
.system {height:99%; overflow:hidden;}
.data-info .board{ height:300px; display:grid; grid-template-columns:auto 200px 200px; gap:20px; align-items:center}
/*.board { background-image: linear-gradient(to bottom, #ffffff,#ffffff,#f3f3f3,#d3d3d3d3)}*/
.number {font-size:48px; font-family:courier;padding:8px; ;}
.etl {display:grid; grid-template-columns: 250px auto; gap:2;}
.chart {box-shadow : 0px 1px 4px 2px #d3d3d3; width:200px; height:250px;
display:grid; align-items:center;
background-image: linear-gradient(to bottom,#f3f3f3,#ffffff);
overflow:hidden;
}
.dialog { width:450px; min-height:200px; display:grid; grid-template-rows: 40px 80% auto; gap:4px}
.dialog .title-bar { border-top-left-radius: 8px; border-top-right-radius: 8px ; padding:4px; background-color:#f3f3f3; gap:2px; display:grid; grid-template-columns: auto 32px; align-items:center}
.dialog .action {display:grid; align-items: flex-end; padding-left:25%; padding-right:25%;}
.dialog .message {display:grid; align-items: center; grid-template-columns: 20% auto;}
.dialog .message .text {line-height:2; text-transform: capitalize;}
.fa-exclamation-triangle {color:orange}
.fa-question-circle{color:#009df7}
</style>
<script src="https://cdn.socket.io/socket.io-1.3.5.js"></script>
<script src="{{context}}/static/js/io/dialog.js"></script>
<script src="{{context}}/static/js/io/io.js"></script>
<script src="{{context}}/static/js/io/healthcare.js"></script>
<script>
var select = function(node){
var value = $($(node).children()[0]).attr('data-value')
jx.utils.patterns.visitor($('.tab'),function(_item){
var button = $(_item).children()[0]
$(_item).removeClass('selected')
//alert([$(button).attr('data-value'),value])
if($(button).attr('data-value') == value){
$(node).addClass('selected')
$('.'+value).show()
}else{
var m = '.'+ $(button).attr('data-value')
$(m).hide()
}
})
}
var monitor = {}
monitor.listen = {handler:null}
monitor.data = function(){
var http = HttpClient.instance()
http.get("/data",function(x){
var r = JSON.parse(x.responseText)
var keys = jx.utils.keys(r) //-- process,files
for (var i in keys){
var prefix = keys[i]
if(prefix == 'process'){
if(r[prefix].counts != 0){
//
// We should insure the listeners are enabled
if(monitor.listen.handler == null){
monitor.listen.handler = setInterval(
function(){
console.log('running ...')
monitor.data()},5000)
}
}else{
if (monitor.listen.handler != null){
clearInterval(monitor.listen.handler)
}
dialog.close()
}
}
monitor.render(prefix,r[prefix])
}
})
}
monitor.render = function(prefix,r){
prefix = '.'+prefix
var div = jx.dom.get.instance('DIV')
var label = jx.dom.get.instance('DIV')
div.align = 'center'
div.innerHTML = r.counts
div.className = 'number'
label.innerHTML = prefix.replace(/\./,'')
label.style.textTransform = 'capitalize'
label.className = 'small bold border-top'
div.append(label)
$(prefix + ' .board').empty()
$(prefix+' .board').append(div)
var charts = jx.utils.patterns.visitor(r.chart,function(option){
var div = jx.dom.get.instance('div')
div.className = 'chart'
div.align='center'
$(prefix+' .board').append(div)
var chart = new ApexCharts($(div)[0],option)
//chart.render()
div.chart = chart
return chart
})
var observers = jx.utils.patterns.visitor(charts,function(_item){
var m = function(_chart){
this.chart = _chart ;
this.apply = function(caller){this.chart.render();
caller.notify()
}
}
return new m(_item)
})
jx.utils.patterns.observer(observers,'apply')
//jx.utils.patterns.iterator(charts,'render')
/*setTimeout(function(){
jx.utils.patterns.visitor(charts,function(_item){_item.render()})
},1000) */
}
var setup = {}
setup.open = function(){
$('.dashboard').slideUp(
function(){
$('.setup').slideDown()
}
)
}
$(document).ready(function(){
/*var shandler = new io();
if (shandler.disconnected ==false){
shandler.disconnect()
}
var socket = io.connect()
socket.on('connect',function(e){
socket.emit('connect',{name:'steve'})
})
socket.on('update',function(e){
console.log(e)
console.log()
})
var socket = io.connect('http://localhost:81',{cors:{AccessControlAllowOrigin:'*'}}) //('http://localhost:81/stream')
socket.on('procs',function(e){
})
socket.on('data',function(e){
$('.logs').empty()
var div = $('.logs')
var option = e.apex
option.plotOptions.pie.size = 220
option.plotOptions.pie = {dataLabels: {show:true,name:{show:true},value:{show:true}}}
option.legend.show = false
console.log(option)
c = new ApexCharts(div[0],option)
c.render()
socket.emit("procs",{"name":"steve"})
})*/
select($('.tab')[0])
monitor.data()
$('.email').text($('#email').val())
})
</script>
<div class="system setup">
<div class="status">
{% if not store.type %}
<div >
<span class="caption bold border-bottom" style="padding-right:10">Current Configuration</span>
<p></p>
<div style="display:grid; align-items:center; grid-template-columns:32px auto;">
<i class="fa fa-times" style="font-size:28; margin:4px;"></i> <span>System needs to be initialized !</span>
</div>
<p></p>
<div class="active-button border-round" style="width:50%">
<div class="icon">
<i class="fas fa-cog" style="font-size:28"></i>
</div>
<div class="bold" align="center">Initialize</div>
</div>
</div>
{% else %}
<div class="border-right">
<span class="caption bold border-bottom" style="padding-right:10">Current Configuration</span>
<p>
</p>
<div class="item" style="display:grid; align-items:center">
<div class="bold" style="text-transform: capitalize;">Owner </div><div class="bold">:</div>
<div>
<input type="text" id="email" value="{{owner}}">
</div>
</div>
<div class="item">
<div class="bold" style="text-transform: capitalize;">store</div><div class="bold">:</div>
<div class="">{{store.type}}</div>
</div>
<p></p>
<div class="active-button border-round" style="width:50%" onclick="healthcare.io.update()">
<div class="icon">
<I class="fas fa-download" style="font-size:28"></I>
</div>
<div class="bold" align="center">Update Config</div>
</div>
<p>
<div class="code">
#<br>
healthcare-io.py --init <span class="email"></span> --store mongo
</div>
</p>
</div>
{%endif%}
<p></p>
<br>
<div class="border-right">
<span class="caption bold border-bottom" style="padding-right:10">Manage Plan</span>
<p></p>
<div style="line-height: 2;">Insure your account is tied to a cloud service provider.
<br>We support <span class="bold">google-drive, dropbox, one-drive or box. </span>
</div>
<p>
<div class="bold active-button border-round" style="width:50%" onclick="jx.modal.show({url:'https://healthcareio.the-phi.com/store/healthcareio/plans'})">
<div>
<img src="{{context}}/static/img/logo.svg" />
</div>
<div align="center">Open Plan Console</div>
</div>
</p>
</div>
<br>
<div class="border-right" style="height:30%"></div>
</div>
<div class="_border-right"></div>
<div >
<span class="caption bold border-bottom" style="padding-right:10">
Manage Processes</span>
<p>
<div class="input-form" style="grid-template-columns: 30% auto;">
<div class="item" style="grid-row:1; grid-column:1; ">
<div class="label">Process #</div>
<input type="text" class="procs batch"placeholder="#" style="width:64px; text-align:right" value="{{args.batch}}" onchange="healthcare.io.apply()"/>
</div>
<div class="item" style="grid-row:1; grid-column:2 ">
<div class="label">Folder #</div>
<input type="text" placeholder="Process counts" value="/data"/ class="data folder" disabled>
</div>
</div>
<div style="display:grid; grid-template-columns:repeat(2,215px); gap:2px;">
<div class="active-button border-round bold io-apply" style="margin-top:32; display:none" onclick="healthcare.io.apply()">
<div class="icon"><i class="far fa-save" style="font-size:28; color:#4682B4"></i></div>
<div align="center">Apply</div>
</div>
<div class="active-button border-round bold" style="margin-top:32" onclick="healthcare.io.stop()">
<i class="far fa-stop-circle" style="font-size:28; color:maroon"></i>
<div align="center">Stop</div>
</div>
<div class="active-button border-round bold" style="margin-top:32" onclick="healthcare.io.run()">
<i class="fas fa-running" style="font-size:28; color:green"></i>
<div align="center">Run</div>
</div>
</div>
</p>
<p>
<div class="code">
<div class="bold"># The command that will be executed</div>
<div>healthcare-io.py --parse --folder /data --batch <span class="batch">{{args.batch}}</span></div>
</div>
</p>
<p>
<div style="display:grid; grid-template-columns:auto 48px ; gap:2px">
<div class="bold caption border-bottom">Process Monitoring</div>
<div class="active" align="center" title="reload" onclick="monitor.data()"><i class="fas fa-sync"></i></div>
</div>
<div class="small">Powered by smart-top</div>
<p></p>
<div class="tabs">
<div class="tab selected" onclick="select(this)">
<div class="active" data-value="process" >
<i class="far fa-clock" style="color:maroon"></i>
<span>Process</span>
</div>
</div>
<div class="tab" onclick="select(this)">
<div class="active" data-value="files"><i class="fas fa-file-alt"></i> Files</div>
</div>
<div class="tab" onclick="select(this)">
<div class="active" data-value="export"><i class="fas fa-upload"></i> Export</div>
</div>
</div>
<div class="data-info">
<div class="process ">
<div class="board"></div>
<div class="small" align="center">
<div class="border-top bold" style="color:#4682B4;">Running Processes and resource usage</div>
</div>
</div>
<div class="files">
<div class="board"></div>
<div class="small" align="center">
<div class="border-top bold" style="color:#4682B4;">Summary of files found and processed</div>
</div>
</div>
<div class="export">
<p></p>
<div class="etl">
<div class="" >
<div class="menu" style="position:absolute; width:200">
<div class="items ">
<div class="bold active" style="display:grid; grid-template-columns:80% auto;">
<span>
<i class="fas fa-cloud"></i>
Cloud</span>
<span class="glyph">
<i class="fas fa-angle-down"></i>
</span>
</div>
<div class="item-group border-round border small">
<div class="item" onclick="healthcare.io.publish.database.init('s3')">AWS S3</div>
<div class="item" onclick="healthcare.io.publish.database.init('bigquery')">Google Bigquery</div>
</div>
</div>
<div class="items ">
<div class="bold active"style="display:grid; grid-template-columns:80% auto;">
<span>
<i class="fas fa-database"></i>
Database</span>
<span class="glyph">
<i class="fas fa-angle-down"></i>
</span>
</div>
<div class="item-group border-round border small">
<div class="bold">SQL</div>
<div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('postgresql')">PostgreSQL</div>
<div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('mysql')">MySQL</div>
<div class="bold">NoSQL</div>
<div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('mongodb')">Mongodb</div>
<div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('couchdb')">Couchdb</div>
</div>
</div>
</div>
</div>
<div>
<div class="active-button border-round" style="width:50%">
<div class="icon"><i class="fas fa-running" style="font-size:28"></i></div> <div class="bold" align="center">Start</div>
</div>
</div>
</div>
</div>
</div>
</p>
</div>
</div>

@ -0,0 +1,78 @@
<link href="{{context}}/static/css/fa/css/all.css" type="text/css" rel="stylesheet">
<link rel="stylesheet" href="{{context}}/static/css/default.css" type="text/css">
<link href="{{context}}/static/css/borders.css" type="text/css" rel="stylesheet">
<style>
.form , input{font-family:sans-serif; font-size:18px}
.form .small {font-size:14px; line-height:2}
.mongo, .couchdb{
display:grid;
gap:2px;
}
.grid-full {display:grid; grid-template-columns: 100%;}
.grid-split-half {display:grid; grid-template-columns: 50% 50%; gap:2px;}
.store .title-bar {display:grid; align-items:center; grid-template-columns: auto 32px; padding:8px;}
.file {display:grid; align-items:center; grid-template-columns: 40% auto; gap:2px; font-family:sans-serif; padding:8px}
.file input {display:none}
.file label {padding:8px;}
</style>
<div class="store">
<div class="title-bar">
<div class="caption">Export Module <span class="bold id"></span></div>
<div class="active" align="center" onclick="jx.modal.close()">
<i class="fas fa-times"></i>
</div>
</div>
<p></p>
<div class="cloud form border-round">
<div class="small border-round">
Please provide select access-key file or service account key file to perform ETL
to <span class="bold id"></span>. <br>The files will be re-created in JSON format
</div>
<div class="file">
<label class="active-button border-round">
<span style="font-family:sans-serif">
<i class="icon fas fa-file-upload" style="font-size:24px"></i>
</span>
<div>Select key file</div>
<input type="file" id="file" aria-label="File browser example" onchange="publish.set.file()">
</label>
<div class="name small black bold" style="padding-left:24px"></div>
</div>
</div>
<div class="database form border-round">
<div class="small border-round">
Tables / collections will be automatically inferred in the <span class="bold id"></span>
</div>
<div class="mongo">
<div class="grid-split-half">
<input type="text" class="host" placeholder="host:port"/>
<input type="text" class="host" placeholder="database"/>
</div>
<div class="grid-split-half">
<input type="text" placeholder="user"/>
<input type="text" placeholder="password"/>
</div>
</div>
<p></p>
<div>
<br>
<div class="active-button border-round" style="margin-left:30%; margin-right:30%">
<div class="icon">
<i class="fas fa-check"></i>
</div>
<div class="bold" align="center">Export Now</div>
</div>
</div>
</div>
</div>

@ -49,8 +49,9 @@ class Formatters :
"""
This function is designed to split an x12 row and
"""
value = []
if row.startswith(prefix) is False:
value = []
for row_value in row.replace('~','').split(sep) :
@ -65,10 +66,12 @@ class Formatters :
else :
value.append(row_value.replace('\n',''))
return [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep)
value = [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep)
else:
return [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ]
value = [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ]
return value if type(value) == list and type(value[0]) != list else value[0]
def get_config(self,config,row):
"""
This function will return the meaningfull parts of the configuration for a given item
@ -130,7 +133,7 @@ class Formatters :
terms = value[1].split('>')
return {'type':terms[0],'code':terms[1],"amount":float(value[2])}
else:
return {"code":value[2],"type":value[1],"amount":float(value[3])}
def sv2(self,value):
#
@ -191,6 +194,9 @@ class Parser (Process):
self.files = []
self.set = void()
self.set.files = self.set_files
self.emit = void()
self.emit.pre = None
self.emit.post = None
def set_files(self,files):
self.files = files
def get_map(self,row,config,version=None):
@ -279,11 +285,12 @@ class Parser (Process):
row = util.split(row.replace('\n','').replace('~',''))
_info = util.get.config(self.config[_code][0],row)
if _info :
try:
tmp = self.get.value(row,_info)
# if 'P1080351470' in content[0] and 'PLB' in row:
# print (_info)
# print (row)
@ -300,10 +307,10 @@ class Parser (Process):
else:
if label not in value:
value[label] = [tmp]
elif len(list(tmp.keys())) == 1 :
# print "\t",len(claim[label]),tmp
index = len(value[label]) -1
value[label][index] = dict(value[label][index],**tmp)
# elif len(list(tmp.keys())) == 1 :
# # print "\t",len(claim[label]),tmp
# index = len(value[label]) -1
# value[label][index] = dict(value[label][index],**tmp)
else:
value[label].append(tmp)
tmp['_index'] = len(value[label]) -1
@ -327,7 +334,7 @@ class Parser (Process):
pass
except Exception as e :
print ('__',e.args)
pass
@ -432,6 +439,9 @@ class Parser (Process):
# self.finish(claims,logs,_code)
return claims,logs,_code
def run(self):
if self.emit.pre :
self.emit.pre()
for filename in self.files :
content,logs,_code = self.read(filename)
self.finish(content,logs,_code)
@ -441,14 +451,22 @@ class Parser (Process):
if args['type'] == 'mongo.MongoWriter' :
args['args']['doc'] = 'claims' if _code == '837' else 'remits'
_args['args']['doc'] = 'logs'
else:
args['args']['table'] = 'claims' if _code == '837' else 'remits'
_args['args']['table'] = 'logs'
if content :
writer = transport.factory.instance(**args)
writer.write(content)
writer.close()
if logs :
logger = transport.factory.instance(**_args)
logger.write(logs)
logger.close()
if self.emit.post :
self.emit.post(content,logs)

@ -8,7 +8,7 @@ import sys
def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = {
"name":"healthcareio","version":"1.3.7",
"name":"healthcareio","version":"1.4.0",
"author":"Vanderbilt University Medical Center",
"author_email":"steve.l.nyemba@vumc.org",
"license":"MIT",

Loading…
Cancel
Save