From 2d976cd607fae10e412fe6f3221f48691ec5c91e Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Tue, 6 Feb 2024 10:58:08 -0600 Subject: [PATCH] bug fix, logging --- healthcareio/__init__.py | 2 +- healthcareio/x12/__init__.py | 1278 +++++++++++++++++----------------- 2 files changed, 640 insertions(+), 640 deletions(-) diff --git a/healthcareio/__init__.py b/healthcareio/__init__.py index 43f60d1..0c358ce 100644 --- a/healthcareio/__init__.py +++ b/healthcareio/__init__.py @@ -21,6 +21,6 @@ Usage : import healthcareio.x12 as x12 from healthcareio.x12 import publish, plugins import healthcareio.params as params - +from healthcareio import logger # from healthcareio import server diff --git a/healthcareio/x12/__init__.py b/healthcareio/x12/__init__.py index 287bc96..ea0b390 100644 --- a/healthcareio/x12/__init__.py +++ b/healthcareio/x12/__init__.py @@ -17,770 +17,770 @@ - EMBEDDED """ -import hashlib -import json -import os -import sys -# version 2.0 -# import util -# from parser import X12Parser -#-- end - -from itertools import islice -from multiprocessing import Process -import transport -from transport import providers -import jsonmerge -# import plugins - -import copy - -class void : - pass -class Formatters : - def __init__(self): - # self.config = config - self.get = void() - self.get.config = self.get_config +# import hashlib +# import json +# import os +# import sys +# # version 2.0 +# # import util +# # from parser import X12Parser +# #-- end + +# from itertools import islice +# from multiprocessing import Process +# import transport +# from transport import providers +# import jsonmerge +# # import plugins + +# import copy + +# class void : +# pass +# class Formatters : +# def __init__(self): +# # self.config = config +# self.get = void() +# self.get.config = self.get_config - self.parse = void() - self.parse.sv3 = self.sv3 - self.parse.sv2 = self.sv2 - self.sv2_parser = self.sv2 - self.sv3_parser = self.sv3 - self.sv3_parse = self.sv3 - self.format_proc = self.procedure - self.format_diag = self.diagnosis - self.parse.procedure = self.procedure - self.parse.diagnosis = self.diagnosis - self.parse.date = self.date - self.format_date = self.date - self.format_pos = self.pos - self.format_time = self.time - def split(self,row,sep='*',prefix='HI') : - """ - This function is designed to split an x12 row and - """ +# self.parse = void() +# self.parse.sv3 = self.sv3 +# self.parse.sv2 = self.sv2 +# self.sv2_parser = self.sv2 +# self.sv3_parser = self.sv3 +# self.sv3_parse = self.sv3 +# self.format_proc = self.procedure +# self.format_diag = self.diagnosis +# self.parse.procedure = self.procedure +# self.parse.diagnosis = self.diagnosis +# self.parse.date = self.date +# self.format_date = self.date +# self.format_pos = self.pos +# self.format_time = self.time +# def split(self,row,sep='*',prefix='HI') : +# """ +# This function is designed to split an x12 row and +# """ - value = [] - if row.startswith(prefix) is False: +# value = [] +# if row.startswith(prefix) is False: - for row_value in row.replace('~','').split(sep) : +# for row_value in row.replace('~','').split(sep) : - if '>' in row_value and not row_value.startswith('HC'): - # if row_value.startswith('HC') or row_value.startswith('AD'): - if row_value.startswith('AD'): +# if '>' in row_value and not row_value.startswith('HC'): +# # if row_value.startswith('HC') or row_value.startswith('AD'): +# if row_value.startswith('AD'): - value += row_value.split('>')[:2] - pass - else: +# value += row_value.split('>')[:2] +# pass +# else: - value += [row_value] - # value += row_value.split('>') if row.startswith('CLM') is False else [row_value] +# value += [row_value] +# # value += row_value.split('>') if row.startswith('CLM') is False else [row_value] - else : +# else : - value.append(row_value.replace('\n','')) - value = [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep) - else: +# value.append(row_value.replace('\n','')) +# value = [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep) +# else: - value = [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ] +# value = [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ] - return value if type(value) == list and type(value[0]) != list else value[0] - def get_config(self,config,row): - """ - This function will return the meaningfull parts of the configuration for a given item - """ +# return value if type(value) == list and type(value[0]) != list else value[0] +# def get_config(self,config,row): +# """ +# This function will return the meaningfull parts of the configuration for a given item +# """ - _row = list(row) if type(row[0]) == str else list(row[0]) +# _row = list(row) if type(row[0]) == str else list(row[0]) - _info = config[_row[0]] if _row[0] in config else {} - _rinfo = {} - key = None - if '@ref' in _info: - keys = list(set(_row) & set(_info['@ref'].keys())) - if keys : - _rinfo = {} - for key in keys : - _rinfo = jsonmerge.merge(_rinfo,_info['@ref'][key]) - return _rinfo - # key = key[0] - # return _info['@ref'][key] - else: - return {} +# _info = config[_row[0]] if _row[0] in config else {} +# _rinfo = {} +# key = None +# if '@ref' in _info: +# keys = list(set(_row) & set(_info['@ref'].keys())) +# if keys : +# _rinfo = {} +# for key in keys : +# _rinfo = jsonmerge.merge(_rinfo,_info['@ref'][key]) +# return _rinfo +# # key = key[0] +# # return _info['@ref'][key] +# else: +# return {} - if not _info and 'SIMILAR' in config: - # - # Let's look for the nearest key using the edit distance - if _row[0] in config['SIMILAR'] : - key = config['SIMILAR'][_row[0]] - _info = config[key] - return _info +# if not _info and 'SIMILAR' in config: +# # +# # Let's look for the nearest key using the edit distance +# if _row[0] in config['SIMILAR'] : +# key = config['SIMILAR'][_row[0]] +# _info = config[key] +# return _info - def hash(self,value): - salt = os.environ['HEALTHCAREIO_SALT'] if 'HEALTHCAREIO_SALT' in os.environ else '' - _value = str(value)+ salt - if sys.version_info[0] > 2 : - return hashlib.md5(_value.encode('utf-8')).hexdigest() - else: - return hashlib.md5(_value).hexdigest() - - def suppress (self,value): - return 'N/A' - def date(self,value): - value = value if type(value) != list else "-".join(value) - if len(value) > 8 or '-' in value: - # - # This is the case of a thru date i.e the first part should be provided in a 435 entry - # - fdate = "-".join([value[:8][:4],value[:8][4:6],value[:8][6:8]]) - tdate = "-".join([value[9:][:4],value[9:][4:6],value[9:][6:8]]) +# def hash(self,value): +# salt = os.environ['HEALTHCAREIO_SALT'] if 'HEALTHCAREIO_SALT' in os.environ else '' +# _value = str(value)+ salt +# if sys.version_info[0] > 2 : +# return hashlib.md5(_value.encode('utf-8')).hexdigest() +# else: +# return hashlib.md5(_value).hexdigest() + +# def suppress (self,value): +# return 'N/A' +# def date(self,value): +# value = value if type(value) != list else "-".join(value) +# if len(value) > 8 or '-' in value: +# # +# # This is the case of a thru date i.e the first part should be provided in a 435 entry +# # +# fdate = "-".join([value[:8][:4],value[:8][4:6],value[:8][6:8]]) +# tdate = "-".join([value[9:][:4],value[9:][4:6],value[9:][6:8]]) - return {"from":fdate,"to":tdate} - - if len(value) == 8 : - year = value[:4] - month = value[4:6] - day = value[6:] - return "-".join([year,month,day])[:10] #{"year":year,"month":month,"day":day} - elif len(value) == 6 : - year = '20' + value[:2] - month = value[2:4] - day = value[4:] - elif value.isnumeric() and len(value) >= 10: - # - # Here I a will assume we have a numeric vale - year = value[:4] - month= value[4:6] - day = value[6:8] - else: +# return {"from":fdate,"to":tdate} + +# if len(value) == 8 : +# year = value[:4] +# month = value[4:6] +# day = value[6:] +# return "-".join([year,month,day])[:10] #{"year":year,"month":month,"day":day} +# elif len(value) == 6 : +# year = '20' + value[:2] +# month = value[2:4] +# day = value[4:] +# elif value.isnumeric() and len(value) >= 10: +# # +# # Here I a will assume we have a numeric vale +# year = value[:4] +# month= value[4:6] +# day = value[6:8] +# else: - # - # We have a date formatting issue - return value - return "-".join([year,month,day]) - def time(self,value): - pass - def sv3(self,value): - if '>' in value [1]: - terms = value[1].split('>') - return {'type':terms[0],'code':terms[1],"amount":float(value[2])} - else: +# # +# # We have a date formatting issue +# return value +# return "-".join([year,month,day]) +# def time(self,value): +# pass +# def sv3(self,value): +# if '>' in value [1]: +# terms = value[1].split('>') +# return {'type':terms[0],'code':terms[1],"amount":float(value[2])} +# else: - return {"code":value[2],"type":value[1],"amount":float(value[3])} - def sv2(self,value): - # - # @TODO: Sometimes there's a suffix (need to inventory all the variations) - # - if '>' in value or ':' in value: - xchar = '>' if '>' in value else ':' - _values = value.split(xchar) - modifier = {} +# return {"code":value[2],"type":value[1],"amount":float(value[3])} +# def sv2(self,value): +# # +# # @TODO: Sometimes there's a suffix (need to inventory all the variations) +# # +# if '>' in value or ':' in value: +# xchar = '>' if '>' in value else ':' +# _values = value.split(xchar) +# modifier = {} - if len(_values) > 2 : - - modifier= {"code":_values[2]} - if len(_values) > 3 : - modifier['type'] = _values[3] - _value = {"code":_values[1],"type":_values[0]} - if modifier : - _value['modifier'] = modifier - - return _value - else: - return value +# if len(_values) > 2 : + +# modifier= {"code":_values[2]} +# if len(_values) > 3 : +# modifier['type'] = _values[3] +# _value = {"code":_values[1],"type":_values[0]} +# if modifier : +# _value['modifier'] = modifier + +# return _value +# else: +# return value - def procedure(self,value): +# def procedure(self,value): - for xchar in [':','<','|','>'] : +# for xchar in [':','<','|','>'] : - if xchar in value and len(value.split(xchar)) > 1 : - #_value = {"type":value.split(':')[0].strip(),"code":value.split(':')[1].strip()} - _value = {"type":value.split(xchar)[0].strip(),"code":value.split(xchar)[1].strip()} +# if xchar in value and len(value.split(xchar)) > 1 : +# #_value = {"type":value.split(':')[0].strip(),"code":value.split(':')[1].strip()} +# _value = {"type":value.split(xchar)[0].strip(),"code":value.split(xchar)[1].strip()} - if len(value.split(xchar)) >2 : - index = 1; - for modifier in value.split(xchar)[2:] : - _value['modifier_'+str(index)] = modifier - index += 1 - break - - else: - _value = str(value) - return _value - def diagnosis(self,value): - return [ {"code":item[2], "type":item[1]} for item in value if len(item) > 1] - def parse_loc(self,value): - if ':' in value : - return dict(zip(['place_of_service','claim_indicator','claim_frequency'],value.split(':'))) - def pos(self,value): - """ - formatting place of service information within a segment (REF) - @TODO: In order to accomodate the other elements they need to be specified in the configuration - Otherwise it causes problems on export - """ +# if len(value.split(xchar)) >2 : +# index = 1; +# for modifier in value.split(xchar)[2:] : +# _value['modifier_'+str(index)] = modifier +# index += 1 +# break + +# else: +# _value = str(value) +# return _value +# def diagnosis(self,value): +# return [ {"code":item[2], "type":item[1]} for item in value if len(item) > 1] +# def parse_loc(self,value): +# if ':' in value : +# return dict(zip(['place_of_service','claim_indicator','claim_frequency'],value.split(':'))) +# def pos(self,value): +# """ +# formatting place of service information within a segment (REF) +# @TODO: In order to accomodate the other elements they need to be specified in the configuration +# Otherwise it causes problems on export +# """ - xchar = '>' if '>' in value else ':' - x = value.split(xchar) - x = {"place_of_service":x[0],"indicator":x[1],"frequency":x[2]} if len(x) == 3 else {"place_of_service":x[0],"indicator":None,"frequency":None} - return x -class Parser (Process): - @staticmethod - def setup (path): - # self.config = _config['parser'] - config = json.loads(open(path).read()) - _config = config['parser'] - # - # The parser may need some editing provided, this allows ease of developement and using alternate configurations - # - if type(_config['837']) == str or type(_config['835']) == str : - for _id in ['837','835'] : - if type(_config[_id]) == str and os.path.exists(_config[_id]): - _config[_id] = json.loads(open(_config[_id]).read()) - if type(_config[_id]) == dict : - _config[_id] = [_config[_id]] - config['parser'] = _config - return config - @staticmethod - def init(**_args): - """ - This function allows to initialize the database that will store the claims if need be - :path configuration file - """ - PATH = os.sep.join([os.environ['HOME'],'.healthcareio']) - filename = os.sep.join([PATH,'config.json']) - - filename = _args['path'] if 'path' in _args else filename - info = None - if os.path.exists(filename): - # - # Loading the configuration file (JSON format) - file = open(filename) - info = json.loads(file.read()) +# xchar = '>' if '>' in value else ':' +# x = value.split(xchar) +# x = {"place_of_service":x[0],"indicator":x[1],"frequency":x[2]} if len(x) == 3 else {"place_of_service":x[0],"indicator":None,"frequency":None} +# return x +# class Parser (Process): +# @staticmethod +# def setup (path): +# # self.config = _config['parser'] +# config = json.loads(open(path).read()) +# _config = config['parser'] +# # +# # The parser may need some editing provided, this allows ease of developement and using alternate configurations +# # +# if type(_config['837']) == str or type(_config['835']) == str : +# for _id in ['837','835'] : +# if type(_config[_id]) == str and os.path.exists(_config[_id]): +# _config[_id] = json.loads(open(_config[_id]).read()) +# if type(_config[_id]) == dict : +# _config[_id] = [_config[_id]] +# config['parser'] = _config +# return config +# @staticmethod +# def init(**_args): +# """ +# This function allows to initialize the database that will store the claims if need be +# :path configuration file +# """ +# PATH = os.sep.join([os.environ['HOME'],'.healthcareio']) +# filename = os.sep.join([PATH,'config.json']) + +# filename = _args['path'] if 'path' in _args else filename +# info = None +# if os.path.exists(filename): +# # +# # Loading the configuration file (JSON format) +# file = open(filename) +# info = json.loads(file.read()) - OUTPUT_FOLDER = info['out-folder'] - if 'output-folder' not in info and not os.path.exists(OUTPUT_FOLDER) : - os.mkdir(OUTPUT_FOLDER) - elif 'output-folder' in info and not os.path.exists(info['out-folder']) : - os.mkdir(info['out-folder']) - # if 'type' in info['store'] : - lwriter = None - IS_SQL = False - if'type' in info['store'] and info['store']['type'] == 'disk.SQLiteWriter' : - lwriter = transport.factory.instance(**info['store']) - IS_SQL = True - elif 'provider' in info['store'] and info['store']['provider'] == 'sqlite' : - lwriter = transport.instance(**info['store']) ; - IS_SQL = [providers.SQLITE,providers.POSTGRESQL,providers.NETEZZA,providers.MYSQL,providers.MARIADB] +# OUTPUT_FOLDER = info['out-folder'] +# if 'output-folder' not in info and not os.path.exists(OUTPUT_FOLDER) : +# os.mkdir(OUTPUT_FOLDER) +# elif 'output-folder' in info and not os.path.exists(info['out-folder']) : +# os.mkdir(info['out-folder']) +# # if 'type' in info['store'] : +# lwriter = None +# IS_SQL = False +# if'type' in info['store'] and info['store']['type'] == 'disk.SQLiteWriter' : +# lwriter = transport.factory.instance(**info['store']) +# IS_SQL = True +# elif 'provider' in info['store'] and info['store']['provider'] == 'sqlite' : +# lwriter = transport.instance(**info['store']) ; +# IS_SQL = [providers.SQLITE,providers.POSTGRESQL,providers.NETEZZA,providers.MYSQL,providers.MARIADB] - if lwriter and IS_SQL: - for key in info['schema'] : - if key != 'logs' : - _id = 'claims' if key == '837' else 'remits' - else: - _id = key +# if lwriter and IS_SQL: +# for key in info['schema'] : +# if key != 'logs' : +# _id = 'claims' if key == '837' else 'remits' +# else: +# _id = key - if not lwriter.has(table=_id) : - lwriter.apply(info['schema'][key]['create']) - - # [lwriter.apply( info['schema'][key]['create']) for key in info['schema'] if not lwriter.has(table=key)] - lwriter.close() - - return info - - def __init__(self,path): - """ - :path path of the configuration file (it can be absolute) - """ - Process.__init__(self) - self.utils = Formatters() - self.get = void() - self.get.value = self.get_map - self.get.default_value = self.get_default_value - # _config = json.loads(open(path).read()) - self._custom_config = self.get_custom(path) - # self.config = _config['parser'] - # # - # # The parser may need some editing provided, this allows ease of developement and using alternate configurations - # # - # if type(self.config['837']) == str or type(self.config['835']) == str : - # for _id in ['837','835'] : - # if type(self.config[_id]) == str: - # self.config[_id] = json.loads(open(self.config[_id]).read()) - # if type(self.config[_id]) == dict : - # self.config[_id] = [self.config[_id]] - _config = Parser.setup(path) - self.config = _config['parser'] +# if not lwriter.has(table=_id) : +# lwriter.apply(info['schema'][key]['create']) + +# # [lwriter.apply( info['schema'][key]['create']) for key in info['schema'] if not lwriter.has(table=key)] +# lwriter.close() + +# return info + +# def __init__(self,path): +# """ +# :path path of the configuration file (it can be absolute) +# """ +# Process.__init__(self) +# self.utils = Formatters() +# self.get = void() +# self.get.value = self.get_map +# self.get.default_value = self.get_default_value +# # _config = json.loads(open(path).read()) +# self._custom_config = self.get_custom(path) +# # self.config = _config['parser'] +# # # +# # # The parser may need some editing provided, this allows ease of developement and using alternate configurations +# # # +# # if type(self.config['837']) == str or type(self.config['835']) == str : +# # for _id in ['837','835'] : +# # if type(self.config[_id]) == str: +# # self.config[_id] = json.loads(open(self.config[_id]).read()) +# # if type(self.config[_id]) == dict : +# # self.config[_id] = [self.config[_id]] +# _config = Parser.setup(path) +# self.config = _config['parser'] - self.store = _config['store'] - self.cache = {} - self.files = [] - self.set = void() - self.set.files = self.set_files - self.emit = void() - self.emit.pre = None - self.emit.post = None - def get_custom(self,path) : - """ - :path path of the configuration file (it can be absolute) - """ - # - # - _path = path.replace('config.json','') - if _path.endswith(os.sep) : - _path = _path[:-1] +# self.store = _config['store'] +# self.cache = {} +# self.files = [] +# self.set = void() +# self.set.files = self.set_files +# self.emit = void() +# self.emit.pre = None +# self.emit.post = None +# def get_custom(self,path) : +# """ +# :path path of the configuration file (it can be absolute) +# """ +# # +# # +# _path = path.replace('config.json','') +# if _path.endswith(os.sep) : +# _path = _path[:-1] - _config = {} - _path = os.sep.join([_path,'custom']) - if os.path.exists(_path) : +# _config = {} +# _path = os.sep.join([_path,'custom']) +# if os.path.exists(_path) : - files = os.listdir(_path) - if files : - fullname = os.sep.join([_path,files[0]]) - _config = json.loads ( (open(fullname)).read() ) - return _config - - def set_files(self,files): - self.files = files - def get_map(self,row,config,version=None): +# files = os.listdir(_path) +# if files : +# fullname = os.sep.join([_path,files[0]]) +# _config = json.loads ( (open(fullname)).read() ) +# return _config + +# def set_files(self,files): +# self.files = files +# def get_map(self,row,config,version=None): - # label = config['label'] if 'label' in config else None - handler = Formatters() +# # label = config['label'] if 'label' in config else None +# handler = Formatters() - if 'map' not in config and hasattr(handler,config['apply']): +# if 'map' not in config and hasattr(handler,config['apply']): - pointer = getattr(handler,config['apply']) +# pointer = getattr(handler,config['apply']) - object_value = pointer(row) - return object_value - # - # Pull the goto configuration that skips rows - # - omap = config['map'] if not version or version not in config else config[version] - anchors = config['anchors'] if 'anchors' in config else [] - rewrite = config['rewrite'] if 'rewrite' in config else {} - if len(row) == 2 and row[0] == 'HI' : - row = ([row[0]] + row[1].split(':')) - if type(row[0]) == str: - object_value = {} - for key in omap : +# object_value = pointer(row) +# return object_value +# # +# # Pull the goto configuration that skips rows +# # +# omap = config['map'] if not version or version not in config else config[version] +# anchors = config['anchors'] if 'anchors' in config else [] +# rewrite = config['rewrite'] if 'rewrite' in config else {} +# if len(row) == 2 and row[0] == 'HI' : +# row = ([row[0]] + row[1].split(':')) +# if type(row[0]) == str: +# object_value = {} +# for key in omap : - index = omap[key] - if anchors and set(anchors) & set(row): - _key = list(set(anchors) & set(row))[0] +# index = omap[key] +# if anchors and set(anchors) & set(row): +# _key = list(set(anchors) & set(row))[0] - aindex = row.index(_key) - index = aindex + index - - if index < len(row) : - value = row[index] - if 'cast' in config and key in config['cast'] and value.strip() != '' : - if config['cast'][key] in ['float','int']: - try: - value = eval(config['cast'][key])(value) - except Exception as e: - pass - # - # Sometimes shit hits the fan when the anchor is missing - # This is typical but using the hardened function helps circumvent this (SV2,SV3) - # - elif hasattr(handler,config['cast'][key]): +# aindex = row.index(_key) +# index = aindex + index + +# if index < len(row) : +# value = row[index] +# if 'cast' in config and key in config['cast'] and value.strip() != '' : +# if config['cast'][key] in ['float','int']: +# try: +# value = eval(config['cast'][key])(value) +# except Exception as e: +# pass +# # +# # Sometimes shit hits the fan when the anchor is missing +# # This is typical but using the hardened function helps circumvent this (SV2,SV3) +# # +# elif hasattr(handler,config['cast'][key]): - pointer = getattr(handler,config['cast'][key]) - value = pointer(value) +# pointer = getattr(handler,config['cast'][key]) +# value = pointer(value) - else: - print ("Missing Pointer ",key,config['cast']) +# else: +# print ("Missing Pointer ",key,config['cast']) - if type(value) == dict : - for objkey in value : +# if type(value) == dict : +# for objkey in value : - if type(value[objkey]) == dict : - continue - if 'syn' in config and value[objkey] in config['syn'] : - # value[objkey] = config['syn'][ value[objkey]] - pass +# if type(value[objkey]) == dict : +# continue +# if 'syn' in config and value[objkey] in config['syn'] : +# # value[objkey] = config['syn'][ value[objkey]] +# pass - if key in rewrite : +# if key in rewrite : - _key = rewrite[key] - if _key in value : - value = value[_key] - else: - value = "" - value = {key:value} if key not in value else value - else: +# _key = rewrite[key] +# if _key in value : +# value = value[_key] +# else: +# value = "" +# value = {key:value} if key not in value else value +# else: - if 'syn' in config and value in config['syn'] : - # value = config['syn'][value] - pass +# if 'syn' in config and value in config['syn'] : +# # value = config['syn'][value] +# pass - if type(value) == dict : - object_value = jsonmerge.merge(object_value, value) - else: - object_value[key] = value - else: - # - # we are dealing with a complex object - object_value = [] +# if type(value) == dict : +# object_value = jsonmerge.merge(object_value, value) +# else: +# object_value[key] = value +# else: +# # +# # we are dealing with a complex object +# object_value = [] - for row_item in row : +# for row_item in row : - value = self.get.value(row_item,config,version) - object_value.append(value) - return object_value - def set_cache(self,tmp,_info) : - """ - insert into cache a value that the, these are in reference to a loop - """ - if 'cache' in _info : - key = _info['cache']['key'] - value=_info['cache']['value'] - field = _info['cache']['field'] - if value in tmp : - - self.cache [key] = {field:tmp[value]} - pass - def get_cache(self,row) : - """ - retrieve cache element for a current - """ - key = row[0] - return self.cache[key] if key in self.cache else {} +# value = self.get.value(row_item,config,version) +# object_value.append(value) +# return object_value +# def set_cache(self,tmp,_info) : +# """ +# insert into cache a value that the, these are in reference to a loop +# """ +# if 'cache' in _info : +# key = _info['cache']['key'] +# value=_info['cache']['value'] +# field = _info['cache']['field'] +# if value in tmp : + +# self.cache [key] = {field:tmp[value]} +# pass +# def get_cache(self,row) : +# """ +# retrieve cache element for a current +# """ +# key = row[0] +# return self.cache[key] if key in self.cache else {} - def apply(self,content,_code) : - """ - :content content of a file i.e a segment with the envelope - :_code 837 or 835 (helps get the appropriate configuration) - """ - util = Formatters() - # header = default_value.copy() - value = {} +# def apply(self,content,_code) : +# """ +# :content content of a file i.e a segment with the envelope +# :_code 837 or 835 (helps get the appropriate configuration) +# """ +# util = Formatters() +# # header = default_value.copy() +# value = {} - for row in content[:] : - row = util.split(row.replace('\n','').replace('~','')) - _info = util.get.config(self.config[_code][0],row) - if self._custom_config and _code in self._custom_config: - _cinfo = util.get.config(self._custom_config[_code],row) - else: - _cinfo = {} +# for row in content[:] : +# row = util.split(row.replace('\n','').replace('~','')) +# _info = util.get.config(self.config[_code][0],row) +# if self._custom_config and _code in self._custom_config: +# _cinfo = util.get.config(self._custom_config[_code],row) +# else: +# _cinfo = {} - if _info or _cinfo: +# if _info or _cinfo: - try: +# try: - _info = jsonmerge.merge(_info,_cinfo) - tmp = self.get.value(row,_info) +# _info = jsonmerge.merge(_info,_cinfo) +# tmp = self.get.value(row,_info) - if not tmp : - continue - # - # At this point we have the configuration and the row parsed into values - # We should check to see if we don't have anything in the cache to be added to it - # - if row[0] in self.cache : - tmp = jsonmerge.merge(tmp,self.get_cache(row)) +# if not tmp : +# continue +# # +# # At this point we have the configuration and the row parsed into values +# # We should check to see if we don't have anything in the cache to be added to it +# # +# if row[0] in self.cache : +# tmp = jsonmerge.merge(tmp,self.get_cache(row)) - if 'label' in _info : - label = _info['label'] +# if 'label' in _info : +# label = _info['label'] - if type(tmp) == list : +# if type(tmp) == list : - value[label] = tmp if label not in value else value[label] + tmp +# value[label] = tmp if label not in value else value[label] + tmp - else: - # if 'DTM' in row : - # print ([label,tmp,label in value]) - if label not in value : - value[label] = [] - value[label].append(tmp) - # if label not in value: +# else: +# # if 'DTM' in row : +# # print ([label,tmp,label in value]) +# if label not in value : +# value[label] = [] +# value[label].append(tmp) +# # if label not in value: - # value[label] = [tmp] - # else: - # value[label].append(tmp) +# # value[label] = [tmp] +# # else: +# # value[label].append(tmp) - if '_index' not in tmp : - # - # In case we asked it to be overriden, then this will not apply - # X12 occasionally requires references to other elements in a loop (alas) - # - tmp['_index'] = len(value[label]) -1 +# if '_index' not in tmp : +# # +# # In case we asked it to be overriden, then this will not apply +# # X12 occasionally requires references to other elements in a loop (alas) +# # +# tmp['_index'] = len(value[label]) -1 - elif 'field' in _info : +# elif 'field' in _info : - name = _info['field'] - # value[name] = tmp - # value = jsonmerge.merge(value,{name:tmp}) - if name not in value : - value = dict(value,**{name:tmp}) - else: - value[name] = dict(value[name],**tmp) +# name = _info['field'] +# # value[name] = tmp +# # value = jsonmerge.merge(value,{name:tmp}) +# if name not in value : +# value = dict(value,**{name:tmp}) +# else: +# value[name] = dict(value[name],**tmp) - else: - value = dict(value,**tmp) +# else: +# value = dict(value,**tmp) - pass - except Exception as e : - print (e.args[0]) - # print ('__',(dir(e.args))) - pass - # - # At this point the object is completely built, - # if there ar any attributes to be cached it will be done here - # +# pass +# except Exception as e : +# print (e.args[0]) +# # print ('__',(dir(e.args))) +# pass +# # +# # At this point the object is completely built, +# # if there ar any attributes to be cached it will be done here +# # - if 'cache' in _info : +# if 'cache' in _info : - self.set_cache(tmp,_info) +# self.set_cache(tmp,_info) - return value if value else {} +# return value if value else {} - def get_default_value(self,content,_code): +# def get_default_value(self,content,_code): - util = Formatters() - TOP_ROW = content[1].split('*') +# util = Formatters() +# TOP_ROW = content[1].split('*') - SUBMITTED_DATE = util.parse.date(TOP_ROW[4]) +# SUBMITTED_DATE = util.parse.date(TOP_ROW[4]) - CATEGORY= content[2].split('*')[1].strip() +# CATEGORY= content[2].split('*')[1].strip() - VERSION = content[1].split('*')[-1].replace('~','').replace('\n','') - SENDER_ID = TOP_ROW[2] +# VERSION = content[1].split('*')[-1].replace('~','').replace('\n','') +# SENDER_ID = TOP_ROW[2] - row = util.split(content[3]) +# row = util.split(content[3]) - _info = util.get_config(self.config[_code][0],row) +# _info = util.get_config(self.config[_code][0],row) - value = self.get.value(row,_info,VERSION) if _info else {} - value['category'] = {"setid": _code,"version":'X'+VERSION.split('X')[1],"id":VERSION.split('X')[0].strip()} - value["submitted"] = SUBMITTED_DATE - value['sender_id'] = SENDER_ID +# value = self.get.value(row,_info,VERSION) if _info else {} +# value['category'] = {"setid": _code,"version":'X'+VERSION.split('X')[1],"id":VERSION.split('X')[0].strip()} +# value["submitted"] = SUBMITTED_DATE +# value['sender_id'] = SENDER_ID - # value = dict(value,**self.apply(content,_code)) - value = jsonmerge.merge(value,self.apply(content,_code)) - # Let's parse this for default values - return value #jsonmerge.merge(value,self.apply(content,_code)) - - def read(self,filename) : - """ - :formerly get_content - This function returns the of the EDI file parsed given the configuration specified. it is capable of identifying a file given the content - :section loop prefix (HL, CLP) - :config configuration with formatting rules, labels ... - :filename location of the file - """ - # section = section if section else config['SECTION'] - logs = [] - claims = [] - _code = 'UNKNOWN' - try: - self.cache = {} - file = open(filename.strip()) - file = file.read().split('CLP') - _code = '835' - section = 'CLP' +# # value = dict(value,**self.apply(content,_code)) +# value = jsonmerge.merge(value,self.apply(content,_code)) +# # Let's parse this for default values +# return value #jsonmerge.merge(value,self.apply(content,_code)) + +# def read(self,filename) : +# """ +# :formerly get_content +# This function returns the of the EDI file parsed given the configuration specified. it is capable of identifying a file given the content +# :section loop prefix (HL, CLP) +# :config configuration with formatting rules, labels ... +# :filename location of the file +# """ +# # section = section if section else config['SECTION'] +# logs = [] +# claims = [] +# _code = 'UNKNOWN' +# try: +# self.cache = {} +# file = open(filename.strip()) +# file = file.read().split('CLP') +# _code = '835' +# section = 'CLP' - if len(file) == 1 : +# if len(file) == 1 : - file = file[0].split('CLM') #.split('HL') - _code = '837' - section = 'CLM' #'HL' +# file = file[0].split('CLM') #.split('HL') +# _code = '837' +# section = 'CLM' #'HL' - INITIAL_ROWS = file[0].split(section)[0].split('\n') +# INITIAL_ROWS = file[0].split(section)[0].split('\n') - if len(INITIAL_ROWS) == 1 : +# if len(INITIAL_ROWS) == 1 : - INITIAL_ROWS = INITIAL_ROWS[0].split('~') +# INITIAL_ROWS = INITIAL_ROWS[0].split('~') - # for item in file[1:] : - # item = item.replace('~','\n') - # print (INITIAL_ROWS) +# # for item in file[1:] : +# # item = item.replace('~','\n') +# # print (INITIAL_ROWS) - DEFAULT_VALUE = self.get.default_value(INITIAL_ROWS,_code) - DEFAULT_VALUE['name'] = filename.strip() +# DEFAULT_VALUE = self.get.default_value(INITIAL_ROWS,_code) +# DEFAULT_VALUE['name'] = filename.strip() - file = section.join(file).split('\n') - if len(file) == 1: +# file = section.join(file).split('\n') +# if len(file) == 1: - file = file[0].split('~') - # - # In the initial rows, there's redundant information (so much for x12 standard) - # index 1 identifies file type i.e CLM for claim and CLP for remittance - segment = [] - index = 0; - _toprows = [] - _default = None +# file = file[0].split('~') +# # +# # In the initial rows, there's redundant information (so much for x12 standard) +# # index 1 identifies file type i.e CLM for claim and CLP for remittance +# segment = [] +# index = 0; +# _toprows = [] +# _default = None - for row in file : +# for row in file : - row = row.replace('\r','') - # if not segment and not row.startswith(section): - # _toprows += [row] +# row = row.replace('\r','') +# # if not segment and not row.startswith(section): +# # _toprows += [row] - if row.startswith(section) and not segment: +# if row.startswith(section) and not segment: - segment = [row] +# segment = [row] - continue +# continue - elif segment and not row.startswith(section): +# elif segment and not row.startswith(section): - segment.append(row) +# segment.append(row) - if len(segment) > 1 and row.startswith(section): - # - # process the segment somewhere (create a thread maybe?) - # +# if len(segment) > 1 and row.startswith(section): +# # +# # process the segment somewhere (create a thread maybe?) +# # - _claim = self.apply(segment,_code) +# _claim = self.apply(segment,_code) - if _claim : - _claim['index'] = index #len(claims) - # claims.append(dict(DEFAULT_VALUE,**_claim)) - # - # schema = [ {key:{"mergeStrategy":"append" if list( type(_claim[key])) else "overwrite"}} for key in _claim.keys()] # if type(_claim[key]) == list] - # _schema = set(DEFAULT_VALUE.keys()) - schema - # if schema : - # schema = {"properties":dict.fromkeys(schema,{"mergeStrategy":"append"})} +# if _claim : +# _claim['index'] = index #len(claims) +# # claims.append(dict(DEFAULT_VALUE,**_claim)) +# # +# # schema = [ {key:{"mergeStrategy":"append" if list( type(_claim[key])) else "overwrite"}} for key in _claim.keys()] # if type(_claim[key]) == list] +# # _schema = set(DEFAULT_VALUE.keys()) - schema +# # if schema : +# # schema = {"properties":dict.fromkeys(schema,{"mergeStrategy":"append"})} - # else: +# # else: - # schema = {"properties":{}} +# # schema = {"properties":{}} - # schema = jsonmerge.merge(schema['properties'],dict.fromkeys(_schema,{"mergeStrategy":"overwrite"})) - schema = {"properties":{}} - for attr in _claim.keys() : - schema['properties'][attr] = {"mergeStrategy": "append" if type(_claim[attr]) == list else "overwrite" } +# # schema = jsonmerge.merge(schema['properties'],dict.fromkeys(_schema,{"mergeStrategy":"overwrite"})) +# schema = {"properties":{}} +# for attr in _claim.keys() : +# schema['properties'][attr] = {"mergeStrategy": "append" if type(_claim[attr]) == list else "overwrite" } - merger = jsonmerge.Merger(schema) - _baseclaim = None - _baseclaim = merger.merge(_baseclaim,copy.deepcopy(DEFAULT_VALUE)) - _claim = merger.merge(_baseclaim,_claim) +# merger = jsonmerge.Merger(schema) +# _baseclaim = None +# _baseclaim = merger.merge(_baseclaim,copy.deepcopy(DEFAULT_VALUE)) +# _claim = merger.merge(_baseclaim,_claim) - # _claim = merger.merge(DEFAULT_VALUE.copy(),_claim) +# # _claim = merger.merge(DEFAULT_VALUE.copy(),_claim) - claims.append( _claim) - segment = [row] +# claims.append( _claim) +# segment = [row] - index += 1 +# index += 1 - pass - # - # Handling the last claim found +# pass +# # +# # Handling the last claim found - if segment and segment[0].startswith(section) : - # default_claim = dict({"name":index},**DEFAULT_VALUE) - - claim = self.apply(segment,_code) - if claim : - claim['index'] = len(claims) - # schema = [key for key in claim.keys() if type(claim[key]) == list] - # if schema : - # schema = {"properties":dict.fromkeys(schema,{"mergeStrategy":"append"})} +# if segment and segment[0].startswith(section) : +# # default_claim = dict({"name":index},**DEFAULT_VALUE) + +# claim = self.apply(segment,_code) +# if claim : +# claim['index'] = len(claims) +# # schema = [key for key in claim.keys() if type(claim[key]) == list] +# # if schema : +# # schema = {"properties":dict.fromkeys(schema,{"mergeStrategy":"append"})} - # else: - # print (claim.keys()) - # schema = {} - # - # @TODO: Fix merger related to schema (drops certain fields ... NOT cool) +# # else: +# # print (claim.keys()) +# # schema = {} +# # +# # @TODO: Fix merger related to schema (drops certain fields ... NOT cool) - # merger = jsonmerge.Merger(schema) - # top_row_claim = self.apply(_toprows,_code) +# # merger = jsonmerge.Merger(schema) +# # top_row_claim = self.apply(_toprows,_code) - # claim = merger.merge(claim,self.apply(_toprows,_code)) - # claims.append(dict(DEFAULT_VALUE,**claim)) - schema = {"properties":{}} - for attr in claim.keys() : - schema['properties'][attr] = {"mergeStrategy": "append" if type(claim[attr]) == list else "overwrite" } - merger = jsonmerge.Merger(schema) - _baseclaim = None - _baseclaim = merger.merge(_baseclaim,copy.deepcopy(DEFAULT_VALUE)) - claim = merger.merge(_baseclaim,claim) - claims.append(claim) - - # claims.append(merger.merge(DEFAULT_VALUE.copy(),claim)) - if type(file) != list : - file.close() - - # x12_file = open(filename.strip(),errors='ignore').read().split('\n') - except Exception as e: +# # claim = merger.merge(claim,self.apply(_toprows,_code)) +# # claims.append(dict(DEFAULT_VALUE,**claim)) +# schema = {"properties":{}} +# for attr in claim.keys() : +# schema['properties'][attr] = {"mergeStrategy": "append" if type(claim[attr]) == list else "overwrite" } +# merger = jsonmerge.Merger(schema) +# _baseclaim = None +# _baseclaim = merger.merge(_baseclaim,copy.deepcopy(DEFAULT_VALUE)) +# claim = merger.merge(_baseclaim,claim) +# claims.append(claim) + +# # claims.append(merger.merge(DEFAULT_VALUE.copy(),claim)) +# if type(file) != list : +# file.close() + +# # x12_file = open(filename.strip(),errors='ignore').read().split('\n') +# except Exception as e: - logs.append ({"parse":_code,"completed":False,"name":filename,"msg":e.args[0]}) - return [],logs,None +# logs.append ({"parse":_code,"completed":False,"name":filename,"msg":e.args[0]}) +# return [],logs,None - rate = 0 if len(claims) == 0 else (1 + index)/len(claims) - logs.append ({"parse":"claims" if _code == '837' else 'remits',"completed":True,"name":filename,"rate":rate}) - # self.finish(claims,logs,_code) - return claims,logs,_code - def run(self): - if self.emit.pre : - self.emit.pre() - - for filename in self.files : - content,logs,_code = self.read(filename) +# rate = 0 if len(claims) == 0 else (1 + index)/len(claims) +# logs.append ({"parse":"claims" if _code == '837' else 'remits',"completed":True,"name":filename,"rate":rate}) +# # self.finish(claims,logs,_code) +# return claims,logs,_code +# def run(self): +# if self.emit.pre : +# self.emit.pre() + +# for filename in self.files : +# content,logs,_code = self.read(filename) - self.finish(content,logs,_code) - def finish(self,content,logs,_code) : - args = self.store - _args = json.loads(json.dumps(self.store)) - ISNEW_MONGO = 'provider' in args and args['provider'] in ['mongo', 'mongodb'] - ISLEG_MONGO = ('type' in args and args['type'] == 'mongo.MongoWriter') - if ISLEG_MONGO or ISNEW_MONGO: - if ISLEG_MONGO: - # Legacy specification ... - args['args']['doc'] = 'claims' if _code == '837' else 'remits' - _args['args']['doc'] = 'logs' - else: - args['doc'] = 'claims' if _code == '837' else 'remits' - _args['doc'] = 'logs' +# self.finish(content,logs,_code) +# def finish(self,content,logs,_code) : +# args = self.store +# _args = json.loads(json.dumps(self.store)) +# ISNEW_MONGO = 'provider' in args and args['provider'] in ['mongo', 'mongodb'] +# ISLEG_MONGO = ('type' in args and args['type'] == 'mongo.MongoWriter') +# if ISLEG_MONGO or ISNEW_MONGO: +# if ISLEG_MONGO: +# # Legacy specification ... +# args['args']['doc'] = 'claims' if _code == '837' else 'remits' +# _args['args']['doc'] = 'logs' +# else: +# args['doc'] = 'claims' if _code == '837' else 'remits' +# _args['doc'] = 'logs' - else: - if 'type' in args : - # Legacy specification ... - args['args']['table'] = 'claims' if _code == '837' else 'remits' - _args['args']['table'] = 'logs' - table = args['args']['table'] - else: - args['table']= 'claims' if _code == '837' else 'remits' - _args['table'] = 'logs' - table = args['table'] +# else: +# if 'type' in args : +# # Legacy specification ... +# args['args']['table'] = 'claims' if _code == '837' else 'remits' +# _args['args']['table'] = 'logs' +# table = args['args']['table'] +# else: +# args['table']= 'claims' if _code == '837' else 'remits' +# _args['table'] = 'logs' +# table = args['table'] - writer = transport.factory.instance(**args) - IS_SQLITE = type(writer) == transport.disk.SQLiteWriter - if content: - if IS_SQLITE : - for row in content : - writer.apply("""insert into :table(data) values (':values')""".replace(":values",json.dumps(row)).replace(":table",table) ) - else: - writer.write(content) - writer.close() - if logs : - logger = transport.factory.instance(**_args) - if IS_SQLITE: - for row in logs: - logger.apply("""insert into logs values (':values')""".replace(":values",json.dumps(row))) - else: - logger.write(logs) +# writer = transport.factory.instance(**args) +# IS_SQLITE = type(writer) == transport.disk.SQLiteWriter +# if content: +# if IS_SQLITE : +# for row in content : +# writer.apply("""insert into :table(data) values (':values')""".replace(":values",json.dumps(row)).replace(":table",table) ) +# else: +# writer.write(content) +# writer.close() +# if logs : +# logger = transport.factory.instance(**_args) +# if IS_SQLITE: +# for row in logs: +# logger.apply("""insert into logs values (':values')""".replace(":values",json.dumps(row))) +# else: +# logger.write(logs) - logger.close() - if self.emit.post : - self.emit.post(content,logs) +# logger.close() +# if self.emit.post : +# self.emit.post(content,logs)