new: resume functionality

healthcareio/healthcare-io.py

@@ -144,7 +144,20 @@ def parse(**args):
 
         
         return get_content(args['filename'],CONFIG,SECTION)
-
+def resume (files,id,config):
+    _args = config['store'].copy()
+    if 'mongo' in config['store']['type'] :
+        _args['type'] = 'mongo.MongoReader'
+        reader = factory.instance(**_args)
+    _files = []
+    if 'resume' in config['analytics'] :
+        _args = config['analytics']['resume'][id]
+        _files = reader.read(**_args)
+        _files = [item['name'] for item in _files if item['name'] != None]
+        return list(set(files) - set(_files))
+        
+    return files
+    pass
 def apply(files,store_info,logger_info=None):
     """
         :files          list of files to be processed in this given thread/process
@@ -166,6 +179,7 @@ def apply(files,store_info,logger_info=None):
         # 
         try:              
             content,logs = parse(filename = filename,type=SYS_ARGS['parse'])
+            
             if content :                        
                 writer.write(content)
             if logs :
@@ -173,6 +187,7 @@ def apply(files,store_info,logger_info=None):
             else:
                 logger.write({"parse":SYS_ARGS['parse'],"name":filename,"completed":True,"rows":len(content)})
         except Exception as e:
+            
             logger.write({"parse":SYS_ARGS['parse'],"filename":filename,"completed":False,"rows":-1,"msg":e.args[0]})
         # print ([filename,len(content)])
         #
@@ -233,6 +248,12 @@ if __name__ == '__main__' :
             # raise an erro
             pass
         #
+        # if the user has specified to resume, we should look into the logs and pull the files processed and those that haven't
+        #
+        if 'resume' in SYS_ARGS :
+            files = resume(files,SYS_ARGS['parse'],info)
+            print (["Found ",len(files)," files unprocessed"])
+        #
         # @TODO: Log this here so we know what is being processed or not
         SCOPE = None
         
@@ -348,11 +369,12 @@ if __name__ == '__main__' :
         
             healthcare-io.py    --<[signup|init]> <email> --store <sqlite|mongo> [--batch <value>]
             healthcare-io.py    --parse claims --folder <path> [--batch <value>]
-            healthcare-io.py    --parse remits --folder <path> [--batch <value>]
+            healthcare-io.py    --parse remits --folder <path> [--batch <value>] [--resume]
         
         parameters :
             --<[signup|init]>   signup or get a configuration file from a parsing server
             --store             data store mongo or sqlite or mongodb
+            --resume            will attempt to resume if there was an interruption
         """
         print(msg)
         pass

healthcareio/parser.py

@@ -21,6 +21,18 @@ import os
 import sys
 import hashlib
 import json
+class X12 :
+    def split(self,row,sep='*',prefix='HI') :
+        pass
+    def get_config(self,config,row):
+        pass
+    def hash(self,value):
+        pass
+    def suppress (self,value):
+        pass
+    def format_date(self,value):
+        pass
+    
 def split(row,sep='*',prefix='HI'):
     """
     This function is designed to split an x12 row and 

setup.py

@@ -8,7 +8,7 @@ import sys
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read() 
 args = {
-    "name":"healthcareio","version":"1.3.0",
+    "name":"healthcareio","version":"1.3.1",
     "author":"Vanderbilt University Medical Center",
     "author_email":"steve.l.nyemba@vumc.org",
     "license":"MIT",