From b9bc898161f6ee4810dc7c9d30360af2f39a07a1 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 14 Jun 2024 15:30:09 -0500 Subject: [PATCH] bug fix: registry (more usable) and added to factory method --- bin/transport | 33 ++++++++++++++++++++------------- transport/__init__.py | 29 +++++++++++++++++++++++++---- transport/registry.py | 26 +++++++++++++++++++++----- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/bin/transport b/bin/transport index 4f9a7e8..4053c4e 100755 --- a/bin/transport +++ b/bin/transport @@ -47,7 +47,7 @@ def wait(jobs): @app.command(name="apply") def apply (path:Annotated[str,typer.Argument(help="path of the configuration file")], - index:int = typer.Option(help="index of the item of interest, otherwise everything in the file will be processed")): + index:int = typer.Option(default= None, help="index of the item of interest, otherwise everything in the file will be processed")): """ This function applies data transport ETL feature to read data from one source to write it one or several others """ @@ -92,19 +92,23 @@ def version(): @app.command() def generate (path:Annotated[str,typer.Argument(help="path of the ETL configuration file template (name included)")]): - """ - This function will generate a configuration template to give a sense of how to create one - """ - _config = [ - { - "source":{"provider":"http","url":"https://raw.githubusercontent.com/codeforamerica/ohana-api/master/data/sample-csv/addresses.csv"}, - "target": + """ + This function will generate a configuration template to give a sense of how to create one + """ + _config = [ + { + "source":{"provider":"http","url":"https://raw.githubusercontent.com/codeforamerica/ohana-api/master/data/sample-csv/addresses.csv"}, + "target": [{"provider":"files","path":"addresses.csv","delimiter":","},{"provider":"sqlite","database":"sample.db3","table":"addresses"}] } ] - file = open(path,'w') - file.write(json.dumps(_config)) - file.close() + file = open(path,'w') + file.write(json.dumps(_config)) + file.close() + print (f"""{CHECK_MARK} Successfully generated a template ETL file at {path}""" ) + print ("""NOTE: Each line (source or target) is the content of an auth-file""") + + @app.command(name="init") def initregistry (email:Annotated[str,typer.Argument(help="email")], @@ -131,8 +135,11 @@ def register (label:Annotated[str,typer.Argument(help="unique label that will be Learn more about auth-file at https://healthcareio.the-phi.com/data-transport """ try: - transport.registry.set(label=label,auth_file=auth_file, default=default, path=path) - _msg = f"""{CHECK_MARK} Successfully added label "{label}" to data-transport registry""" + if transport.registry.exists(path) : + transport.registry.set(label=label,auth_file=auth_file, default=default, path=path) + _msg = f"""{CHECK_MARK} Successfully added label "{label}" to data-transport registry""" + else: + _msg = f"""{TIMES_MARK} Registry is not initialized, please initialize the registry (check help)""" except Exception as e: _msg = f"""{TIMES_MARK} {e}""" print (_msg) diff --git a/transport/__init__.py b/transport/__init__.py index 2f97c0f..b2ea543 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -48,10 +48,16 @@ def instance (**_args): kwargs These are arguments that are provider/vendor specific """ global PROVIDERS - if not registry.isloaded () : - registry.load() if 'path' not in _args else registry.load(_args['path']) - if 'label' in _args : - _info = registry.load(_args['label']) + # if not registry.isloaded () : + # if ('path' in _args and registry.exists(_args['path'] )) or registry.exists(): + # registry.load() if 'path' not in _args else registry.load(_args['path']) + # print ([' GOT IT']) + # if 'label' in _args and registry.isloaded(): + # _info = registry.get(_args['label']) + # if _info : + # # + # _args = dict(_args,**_info) + if 'auth_file' in _args: if os.path.exists(_args['auth_file']) : # @@ -67,6 +73,17 @@ def instance (**_args): else: filename = _args['auth_file'] raise Exception(f" {filename} was not found or is invalid") + if 'provider' not in _args and 'auth_file' not in _args : + if not registry.isloaded () : + if ('path' in _args and registry.exists(_args['path'] )) or registry.exists(): + registry.load() if 'path' not in _args else registry.load(_args['path']) + if 'label' in _args and registry.isloaded(): + _info = registry.get(_args['label']) + print(_info) + if _info : + # + _args = dict(_args,**_info) + if 'provider' in _args and _args['provider'] in PROVIDERS : _info = PROVIDERS[_args['provider']] _module = _info['module'] @@ -110,6 +127,8 @@ class get : """ @staticmethod def reader (**_args): + if not _args : + _args['label'] = 'default' _args['context'] = 'read' return instance(**_args) @staticmethod @@ -117,6 +136,8 @@ class get : """ This function is a wrapper that will return a writer to a database. It disambiguates the interface """ + if not _args : + _args['label'] = 'default' _args['context'] = 'write' return instance(**_args) @staticmethod diff --git a/transport/registry.py b/transport/registry.py index 9fe942d..f487b54 100644 --- a/transport/registry.py +++ b/transport/registry.py @@ -16,11 +16,20 @@ DATA = {} def isloaded (): return DATA not in [{},None] +def exists (path=REGISTRY_PATH) : + """ + This function determines if there is a registry at all + """ + p = os.path.exists(path) + q = os.path.exists( os.sep.join([path,REGISTRY_FILE])) + print ([p,q, os.sep.join([path,REGISTRY_FILE])]) + return p and q def load (_path=REGISTRY_PATH): global DATA - _path = os.sep.join([_path,REGISTRY_FILE]) - if os.path.exists(_path) : - f = open(_path) + + if exists(_path) : + path = os.sep.join([_path,REGISTRY_FILE]) + f = open(path) DATA = json.loads(f.read()) f.close() def init (email,path=REGISTRY_PATH,override=False): @@ -45,12 +54,19 @@ def init (email,path=REGISTRY_PATH,override=False): raise Exception (f"""Unable to write configuration, Please check parameters (or help) and try again""") else: raise Exception (f"""Invalid Input, {email} is not well formatted, provide an email with adequate format""") - +def lookup (label): + global DATA + return label in DATA def get (label='default') : global DATA return copy.copy(DATA[label]) if label in DATA else {} def set (label, auth_file, default=False,path=REGISTRY_PATH) : + """ + This function will add a label (auth-file data) into the registry and can set it as the default + """ + if label == 'default' : + raise Exception ("""Invalid label name provided, please change the label name and use the switch""") reg_file = os.sep.join([path,REGISTRY_FILE]) if os.path.exists (auth_file) and os.path.exists(path) and os.path.exists(reg_file): f = open(auth_file) @@ -73,7 +89,7 @@ def set (label, auth_file, default=False,path=REGISTRY_PATH) : f.write(json.dumps(_config)) f.close() else: - _msg = f"""Unable to load file locate at {path},\nLearn how to generate auth-file with wizard found at https://healthcareio.the-phi.com/data-transport""" + raise Exception( f"""Unable to load file locate at {path},\nLearn how to generate auth-file with wizard found at https://healthcareio.the-phi.com/data-transport""") pass else: pass