You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
116 lines
3.1 KiB
Python
116 lines
3.1 KiB
Python
"""
|
|
Data Transport - 1.0
|
|
Steve L. Nyemba, The Phi Technology LLC
|
|
|
|
This module is designed to serve as a wrapper to a set of supported data stores :
|
|
- couchdb
|
|
- mongodb
|
|
- Files (character delimited)
|
|
- Queues (Rabbmitmq)
|
|
- Session (Flask)
|
|
- s3
|
|
The supported operations are read/write and providing meta data to the calling code
|
|
Requirements :
|
|
pymongo
|
|
boto
|
|
couldant
|
|
@TODO:
|
|
Enable read/writing to multiple reads/writes
|
|
"""
|
|
__author__ = 'The Phi Technology'
|
|
import numpy as np
|
|
import json
|
|
import importlib
|
|
from multiprocessing import RLock
|
|
# import couch
|
|
# import mongo
|
|
class IO:
|
|
def init(self,**args):
|
|
"""
|
|
This function enables attributes to be changed at runtime. Only the attributes defined in the class can be changed
|
|
Adding attributes will require sub-classing otherwise we may have an unpredictable class ...
|
|
"""
|
|
allowed = list(vars(self).keys())
|
|
for field in args :
|
|
if field not in allowed :
|
|
continue
|
|
value = args[field]
|
|
setattr(self,field,value)
|
|
class Reader (IO):
|
|
"""
|
|
This class is an abstraction of a read functionalities of a data store
|
|
"""
|
|
def __init__(self):
|
|
pass
|
|
def meta(self,**_args):
|
|
"""
|
|
This function is intended to return meta-data associated with what has just been read
|
|
@return object of meta data information associated with the content of the store
|
|
"""
|
|
raise Exception ("meta function needs to be implemented")
|
|
def read(self,**args):
|
|
"""
|
|
This function is intended to read the content of a store provided parameters to be used at the discretion of the subclass
|
|
"""
|
|
raise Exception ("read function needs to be implemented")
|
|
|
|
|
|
class Writer(IO):
|
|
def __init__(self):
|
|
self.cache = {"default":[]}
|
|
def log(self,**args):
|
|
self.cache[id] = args
|
|
def meta (self,id="default",**args):
|
|
raise Exception ("meta function needs to be implemented")
|
|
def format(self,row,xchar):
|
|
if xchar is not None and isinstance(row,list):
|
|
return xchar.join(row)+'\n'
|
|
elif xchar is None and isinstance(row,dict):
|
|
row = json.dumps(row)
|
|
return row
|
|
def write(self,**args):
|
|
"""
|
|
This function will write content to a store given parameters to be used at the discretion of the sub-class
|
|
"""
|
|
raise Exception ("write function needs to be implemented")
|
|
|
|
def archive(self):
|
|
"""
|
|
It is important to be able to archive data so as to insure that growth is controlled
|
|
Nothing in nature grows indefinitely neither should data being handled.
|
|
"""
|
|
raise Exception ("archive function needs to be implemented")
|
|
def close(self):
|
|
"""
|
|
This function will close the persistent storage connection/handler
|
|
"""
|
|
pass
|
|
class ReadWriter(Reader,Writer) :
|
|
"""
|
|
This class implements the read/write functions aggregated
|
|
"""
|
|
pass
|
|
class Console(Writer):
|
|
lock = RLock()
|
|
def __init__(self,**_args):
|
|
self.lock = _args['lock'] if 'lock' in _args else False
|
|
self.info = self.write
|
|
self.debug = self.write
|
|
self.log = self.write
|
|
pass
|
|
def write (self,**_args):
|
|
if self.lock :
|
|
Console.lock.acquire()
|
|
try:
|
|
if type(_args) == list:
|
|
for row in _args :
|
|
print (row)
|
|
else:
|
|
print (_args)
|
|
except Exception as e :
|
|
print (e)
|
|
finally:
|
|
if self.lock :
|
|
Console.lock.release()
|
|
|