""" Data Transport, The Phi Technology LLC Steve L. Nyemba, steve@the-phi.com This library is designed to serve as a wrapper to a set of supported data stores : - couchdb - mongodb - Files (character delimited) - Queues (RabbmitMq) - Session (Flask) - s3 - sqlite The supported operations are read/write and providing meta data to the calling code Requirements : pymongo boto couldant The configuration for the data-store is as follows : e.g: mongodb provider:'mongodb',[port:27017],[host:localhost],db:,doc:<_name>,context: """ __author__ = 'The Phi Technology' import pandas as pd import numpy as np import json import importlib import sys if sys.version_info[0] > 2 : from transport.common import Reader, Writer #, factory from transport import disk from transport import s3 as s3 from transport import rabbitmq as queue from transport import couch as couch from transport import mongo as mongo from transport import sql as sql else: from common import Reader, Writer #, factory import disk import queue import couch import mongo import s3 import sql import psycopg2 as pg import mysql.connector as my from google.cloud import bigquery as bq import nzpy as nz #--- netezza drivers import os class factory : TYPE = {"sql":{"providers":["postgresql","mysql","neteeza","bigquery","mariadb","redshift"]}} PROVIDERS = { "file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}}, "sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}}, "postgresql":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"}}, "redshift":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"}}, "bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}}, "mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"}}, "mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"}}, "mongo":{"port":27017,"host":"localhost","class":{"read":mongo.MongoReader,"write":mongo.MongoWriter}}, "couch":{"port":5984,"host":"localhost","class":{"read":couch.CouchReader,"write":couch.CouchWriter}}, "netezza":{"port":5480,"driver":nz,"default":{"type":"VARCHAR(256)"}}} # # creating synonyms PROVIDERS['mongodb'] = PROVIDERS['mongo'] PROVIDERS['couchdb'] = PROVIDERS['couch'] PROVIDERS['sqlite3'] = PROVIDERS['sqlite'] @staticmethod def instance(**_args): if 'type' in _args : # # Legacy code being returned return factory._instance(**_args); else: return instance(**_args) @staticmethod def _instance(**args): """ This class will create an instance of a transport when providing :type name of the type we are trying to create :args The arguments needed to create the instance """ source = args['type'] params = args['args'] anObject = None if source in ['HttpRequestReader','HttpSessionWriter']: # # @TODO: Make sure objects are serializable, be smart about them !! # aClassName = ''.join([source,'(**params)']) else: stream = json.dumps(params) aClassName = ''.join([source,'(**',stream,')']) try: anObject = eval( aClassName) #setattr(anObject,'name',source) except Exception as e: print(['Error ',e]) return anObject import time def instance(**_args): """ @param provider {file,sqlite,postgresql,redshift,bigquery,netezza,mongo,couch ...} @param context read|write|rw @param _args argument to got with the datastore (username,password,host,port ...) """ provider = _args['provider'] context = _args['context']if 'context' in _args else None _id = context if context in ['read','write'] else 'read' if _id : args = {'provider':_id} for key in factory.PROVIDERS[provider] : if key == 'class' : continue value = factory.PROVIDERS[provider][key] args[key] = value # # args = dict(args,**_args) # print (provider in factory.PROVIDERS) if 'class' in factory.PROVIDERS[provider]: pointer = factory.PROVIDERS[provider]['class'][_id] else: pointer = sql.SQLReader if _id == 'read' else sql.SQLWriter return pointer(**args) return None