data-transport/transport/__init__.py

"""
Data Transport - 1.0
Steve L. Nyemba, The Phi Technology LLC

This module is designed to serve as a wrapper to a set of supported data stores :
    - couchdb
    - mongodb
    - Files (character delimited)
    - Queues (RabbmitMq)
    - Session (Flask)
    - s3
The supported operations are read/write and providing meta data to the calling code
Requirements :
	pymongo
	boto
	couldant
The configuration for the data-store is as follows :
	couchdb:
		{
			args:{
				url:<url>,
				username:<username>,
				password:<password>,
				dbname:<database>,
				doc:<document id>
			}
		}
	RabbitMQ:
		{
			
		}
	Mongodb:
	{
		args:{
			host:<url>, #localhost:27017
			username:<username>,
			password:<password>,
			dbname:<database>,
			doc:<document id>s

		}
	}
"""
__author__ = 'The Phi Technology'
import pandas 	as pd
import numpy 	as np
import json
import importlib 
import sys 

if sys.version_info[0] > 2 : 
    from transport.common import Reader, Writer #, factory
    from transport import disk

    from transport import s3 as s3
    from transport import rabbitmq as queue
    from transport import couch as couch
    from transport import mongo as mongo
    from transport import sql as sql
else:
    from common import Reader, Writer #, factory
    import disk
    import queue
    import couch
    import mongo
    import s3
    import sql
import psycopg2 as pg
import mysql.connector as my
from google.cloud import bigquery as bq
import nzpy as nz   #--- netezza drivers
import os

RDBMS = {
	
    "postgresql":{"port":"5432","driver":pg},
    "redshift":{"port":"5432","driver":pg},
    "netezza":{"port":"5480","driver":nz},
	"mysql":{"port":"3306","driver":my},
	"mariadb":{"port":"3306","driver":my},
	"mongodb":{"port":"27017","class":{"read"}},
	"couchdb":{"port":"5984"}
}
class factory :
	TYPE = {"sql":{"providers":["postgresql","mysql","neteeza","bigquery","mariadb","redshift"]}}
	PROVIDERS = {
		"file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}},
		"sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}},
        "postgresql":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"}},
        "redshift":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"}},
        "bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}},
        "mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"}},
        "mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"}},
		"mongo":{"port":27017,"host":"localhost","class":{"read":mongo.MongoReader,"write":mongo.MongoWriter}},
		"couch":{"port":5984,"host":"localhost","class":{"read":couch.CouchReader,"write":couch.CouchWriter}},
        "netezza":{"port":5480,"driver":nz,"default":{"type":"VARCHAR(256)"}}}

	@staticmethod
	def instance(**args):
		"""
		This class will create an instance of a transport when providing 
		:type	name of the type we are trying to create
		:args	The arguments needed to create the instance
		"""
		source = args['type']		
		params = args['args']
		anObject = None
		
		if source in ['HttpRequestReader','HttpSessionWriter']:
			#
			# @TODO: Make sure objects are serializable, be smart about them !!
			#
			aClassName = ''.join([source,'(**params)'])
			

		else:
			
			stream = json.dumps(params)
			aClassName = ''.join([source,'(**',stream,')'])
			
		try:
			anObject = eval( aClassName)
			#setattr(anObject,'name',source)
		except Exception as e:
			print(['Error ',e])
		return anObject

import time
def instance(provider,context,**_args):
	"""

	@param provider	{file,sqlite,postgresql,redshift,bigquery,netezza,mongo,couch ...}
	@param context	read|write|rw
	@param _args	argument to got with the datastore (username,password,host,port ...)
	"""
	_id = context if context in ['read','write'] else None
	if _id :
		args = {'provider':_id}
		for key in factory.PROVIDERS[provider] :
			if key == 'class' :
				continue 
			value = factory.PROVIDERS[provider][key]
			args[key] = value
		#
		#
		args = dict(args,**_args)
		
		# print (provider in factory.PROVIDERS)
		if 'class' in factory.PROVIDERS[provider]:
			pointer = factory.PROVIDERS[provider]['class'][_id] 
		else:
			pointer = sql.SQLReader if _id == 'read' else sql.SQLWriter
		return pointer(**args)

	return None