From 8ca04e8c48a86956d709db854863cb55a2c19956 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Sat, 25 Jun 2022 13:59:49 -0500 Subject: [PATCH] bug fix: environment variable on default database --- setup.py | 2 +- transport/__init__.py | 4 +-- transport/common.py | 2 +- transport/mongo.py | 77 +++++++++++++++++++++++++++++-------------- 4 files changed, 57 insertions(+), 28 deletions(-) diff --git a/setup.py b/setup.py index ad835f2..e4b84a5 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() args = { "name":"data-transport", - "version":"1.5.5", + "version":"1.5.6", "author":"The Phi Technology LLC","author_email":"info@the-phi.com", "license":"MIT", "packages":["transport"]} diff --git a/transport/__init__.py b/transport/__init__.py index 15f8f8d..2f6f5ac 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -61,8 +61,8 @@ class factory : "console":{"class":{"write":Console,"read":Console}}, "file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}}, "sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}}, - "postgresql":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, - "redshift":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, + "postgresql":{"port":5432,"host":"localhost","driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, + "redshift":{"port":5432,"host":"localhost","driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, "bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}}, "mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, "mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, diff --git a/transport/common.py b/transport/common.py index e6578a6..8234290 100644 --- a/transport/common.py +++ b/transport/common.py @@ -98,7 +98,7 @@ class Console(Writer): self.debug = self.write self.log = self.write pass - def write (self,**_args): + def write (self,logs,**_args): if self.lock : Console.lock.acquire() try: diff --git a/transport/mongo.py b/transport/mongo.py index f6fa12b..50f463f 100644 --- a/transport/mongo.py +++ b/transport/mongo.py @@ -33,37 +33,59 @@ class Mongo : :username username for authentication :password password for current user """ - port = str(args['port']) if 'port' in args else '27017' - host = args['host'] if 'host' in args else 'localhost' - host = ":".join([host,port]) #-- Formatting host information here - self.uid = args['doc'] if 'doc' in args else None #-- document identifier - self.dbname = args['dbname'] if 'dbname' in args else args['db'] - authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism'] - authSource=(args['authSource'] if 'authSource' in args else self.dbname) + # port = str(args['port']) if 'port' in args else '27017' + # host = args['host'] if 'host' in args else 'localhost' + # host = ":".join([host,port]) #-- Formatting host information here + # self.uid = args['doc'] if 'doc' in args else None #-- document identifier + # self.dbname = args['dbname'] if 'dbname' in args else args['db'] + self.authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism'] + # authSource=(args['authSource'] if 'authSource' in args else self.dbname) self._lock = False if 'lock' not in args else args['lock'] username = password = None - if 'username' in args and 'password' in args: - username = args['username'] - password=args['password'] + # if 'username' in args and 'password' in args: + # username = args['username'] + # password=args['password'] if 'auth_file' in args : _info = json.loads((open(args['auth_file'])).read()) - username = _info['username'] - password = _info['password'] - if 'mechanism' in _info: - authMechanism = _info['mechanism'] - if 'authSource' in _info: - authSource = _info['authSource'] - - + # username = _info['username'] + # password = _info['password'] + # if 'mechanism' in _info: + # authMechanism = _info['mechanism'] + # if 'authSource' in _info: + # authSource = _info['authSource'] + # # + # # We are allowing the authentication file to set collection and databases too + # if 'db' in _info : + # self.dbname = _info['db'] + # if 'doc' in _info : + # self.uid = _info['doc'] + + else: + _info = {} + _args = dict(args,**_info) + for key in _args : + if key in ['username','password'] : + username = _args['username'] if key=='username' else username + password = _args['password'] if key == 'password' else password + continue + value = _args[key] + + self.setattr(key,value) + # + # Let us perform aliasing in order to remain backwards compatible + + self.dbname = self.db if hasattr(self,'db')else self.dbname + self.uid = _args['table'] if 'table' in _args else (_args['doc'] if 'doc' in _args else (_args['collection'] if 'collection' in _args else None)) if username and password : - self.client = MongoClient(host, + self.client = MongoClient(self.host, username=username, password=password , - authSource=authSource, - authMechanism=authMechanism) + authSource=self.authSource, + authMechanism=self.authMechanism) + else: - self.client = MongoClient(host,maxPoolSize=10000) + self.client = MongoClient(self.host,maxPoolSize=10000) self.db = self.client[self.dbname] @@ -71,6 +93,11 @@ class Mongo : p = self.dbname in self.client.list_database_names() q = self.uid in self.client[self.dbname].list_collection_names() return p and q + def setattr(self,key,value): + _allowed = ['host','port','db','doc','authSource'] + if key in _allowed : + setattr(self,key,value) + pass def close(self): self.client.close() @@ -110,7 +137,9 @@ class MongoReader(Mongo,Reader): else: collection = self.db[self.uid] _filter = args['filter'] if 'filter' in args else {} - return collection.find(_filter) + _df = pd.DataFrame(collection.find(_filter)) + columns = _df.columns.tolist()[1:] + return _df[columns] def view(self,**args): """ This function is designed to execute a view (map/reduce) operation @@ -162,7 +191,7 @@ class MongoWriter(Mongo,Writer): @param info new record in the collection to be added """ # document = self.db[self.uid].find() - collection = self.db[self.uid] + #collection = self.db[self.uid] # if type(info) == list : # self.db[self.uid].insert_many(info) # else: