bug fix mongodb,bigquery

pull/1/head
Steve Nyemba 2 years ago
parent 2c675d5656
commit b8fb538ec7

@ -8,7 +8,7 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = {
"name":"data-transport",
"version":"1.6.0",
"version":"1.6.1",
"author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":"MIT",
"packages":["transport"]}

@ -34,7 +34,7 @@ class Mongo :
:password password for current user
"""
self.authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
self.mechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
# authSource=(args['authSource'] if 'authSource' in args else self.dbname)
self._lock = False if 'lock' not in args else args['lock']
@ -64,7 +64,7 @@ class Mongo :
username=username,
password=password ,
authSource=self.authSource,
authMechanism=self.authMechanism)
authMechanism=self.mechanism)
else:
self.client = MongoClient(self.host,maxPoolSize=10000)
@ -76,7 +76,7 @@ class Mongo :
q = self.uid in self.client[self.dbname].list_collection_names()
return p and q
def setattr(self,key,value):
_allowed = ['host','port','db','doc','authSource']
_allowed = ['host','port','db','doc','authSource','mechanism']
if key in _allowed :
setattr(self,key,value)
pass
@ -92,10 +92,10 @@ class MongoReader(Mongo,Reader):
Mongo.__init__(self,**args)
def read(self,**args):
if 'mongo' in args :
if 'mongo' in args or 'cmd' :
#
# @TODO:
cmd = args['mongo']
cmd = args['mongo'] if 'mongo' in args else args['cmd']
if "aggregate" in cmd :
if "allowDiskUse" not in cmd :
cmd["allowDiskUse"] = True

@ -430,6 +430,9 @@ class BQWriter(BigQuery,Writer):
self.mode['destination_table'] = _args['table'].strip()
if 'schema' in _args :
self.mode['table_schema'] = _args['schema']
#
# Let us insure that the types are somewhat compatible ...
# _map = {'INTEGER':np.int64,'DATETIME':'datetime64[ns]','TIMESTAMP':'datetime64[ns]','FLOAT':np.float64,'DOUBLE':np.float64,'STRING':str}
# _mode = copy.deepcopy(self.mode)
_mode = self.mode
# _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)

Loading…
Cancel
Save