bug fix: pandas dtypes

pull/1/head
Steve L. Nyemba 3 years ago
parent cebc784494
commit 91434ec32d

@ -185,6 +185,7 @@ class BigQuery:
self.credentials = service_account.Credentials.from_service_account_file(path) self.credentials = service_account.Credentials.from_service_account_file(path)
self.dataset = _args['dataset'] if 'dataset' in _args else None self.dataset = _args['dataset'] if 'dataset' in _args else None
self.path = path self.path = path
self.dtypes = _args['dtypes'] if 'dtypes' in _args else None
def meta(self,**_args): def meta(self,**_args):
""" """
This function returns meta data for a given table or query with dataset/table properly formatted This function returns meta data for a given table or query with dataset/table properly formatted
@ -225,7 +226,13 @@ class BQReader(BigQuery,Reader) :
SQL += " LIMIT "+str(_args['limit']) SQL += " LIMIT "+str(_args['limit'])
if (':dataset' in SQL or ':DATASET' in SQL) and self.dataset: if (':dataset' in SQL or ':DATASET' in SQL) and self.dataset:
SQL = SQL.replace(':dataset',self.dataset).replace(':DATASET',self.dataset) SQL = SQL.replace(':dataset',self.dataset).replace(':DATASET',self.dataset)
return pd.read_gbq(SQL,credentials=self.credentials,dialect='standard') if SQL else None _info = {'credentials':self.credentials,'dialect':'standard'}
if 'dtypes' in _args or self.dtypes :
self.dtypes = _args ['dtypes'] if 'dtypes' in self.dtypes else None
if self.dtypes :
_info['dtypes'] = self.dtypes
return pd.read_gbq(SQL,*_info) if SQL else None
# return pd.read_gbq(SQL,credentials=self.credentials,dialect='standard') if SQL else None
class BQWriter(BigQuery,Writer): class BQWriter(BigQuery,Writer):
lock = Lock() lock = Lock()
def __init__(self,**_args): def __init__(self,**_args):
@ -237,7 +244,7 @@ class BQWriter(BigQuery,Writer):
def write(self,_info,**_args) : def write(self,_info,**_args) :
try: try:
if self.parallel : if self.parallel or 'lock' in _args :
BQWriter.lock.acquire() BQWriter.lock.acquire()
self._write(_info,**_args) self._write(_info,**_args)
finally: finally:

Loading…
Cancel
Save