dev #8

Merged
steve merged 2 commits from dev into master 10 months ago

5
.gitignore vendored

@ -1 +1,6 @@
build
*.pyc *.pyc
*.csv
*.json
*.swp
*.egg-info

@ -32,7 +32,7 @@ import nzpy as nz #--- netezza drivers
import sqlite3 import sqlite3
import copy import copy
import os import os
import time
class SQLRW : class SQLRW :
lock = RLock() lock = RLock()
@ -357,7 +357,7 @@ class SQLWriter(SQLRW,Writer):
# #
# Writing with schema information ... # Writing with schema information ...
rows = _info.iloc[i].to_sql(self.table,self._engine,schema=self.schema,if_exists='append',index=False) rows = _info.iloc[i].to_sql(self.table,self._engine,schema=self.schema,if_exists='append',index=False)
time.sleep(1)
else: else:
_fields = ",".join(self.fields) _fields = ",".join(self.fields)
_sql = _sql.replace(":fields",_fields) _sql = _sql.replace(":fields",_fields)
@ -495,10 +495,11 @@ class BQWriter(BigQuery,Writer):
# _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000) # _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)
# #
# Let us adjust the chunking here # Let us adjust the chunking here
self._chunkks = 10 if _df.shape[0] > SQLRW.MAX_CHUNK and self._chunks == 1 else self._chunks self._chunks = 10 if _df.shape[0] > SQLRW.MAX_CHUNK and self._chunks == 1 else self._chunks
_indexes = np.array_split(np.arange(_df.shape[0]),self._chunks) _indexes = np.array_split(np.arange(_df.shape[0]),self._chunks)
for i in _indexes : for i in _indexes :
_df.iloc[i].to_gbq(**self.mode) _df.iloc[i].to_gbq(**self.mode)
time.sleep(1)
pass pass
# #
# Aliasing the big query classes allowing it to be backward compatible # Aliasing the big query classes allowing it to be backward compatible

@ -1,2 +1,2 @@
__author__ = 'The Phi Technology' __author__ = 'The Phi Technology'
__version__= '1.9.2' __version__= '1.9.3'

Loading…
Cancel
Save