From 835e1253a50916fa8e77580c1290a3994f459a99 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Mon, 19 Jul 2021 19:07:06 -0500 Subject: [PATCH] bug fix: filereader --- setup.py | 4 ++-- transport/disk.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index a974ef7..f26d1ad 100644 --- a/setup.py +++ b/setup.py @@ -8,12 +8,12 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() args = { "name":"data-transport", - "version":"1.3.8.8", + "version":"1.3.9.0", "author":"The Phi Technology LLC","author_email":"info@the-phi.com", "license":"MIT", "packages":["transport"]} args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite'] -args["install_requires"] = ['pymongo','numpy','cloudant','pika','nzpy','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python'] +args["install_requires"] = ['pymongo','pandas','numpy','cloudant','pika','nzpy','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python'] args["url"] = "https://healthcareio.the-phi.com/git/code/transport.git" args['scripts'] = ['bin/transport'] if sys.version_info[0] == 2 : diff --git a/transport/disk.py b/transport/disk.py index 64cabd4..7f73cf7 100644 --- a/transport/disk.py +++ b/transport/disk.py @@ -26,10 +26,13 @@ class DiskReader(Reader) : def isready(self): return os.path.exists(self.path) def read(self,**args): + return pd.read_csv(self.path,delimiter=self.delimiter) + def stream(self,**args): """ This function reads the rows from a designated location on disk @param size number of rows to be read, -1 suggests all rows """ + size = -1 if 'size' not in args else int(args['size']) f = open(self.path,'rU') i = 1 @@ -39,7 +42,7 @@ class DiskReader(Reader) : if size == i: break if self.delimiter : - yield row.split(self.char) + yield row.split(self.delimiter) yield row f.close() class DiskWriter(Writer):