From b2a2e498589838d8d1c97a0b64c39bf48a9b6c1d Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Sun, 1 Dec 2024 23:13:00 -0600 Subject: [PATCH] bug fix --- transport/warehouse/iceberg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/transport/warehouse/iceberg.py b/transport/warehouse/iceberg.py index 7d7297a..b6926c9 100644 --- a/transport/warehouse/iceberg.py +++ b/transport/warehouse/iceberg.py @@ -3,7 +3,7 @@ dependency: - spark and SPARK_HOME environment variable must be set NOTE: When using streaming option, insure that it is inline with default (1000 rows) or increase it in spark-defaults.conf - + """ from pyspark.sql import SparkSession from pyspark import SparkContext @@ -110,7 +110,6 @@ class Writer (Iceberg): _prefix = self._getPrefix(**_args) if 'table' not in _args and not self._table : raise Exception (f"Table Name should be specified for catalog/database {_prefix}") - rdd = self._session.createDataFrame(_data,verifySchema=False) _mode = self._mode if 'mode' not in _args else _args['mode'] _table = self._table if 'table' not in _args else _args['table']