diff --git a/transport/warehouse/iceberg.py b/transport/warehouse/iceberg.py index 7d7297a..b6926c9 100644 --- a/transport/warehouse/iceberg.py +++ b/transport/warehouse/iceberg.py @@ -3,7 +3,7 @@ dependency: - spark and SPARK_HOME environment variable must be set NOTE: When using streaming option, insure that it is inline with default (1000 rows) or increase it in spark-defaults.conf - + """ from pyspark.sql import SparkSession from pyspark import SparkContext @@ -110,7 +110,6 @@ class Writer (Iceberg): _prefix = self._getPrefix(**_args) if 'table' not in _args and not self._table : raise Exception (f"Table Name should be specified for catalog/database {_prefix}") - rdd = self._session.createDataFrame(_data,verifySchema=False) _mode = self._mode if 'mode' not in _args else _args['mode'] _table = self._table if 'table' not in _args else _args['table']