v2.4
Steve Nyemba 2 months ago
parent 0fd29207cf
commit b2a2e49858

@ -3,7 +3,7 @@ dependency:
- spark and SPARK_HOME environment variable must be set - spark and SPARK_HOME environment variable must be set
NOTE: NOTE:
When using streaming option, insure that it is inline with default (1000 rows) or increase it in spark-defaults.conf When using streaming option, insure that it is inline with default (1000 rows) or increase it in spark-defaults.conf
""" """
from pyspark.sql import SparkSession from pyspark.sql import SparkSession
from pyspark import SparkContext from pyspark import SparkContext
@ -110,7 +110,6 @@ class Writer (Iceberg):
_prefix = self._getPrefix(**_args) _prefix = self._getPrefix(**_args)
if 'table' not in _args and not self._table : if 'table' not in _args and not self._table :
raise Exception (f"Table Name should be specified for catalog/database {_prefix}") raise Exception (f"Table Name should be specified for catalog/database {_prefix}")
rdd = self._session.createDataFrame(_data,verifySchema=False) rdd = self._session.createDataFrame(_data,verifySchema=False)
_mode = self._mode if 'mode' not in _args else _args['mode'] _mode = self._mode if 'mode' not in _args else _args['mode']
_table = self._table if 'table' not in _args else _args['table'] _table = self._table if 'table' not in _args else _args['table']

Loading…
Cancel
Save