v2.4
Steve Nyemba 2 months ago
parent 0fd29207cf
commit b2a2e49858

@ -3,7 +3,7 @@ dependency:
- spark and SPARK_HOME environment variable must be set
NOTE:
When using streaming option, insure that it is inline with default (1000 rows) or increase it in spark-defaults.conf
"""
from pyspark.sql import SparkSession
from pyspark import SparkContext
@ -110,7 +110,6 @@ class Writer (Iceberg):
_prefix = self._getPrefix(**_args)
if 'table' not in _args and not self._table :
raise Exception (f"Table Name should be specified for catalog/database {_prefix}")
rdd = self._session.createDataFrame(_data,verifySchema=False)
_mode = self._mode if 'mode' not in _args else _args['mode']
_table = self._table if 'table' not in _args else _args['table']

Loading…
Cancel
Save