|
|
@ -3,7 +3,7 @@ dependency:
|
|
|
|
- spark and SPARK_HOME environment variable must be set
|
|
|
|
- spark and SPARK_HOME environment variable must be set
|
|
|
|
NOTE:
|
|
|
|
NOTE:
|
|
|
|
When using streaming option, insure that it is inline with default (1000 rows) or increase it in spark-defaults.conf
|
|
|
|
When using streaming option, insure that it is inline with default (1000 rows) or increase it in spark-defaults.conf
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
from pyspark.sql import SparkSession
|
|
|
|
from pyspark.sql import SparkSession
|
|
|
|
from pyspark import SparkContext
|
|
|
|
from pyspark import SparkContext
|
|
|
@ -110,7 +110,6 @@ class Writer (Iceberg):
|
|
|
|
_prefix = self._getPrefix(**_args)
|
|
|
|
_prefix = self._getPrefix(**_args)
|
|
|
|
if 'table' not in _args and not self._table :
|
|
|
|
if 'table' not in _args and not self._table :
|
|
|
|
raise Exception (f"Table Name should be specified for catalog/database {_prefix}")
|
|
|
|
raise Exception (f"Table Name should be specified for catalog/database {_prefix}")
|
|
|
|
|
|
|
|
|
|
|
|
rdd = self._session.createDataFrame(_data,verifySchema=False)
|
|
|
|
rdd = self._session.createDataFrame(_data,verifySchema=False)
|
|
|
|
_mode = self._mode if 'mode' not in _args else _args['mode']
|
|
|
|
_mode = self._mode if 'mode' not in _args else _args['mode']
|
|
|
|
_table = self._table if 'table' not in _args else _args['table']
|
|
|
|
_table = self._table if 'table' not in _args else _args['table']
|
|
|
|