You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
"""
|
|
|
|
|
dependency:
|
|
|
|
|
- spark and SPARK_HOME environment variable must be set
|
|
|
|
|
"""
|
|
|
|
|
else:
|
|
|
|
|
#
|
|
|
|
|
# Should we set the default as the first one if available ?
|
|
|
|
|
#
|
|
|
|
|
pass
|
|
|
|
|
self._catalogName = self._catalog.currentCatalog()
|
|
|
|
|
self._databaseName = self._catalog.currentDatabase()
|
|
|
|
|
def meta (self,**_args) :
|
|
|
|
|
"""
|
|
|
|
|
This function should return the schema of a table (only)
|
|
|
|
|
"""
|
|
|
|
|
_schema = []
|
|
|
|
|
try:
|
|
|
|
|
_tableName = self._getPrefix(**_args) + f".{_args['table']}"
|
|
|
|
|
_tmp = self._session.table(_tableName).schema
|
|
|
|
|
_schema = _tmp.jsonValue()['fields']
|
|
|
|
|
for _item in _schema :
|
|
|
|
|
del _item['nullable'],_item['metadata']
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
return _schema
|
|
|
|
|
def _getPrefix (self,**_args):
|
|
|
|
|
_catName = self._catalogName if 'catalog' not in _args else _args['catalog']
|
|
|
|
|
_datName = self._databaseName if 'database' not in _args else _args['database']
|
|
|
|
|
|
|
|
|
|
return '.'.join([_catName,_datName])
|
|
|
|
|
def close(self):
|
|
|
|
|
# # _mode = 'overwrite'
|
|
|
|
|
# # rdd.write.format('iceberg').mode(_mode).saveAsTable(_table)
|
