diff --git a/cms/cloud.py b/cms/cloud.py index 4b6df8d..7beed1d 100644 --- a/cms/cloud.py +++ b/cms/cloud.py @@ -26,6 +26,24 @@ def _format_root_folder (_root): _root = _root[:-1] return _root.replace('//','/') +def list_files(folder,_config) : + """ + List the content of a folder (html/md) for now + """ + _authfile = _config['system']['source']['auth'] + _handler = login(_authfile) + _files = _handler.list(folder,50) + + _content = [] + for _item in _files : + if _item.file_type == 'file' and _item.get_content_type() in ['text/markdown','text/html'] : + _uri = '/'.join(_item.path.split('/')[2:]) + _uri = _item.path + # _content.append({'text':_item.name.split('.')[0],'uri':_uri}) + _content.append(_item.name) + + return _content + def content(_args): """ :url @@ -49,10 +67,18 @@ def content(_args): _menu = {} #dict.fromkeys(_menu,[]) for _item in _files : _folder = _item.path.split(_item.name)[0].strip() - _folder = _folder.replace(_root,'').replace('/','') + _folder = _folder.replace(_root,'').replace('//','') + + # + # The following lines are intended to prevent an irradict recursive read of a folder content + # We want to keep things simple as we build the menu + # + if len (_folder.split('/')) > 2: + continue + else: + _folder = _folder.replace('/','') if _item.name[0] in ['.','_'] or _folder == '': continue ; - if _item.file_type == 'file' and _item.get_content_type() in ['text/markdown','text/html'] : # _folder = _item.path.split(_item.name)[0].strip() # _folder = _folder.replace(_root,'').replace('//','') @@ -61,12 +87,6 @@ def content(_args): _folder = _folder.replace('/' ,' ').strip() if _folder not in _menu : _menu [_folder] = [] - # print ([_item.name,_key, _key in _menu]) - - # _menuItem = _ref[_key] - # uri = '/'.join([_args['url'],_item.path]) - # uri = _item - # print ([_menuItem, _menuItem in _menu]) uri = '/'.join(_item.path.split('/')[2:]) uri = _item.path _menu[_folder].append({'text':_item.name.split('.')[0],'uri':uri}) @@ -130,9 +150,15 @@ def download(**_args): else: _stream = _handler.get_file_contents(_request.args['doc']) _handler.logout() + return _stream pass - +def _format (uri,_config) : + """ + This function does nothing but is used to satisfy the demands of a design pattern + @TODO: revisit the design pattern + """ + return uri def plugins (): """ This function publishes the plugins associated with this module diff --git a/cms/disk.py b/cms/disk.py index 7710811..98230e9 100644 --- a/cms/disk.py +++ b/cms/disk.py @@ -2,24 +2,39 @@ This file pulls the content from the disk """ import os -def folders (_path): +import importlib +import importlib.util +import copy +from mistune import markdown + + +def folders (_path,_config): """ This function reads the content of a folder (no depth, it must be simple) """ _content = os.listdir(_path) return [_name for _name in _content if os.path.isdir(os.sep.join([_path,_name])) if not _name.startswith('_')] - -def content(_folder): +def content(_folder,_config): """ :content of the folder """ - + _layout = _config['layout'] + if 'location' in _layout : + _uri = os.sep.join([_layout['root'] ,_folder.split(os.sep)[-1]]) + _path = os.sep.join([_layout['root'],_folder.split(os.sep)[-1]]) + else: + + _path = _folder if os.path.exists(_folder) : - _menuItems = os.listdir(_folder) + _menuItems = list_files(_folder,_config) #os.listdir(_folder) # return [{'text':_name.split('.')[0].replace('_', ' ').replace('-',' ').strip(),'uri': os.sep.join([_folder,_name])} for _name in os.listdir(_folder) if not _name.startswith('_') and os.path.isfile( os.sep.join([_folder,_name]))] - return [{'text':_name.split('.')[0].replace('_', ' ').replace('-',' ').strip(),'uri': os.sep.join([_folder,_name])} for _name in os.listdir(_folder) if not _name.startswith('_') and os.path.isfile( os.sep.join([_folder,_name]))] + + return [{'text':_name.split('.')[0].replace('_', ' ').replace('-',' ').strip(),'uri': os.sep.join([_path,_name])} for _name in os.listdir(_folder) if not _name[0] in ['.','_'] and os.path.isfile( os.sep.join([_folder,_name]))] else: return [] +def list_files(_folder,_config): + + return [name for name in os.listdir(_folder) if name[0] not in ['.','_']] def build (_config): #(_path,_content): """ building the menu for the site given the content is on disk @@ -27,20 +42,104 @@ def build (_config): #(_path,_content): :config configuration associated with the """ _path = _config['layout']['root'] - _items = folders(_path) - _subItems = [ content (os.sep.join([_path,_name]))for _name in _items ] + # if 'location' in _config['layout'] : + # _path = _config['layout']['location'] + _path = _realpath(_path,_config) + # print (_path) + _items = folders(_path,_config) + _subItems = [ content (os.sep.join([_path,_name]),_config)for _name in _items ] _r = {} for _name in _items : _index = _items.index(_name) + if _name.startswith('_') or len(_subItems[_index]) == 0: + continue + # print ([_name,_subItems[_index]]) if _name not in _r : _r[_name] = [] _r[_name] += _subItems[_index] + # _r = [_r[_key] for _key in _r if len(_r[_key]) > 0] return _r # return dict.fromkeys(_items,_subItems) -def html(uri) : - _html = (open(uri)).read() +def _realpath (uri,_config) : + _layout = _config['layout'] + + _uri = copy.copy(uri) + if 'location' in _layout : + _uri = os.sep.join([_layout['location'],_uri]) + return _uri + +def _format (uri,_config): + _layout = _config['layout'] + if 'location' in _layout : + return 'api/disk/read?uri='+uri + return uri +def read (**_args): + """ + This will read binary files from disk, and allow the location or not to be read + @TODO: add permissions otherwise there can be disk-wide reads + """ + request = _args['request'] + _layout = _args['config']['layout'] + + _uri = request.args['uri'] # if 'location' in _layout : + # _uri = os.sep.join([_layout['location'],_uri]) + _uri = _realpath(_uri, _args['config']) + + if os.path.exists(_uri): + f = open(_uri,mode='rb') + _stream = f.read() + f.close() + + return _stream + return None +def exists(**_args): + _path = _realpath(_args['uri'],_args['config']) + + # _layout = _args['config']['layout'] + # if 'location' in _layout : + # _path = os.sep.join([_layout['location'],_path]) + return os.path.exists(_path) +def html(_uri,_config) : + # _html = (open(uri)).read() + _path = _realpath(_uri,_config) + + _html = ( open(_path)).read() + _layout = _config['layout'] + if 'location' in _layout : + + _api = os.sep.join(['api/disk/read?uri=',_layout['root']]) + _html = _html.replace(_layout['root'],_api) + _html = markdown(_html) if _uri[-2:] in ['md','MD','Md','mD'] else _html return _html -def plugins (): - return {} +def plugins (**_args): + """ + This function will load plugins from disk given where they come from + :path path of the files + :name name of the module + """ + if 'path' not in _args : + return {'api/disk/read':read} + _path = _args['path'] #os.sep.join([_args['root'],'plugin']) + if os.path.isdir(_path): + files = os.listdir(_path) + if files : + files = [name for name in files if name.endswith('.py')] + if files: + _path = os.sep.join([_path,files[0]]) + else: + return None + else: + # + # LOG: not a file + return None + #-- We have a file ... + _name = _args['name'] + spec = importlib.util.spec_from_file_location(_name, _path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # + # LOG This plugin .... + return getattr(module,_name) if hasattr(module,_name) else None \ No newline at end of file diff --git a/cms/engine/__init__.py b/cms/engine/__init__.py index 03aae08..f6200cc 100644 --- a/cms/engine/__init__.py +++ b/cms/engine/__init__.py @@ -9,9 +9,14 @@ from jinja2 import Environment, BaseLoader, FileSystemLoader import importlib import importlib.util from cms import disk, cloud - +from . import basic class Loader : + """ + This class is designed to exclusively load configuration from disk into an object + :path path to the configuraiton file + :location original location (caller) + """ def __init__(self,**_args): self._path = _args['path'] self._original_location = None if 'location' not in _args else _args['location'] @@ -20,6 +25,7 @@ class Loader : self._menu = {} self._plugins={} self.load() + def load(self): """ @@ -32,19 +38,34 @@ class Loader : """ Initialize & loading configuration from disk """ - f = open (self._path) + f = open (self._path) self._config = json.loads(f.read()) - if self._caller : self._location = self._original_location.split(os.sep) # needed for plugin loading self._location = os.sep.join(self._location[:-1]) self._config['system']['portal'] = self._caller != None # - # let's see if we have a location for a key in the configuration + # let's see if we have a location for a key (i.e security key) in the configuration # + self.update_config() + # _system = self._config['system'] + # if 'source' in _system and 'key' in _system['source'] : + # _path = _system['source']['key'] + # if os.path.exists(_path): + # f = open(_path) + # _system['source']['key'] = f.read() + # f.close() + # self._system = _system + # self._config['system'] = _system + def update_config(self): + """ + We are going to update the configuration (source.key, layout.root) + """ _system = self._config['system'] + # + # updating security-key that allows the application to update on-demand if 'source' in _system and 'key' in _system['source'] : _path = _system['source']['key'] if os.path.exists(_path): @@ -53,6 +74,22 @@ class Loader : f.close() self._system = _system self._config['system'] = _system + _layout = self._config['layout'] + # + # update root so that the app can be launched from anywhere + # This would help reduce the footprint of the app/framework + _path = os.sep.join(self._path.split(os.sep)[:-1]) + _p = 'source' not in _system + _q = 'source' in _system and _system['source']['id'] != 'cloud' + _r = os.path.exists(_layout['root']) + if not _r and (_p or _q) : + # + # If we did running this app from installed framework (this should not apply to dependent apps) + # + _root = os.sep.join([_path,_layout['root']]) + self._config['layout']['root'] = _root + self._config['layout']['root_prefix'] = _root + def init_menu(self): """ This function will read menu and sub-menu items from disk structure, @@ -252,7 +289,8 @@ class Getter (Loader): _html = cloud.html(uri,dict(_args,**{'system':_system})) else: - _html = disk.html(uri) + + _html = disk.html(uri,self.layout()) # _html = (open(uri)).read() @@ -313,14 +351,19 @@ class Getter (Loader): else: _system= _data return _system + def layout(self): + return self._config['layout'] def get_app(self): return self._config['system']['app'] class Router : def __init__(self,**_args) : - path = _args['path'] - _app = Getter (path = path) + + # _app = Getter (path = path) + _app = Getter (**_args) + + self._id = 'main' # _app.load() self._apps = {} @@ -330,8 +373,8 @@ class Router : for _name in _system : _path = _system[_name]['path'] self._apps[_name] = Getter(path=_path,caller=_app,location=_path) - print ([_name, self._apps[_name].plugins().keys()]) - self._apps['main'] = _app + self._apps['main'] = _app + def set(self,_id): self._id = _id def get(self): diff --git a/cms/engine/basic.py b/cms/engine/basic.py new file mode 100644 index 0000000..60ebfdc --- /dev/null +++ b/cms/engine/basic.py @@ -0,0 +1,435 @@ +import json +import os +import io +import copy +from cms import disk, cloud +from jinja2 import Environment, BaseLoader, FileSystemLoader +import importlib +import importlib.util + + +class Initializer : + """ + This class handles initialization of all sorts associated with "cms engine" + :path + :location + :shared + """ + def __init__(self,**_args): + self._config = {'system':{},'layout':{},'plugins':{}} + self._shared = False if not 'shared' in _args else _args['shared'] + self._location= _args['location'] if 'location' in _args else None + self._menu = {} + # _source = self._config ['system']['source'] if 'source' in self._config['system'] else {} + # self._ISCLOUD = 'source' in self._config['system'] and self._config['system']['source']['id'] == 'cloud' + # print ([self._ISCLOUD,self._config['system'].keys()]) + self._ISCLOUD = False + self._caller = None if 'caller' not in _args else _args['caller'] + + # + # actual initialization of the CMS components + # self._iconfig(**_args) + # self._uconfig(**_args) + # self._isource() + # self._imenu() + # self._iplugins() + + self._args = _args + self.reload() + + def reload(self): + self._iconfig(**self._args) + self._uconfig(**self._args) + self._isource() + self._imenu() + self._iplugins() + + + # self._ISCLOUD = 'source' in self._config['system'] and self._config['system']['source']['id'] == 'cloud' + + def _handler (self): + """ + This function returns the appropriate handler to the calling code, The handler enables read/write from a location + """ + if self._ISCLOUD: #'source' in self._config['system'] and self._config['system']['source']['id'] == 'cloud' : + return cloud + else: + return disk + + def _imenu(self,**_args) : + pass + def _iplugins(self,**_args) : + """ + Initialize plugins from disk (always) + :plugins + """ + _config = self._config + PATH= os.sep.join([_config['layout']['root'],'_plugins']) + + # if not os.path.exists(PATH) : + # # + # # we need to determin if there's an existing + # print ([' **** ',PATH]) + # PATH = os.sep.join(self._path.split(os.sep)[:-1]+ [PATH] ) + if not os.path.exists(PATH) and self._location and os.path.exists(self._location) : + # + # overriding the location of plugins ... + if os.path.isfile(self._location) : + _location = os.sep.join(self._location.split(os.sep)[:-1]) + else: + _location = self._location + PATH = os.sep.join([_location, _config['layout']['root'],'_plugins']) + + _map = {} + # if not os.path.exists(PATH) : + # return _map + if 'plugins' not in _config : + _config['plugins'] = {} + _conf = _config['plugins'] + + for _key in _conf : + + _path = os.sep.join([PATH,_key+".py"]) + if not os.path.exists(_path): + continue + for _name in _conf[_key] : + _pointer = disk.plugins(path=_path,name=_name) + if _pointer : + _uri = "/".join(["api",_key,_name]) + _map[_uri] = _pointer + # + # We are adding some source specific plugins to the user-defined plugins + # This is intended to have out-of the box plugins... + # + + if self._ISCLOUD : + _plugins = cloud.plugins() + else: + _plugins = disk.plugins() + # + # If there are any plugins found, we should load them and use them + + if _plugins : + _map = dict(_map,**_plugins) + else: + pass + + self._plugins = _map + self._config['plugins'] = self._plugins + + + def _isource (self): + """ + Initializing the source of the data, so we can read/write load from anywhere + """ + if 'source' not in self._config['system'] : + return + # + # + self._ISCLOUD = 'source' in self._config['system'] and self._config['system']['source']['id'] == 'cloud' + _source = self._config['system']['source'] + if 'key' in _source : + # + _path = _source['key'] + if os.path.exists(_path) : + f = open(_path) + _source['key'] = f.read() + f.close() + self._config['system']['source'] = _source + def _ilayout(self,**_args): + """ + Initialization of the layout section (should only happen if ) being called via framework + :path path to the dependent apps + """ + _layout = self._config['layout'] + _path = os.sep.join(_args['path'].split(os.sep)[:-1]) + # + # find the new root and the one associated with the dependent app + # + + pass + def _imenu(self,**_args): + _gshandler = self._handler() + _object = _gshandler.build(self._config) #-- this will build the menu + # + # post-processing menu, overwrite items and behaviors + # + _layout = copy.deepcopy(self._config['layout']) + _overwrite = _layout['overwrite'] if 'overwrite' in _layout else {} + + for _name in _object : + _submenu = _object[_name] + _index = 0 + for _item in _submenu : + text = _item['text'].strip() + if text in _overwrite : + if 'uri' in _item and 'url' in _overwrite[text] : + del _item['uri'] + _item = dict(_item,**_overwrite[text]) + + if 'uri' in _item and 'type' in _item and _item['type'] != 'open': + _item['uri'] = _item['uri'] #.replace(_layout['root'],'') + # _item['uri'] = _gshandler._format(_item['uri'],self._config) + + _submenu[_index] = _item + _index += 1 + # + # updating menu _items as it relates to apps, configuration and the order in which they appear + # + _layout['menu'] = _object + self._menu = _object + self._config['layout'] = _layout + self._iapps() + self._iorder() + pass + + def _iorder (self): + _config = self._config + if 'order' in _config['layout'] and 'menu' in _config['layout']['order']: + _sortedmenu = {} + _menu = self._menu + for _name in _config['layout']['order']['menu'] : + if _name in _menu : + _sortedmenu[_name] = _menu[_name] + + _menu = _sortedmenu if _sortedmenu else _menu + # + # If there are missing items in the sorting + _missing = list(set(self._menu.keys()) - set(_sortedmenu)) + if _missing : + for _name in _missing : + _menu[_name] = self._menu[_name] + _config['layout']['menu'] = _menu #cms.components.menu(_config) + self._menu = _menu + self._config = _config + def _iapps (self): + """ + Initializing dependent applications into a menu area if need be + """ + _layout = self._config['layout'] + _menu = _layout['menu'] if 'menu' in _layout else {} + _system = self._config['system'] + _context= _system['context'] + if 'routes' in _system : + # _items = [] + _overwrite = {} if 'overwrite' not in self._config['layout'] else self._config['layout']['overwrite'] + for _text in _system['routes'] : + _item = _system['routes'][_text] + if 'menu' not in _item : + continue + uri = f'{_context}/{_text}' + # _items.append ({"text":_text,'uri':uri,'type':'open'}) + _label = _item['menu'] + if _label not in _menu : + _menu [_label] = [] + _menu[_label].append ({"text":_text,'uri':uri,'type':'open'}) + # + # update the menu items and the configuration + # + _layout['menu'] = _menu + self._config['layout'] = _layout + def _ilogo(self): + _gshandler = self._handler() + + pass + def _iconfig(self,**_args): + """ + Implement this in a base class + :path or uri + """ + raise Exception ("Configuration Initialization is NOT implemented") + def _uconfig(self,**_args): + """ + This file will update the configuration provided the CMS is run in shared mode (framework level) + """ + if not self._location : + return ; + _path = os.sep.join(self._location.split(os.sep)[:-1]) + _layout = self._config['layout'] + _oroot = _layout['root'] + _orw = _layout['overwrite'] + _index = _layout['index'] + _newpath = os.sep.join([_path,_oroot]) + self._config['system']['portal'] = self._caller != None + + if self._caller : + # + self._config['system']['caller'] = {'icon':'/caller/main/' + self._caller.system()['icon']} + # self._config['system']['caller'] = {'icon': self._caller.icon()} + + + if os.path.exists(_newpath) and not self._ISCLOUD: + # + # LOG: rewrite due to the mode in which the site is being run + # + _api = 'api/disk/read?uri='+_oroot + _stream = json.dumps(self._config) + _stream = _stream.replace(_oroot,_api) + # self._config = json.loads(_stream) + self._config['layout']['root'] = _oroot + + # self._config['layout']['overwrite'] = _orw + # + # We need to update the logo/icon + _logo = self._config['system']['logo'] + if self._ISCLOUD: + + _icon = f'/api/cloud/download?doc=/{_logo}' + + + else: + + _icon = f'api/disk/read?uri={_logo}' + if disk.exists(uri=_logo,config=self._config): + _icon = _logo + if self._location : + self._config['layout']['location'] = _path + + self._config['system']['icon'] = _icon + self._config['system']['logo'] = _logo + + # self.set('layout.root',os.sep.join([_path,_oroot])) + pass +class Accessor (Initializer): + """ + This is a basic structure for an application working in either portal or app mode + """ + def __init__(self,**_args): + super().__init__(**_args) + pass + def _iconfig(self, **_args): + """ + initialization of the configuration file i.e loading the files and having a baseline workable structure + :path|stream path of the configuration file + or stream of JSON configuration file + """ + if 'path' in _args : + f = open(_args['path']) + self._config = json.loads(f.read()) + f.close() + elif 'stream' in _args : + _stream = _args['stream'] + if type(_stream) == 'str' : + self._config = json.loads(_stream) + elif type(_stream) == io.StringIO : + self._config = json.loads( _stream.read()) + self._ISCLOUD = 'source' in self._config['system'] and self._config['system']['source']['id'] == 'cloud' + # + # + # self._name = self._config['system']['name'] if 'name' in self._config['system'] else _args['name'] + def system (self,skip=[]): + """ + This function returns system attributes without specific components + """ + _data = copy.deepcopy(self._config['system']) + exclude = skip + _system = {} + if exclude and _system: + for key in _data.keys() : + if key not in exclude : + _system[key] = _data[key] + else: + _system= _data + return _system + def layout (self): + return copy.copy(self._config['layout']) + def plugins (self): + return copy.copy(self._config['plugins']) + def config (self): + + return copy.copy(self._config) + def app(self): + _system = self.system() + return _system['app'] + def set(self,key,value): + """ + This function will update/set an attribute with a given value + :key + """ + _keys = key.split('.') + _found = 0 + if _keys[0] in self._config : + _object = self._config[_keys[0]] + for _akey in _object.keys() : + if _akey == _keys[-1] : + _object[_akey] = value + _found = 1 + break + + # + # + return _found + # +class MicroService (Accessor): + """ + This is a CMS MicroService class that is capable of initializing a site and exposing accessor functions + """ + def __init__(self,**_args): + super().__init__(**_args) + def format(_content,mimetype): + pass + def html (self,uri, id) : + _system = self.system() + _gshandler = self._handler() + # + #@TODO: + # The uri here must be properly formatted, We need to define the conditions for this + # + _html = _gshandler.html(uri,self._config) + return " ".join([f'