diff --git a/config.json b/config.json index 36f6c3b..4f1a7c4 100755 --- a/config.json +++ b/config.json @@ -1,6 +1,7 @@ { -"id":"debug", +"id":"osx-sierra", +"api":"http://localhost/monitor", "key":"c259e8b1-e2fb-40df-bf03-f521f8ee352d", -"apps":["chrome","itunes","firefox"], -"folders":["/Users/steve/Music"] +"apps":["iTerm2","monitor/server","firefox","itunes"], +"folders":["/Users/steve/git/monitor/client","/Users/steve/Downloads","/Users/steve/the-phi"] } diff --git a/init.sh b/init.sh index 6f481aa..6bb70ab 100755 --- a/init.sh +++ b/init.sh @@ -16,9 +16,9 @@ install(){ } upgrade(){ git pull - count=`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|wc -l` + count=`$PWD/sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|wc -l` if [ ! "$count" = "0" ]; then - `sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade` + `$PWD/sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade` else echo "No Upgrade required for sandbox" fi @@ -26,7 +26,7 @@ upgrade(){ } start(){ - sandbox/bin/python src/data-collector.py --path $PWD/config.json + $PWD/sandbox/bin/python src/data-collector.py --path $PWD/config.json } stop(){ diff --git a/requirements.txt b/requirements.txt index 4fba40d..5843af7 100755 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,10 @@ python-socketio==1.6.2 pytz==2016.10 requests==2.18.3 restkit==4.2.2 +scipy==1.0.0 six==1.10.0 socketpool==0.5.3 urllib3==1.22 Werkzeug==0.11.11 +xmljson +xmltodict diff --git a/src/data-collector.py b/src/data-collector.py index 85cba33..09c88ae 100755 --- a/src/data-collector.py +++ b/src/data-collector.py @@ -46,8 +46,10 @@ class Collector(Thread) : #headers['content-type'] = 'application/json' try: self.key = SYS_ARGS['key'] - Logger.log(subject='Collector',object='api',action='request',value=ENDPOINT) - url = "/".join([ENDPOINT,"init/collector"]) + # Logger.log(subject='Collector',object='api',action='request',value=ENDPOINT) + # url = "/".join([ENDPOINT,"init/collector"]) + Logger.log(subject='Collector',object='api',action='request',value=SYS_ARGS['api']) + url = "/".join([SYS_ARGS['api'],"init/collector"]) r = requests.post(url,headers=headers) @@ -96,6 +98,9 @@ class Collector(Thread) : config = {"store":self.store,"plan":self.plan} + #@TODO: add SYS_ARGS content so apps that are configured reboot/delete folders that are marked + # This is an important security measure! + # self.manager = Manager() self.manager.init(node=SYS_ARGS['id'],agents=_agents,actors=_actors,config=config,key=self.key,host=SYS_ARGS['host']) diff --git a/src/utils/agents/actor.py b/src/utils/agents/actor.py index fadf018..864ae5f 100755 --- a/src/utils/agents/actor.py +++ b/src/utils/agents/actor.py @@ -122,6 +122,8 @@ class Apps(Actor) : @TODO We need to find the command in case the app has crashed """ try: + print"" + print cmd os.system(cmd +" &") self.log(action='startup',value=cmd) except Exception, e: diff --git a/src/utils/agents/manager.py b/src/utils/agents/manager.py index f96051f..f7cc5bd 100755 --- a/src/utils/agents/manager.py +++ b/src/utils/agents/manager.py @@ -41,6 +41,7 @@ class Manager() : _args={"host":"dev.the-phi.com","qid":self.id,"uid":self.key} # # Connecting to the messaging service + self.qlistener = self.factory.instance(type="QueueListener",args=_args) self.qlistener.callback = self.callback self.qlistener.init(self.id) @@ -48,6 +49,7 @@ class Manager() : # self.qlistener.read() thread = (Thread(target=self.qlistener.read)) thread.start() + def update(self) : """ This method inspect the plans for the current account and makes sure it can/should proceed @@ -80,47 +82,53 @@ class Manager() : self.actors = self.filter('actors',meta,self.actors) self.setup(meta) - def filter_collectors(self,meta) : - """ - remove collectors that are not specified by the plan - Note that the agents (collectors) have already been initialized ? - """ - values = meta['agents'].replace(' ','').split(',') - self.agents = [agent for agent in self.agents if agent.getName() in values] + # def filter_collectors(self,meta) : + # """ + # remove collectors that are not specified by the plan + # Note that the agents (collectors) have already been initialized ? + # """ + # values = meta['agents'].replace(' ','').split(',') + # self.agents = [agent for agent in self.agents if agent.getName() in values] - def filter_actors(self,meta): - """ - removes actors that are NOT specified by the subscription plan - Note that the actor have already been instatiated and need initialization - """ - values = meta['actors'].replace(' ','').split('.') - self.actors = [actor for actor in self.actors if actor.getName() in values] + # def filter_actors(self,meta): + # """ + # removes actors that are NOT specified by the subscription plan + # Note that the actor have already been instatiated and need initialization + # """ + # values = meta['actors'].replace(' ','').split('.') + # self.actors = [actor for actor in self.actors if actor.getName() in values] def filter(self,id,meta,objects): + """ + This function filters the agents/actors given what is available in the user's plan + + """ values = meta[id].replace(' ','').split(',') return [item for item in objects if item.getName() in values] def setup(self,meta) : - conf = {"folders":None,"apps":None} - read_class = self.config['store']['class']['read'] - read_args = self.config['store']['args'] + # conf = {"folders":None,"apps":None} + # read_class = self.config['store']['class']['read'] + # read_args = self.config['store']['args'] - args = None - couchdb = self.factory.instance(type=read_class,args=read_args) - args = couchdb.view('config/apps',key=self.key) - if len(args.keys()) > 0 : - self.apply_setup('apps',args) - args = couchdb.view('config/folders',key=self.key) - - if 'folder_size' not in meta : - args['threshold'] = meta['folder_size'] - self.apply_setup('folders',args) + # args = None + # couchdb = self.factory.instance(type=read_class,args=read_args) + # args = couchdb.view('config/apps',key=self.key) + # if len(args.keys()) > 0 : + # self.apply_setup('apps',args) + # args = couchdb.view('config/folders',key=self.key) + # if 'folder_size' not in meta : + # # args['threshold'] = meta['folder_size'] + # self.apply_setup('folders',meta) + #self.apply_setup('folders',meta) + #@TODO: For now app actors don't need any particular initialization + pass def apply_setup(self,name,args) : for actor in self.actors : - if args is not None and actor.getName() == name and len(args.keys()) > 0: + if args is not None and actor.getName() == name and len(args.keys()) > 0: actor.init(args) def isvalid(self): @@ -148,6 +156,7 @@ class Manager() : if 'node' in message and message['node'] == self.id : action = message['action'] params = message['params'] + # params['plan'] = self.plan['metadata'] self.delegate(action,params) def delegate(self,action,params): @@ -193,7 +202,6 @@ class Manager() : if type(row)==list and len(row) == 0 : continue - print get.getName(),len(row) # # index = self.agents.index(agent) diff --git a/src/utils/services.py b/src/utils/services.py new file mode 100755 index 0000000..b58c03b --- /dev/null +++ b/src/utils/services.py @@ -0,0 +1,598 @@ +""" + CloudView Engine 2.0 + The Phi Technology LLC - Steve L. Nyemba + + This is a basic cloud view engine that is designed to be integrated into any service and intended to work for anyone provided they have signed up with the cloud service provider + The intent is to make the engine a general purpose engine that can be either deployed as a service (3-launchpad) or integrated as data feed for a third party utility + +""" +from __future__ import division +from threading import Thread +import json +import requests +from xmljson import yahoo as bf +from xml.etree.ElementTree import Element, tostring, fromstring, ElementTree as ET +import xmltodict +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from StringIO import StringIO +class Cloud: + BYTES_TO_GB = 1000000000 + Config = None + STREAMING_URI = None + @staticmethod + def instance(id,**args): + id = id.strip() + if id == 'skydrive' : + id = 'one-drive' + + handler = None + path = args['path'] if 'path' in args else None + if not Cloud.Config and path: + f = open(path) + Cloud.Config = json.loads(f.read()) + Cloud.STREAMING_URI = str(Cloud.Config['api']) + Cloud.Config = Cloud.Config['cloud'] + f.close() + if path and id in Cloud.Config : + context = Cloud.Config[id] + className = context['class'] + config = json.dumps(context['config']) + handler = eval( "".join([className,"(",config,")"])) + + # + # In case a stream was passed in ... + # + if 'stream' in args: + stream = args['stream'] + context = Cloud.Config[id] + className = context['class'] + + handler = eval("".join([className,"(None)"])) + handler.from_json(stream) + # + # Once the handler is rovided we must retrieve the service given the key + # The key provides information about what files to extract as well as the preconditions + # @TODO: + # - Keys are maintained within the stripe account/couchdb + # - + return handler + + def __init__(self): + self.access_token = None + self.refresh_token= None + self.files = [] + self.client_id = None + self.secret = None + self.mfiles = {} + self.folders={} + + def to_json(self): + object = {} + keys = vars(self) + for key in keys: + value = getattr(self,key) + object[key] = value + return json.dumps(object) + def from_json(self,stream): + ref = json.loads(stream) ; + for key in ref.keys() : + value = ref[key] + setattr(self,key,value) + # self.access_token = ref['access_token'] + # self.refesh_token = ref['refresh_token'] + # self.files = ref['files'] + """ + This function matches a name with a list of possible features/extensions + + """ + def match(self,filename,filters): + if isinstance(filters,str): + filters = [filters] + return len(set(filename.lower().split('.')) & set(filters)) > 0 + + def getName(self): + return self.__class__.__name__.lower() + def get_authURL(self): + + config = Cloud.Config[self.getName()]['config'] + url = config['authURL'] + + if '?' in url == False: + url += '?' + keys=['client_id','redirect_uri'] + p = [] + for id in keys: + value = config[id] + p.append(id+'='+value) + url = url +"&"+ "&".join(p) + + return url +Cloud.Config = {} + +class Google(Cloud): + def __init__(self,conf=None): + Cloud.__init__(self) + def getName(self): + return 'google-drive' + def init(self,token): + self.refresh_token = token + self._refresh() + + def _refresh(self,code=None): + url = "https://accounts.google.com/o/oauth2/token" + headers = {"Content-Type":"application/x-www-form-urlencoded"} + data = {"client_id":self.client_id,"client_secret":self.secret} + if code : + grant_type = 'authorization_code' + data['code'] = code + else: + data['refresh_token'] = self.refresh_token + grant_type = 'refresh_token' + + data['grant_type'] = grant_type + data['redirect_uri'] = self.redirect_uri + + resp = requests.post(url,headers=headers,data=data) + r = json.loads(resp.text) + if 'access_token' in r: + self.access_token = r['access_token'] + self.refresh_token = r['refresh_token'] if 'refresh_token' in r else r['access_token'] + self.id_token = r['id_token'] + + + def create_file(self,**args): + url = "https://www.googleapis.com/upload/drive/v2/files" ; + headers = {"Authorization":"Bearer "+self.access_token} + headers['Content-Type'] = args['mimetype'] + params = args['params'] + if 'data' not in args : + r = requests.post(url,params = params,headers=headers) + else: + data = args['data'] + r = requests.post(url,data=data,params = params,headers=headers) + return r.json() + def update_metadata(self,id,metadata) : + url = "https://www.googleapis.com/drive/v2/files" + headers = {"Authorization":"Bearer "+self.access_token} + headers['Content-Type'] = 'application/json; charset=UTF-8' + + if id is not None : + url += ("/"+id) + r = requests.put(url,json=metadata,headers=headers) + else: + # url += ("/?key="+self.secret) + r = requests.post(url,data=json.dumps(metadata),headers=headers) + + return r.json() + + def upload(self,folder,mimetype,file): + """ + This function will upload a file to a given folder and will provide + If the folder doesn't exist it will be created otherwise the references will be fetched + This allows us to avoid having to create several folders with the same name + """ + r = self.get_files(folder) + + if len(r) == 0 : + info = {"name":folder, "mimeType":"application/vnd.google-apps.folder"} + r = self.update_metadata(None,{"name":folder,"title":folder, "mimeType":"application/vnd.google-apps.folder"}) + else: + r = r[0] + parent = r + parent = {"kind":"drive#file","name":folder,"id":parent['id'],"mimeType":"application/vnd.google-apps.folder"} + + + r = self.create_file(data=file.read(),mimetype=mimetype,params={"uploadType":"media"}) + info = {"title":file.filename,"description":"Create by Cloud View"} + info['parents'] = [parent] + + r = self.update_metadata(r['id'],metadata=info) + return r + + +""" + This class is designed to allow users to interact with one-drive +""" +class OneDrive(Cloud): + def __init__(self,conf): + Cloud.__init__(self) + def getName(self): + return 'one-drive' + def init(self,token): + self.refresh_token = token + self._refresh() + + def _refresh(self,code=None): + url = "https://login.live.com/oauth20_token.srf" + #url="https://login.microsoftonline.com/common/oauth2/v2.0/token" + + headers = {"Content-Type":"application/x-www-form-urlencoded"} + form = {"client_id":self.client_id,"client_secret":self.secret} + if code: + grant_type = 'authorization_code' + form['code'] = str(code) + else: + grant_type = 'refresh_token' + form['refresh_token'] = self.refresh_token + form['grant_type'] = grant_type + if self.redirect_uri: + form['redirect_uri'] = self.redirect_uri + r = requests.post(url,headers=headers,data=form) + r = json.loads(r.text) + if 'access_token' in r: + self.access_token = r['access_token'] + self.refresh_token = r['refresh_token'] + + def upload(self,folder,mimetype,file): + """ + @param folder parent.id + @param name name of the file with extension + @param stream file content + + """ + path = folder+"%2f"+file.filename + url = "https://apis.live.net/v5.0/me/skydrive/files/:name?access_token=:token".replace(":name",path).replace(":token",self.access_token) ; + + header = {"Authorization": "Bearer "+self.access_token,"Content-Type":mimetype} + header['Content-Type']= mimetype + r = requests.put(url,header=header,files=file) + r = r.json() + return r + +""" + This class uses dropbox version 2 API +""" +class Dropbox(Cloud): + def __init__(self): + Cloud.__init__(self) + def init(self,access_token): + self.access_token = access_token + def upload(self,folder,mimetype,file): + """ + @param folder parent.id + @param name name of the file with extension + @param stream file content + + @TODO: This upload will only limit itself to 150 MB, it is possible to increase this size + """ + url = "https://content.dropboxapi.com/2/files/upload" + folder = folder if folder is not None else "" + path = "/"+folder+"/"+file.name.split('/')[-1] + path = path.replace("//","/") + header = {"Authorization":"Bearer "+self.access_token,"Content-Type":mimetype} + #header['autorename']= "false" + header['mode'] = "add" + #header['mute'] = "false" + header['Dropbox-API-Arg'] = json.dumps({"path":path}) + r = requests.post(url,headers=header,data=file.read()) + print r.text + r = r.json() + return r + + +""" + This class implements basic interactions with box (cloud service providers) + Available functionalities are: authentication, file access,share and stream/download +""" +class Box(Cloud) : + def __init__(self,conf): + Cloud.__init__(self); + if conf is not None: + self.client_id = conf['client_id'] + self.secret = conf['secret'] + self.redirect_uri = conf['redirect_uri'] if 'redirect_uri' in conf else None + def init(self,token): + self.refresh_token = token + def set(self,code) : + self._access(code) + return 1 if self.access_token else 0 + + def _access(self,code): + body = {"client_id":self.client_id,"client_secret":self.secret,"grant_type":"authorization_code","code":code,"redirect_uri":self.redirect_uri} + headers = {"Content-Type":"application/x-www-form-urlencoded"} + url = "https://app.box.com/api/oauth2/token" + r = requests.post(url,headers=headers,data=body) + r = json.loads(r.text) + if 'error' not in r: + self.access_token = r['access_token'] + self.refresh_token= r['refresh_token'] + def _refresh(self,authToken) : + body = {"client_id":self.client_id,"client_secret":self.secret,"grant_type":"refresh_token"} + url = "https://app.box.com/api/oauth2/token"; + headers = {"Content-Type":"application/x-www-form-urlencoded"} + r = requests.post(url,headers=headers,data=body) + r = json.loads(r.text) + if 'error' not in r : + self.access_token = r['access_token'] + def get_user(self): + url = "https://api.box.com/2.0/users/me" + headers = {"Authorization":"Bearer "+self.access_token} + r = requests.get(url,headers=headers) + r = json.loads(r.text) + if 'login' in r : + #BYTES_TO_GB = 1000000000 + user = {"uii":r['name'],"uid":r['login']} + usage = {"size":r['space_amount']/Cloud.BYTES_TO_GB,"used":r['space_used']/Cloud.BYTES_TO_GB,"units":"GB"} + user['usage'] = usage + return user + else: + return None + + def format(self,item) : + file = {"name":item['name'],"origin":"box","id":item['id'],"url":""} + meta = {"last_modified":item['content_modified_at']} + return file + def get_files(self,ext,url=None): + ext = " ".join(ext) + url = "https://api.box.com/2.0/search?query=:filter&type=file" + url = url.replace(":filter",ext) + headers = {"Authorization":"Bearer "+self.access_token} + + r = requests.get(url,headers=headers) ; + r = json.loads(r.text) + if 'entries' in r: + #self.files = [ self.format(file) for file in r['entries'] if file['type'] == 'file' and 'id' in file] + for item in r : + if item['type'] == 'file' and 'id' in item : + self.files.append( self.format(item)) + else: + # + # We are dealing with a folder, this is necessary uploads + # + self.folder[item['name']] = item["id"] + + return self.files + def stream(self,url): + headers = {"Authorization":"Bearer "+self.access_token} + r = requests.get(url,headers=headers,stream=True) + yield r.content + def share(self,id): + url = "https://api.box.com/2.0/files/:id".replace(":id",id); + headers = {"Authorization":"Bearer "+self.access_token,"Content-Type":"application/json"} + body = {"shared_link":{"access":"open","permissions":{"can_download":True}}} + r = requests.put(url,headers=headers,data=json.dumps(body)) + r = json.loads(r.text) + if 'shared_link' in r: + return r['shared_link']['download_url'] + + return None + def upload(self,folder,mimetype,file): + """ + @param folder parent.id + @param name name of the file with extension + @param stream file content + """ + if folder not in self.folders : + # + # Let us create the folder now + # + url = "https://api.box.com/2.0/folders" + header = {"Authorization":"Bearer "+self.access_token} + pid = self.folders["/"] if "/" in self.folders else self.folders[""] + data = {"parent":{"id":str(pid)}} + + r = requests.post(url,header=header,data=data) + r = r.json() + pid = r["id"] + else: + pid = self.folders[folder] + url = "https://upload.box.com/api/2.0/files/content" + header = {"Authorization Bearer ":self.access_token,"Content-Type":mimetype} + r = requests.post(url,header=header,file=file) + r = r.json() + return r + + +class SugarSync(Cloud): + def __init__(self): + Cloud.__init__(self) + + def __init__(self,conf): + Cloud.__init__(self); + if conf is not None: + self.client_id = conf['app_id'] + self.private_key = conf['private_key'] + self.access_key = conf['access_key'] + #self.access_token = None + #self.refresh_token= None + # self.redirect_uri = conf['redirect_uri'] if 'redirect_uri' in conf else None + #self.files = [] + def init(self,token): + self.refresh_token = token + self._refresh() + def login(self,email,password): + xml = ':username:password:app_id:accesskey:privatekey' + xml = xml.replace(":app_id",self.app_id).replace(":privatekey",self.private_key).replace(":accesskey",self.access_key).replace(":username",email).replace(":password",password) + headers = {"Content-Type":"application/xml","User-Agent":"The Phi Technology"} + r = requests.post(url,headers=headers,data=xml) + self.refresh_token = r.headers['Location'] + + + def _refresh(self): + xml = ':accesskey:privatekey:authtoken' + xml = xml.replace(":accesskey",self.access_key).replace(":privatekey",self.private_key).replace(":authtoken",self.refresh_token) + + headers = {"Content-Type":"application/xml","User-Agent":"The Phi Technology LLC"} + url = "https://api.sugarsync.com/authorization" + r = requests.post(url,data=xml,headers=headers) + + self.access_token = r.headers['Location'] + def format(self,item): + file = {} + file['name'] = item['displayName'] + file['url'] = item['fileData'] + file['id'] = item['ref'] + meta = {} + meta['last_modified'] = item['lastModified'] + file['meta'] = meta + return file + + def get_files(self,ext,url=None) : + if url is None: + url = "https://api.sugarsync.com/folder/:sc:3989243:2/contents"; + headers = {"Authorization":self.access_token,"User-Agent":"The Phi Technology LLC","Content-Type":"application/xml;charset=utf-8"} + r = requests.get(url,headers=headers) + stream = r.text #.encode('utf-8') + r = xmltodict.parse(r.text) + + if 'collectionContents' in r: + + r = r['collectionContents'] + # + # Extracting files in the current folder then we will see if there are any subfolders + # The parser has weird behaviors that leave inconsistent objects (field names) + # This means we have to filter it out by testing the item being processed + if 'file' in r: + if isinstance(r['file'],dict): + self.files += [ self.format(r['file']) ] + else: + + #self.files += [self.format(item) for item in r['file'] if isinstance(item,(str, unicode)) == False and item['displayName'].endswith(ext)] + self.files += [self.format(item) for item in r['file'] if isinstance(item,(str, unicode)) == False and self.match(item['displayName'],ext)] + + if 'collection' in r: + if isinstance(r['collection'],dict) : + # + # For some unusual reason the parser handles single instances as objects instead of collection + # @NOTE: This is a behavior that happens when a single item is in the collection + # + self.get_files(ext,r['collection']['contents']) + for item in r['collection'] : + if 'contents' in item: + if isinstance(item,(str, unicode)) == False: + self.files += self.get_files(ext,item['contents']) + #[ self.get_files(ext,item['contents']) for item in r['collection'] if item['type'] == 'folder'] + return self.files + + def get_user(self): + url = "https://api.sugarsync.com/user" + headers = {"Authorization":self.access_token,"User-Agent":"The Phi Technology LLC","Content-Type":"application/xml;charset=utf-8"} + r = requests.get(url,headers=headers) + + r = xmltodict.parse(r.text) + r = r['user'] + + + if 'username' in r and 'quota' in r: + user = {"uid":r['username'],"uii":r['nickname']} + size = long(r['quota']['limit']) + used = long(r['quota']['usage']) + usage = {"size":size/Cloud.BYTES_TO_GB,"used":used/Cloud.BYTES_TO_GB,"units":"GB"} + user['usage'] = usage + return user + else: + return None + def stream(self,url): + headers = {"Authorization":self.access_token} + r = requests.get(url,headers=headers,stream=True) + yield r.content + """ + This function will create a public link and share it to designated parties + """ + def share(self,id): + url = "https://api.sugarsync.com/file/:id".replace(":id",id); + xml = ''; + headers = {"Content-Type":"application/xml","Authorization":self.access_token,"User-Agent":"The Phi Technology LLC"} + r = requests.put(url,header=header,data=xml) + r = xmltodict.parse(r.text) + if 'file' in r: + return r['file']['publicLink']['content']+"?directDownload=true" + else: + return None + + def upload(self,folder,mimetype,file): + + name = foler+"/"+file.filename + xml = ':name:type' + xml = xml.replace(':name',name).replace(':type',mimetype) + header = {"content-type":"application/xml","User-Agent":"The Phi Technology LLC"} + header['Authorization'] = self.access_token + + r = requests.post(url,headers=header,files=file,data=xml) + pass + +class iTunes(Cloud): + def __init__(self): + Cloud.__init__(self) + self.url_topsongs = "http://ax.itunes.apple.com/WebObjects/MZStoreServices.woa/ws/RSS/topsongs/limit=:limit/explicit=false/json" + self.url_search = "http://itunes.apple.com/search?term=:keyword&limit=:limit&media=music" + def parse_search(self,obj): + + files = [] + try: + logs = obj['results'] + + for item in logs : + + file = {} + file['id'] = item['trackId'] + file['name'] = item['trackName'] + file['id3'] = {} + file['id3']['track'] = item['trackName'] + file['id3']['title'] = item['trackName'] + file['id3']['artist']= item['artistName'] + file['id3']['album'] = item['collectionName'] + file['id3']['genre'] = item['primaryGenreName'] + file['id3']['poster']= item['artworkUrl100'] + file['url'] = item['previewUrl'] + + files.append(file) + except Exception,e: + print e + return files + def parse_chart(self,obj): + """ + This function will parse the tonsongs returned by the itunes API + """ + files = [] + + try: + logs = obj['feed']['entry'] + if isinstance(logs,dict) : + logs = [logs] + + for item in logs : + + file = {'name':item['im:name']['label'],'id3':{}} + file['id'] = item['id']['attributes']['im:id'] + file['id3'] = {} + file['id3']['artist'] = item['im:artist']['label'] + file['id3']['track'] = item['title']['label'] + file['id3']['title'] = item['title']['label'] + file['id3']['album'] = item['im:collection']['im:name']['label'] + file['id3']['genre'] = item['category']['attributes']['term'] + index = len(item['im:image'])-1 + file['id3']['poster'] = item['im:image'][index]['label'] + url = [link['attributes']['href'] for link in item['link'] if 'im:assetType' in link['attributes'] and link['attributes']['im:assetType']=='preview'] + + if len(url) > 0: + url = url[0] + file['url'] = url #item['link'][1]['attributes']['href'] //'im:assetType' == 'preview' and 'im:duration' is in the sub-item + files.append(file) + + else: + continue + except Exception,e: + print e + # + # @TODO: Log the error somewhere to make it useful + + return files + + def parse(self,obj) : + if 'feed' in obj and 'entry' in obj['feed']: + return self.parse_chart(obj) + elif 'results' in obj : + return self.parse_search(obj) + else: + return [] + def get_files(self,keyword=None,limit="1") : + url = self.url_search if keyword is not None else self.url_topsongs + keyword = "" if keyword is None else keyword + # limit = "50" if keyword == "" else "1" + + url = url.replace(":keyword",keyword.replace(' ','+')).replace(':limit',limit) + r = requests.get(url) + r= r.json() + return self.parse(r) diff --git a/test/TestActors.py b/test/TestActors.py new file mode 100644 index 0000000..3f32507 --- /dev/null +++ b/test/TestActors.py @@ -0,0 +1,36 @@ +from utils.agents.actor import Actor +import json +import unittest +f = open ('config.json') +CONFIG = json.loads(f.read()) +f.close() +class TestActor(unittest.TestCase): + @staticmethod + def message(): + m = {} + return m + def test_InstanceName(self) : + name = 'Apps' + o = Actor.instance('Apps',CONFIG) + self.assertIsNotNone(o) + + def test_InstanceList(self) : + name = ['Apps','Folders','Mailer'] + o = Actor.instance(name,CONFIG) + self.assertTrue(isinstance(o,list)) + self.assertTrue(len(o) > 0) + def test_AppKill(self) : + m = {'label':'firefox','action':'kill'} + app = Actor.instance('Apps',CONFIG) + app.init('kill',m) + app.run() + def test_AppReboot(self): + m = {'label':'firefox','cmd':'/Applications/Firefox.app/Contents/MacOS/firefox'} + app = Actor.instance('Apps',CONFIG) + app.init('start',m) + app.run() + + pass +if __name__ == '__main__': + unittest.main() +