""" This program is designed to inspect an application environment This program should only be run on unix friendly systems We enable the engines to be able to run a several configurations Similarly to what a visitor design-pattern would do """ from __future__ import division import os import subprocess from sets import Set import re import datetime import urllib2 as http, base64 from threading import Thread, RLock import time import numpy as np from utils.ml import ML class Analysis: def __init__(self): self.logs = [] pass def post(self,object): self.logs.append(object) def init(self): d = datetime.datetime.now() self.now = {"month":d.month,"year":d.year, "day":d.day,"hour":d.hour,"minute":d.minute} def getNow(self): d = datetime.datetime.now() return {"month":d.month,"year":d.year, "day":d.day,"hour":d.hour,"minute":d.minute} def getName(self): return self.__class__.__name__ def cleanup(self,text): return re.sub('[^a-zA-Z0-9\s:]',' ',str(text)).strip() """ This class is designed to analyze environment variables. Environment variables can either be folders, files or simple values The class returns a quantifiable assessment of the environment variables (expected 100%) """ class Env(Analysis): def __init__(self): Analysis.__init__(self) def init(self,values): #Analysis.init(self) self.values = values """ This function evaluate the validity of an environment variable by returning a 1 or 0 (computable) The function will use propositional logic (https://en.wikipedia.org/wiki/Propositional_calculus) """ def evaluate(self,id): if id in os.environ : # # We can inspect to make sure the environment variable is not a path or filename. # Using propositional logic we proceed as follows: # - (p) We determine if the value is an folder or file name (using regex) # - (q) In case of a file or folder we check for existance # The final result is a conjuction of p and q # value = os.environ[id] expressions = [os.sep,'(\\.\w+)$'] p = sum([ re.search(xchar,value) is not None for xchar in expressions]) q = os.path.exists(value) return int(p and q) else: return 0 def composite (self): #Analysis.init(self) r = [ self.evaluate(id) for id in self.values] ; N = len(r) n = sum(r) value = 100 * round(n/N,2) missing = [self.values[i] for i in range(0,N) if r[i] == 0] return dict(self.getNow(),**{"value":value,"missing":missing}) """ This class is designed to handle analaysis of the a python virtual environment i.e deltas between requirments file and a virtualenv @TODO: update the virtual environment """ class Sandbox(Analysis): def __init__(self): Analysis.__init__(self) def init(self,conf): #Analysis.init(self) if os.path.exists(conf['sandbox']) : self.sandbox_path = conf['sandbox'] else: self.sandbox_path = None if os.path.exists(conf['requirements']) : self.requirements_path = conf['requirements'] else: self.requirements_path = None def get_requirements (self): f = open(self.requirements_path) return [ name.replace('-',' ').replace('_',' ') for name in f.read().split('\n') if name != ''] """ This function will return the modules installed in the sandbox (virtual environment) """ def get_sandbox_requirements(self): cmd = ['freeze'] xchar = ''.join([os.sep]*2) pip_vm = ''.join([self.sandbox_path,os.sep,'bin',os.sep,'pip']).replace(xchar,os.sep) cmd = [pip_vm]+cmd r = subprocess.check_output(cmd).split('\n') return [row.replace('-',' ').replace('_',' ') for row in r if row.strip() != ''] def evaluate(self): pass """ This function returns the ratio of existing modules relative to the ones expected """ def composite(self): Analysis.init(self) if self.sandbox_path and self.requirements_path : required_modules= self.get_requirements() sandbox_modules = self.get_sandbox_requirements() N = len(required_modules) n = len(Set(required_modules) - Set(sandbox_modules)) value = round(1 - (n/N),2)*100 missing = list(Set(required_modules) - Set(sandbox_modules)) return dict(self.getNow(),**{"value":value,"missing":missing}) else: return None """ This class performs the analysis of a list of processes and determines The class provides a quantifiable measure of how many processes it found over all """ class ProcessCounter(Analysis): def __init__(self): Analysis.__init__(self) def init(self,names): #Analysis.init(self) self.names = names def evaluate(self,name): cmd = "".join(['ps -eo comm |grep ',name,' |wc -l']) handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) return int(handler.communicate()[0].replace("\n","") ) def composite(self): #Analysis.init(self) r = {} for name in self.names : r[name] = self.evaluate(name) #N = len(r) #n = sum(r) #return n/N return dict(self.getNow(),**r) """ This class returns an application's both memory and cpu usage """ class DetailProcess(Analysis): def __init__(self): Analysis.__init__(self) def init (self,names): #Analysis.init(self) self.names = names; def getName(self): return "apps" def split(self,name,stream): pattern = "(\d+.{0,1}\d*)\x20*(\d+.{0,1}\d*)\x20*(\d+.{0,1}\d*)".replace(":name",name).strip() g = re.match(pattern,stream.strip()) if g : return list(g.groups())+['1']+[name] else: return '' def evaluate(self,name) : cmd = "ps -eo pmem,pcpu,vsize,command|grep -E \":app\"" handler = subprocess.Popen(cmd.replace(":app",name),shell=True,stdout=subprocess.PIPE) ostream = handler.communicate()[0].split('\n') #xstr = ostream ostream = [ self.split(name,row) for row in ostream if row != '' and 'grep' not in row] if len(ostream) == 0 or len(ostream[0]) < 4 : ostream = [['0','0','0','0',name]] r = [] for row in ostream : # # Though the comm should only return the name as specified, # On OSX it has been observed that the fully qualified path is sometimes returned (go figure) # row = [float(value) for value in row if value.strip() != '' and name not in value ] +[re.sub('\$|^','',name)] r.append(row) # # At this point we should aggregate results # The aggregation is intended for applications with several processes (e.g: apache2) # if len(r) > 1: m = None for row in r: if m is None: m = row else: m[3] += row[3] m[0] += row[0] m[1] += row[1] m[2] += row[2] m[0] = round((m[0] / m[3]),2) m[1] = round((m[1] / m[3]),2) m[2] = round((m[2] / m[3]),2) r = [m] return r def status(self,row): x = row['memory_usage'] y = row['cpu_usage'] z = row['memory_available'] if z : if y : return "running" return "idle" else: return "crash" def format(self,row): r= {"memory_usage":row[0],"cpu_usage":row[1],"memory_available":row[2]/1000,"proc_count":row[3],"label":self.cleanup(row[4])} status = self.status(r) r['status'] = status return r def composite(self): ma = [] now = self.getNow() for name in self.names: matrix = self.evaluate(name) ma += [ dict(now, **self.format(row)) for row in matrix] return ma """ This class evaluates a list of folders and provides detailed informaiton about age/size of each file Additionally the the details are summarized in terms of global size, and oldest file. """ class FileWatch(Analysis): def __init__(self): pass def init(self,folders): self.folders = folders; def getName(self): return "folders" def split(self,row): x = row.split(' ') r = {} months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] if x: BYTES_TO_MB = 1000000 size = int(x[0])/BYTES_TO_MB month = months.index(x[1]) + 1 day = int(x[2]) age = -1 hour=minute = 0 if ':' in x[3] : hour,minute = x[3].split(':') now = datetime.datetime.today() if month == now.month : year = now.year else: year = now.year - 1 else: year = int(x[3]) hour = 0 minute = 0 file_date = datetime.datetime(year,month,day,int(hour),int(minute)) # size = round(size,2) #file_date = datetime.datetime(year,month,day,hour,minute) now = datetime.datetime.now() age = (now - file_date ).days return {"size":size,"age":age} return None def evaluate(self,path): cmd = "find :path -print0|xargs -0 ls -ls |awk '{print $6,$7,$8,$9,$10}'".replace(":path",path) handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) ostream = handler.communicate()[0].split('\n') #return [self.split(stream) for stream in ostream if stream.strip() != '' and '.DS_Store' not in stream and 'total' not in stream] return [self.split(stream) for stream in ostream if path not in stream and not set(['','total','.DS_Store']) & set(stream.split(' '))] def composite(self): d = [] #-- vector of details (age,size) now = datetime.datetime.today() for folder in self.folders: if os.path.exists(folder): xo_raw = self.evaluate(folder) xo = np.array(ML.Extract(['size','age'],xo_raw)) if len(xo) == 0: continue name = re.findall("([a-z,A-Z,0-9]+)",folder) name = folder.split(os.sep) if len(name) == 1: name = [folder] else: i = len(name) -1 name = [name[i-1]+' '+name[i]] name = name[0] size = round(np.sum(xo[:,0]),2) if size > 1000 : size = round(size/1000,2) units = ' GB' elif size > 1000000: size = round(size/1000000,2) units = ' TB' else: size = size units = ' MB' size = str(size)+ units age = round(np.mean(xo[:,1]),2) if age > 30 and age <= 365 : age = round(age/30,2) units = ' Months' elif age > 365 : age = round(age/365,2) units = ' Years' else: age = age units = ' Days' age = str(age)+units N = len(xo[:,1]) xo = {"label":folder} #,"details":xo_raw,"summary":{"size":size,"age":age,"count":len(xo[:,1])}} xo = dict(xo,**{"size":size,"age":age,"count":N}) xo["name"] = name xo['day'] = now.day xo['month'] = now.month xo['year'] = now.year xo['date'] = time.mktime(now.timetuple()) d.append(xo) return d # class Monitor (Thread): # def __init__(self,pConfig,pWriter,id='processes') : # Thread.__init__(self) # self.config = pConfig[id] # self.writer = pWriter; # self.logs = [] # self.handler = self.config['class'] # self.mconfig = self.config['config'] # def stop(self): # self.keep_running = False # def run(self): # r = {} # self.keep_running = True # lock = RLock() # while self.keep_running: # lock.acquire() # for label in self.mconfig: # self.handler.init(self.mconfig[label]) # r = self.handler.composite() # self.writer.write(label=label,row = r) # time.sleep(2) # lock.release() # self.prune() # TIME_LAPSE = 60*2 # time.sleep(TIME_LAPSE) # print "Stopped ..." # def prune(self) : # MAX_ENTRIES = 100 # if len(self.logs) > MAX_ENTRIES : # BEG = len(self.logs) - MAX_SIZE -1 # self.logs = self.logs[BEG:]