library
/
smart-top
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug fix redesign

Browse Source
data-collector
Steve L. Nyemba 6 years ago
parent 8ecca833b3
commit ccc52fdaea
2 changed files with 195 additions and 415 deletions
Show Stats Download Patch File Download Diff File

199
src/data-collector.py
Unescape View File

@ -7,146 +7,125 @@ h="""
python data-collector.py --path config.json python data-collector.py --path config.json
The configuration file is structured as JSON object as follows : The configuration file is structured as JSON object as follows :
{ {
id: node identifier id : identifier
key: customer's identification key key: customer's identification key,
apps:"app_1,app_2,...", api: http://localhost/monitor/1/client
folders:"path_1,path_2, ..." folders:[]
} }
NOTE: You can download a sample configuration file from https://the-phi.com/smart-top
""" """
from utils.params import PARAMS as SYS_ARGS, Logger from utils.params import PARAMS as SYS_ARGS, Logger
import os
import requests import requests
import pickle
import json import json
from threading import Thread, RLock # from threading import Thread, RLock
import monitor from monitor import Apps, Folders
import utils.agents.actor as actor import time
from utils.agents.manager import Manager from datetime import datetime
SYS_ARGS['host']='localhost' class Collector :
ENDPOINT="http://:host/monitor".replace(":host",SYS_ARGS['host']) def __init__(self) :
class Collector(Thread) :
def __init__(self):
Thread.__init__(self)
""" """
This function initializes the data collector with critical information The configuration file is passed to the class for basic initialization of variables
The process will include validating the user's account and plan
@param key customer's key
@param id node identifier
""" """
# self.httpclient = requests.Session()
# Let's open the file with the key (nothing else should be in the file
#f = open(SYS_ARGS['key'])
#SYS_ARGS['key'] = f.read()
#f.close()
headers = {"key":SYS_ARGS["key"],"id":SYS_ARGS["id"]} #,"scope":json.dumps(scope)}
self.plan = None
self.store= None
#headers['content-type'] = 'application/json' def init(self):
try: # if 'folders' not in SYS_ARGS :
self.key = SYS_ARGS['key'] # #
# Logger.log(subject='Collector',object='api',action='request',value=ENDPOINT) # # If nothing is set it will monitor the temporary directory
# url = "/".join([ENDPOINT,"init/collector"]) # self.locations = [os.environ[name] for name in ['TEMP','TMP','TMPDIR'] if name in os.environ]
Logger.log(subject='Collector',object='api',action='request',value=SYS_ARGS['api']) # else:
url = "/".join([SYS_ARGS['api'],"init/collector"]) # self.locations = SYS_ARGS['folders']
r = requests.post(url,headers=headers)
r = json.loads(r.text) #
# -- let's get the list of features we are interested .
url = SYS_ARGS['api']+'/1/client/login'
key = SYS_ARGS['key']
id = SYS_ARGS['id'] if 'id' in SYS_ARGS else os.environ['HOSTNAME']
headers = {"key":key,"id":id}
if r :
# #
# Persisting plan and data-store ... #-- what features are allowed
self.plan = r['plan'] r = self.httpclient.post(url,headers=headers)
self.store = r['store']
info = {"store":self.store,"plan":self.plan}
if info['plan'] is not None and info['store'] is not None: if r.status_code == 200 :
info['plan'] = self.plan['name'] r = r.json()
info['store'] = self.store['args']['dbname']
_action = 'init' self.features = r['features']
self.initialize() self.config = r['config'] #-- contains apps and folders
Logger.log(action="login",value=r)
else: else:
info['plan'] = self.plan is not None self.features = None
info['store']= self.store is not None self.config = None
_action = 'init.error' Logger.log(action='login',value='error')
Logger.log(subject='collector',object='api',action=_action,value=info) def callback(self,channel,method,header,stream):
except Exception as e: pass
print(e) def listen(self):
Logger.log(subject='collector',object='api',action='init.error',value=str(e))
self.monitor = None factory = DataSourceFactory()
def initialize(self):
# self.qlistener = factory.instance(type="QueueListener",args=_args)
# self.qlistener.callback = self.callback
# self.qlistener.init(SYS_ARGS['id'])
def post(self,**args) :
""" """
This function creates a monitoring object with the associated parameters from the plan This function will post data to the endpoint
plan.metadata = {"agents":...,"folder_size":...,"delay":...,"limit":...,"actors":...}
""" """
_agents = [monitor.DetailProcess(),monitor.FileWatch()] url = SYS_ARGS['api']+'/1/client/log'
_actors = [actor.Apps(),actor.Folders(),actor.Mailer()] key = SYS_ARGS['key']
# Initialiing the agents with the parameter values we know of id = SYS_ARGS['id'] if 'id' in SYS_ARGS else os.environ['HOSTNAME']
r = [] headers = {"key":key,"id":id,"context":args['context'],"content-type":"application/json"}
for agent in _agents : body = args['data'].to_json(orient='records')
if args.shape[0] > 0 :
if agent.getName() in SYS_ARGS : r = self.httpclient.post(url,headers=headers,data=body)
values = SYS_ARGS[agent.getName()] Logger.log(action="post."+args['context'],value=r.status_code)
# print (["init ",agent.getName(),values]) else:
Logger.log(subject='Collector',object=agent.getName(),action='init.agent',value=values ) Logger.log(action="data.error",value="no data")
agent.init(values)
r.append(agent)
_agents = r
config = {"store":self.store,"plan":self.plan}
#@TODO: add SYS_ARGS content so apps that are configured reboot/delete folders that are marked
# This is an important security measure!
#
self.manager = Manager()
self.manager.init(node=SYS_ARGS['id'],agents=_agents,actors=_actors,config=config,key=self.key,host=SYS_ARGS['host'])
def run(self): def run(self):
""" """
This funtion runs the authorized features and This function will execute the basic functions to monitor folders and apps running on the system
given the configuration specified on the server .
""" """
#self.monitor.start() while True :
try:
self.init()
if self.config and self.features :
ELAPSED_TIME = 60* int(self.features['schedule'].replace("min","").strip())
if 'apps' in self.config :
self.post( data=(Apps()).get(filter=self.config['apps']),context="apps")
if 'folders' in self.config and self.config['folders'] :
folder = Folders()
f = folder.get(path=self.config['folders'])
self.post(data = f ,context="folders")
Logger.log(action='sleeping',value=ELAPSED_TIME)
#
# In case no configuration is provided, the system will simply fall asleep and wait
# @TODO: Evaluate whether to wake up the system or not (security concerns)!
#
time.sleep(ELAPSED_TIME)
# Logger.log(subject='Collector',object='monitor',action='rpc',value=(self.manager is None) ) except Exception,e:
thread = Thread(target=self.manager.run) Logger.log(action='error',value=e.message)
thread.start() print e
# print self.manager break
# print self.monitor.config['store']
# for agent in self.pool :
# try:
# agent = pickle.loads(agent)
# agent.start()
# #p = pickle.loads(self.pool[key])
# #p.init(SYS_ARGS[key].split(','))
# #p.start()
# except Exception,e:
# print e
pass
if __name__ == '__main__' : if __name__ == '__main__' :
# #
# #
if 'path' in SYS_ARGS : if 'path' in SYS_ARGS :
path = SYS_ARGS['path'] path = SYS_ARGS['path']
f = open(path) f = open(path)
p = json.loads(f.read()) SYS_ARGS = json.loads(f.read())
f.close() f.close()
else:
for id in ['apps','folders']:
if id in SYS_ARGS :
SYS_ARGS[id] = SYS_ARGS[id].split(',')
p = dict(SYS_ARGS)
Logger.init('data-collector') Logger.init('data-collector')
SYS_ARGS = dict(SYS_ARGS,** p) collector = Collector()
if 'apps' in SYS_ARGS : collector.run()
SYS_ARGS['apps'] += [__file__]
thread = Collector()
thread.start()
else: else:
print (h) print (h)

399
src/monitor.py
Unescape View File

@ -1,275 +1,113 @@
""" """
This program is designed to inspect an application environment Steve L. Nyemba <steve@the-phi.com>
This program should only be run on unix friendly systems The Phi Technology - Smart Top
We enable the engines to be able to run a several configurations This program is the core for evaluating folders and applications. Each class is specialized to generate a report in a pandas data-frame
Similarly to what a visitor design-pattern would do The classes will focus on Apps, Folders and Protocols
- SmartTop.get(**args)
@TODO:
Protocols (will be used in anomaly detection)
""" """
from __future__ import division from __future__ import division
import os import os
import subprocess import subprocess
from sets import Set
import re
import datetime
import urllib2 as http, base64
from threading import Thread, RLock
import time
import numpy as np import numpy as np
from utils.ml import ML
import sys import sys
from scipy import stats import pandas as pd
import datetime
class SmartTop:
def get(self,**args):
return None
class Analysis: class Apps(SmartTop) :
def __init__(self): def __init__(self):
self.logs = []
pass
def post(self,object):
self.logs.append(object)
def init(self):
d = datetime.datetime.now()
self.now = {"month":d.month,"year":d.year, "day":d.day,"hour":d.hour,"minute":d.minute}
def getNow(self):
d = datetime.datetime.now()
return {"month":d.month,"year":d.year, "day":d.day,"hour":d.hour,"minute":d.minute,"second":d.second}
def getName(self):
return self.__class__.__name__
def reboot(self,row,conf) :
return False
def cleanup(self,text):
return re.sub('[^a-zA-Z0-9\s:]',' ',str(text)).strip()
"""
This class is designed to analyze environment variables. Environment variables can either be folders, files or simple values
The class returns a quantifiable assessment of the environment variables (expected 100%)
"""
class Env(Analysis):
def __init__(self):
Analysis.__init__(self)
def init(self,values):
#Analysis.init(self)
self.values = values
""" """
This function evaluate the validity of an environment variable by returning a 1 or 0 (computable) This class will process a system command and parse the outpout accordingly given a parser
The function will use propositional logic (https://en.wikipedia.org/wiki/Propositional_calculus) @param parse is a parser pointer
""" """
def evaluate(self,id): self.cmd = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'"
self.xchar = ';'
def get_app(self,stream):
index = 1 if os.path.exists(" ".join(stream[:1])) else len(stream)-1
if id in os.environ : cmd = " ".join(stream[:index]) if index > 0 else " ".join(stream)
#
# We can inspect to make sure the environment variable is not a path or filename.
# Using propositional logic we proceed as follows:
# - (p) We determine if the value is an folder or file name (using regex)
# - (q) In case of a file or folder we check for existance
# The final result is a conjuction of p and q
#
value = os.environ[id]
expressions = [os.sep,'(\\.\w+)$']
p = sum([ re.search(xchar,value) is not None for xchar in expressions])
q = os.path.exists(value)
return int(p and q) if ' ' in cmd.split('/')[len(cmd.split('/'))-1] :
p = cmd.split('/')[len(cmd.split('/'))-1].split(' ')
name = p[0]
args = " ".join(p[1:])
else: else:
return 0 name = cmd.split('/')[len(cmd.split('/'))-1]
args = " ".join(stream[index:]) if index > 0 else ""
def composite (self): return [name,cmd,args]
#Analysis.init(self) def to_pandas(self,m):
r = [ self.evaluate(id) for id in self.values] ; """
N = len(r) This function will convert the output of ps to a data-frame
n = sum(r) @param m raw matrix i.e list of values like a csv
value = 100 * round(n/N,2)
missing = [self.values[i] for i in range(0,N) if r[i] == 0] """
return dict(self.getNow(),**{"value":value,"missing":missing}) m = [item for item in m if len(item) != len (m[0])]
""" m = "\n".join(m[1:])
This class is designed to handle analaysis of the a python virtual environment i.e deltas between requirments file and a virtualenv df = pd.read_csv(pd.compat.StringIO(m),sep=self.xchar)
@TODO: update the virtual environment df.columns =['pid','user','mem','cpu','status','started','name','cmd','args']
""" return df
class Sandbox(Analysis): def empty(self,name):
def __init__(self): return pd.DataFrame([{"pid":None,"user":None,"mem":0,"cpu":0,"status":"X","started":None,"name":name,"cmd":None,"args":None}])
Analysis.__init__(self) def parse(self,rows):
def init(self,conf): m = []
#Analysis.init(self) TIME_INDEX = 5
if os.path.exists(conf['sandbox']) : ARGS_INDEX = 6
self.sandbox_path = conf['sandbox']
else: for item in rows :
self.sandbox_path = None if rows.index(item) != 0 :
if os.path.exists(conf['requirements']) : parts = item.split(self.xchar)
self.requirements_path = conf['requirements'] row = parts[:TIME_INDEX]
row.append(' '.join(parts[TIME_INDEX:ARGS_INDEX]))
row += self.get_app(parts[ARGS_INDEX:])
else: else:
self.requirements_path = None row = item.split(self.xchar)
row = (self.xchar.join(row)).strip()
if len(row.replace(";","")) > 0 :
m.append(row)
def get_requirements (self): return m
f = open(self.requirements_path) def get(self,**args):
return [ name.replace('-',' ').replace('_',' ') for name in f.read().split('\n') if name != '']
"""
This function will return the modules installed in the sandbox (virtual environment)
"""
def get_sandbox_requirements(self):
cmd = ['freeze']
xchar = ''.join([os.sep]*2)
pip_vm = ''.join([self.sandbox_path,os.sep,'bin',os.sep,'pip']).replace(xchar,os.sep)
cmd = [pip_vm]+cmd
r = subprocess.check_output(cmd).split('\n')
return [row.replace('-',' ').replace('_',' ') for row in r if row.strip() != '']
def evaluate(self):
pass
def reboot(self,rows,limit=None) :
return sum([ len(item['missing']) for item in rows ]) > 0
""" """
This function returns the ratio of existing modules relative to the ones expected This function returns a the output of a command to the calling code that is piped into the class
The output will be stored in a data frame with columns
@
""" """
def composite(self): try:
Analysis.init(self)
if self.sandbox_path and self.requirements_path : handler = subprocess.Popen(self.cmd,shell=True,stdout=subprocess.PIPE)
required_modules= self.get_requirements() stream = handler.communicate()[0]
sandbox_modules = self.get_sandbox_requirements() rows = stream.split('\n')
N = len(required_modules) df = self.to_pandas(self.parse(rows))
n = len(Set(required_modules) - Set(sandbox_modules)) r = pd.DataFrame()
value = round(1 - (n/N),2)*100 if 'filter' in args :
missing = list(Set(required_modules) - Set(sandbox_modules)) pattern = "|".join(args['filter'])
r = df[df.name.str.contains(pattern)]
return dict(self.getNow(),**{"value":value,"missing":missing}) for name in args['filter'] :
else: filter = "name == '"+name+"'"
if r.query(filter).size == 0 :
r = r.append(self.empty(name))
return r
except Exception,e:
print (e)
return None return None
"""
This class performs the analysis of a list of processes and determines
The class provides a quantifiable measure of how many processes it found over all
"""
class ProcessCounter(Analysis):
def __init__(self):
Analysis.__init__(self)
def init(self,names):
#Analysis.init(self)
self.names = names
def evaluate(self,name):
cmd = "".join(['ps -eo comm |grep ',name,' |wc -l'])
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
return int(handler.communicate()[0].replace("\n","") )
def composite(self):
#Analysis.init(self)
r = {}
for name in self.names :
r[name] = self.evaluate(name)
#N = len(r)
#n = sum(r)
#return n/N
return dict(self.getNow(),**r)
"""
This class returns an application's both memory and cpu usage
"""
class DetailProcess(Analysis):
def __init__(self):
Analysis.__init__(self)
def init (self,names):
#Analysis.init(self)
self.names = names;
def getName(self):
return "apps"
def split(self,name,stream):
pattern = "(\d+.{0,1}\d*)\x20*(\d+.{0,1}\d*)\x20*(\d+.{0,1}\d*)".replace(":name",name).strip() class Folders(SmartTop):
g = re.match(pattern,stream.strip())
if g :
return list(g.groups())+['1']+[name]
else:
return ''
def reboot(self,rows,conf=None) :
return np.sum([int(item['label']=='crash') for item in rows]) > 0
def parse(self,row,fields):
""" """
The last field should be the command in its integrity This class will assess a folder and produce a report in a data-frame that can be later on used for summary statistics
@pre len(fields) > len(row)
""" """
r = {}
now = self.getNow()
r['date'] = now
row = [term for term in row.split() if term.strip() != '']
for name in fields :
index = fields.index(name)
r[name] = row[index] if row else 0
if name not in ['user','cmd','status','pid'] :
r[name] = float(r[name])
r[name] = row[index: ] if row else []
#
# Let's set the status give the data extracted
#
if r['status'] == 0 :
r['status'] = 'crash'
elif 'Z' in r['status'] :
r['status'] = 'zombie'
elif r['memory_usage'] > 0 and r['cpu_usage'] > 0:
r['status'] = 'running'
else:
r['status'] = 'idle'
return r
def evaluate(self,name=None) :
if name is None :
name = ".*"
fields = ["user","pid","memory_usage","cpu_usage","memory_available","status","cmd"]
cmd = "ps -eo user,pid,pmem,pcpu,vsize,stat,command|grep -Ei \":app\"".replace(":app",name)
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
logs = handler.communicate()[0].split('\n')
logs = [row for row in logs if (row.strip() != '') and ('grep -Ei' in row )== False ]
if len(logs) == 0:
return [dict(self.parse('',fields),**{'label':name}) ]
else :
return [dict(self.parse(row,fields),**{'label':name}) for row in logs if row.strip() != '' and 'grep' not in row and '-Ei' not in row]
def status(self,row):
x = row['memory_usage']
y = row['cpu_usage']
z = row['memory_available']
if z :
if y :
return "running"
return "idle"
else:
return "crash"
#def format(self,row):
# r= {"memory_usage":row[0],"cpu_usage":row[1],"memory_available":row[2]/1000,"proc_count":row[3],"label":self.cleanup(row[4])}
# status = self.status(r)
# r['status'] = status
# return r
def composite(self):
ma = []
for name in self.names:
row = self.evaluate(name)
ma += row
return ma
"""
This class evaluates a list of folders and provides detailed informaiton about age/size of each file
Additionally the the details are summarized in terms of global size, and oldest file.
"""
class FileWatch(Analysis):
def __init__(self): def __init__(self):
pass pass
def init(self,folders): def _get(self,dir_path,r=[]):
self.folders = folders;
self.cache = []
def getName(self):
return "folders"
def evaluate(self,dir_path,r=[]):
"""
This function will recursively scan a folder and retrieve file sizes and age of the files.
The data will be returned as an array of {size,age,label} items
"""
for child in os.listdir(dir_path): for child in os.listdir(dir_path):
path = os.path.join(dir_path, child) path = os.path.join(dir_path, child)
if os.path.isdir(path): if os.path.isdir(path):
self.evaluate(path,r) self._get(path,r)
else: else:
size = os.path.getsize(path) size = os.path.getsize(path)
@ -277,62 +115,25 @@ class FileWatch(Analysis):
file_date = datetime.datetime.fromtimestamp(file_date) file_date = datetime.datetime.fromtimestamp(file_date)
now = datetime.datetime.now() now = datetime.datetime.now()
age = (now - file_date ).days age = (now - file_date ).days
r.append({"label":path,"size":size,"age":age,"date":self.getNow()})
return r
def reboot(self,rows,limit) :
return np.sum([ 1 for item in rows if rows['size'] > limit ]) > 0
def composite(self):
out = []
for folder in self.folders :
r = self.evaluate(folder,[])
file_count = len(r)
age_mode = [item[0] for item in stats.mode([item['age'] for item in r])]
size_mode= [item[0] for item in stats.mode([item['size'] for item in r])]
age = {"mode":age_mode,"median":np.median([item['age'] for item in r] ),"mean":np.mean([item['age'] for item in r] ),"var":np.mean([item['age'] for item in r])}
size = {"mode":size_mode,"median":np.median([item['size'] for item in r] ), "mean":np.mean([item['size'] for item in r] ),"var":np.mean([item['size'] for item in r])}
out.append({"label":folder,"date":self.getNow(),"stats":{"age":age,"size":size,"file_count":file_count},"logs":r})
return out
# class Monitor (Thread):
# def __init__(self,pConfig,pWriter,id='processes') :
# Thread.__init__(self)
# self.config = pConfig[id]
# self.writer = pWriter;
# self.logs = []
# self.handler = self.config['class']
# self.mconfig = self.config['config']
name = os.path.basename(path)
r.append({"name":name,"path":path,"size":size,"age":age})
return r
# def stop(self): def get(self,**args):
# self.keep_running = False # path = args['path']
# def run(self):
# r = {}
# self.keep_running = True
# lock = RLock()
# while self.keep_running:
# lock.acquire()
# for label in self.mconfig:
# self.handler.init(self.mconfig[label])
# r = self.handler.composite()
# self.writer.write(label=label,row = r)
# time.sleep(2)
# lock.release()
# self.prune()
# TIME_LAPSE = 60*2
# time.sleep(TIME_LAPSE)
# print "Stopped ..."
# def prune(self) :
# MAX_ENTRIES = 100
# if len(self.logs) > MAX_ENTRIES :
# BEG = len(self.logs) - MAX_SIZE -1
# self.logs = self.logs[BEG:]
if isinstance(args['path'],list) == False:
paths = [args['path']]
else:
paths = args['path']
_out = pd.DataFrame()
for path in paths :
if os.path.exists(path) :
#
# If the folder does NOT exists it should not be treated.
#
r = pd.DataFrame(self._get(path))
r = pd.DataFrame([{"name":path,"files":r.shape[0],"age_in_days":r.age.mean(),"size_in_kb":r.size.sum()}])
_out = _out.append(r)
return _out

Write Preview
Loading…
Cancel
Save