library
/
smart-top
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'community' of http://dev.the-phi.com/git/steve/monitor into community

Browse Source
community
Steve L. Nyemba 7 years ago
parent c6de92f49f 1aad0c4acc
commit 6dc7a4c343
9 changed files with 651 additions and 120 deletions
Show Stats Download Patch File Download Diff File

23
config.json
Unescape View File

@ -1,15 +1,18 @@
{
"virtual-env":{
"class":"Sandbox",
"config":{
"3-launchpad":{"requirements":"/Users/steve/Documents/git/repair-file/required.txt","sandbox":"/Users/steve/Documents/git/sandbox"}
}
"id":"zulu-hacker",
"key":"4q-h8r5-247&!570p=[0v8]x360",
"api":"localhost",
"delay":0.5,
"store":{
"class":{"read":"CouchdbReader","write":"CouchdbWriter"},
"args":{"uri":"http://localhost:5984","dbname":"monitor","uid":"logs"}
},
"processes":{
"class":"DetailProcess",
"config":{
"system":["postgresql","couchdb","httpd"]
}
"procs":["kate","firefox"],
"folders":["/home/steve/tmp","/home/steve/git","/home/steve/Downloads"],
"actions":{
"folders":{"threshold":"10mb","action":"archive"},
"apps":{"firefox":"","kate":"" }
}
}

67
init.sh
Unescape View File

@ -0,0 +1,67 @@
#!/bin/bash
#
# This script is designed to handle various operations related to setting up, starting, stopping the python application
# It will assume the requirements file is at the root (not with the source code)
#
export PYTHONPATH=$PWD/src
pip_upgrade='sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o'
install(){
virtualenv sandbox
sandbox/bin/pip install -r requirements.txt
`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade`
}
upgrade(){
git pull
count=`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|wc -l`
if [ ! "$count" = "0" ]; then
`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade`
else
echo "No Upgrade required for sandbox"
fi
}
start(){
if [ "$1" = "collector" ]; then
sandbox/bin/python src/utils/agents/data-collector.py --path $PWD/config.json
else
sandbox/bin/python src/api/index.py --path $PWD/config.json &
fi
}
stop(){
ps -eo pid,command|grep python|grep -E "$PWD"|grep index.py|grep -E "^ {0,}[0-9]+" -o |xargs kill -9
ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -o |xargs kill -9
}
status(){
pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep index.py|grep -E "^ {0,}[0-9]+" -m 1 -o`
if [ "$pid" = "" ]; then
echo "API IS OFFLINE"
else
echo "API IS ONLINE $pid"
fi
pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -m 1 -o`
if [ "$pid" = "" ]; then
echo "DATA-COLLECTOR IS OFFLINE"
else
echo "DATA-COLLECTOR IS ONLINE $pid"
fi
}
if [ "$1" = "start" ]; then
if [ "$2" = "collector" ]; then
start "collector"
else
start
fi
else
$1
fi

14
readme.md
Unescape View File

@ -6,6 +6,20 @@ The program answers basic questions:
- Is a given program still running
- How much resource (memory/cpu) a program is using up
- The number of processes found
- Folder monitoring ...
#Architecture
The architecture of the system is distributed with a central master node,
{
"id":"",
"key":"",
"apps":[],
"sandbox":[{"path":"","requirements":""}],
"folders":["path-1"],
"store":{}
"actions":{}
}
The agent will perform three basic functions :

15
requirements.txt
Unescape View File

@ -1,13 +1,16 @@
aniso8601==1.2.0
click==6.6
couchdbkit==0.6.5
Flask==0.11.1
Flask-Session==0.3.0
Flask-SocketIO==2.8.2
http-parser==0.8.3
itsdangerous==0.24
Jinja2==2.8
MarkupSafe==0.23
Werkzeug==0.11.11
aniso8601==1.2.0
argparse==1.2.1
click==6.6
couchdbkit==0.6.5
http-parser==0.8.3
itsdangerous==0.24
ngram==3.3.0
numpy==1.11.3
pika==0.10.0
python-dateutil==2.6.0
@ -17,4 +20,4 @@ pytz==2016.10
restkit==4.2.2
six==1.10.0
socketpool==0.5.3
Werkzeug==0.11.11
wsgiref==0.1.2

243
src/api/static/img/uml-design
Unescape View File

@ -0,0 +1,243 @@
<?xml version="1.0" encoding="UTF-8"?>
<XMI xmlns:UML="http://schema.omg.org/spec/UML/1.3" timestamp="2017-04-07T08:29:04" xmi.version="1.2" verified="false">
<XMI.header>
<XMI.documentation>
<XMI.exporter>umbrello uml modeller http://umbrello.kde.org</XMI.exporter>
<XMI.exporterVersion>1.6.9</XMI.exporterVersion>
<XMI.exporterEncoding>UnicodeUTF8</XMI.exporterEncoding>
</XMI.documentation>
<XMI.metamodel href="UML.xml" xmi.name="UML" xmi.version="1.3"/>
</XMI.header>
<XMI.content>
<UML:Model isRoot="false" isSpecification="false" name="UML Model" isAbstract="false" xmi.id="m1" isLeaf="false">
<UML:Namespace.ownedElement>
<UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="folder" isAbstract="false" isLeaf="false" xmi.id="folder" visibility="public"/>
<UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="datatype" isAbstract="false" isLeaf="false" xmi.id="datatype" visibility="public"/>
<UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="agent" isAbstract="false" isLeaf="false" xmi.id="rSZCzDKpjhw5" visibility="public"/>
<UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="interface" isAbstract="false" isLeaf="false" xmi.id="interface" visibility="public"/>
<UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Logical View" isAbstract="false" isLeaf="false" xmi.id="Logical View" visibility="public">
<UML:Namespace.ownedElement>
<UML:Package isRoot="false" stereotype="folder" isSpecification="false" namespace="Logical View" name="Datatypes" isAbstract="false" isLeaf="false" xmi.id="Datatypes" visibility="public">
<UML:Namespace.ownedElement>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="int" isAbstract="false" isLeaf="false" xmi.id="fCoa6Ljfk80G" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="char" isAbstract="false" isLeaf="false" xmi.id="XVF672bO1S1I" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="bool" isAbstract="false" isLeaf="false" xmi.id="HU1uO6O1JAZP" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="float" isAbstract="false" isLeaf="false" xmi.id="UI3ER3SwXh1e" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="double" isAbstract="false" isLeaf="false" xmi.id="hc0i20yJS0Dp" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="short" isAbstract="false" isLeaf="false" xmi.id="6aqNkOnEgoR7" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="long" isAbstract="false" isLeaf="false" xmi.id="pt2Mb3yGdzxf" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="unsigned int" isAbstract="false" isLeaf="false" xmi.id="xvyPvAC0i1NS" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="unsigned short" isAbstract="false" isLeaf="false" xmi.id="P8f5w6BmxD8W" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="unsigned long" isAbstract="false" isLeaf="false" xmi.id="E7nr4kW1fSOb" visibility="public"/>
<UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="string" isAbstract="false" isLeaf="false" xmi.id="abbIcwjz5O41" visibility="public"/>
</UML:Namespace.ownedElement>
</UML:Package>
<UML:Interface isRoot="false" stereotype="interface" isSpecification="false" namespace="Logical View" name="Agent" isAbstract="true" isLeaf="false" xmi.id="RXqBshUB41L5" visibility="public">
<UML:Classifier.feature>
<UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="init" isAbstract="false" isLeaf="false" xmi.id="wViGJvmJF1zV" visibility="public"/>
<UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="execute" isAbstract="false" isLeaf="false" xmi.id="7FOYGyKLJkjG" visibility="public"/>
<UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="status" isAbstract="false" isLeaf="false" xmi.id="q8oPwCisiO61" visibility="public"/>
<UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="instance" isAbstract="false" isLeaf="false" xmi.id="2fGeoj3czd3G" visibility="public" ownerScope="classifier"/>
</UML:Classifier.feature>
</UML:Interface>
<UML:Interface isRoot="false" stereotype="interface" isSpecification="false" namespace="Logical View" name="data-store" isAbstract="true" isLeaf="false" xmi.id="dqWyyHKwPkeW" visibility="public"/>
</UML:Namespace.ownedElement>
<XMI.extension xmi.extender="umbrello">
<diagrams>
<diagram usefillcolor="1" linewidth="0" showpubliconly="0" snapy="25" isopen="1" showatts="1" zoom="100" griddotcolor="#d3d3d3" name="class diagram" canvasheight="0" showpackage="1" canvaswidth="0" type="1" showops="1" xmi.id="J1rdJrVlcu5U" showgrid="0" showattsig="1" backgroundcolor="#ffffff" linecolor="#ff0000" snapcsgrid="0" fillcolor="#ffff00" showattribassocs="1" localid="-1" snapgrid="0" showscope="1" showopsig="1" snapx="25" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" documentation="">
<widgets/>
<messages/>
<associations/>
</diagram>
</diagrams>
</XMI.extension>
</UML:Model>
<UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Use Case View" isAbstract="false" isLeaf="false" xmi.id="Use Case View" visibility="public">
<UML:Namespace.ownedElement/>
</UML:Model>
<UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Component View" isAbstract="false" isLeaf="false" xmi.id="Component View" visibility="public">
<UML:Namespace.ownedElement>
<UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="data-collector" isAbstract="false" isLeaf="false" xmi.id="Tz06QOf0xEHX" visibility="public" executable="0"/>
<UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Learner" isAbstract="false" isLeaf="false" xmi.id="qVP8MjpaiUjw" visibility="public" executable="0"/>
<UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Kill" isAbstract="false" isLeaf="false" xmi.id="IsIXtH00UPZi" visibility="public" executable="0"/>
<UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="rVfXulYAdqhX" visibility="public" client="IsIXtH00UPZi"/>
<UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Start" isAbstract="false" isLeaf="false" xmi.id="1TAcIp7NVdvX" visibility="public" executable="0"/>
<UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="PvoWBjUADfkV" visibility="public" client="1TAcIp7NVdvX"/>
<UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Notify" isAbstract="false" isLeaf="false" xmi.id="HSy87jRVwqsY" visibility="public" executable="0"/>
<UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Mailer" isAbstract="false" isLeaf="false" xmi.id="2X1nlev04t8B" visibility="public" executable="0"/>
<UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="HyCj3nQ3KPr5" visibility="public" client="2X1nlev04t8B"/>
<UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="jpx2Co5CY9GC" visibility="public" client="HSy87jRVwqsY"/>
<UML:Association isSpecification="false" namespace="Component View" name="" xmi.id="xfmTo0d7U3Co" visibility="public">
<UML:Association.connection>
<UML:AssociationEnd isNavigable="false" aggregation="none" isSpecification="false" changeability="changeable" name="" type="Tz06QOf0xEHX" xmi.id="GLVKohqCd1VT" visibility="public"/>
<UML:AssociationEnd isNavigable="true" aggregation="none" isSpecification="false" changeability="changeable" name="" type="RXqBshUB41L5" xmi.id="cgz2pZckFNB1" visibility="public"/>
</UML:Association.connection>
</UML:Association>
<UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="NoSQL" isAbstract="false" isLeaf="false" xmi.id="ZPS2o7GDkc5b" visibility="public" executable="0"/>
<UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="Queue" isAbstract="false" isLeaf="false" xmi.id="yAQh9XwiuMkg" visibility="public" executable="0"/>
<UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="dqWyyHKwPkeW" xmi.id="roUjSuB43yq5" visibility="public" client="yAQh9XwiuMkg"/>
<UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="dqWyyHKwPkeW" xmi.id="up5vlwzBogrF" visibility="public" client="ZPS2o7GDkc5b"/>
<UML:Association isSpecification="false" namespace="Component View" name="" xmi.id="EZF1F7EEwgSu" visibility="public">
<UML:Association.connection>
<UML:AssociationEnd isNavigable="false" aggregation="none" isSpecification="false" changeability="changeable" name="" type="Tz06QOf0xEHX" xmi.id="WZqWR7Y3vUHh" visibility="public"/>
<UML:AssociationEnd isNavigable="true" aggregation="none" isSpecification="false" changeability="changeable" name="" type="dqWyyHKwPkeW" xmi.id="2WNNqxT3Uh2j" visibility="public"/>
</UML:Association.connection>
</UML:Association>
<UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="Dashboard" isAbstract="false" isLeaf="false" xmi.id="mGviuNaRvQvK" visibility="public" executable="0"/>
<UML:Association isSpecification="false" namespace="Component View" name="" xmi.id="Mm0rT2YFGUvA" visibility="public">
<UML:Association.connection>
<UML:AssociationEnd isNavigable="false" aggregation="none" isSpecification="false" changeability="changeable" name="" type="mGviuNaRvQvK" xmi.id="pTZirT2NwOlu" visibility="public"/>
<UML:AssociationEnd isNavigable="true" aggregation="none" isSpecification="false" changeability="changeable" name="" type="dqWyyHKwPkeW" xmi.id="0rp9GA8cw6RN" visibility="public"/>
</UML:Association.connection>
</UML:Association>
<UML:Dependency isSpecification="false" namespace="Component View" name="" supplier="dqWyyHKwPkeW" xmi.id="80aOSVWFy4yp" visibility="public" client="qVP8MjpaiUjw"/>
</UML:Namespace.ownedElement>
<XMI.extension xmi.extender="umbrello">
<diagrams>
<diagram usefillcolor="0" linewidth="0" showpubliconly="0" snapy="25" isopen="1" showatts="1" zoom="100" griddotcolor="#d3d3d3" name="component diagram" canvasheight="614" showpackage="1" canvaswidth="1042" type="7" showops="1" xmi.id="SFlPQuwezjV1" showgrid="0" showattsig="1" backgroundcolor="#ffffff" linecolor="#000000" snapcsgrid="0" fillcolor="#ffff00" showattribassocs="1" localid="-1" snapgrid="0" showscope="1" showopsig="1" snapx="25" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" documentation="">
<widgets>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-678" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="v7rsoLtFIrMg" linewidth="0" linecolor="#000000" xmi.id="Tz06QOf0xEHX" x="-1736" width="159" showstereotype="1"/>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-412" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="OPLKYyaGrclm" linewidth="0" linecolor="#000000" xmi.id="qVP8MjpaiUjw" x="-1550" width="146" showstereotype="1"/>
<interfacewidget usefillcolor="0" isinstance="0" linewidth="0" usesdiagramusefillcolor="0" showpubliconly="0" showoperations="1" height="102" y="-690.5" width="93" showattsigs="601" showpackage="0" xmi.id="RXqBshUB41L5" usesdiagramfillcolor="0" x="-1500.5" linecolor="#000000" fillcolor="#ffff00" localid="bSOrj87QKpFY" showscope="1" showattributes="0" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" showopsigs="601" drawascircle="0"/>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-810" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="Lr1cZksQrfnn" linewidth="0" linecolor="#000000" xmi.id="IsIXtH00UPZi" x="-1276" width="118" showstereotype="1"/>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-687" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="fCxXTyzSroIP" linewidth="0" linecolor="#000000" xmi.id="1TAcIp7NVdvX" x="-1114" width="118" showstereotype="1"/>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-579" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="65" localid="rbXRChMtT5dh" linewidth="0" linecolor="#000000" xmi.id="HSy87jRVwqsY" x="-1293" width="150" showstereotype="1"/>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-803" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="cZfZNX5E6adI" linewidth="0" linecolor="#000000" xmi.id="2X1nlev04t8B" x="-1515" width="118" showstereotype="1"/>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-426" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="nIBN2jAINAlN" linewidth="0" linecolor="#000000" xmi.id="ZPS2o7GDkc5b" x="-1891" width="145" showstereotype="1"/>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-319" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="Rzr9ZxkMFuhF" linewidth="0" linecolor="#000000" xmi.id="yAQh9XwiuMkg" x="-1888" width="147" showstereotype="1"/>
<interfacewidget usefillcolor="0" isinstance="0" linewidth="0" usesdiagramusefillcolor="0" showpubliconly="0" showoperations="1" height="40" y="-410" width="40" showattsigs="601" showpackage="0" xmi.id="dqWyyHKwPkeW" usesdiagramfillcolor="0" x="-1674" linecolor="#000000" fillcolor="#ffff00" localid="GYF8sxpJueQR" showscope="1" showattributes="0" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" showopsigs="601" drawascircle="1">
<floatingtext usefillcolor="1" isinstance="0" linewidth="0" pretext="" usesdiagramusefillcolor="1" height="21" y="66" posttext="" width="78" xmi.id="pwDCepiuLv1D" usesdiagramfillcolor="1" x="-22" text="data-store" linecolor="#000000" fillcolor="none" localid="b8no2T4fJacH" showstereotype="1" textcolor="none" font="Ubuntu,11,-1,5,50,0,0,0,0,0" role="700"/>
</interfacewidget>
<componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-312" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="VTNOxe5xARKz" linewidth="0" linecolor="#000000" xmi.id="mGviuNaRvQvK" x="-1546" width="152" showstereotype="1"/>
<boxwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-817" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="338" localid="deiZieMDdXfx" linewidth="0" linecolor="#000000" xmi.id="TyIE7mFOMqb2" x="-1945" width="998" showstereotype="1"/>
<boxwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-462" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="259" localid="r9osJyLub2TH" linewidth="0" linecolor="#0000ff" xmi.id="gqVHN8Fq6V2C" x="-1945" width="1000" showstereotype="1"/>
<floatingtext usefillcolor="0" isinstance="0" linewidth="0" pretext="" usesdiagramusefillcolor="0" height="32" y="-537" posttext="" width="154" xmi.id="M7Vv9zCpnf7A" usesdiagramfillcolor="0" x="-1933" text=" DETECTION" linecolor="#000000" fillcolor="#ffff00" localid="GFaa3AbvJPnb" showstereotype="1" textcolor="#000000" font="Sans Serif,18,-1,5,50,0,0,0,0,0" role="700"/>
<floatingtext usefillcolor="0" isinstance="0" linewidth="0" pretext="" usesdiagramusefillcolor="0" height="32" y="-248" posttext="" width="163" xmi.id="mD94avfu28pB" usesdiagramfillcolor="0" x="-1116" text="PREDICTION " linecolor="#000000" fillcolor="#ffff00" localid="nKysny43DccE" showstereotype="1" textcolor="#000000" font="Sans Serif,18,-1,5,50,0,0,0,0,0" role="700"/>
</widgets>
<messages/>
<associations>
<assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="rVfXulYAdqhX" widgetaid="IsIXtH00UPZi" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-746" startx="-1223"/>
<endpoint endx="-1407.5" endy="-649"/>
<point y="-649" x="-1223"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="PvoWBjUADfkV" widgetaid="1TAcIp7NVdvX" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-650" startx="-1114"/>
<endpoint endx="-1407.5" endy="-650"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="HyCj3nQ3KPr5" widgetaid="2X1nlev04t8B" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-739" startx="-1467"/>
<endpoint endx="-1467" endy="-690.5"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="jpx2Co5CY9GC" widgetaid="HSy87jRVwqsY" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-579" startx="-1224"/>
<endpoint endx="-1407.5" endy="-649"/>
<point y="-649" x="-1224"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="1" fillcolor="none" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="512" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="xfmTo0d7U3Co" widgetaid="Tz06QOf0xEHX" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-651.5" startx="-1577"/>
<endpoint endx="-1500.5" endy="-651.5"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="511" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="roUjSuB43yq5" widgetaid="yAQh9XwiuMkg" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-319" startx="-1741"/>
<endpoint endx="-1674" endy="-370"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="1" fillcolor="none" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="511" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="up5vlwzBogrF" widgetaid="ZPS2o7GDkc5b" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-410" startx="-1746"/>
<endpoint endx="-1674" endy="-410"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="512" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="EZF1F7EEwgSu" widgetaid="Tz06QOf0xEHX" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-614" startx="-1652"/>
<endpoint endx="-1652" endy="-410"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="1" fillcolor="none" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="512" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="Mm0rT2YFGUvA" widgetaid="mGviuNaRvQvK" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-312" startx="-1546"/>
<endpoint endx="-1634" endy="-370"/>
</linepath>
</assocwidget>
<assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="502" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="80aOSVWFy4yp" widgetaid="qVP8MjpaiUjw" seqnum="" totalcounta="2">
<linepath layout="Polyline">
<startpoint starty="-410" startx="-1550"/>
<endpoint endx="-1634" endy="-410"/>
</linepath>
</assocwidget>
</associations>
</diagram>
</diagrams>
</XMI.extension>
</UML:Model>
<UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Deployment View" isAbstract="false" isLeaf="false" xmi.id="Deployment View" visibility="public">
<UML:Namespace.ownedElement/>
</UML:Model>
<UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Entity Relationship Model" isAbstract="false" isLeaf="false" xmi.id="Entity Relationship Model" visibility="public">
<UML:Namespace.ownedElement/>
</UML:Model>
</UML:Namespace.ownedElement>
</UML:Model>
</XMI.content>
<XMI.extensions xmi.extender="umbrello">
<docsettings uniqueid="wLA7xmMizVkY" viewid="SFlPQuwezjV1" documentation=""/>
<listview>
<listitem id="Views" open="1" type="800">
<listitem id="Component View" open="1" type="821">
<listitem id="SFlPQuwezjV1" label="component diagram" open="0" type="819"/>
<listitem id="mGviuNaRvQvK" open="1" type="822"/>
<listitem id="Tz06QOf0xEHX" open="1" type="822"/>
<listitem id="IsIXtH00UPZi" open="1" type="822"/>
<listitem id="qVP8MjpaiUjw" open="1" type="822"/>
<listitem id="2X1nlev04t8B" open="1" type="822"/>
<listitem id="ZPS2o7GDkc5b" open="1" type="822"/>
<listitem id="HSy87jRVwqsY" open="1" type="822"/>
<listitem id="yAQh9XwiuMkg" open="1" type="822"/>
<listitem id="1TAcIp7NVdvX" open="1" type="822"/>
</listitem>
<listitem id="Deployment View" open="1" type="827"/>
<listitem id="Entity Relationship Model" open="1" type="836"/>
<listitem id="Logical View" open="1" type="801">
<listitem id="RXqBshUB41L5" open="1" type="817">
<listitem id="7FOYGyKLJkjG" open="0" type="815"/>
<listitem id="wViGJvmJF1zV" open="0" type="815"/>
<listitem id="2fGeoj3czd3G" open="0" type="815"/>
<listitem id="q8oPwCisiO61" open="0" type="815"/>
</listitem>
<listitem id="J1rdJrVlcu5U" label="class diagram" open="0" type="807"/>
<listitem id="dqWyyHKwPkeW" open="1" type="817"/>
<listitem id="Datatypes" open="0" type="830">
<listitem id="HU1uO6O1JAZP" open="1" type="829"/>
<listitem id="XVF672bO1S1I" open="1" type="829"/>
<listitem id="hc0i20yJS0Dp" open="1" type="829"/>
<listitem id="UI3ER3SwXh1e" open="1" type="829"/>
<listitem id="fCoa6Ljfk80G" open="1" type="829"/>
<listitem id="pt2Mb3yGdzxf" open="1" type="829"/>
<listitem id="6aqNkOnEgoR7" open="1" type="829"/>
<listitem id="abbIcwjz5O41" open="1" type="829"/>
<listitem id="xvyPvAC0i1NS" open="1" type="829"/>
<listitem id="E7nr4kW1fSOb" open="1" type="829"/>
<listitem id="P8f5w6BmxD8W" open="1" type="829"/>
</listitem>
</listitem>
<listitem id="Use Case View" open="1" type="802"/>
</listitem>
</listview>
<codegeneration>
<codegenerator language="C++"/>
</codegeneration>
</XMI.extensions>
</XMI>

2
src/api/static/js/jx

@ -1 +1 @@
Subproject commit 36f9f10ff44406cc0418cc5934eb475bc77ebb1e
Subproject commit 3d00f5a126574f2277cdac25d60008ee35dc8740

365
src/utils/agents/actor.py
Unescape View File

@ -13,112 +13,291 @@
import json
from threading import Thread
import os
import shutil
import subprocess
from monitor import ProcessCounter
from utils.transport import QueueListener, QueueWriter
from utils.transport import QueueListener, QueueWriter, QueueReader
from utils.params import PARAMS
from ngram import NGram as ng
class Actor(Thread):
def __init__(self,config):
Thread.__init__(self)
self.config = config
self.items = []
self.__id = config['id']
def getIdentifier(self):
return self.__id
def init(self,litems):
self.items = litems
def __init__(self):
Thread.__init__(self)
pass
def getIdentifier(self):
return self.__class__.__name__.lower()
"""
Initializing the class with configuration. The configuration will be specific to each subclass
"""
def init(self,config,item=None):
self.config = config
self.item = item
def process(self,item):
pass
def execute(self,cmd):
stream = None
try:
print self.getIdentifier()
print cmd
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
stream = handler.communicate()[0]
except Exception,e:
pass
return stream
def callback(self,channel,method,header,stream):
print [self.getIdentifier(),stream]
message = json.loads(stream)
content = message['content']
sender = message['from']
if content.lower() == 'quit' :
channel.close()
print " *** ",self.getIdentifier()
elif content.lower() == 'ping':
self.post(to=sender,content="1")
else:
self.process(content)
self.post(to=sender,content=content)
#message = None
#try:
#message = json.loads(stream)
#except Exception, e:
#pass
#if message is not None:
#if 'id' in message :
#if 'payload' in message:
#self.execute(message['payload']
"""
Sending a message to a queue with parameters to,from,content
"""
def post(self,**args):
to = args['to']
content = args['content']
message = {"from":self.getIdentifier(),"to":to,"content":content}
host = self.config['api']
uid = self.config['key']
qid = to#self.config['id']
qwriter = QueueWriter(host=host,uid=uid,qid=qid)
qwriter.init(qid)
qwriter.write(label=qid,row=content)
#qwriter.close()
pass
def run(self):
info = {}
host = self.config['api']
uid = self.config['key']
qid = self.config['id']
qlistener = QueueListener(qid=qid,uid=uid,host=host)
qlistener.callback = self.callback
qlistener.read()
r = [self.process(item) for item in self.items]
def isValid(self,item):
return False
def execute(self,cmd):
stream = None
try:
subprocess.call (cmd,shell=False)
#stream = handler.communicate()[0]
except Exception,e:
pass
def run(self):
if self.item is not None:
self.process(self.item)
"""
Sending a message to a queue with parameters to,from,content
"""
def post(self,**args):
pass
"""
This is designed to handle folders i.e cleaning/archiving the folders
"""
class Folders(Actor):
def init(self,config,item):
Actor.init(self,config,item)
self.lfolders = config['folders']
self.config = config['actions']['folders']
self.threshold = self.get_size(self.config['threshold'])
self.item = item
def archive(self,item):
"""
This function will archive all files in a given folder
@pre : isValid
"""
folder = item['label']
signature='-'.join([str(item['date']),str(item['count']),'-files'])
tarball=os.sep([folder,signature])
shutil.make_archive(tarball,'tar',folder)
self.clean(item)
#
# @TODO: The archive can be uploaded to the cloud or else where
# - This allows the submission of data to a processing engine if there ever were one
#
pass
def clean(self,item):
"""
This function consists in deleting files from a given folder
"""
rpath = item['label']
lists = os.listdir(item['label'])
for name in list() :
path = os.sep([item['label'],name])
if os.path.isdir(path) :
shutil.rmtree(path)
else:
os.remove(path)
#
#
def get_size(self,value):
units = {'MB':1000,'GB':1000000,'TB':1000000000} # converting to kb
key = set(unites) & set(re.split('(\d+)',value.upper()))
if len(key) == 0:
return -1
key = key.pop()
return float(value.upper().replace('MB','').strip()) * units[key]
def isvalid(self,item):
"""
This function returns whether the following :
p : folder exists
q : has_reached threashold
"""
p = os.path.exists(item['label']) and item['label'] in self.lfolders
q = self.get_size(item['size']) >= self.threshold
return p and q
def process(self,item):
if self.isValid(item) :
name = self.config['action']
stream = "".join([name,'(',json.dumps(item),')'])
eval(stream)
class Kill(Actor):
def __init__(self,config):
Actor.__init__(self,config)
def isValid(self,item):
return (item is not None) and (item in self.config)
def process(self,item):
cmd = "".join(["ps -eo pid,command|grep ",item,'|grep -E "^ {0,1}[0-9]+" -o|xargs kill -9'])
self.execute(cmd)
args = "".join(["-eo pid,command|grep ",item,'|grep -E "^ {0,1}[0-9]+" -o|xargs kill -9'])
self.execute(["ps",args])
#
# We need to make sure we can get assess the process on this server
#
class Start(Actor):
def __init__(self,config):
Actor.__init__(self,config)
def process(self,item):
path = item['path']
args = item['args'] if 'args' in item else ''
cmd = " ".join([path,args])
self.execute(cmd)
def __init__(self):
Actor.__init__(self)
self.ng = None
def init(self,config,item):
Actor.init(self,config,item)
self.config = config['apps']
self.ng = ng(self.config.keys())
def isValid(self,name):
items = self.ng.search(name)
if len(items) == 0 :
return False
else:
return items[0][1] > 0.1
def process(self,row):
name = row['label']
items = self.ng.search(name)[0]
app = items[0]
args = self.config[app]
cmd = " ".join([app,args])
self.execute([app,args])
"""
This class is designed to handle applications i.e start/stopping applications
@TODO: Assess if a reboot is required, by looking at the variance/anomaly detection
"""
class Apps(Actor):
def __init__(self):
Actor.__init__(self)
self.crashes = []
self.running = []
def isValid(self,rows):
status = [row['status'] for row in rows]
return 'crash' in status
def classify(self,rows):
self.crashes = []
self.running = []
for row in rows:
if row['status'] == 'crash' :
self.crashes.append(row)
else:
self.running.append(row)
def reboot(self):
for row_run in self.running:
pass
def start(self):
for row_crash in self.crashes:
thread = Start()
thread.init(self.config,row_crash)
thread.daemon = True
thread.start()
def process(self,rows):
self.classify(rows)
if self.crashes :
self.start()
if self.running:
self.reboot()
class Event(Thread):
def __init__(self,config):
pass
def run(self):
"""
The orchestrator class is designed to aggregate actions and communicate back to the caller
Mesage passing is structured as follows {from,to,content} The content is designed to be understood by the actor
The orchestrator is implemented using a simple iterator design-pattern
@TODO: action specifications should be provided remotely
"""
class Orchestrator(Actor):
def __init__(self,config=None):
Actor.__init__(self)
if config is None:
f = open(PARAMS['path'])
config = json.loads(f.read())
f.close()
self.config = config
Actor.__init__(self)
self.actors = {"apps":Apps(),"folders":Folders()}
self.is_master_node = False
self.items = []
#
# If the configuration only has id,key then this is NOT the maestro
#
host = config['api']
qid = config['id']
print "Initialized ***** ",self.getIdentifier(), " as ",config['id']
#
# This object will have to request for the configuration
#
#for id in config['actions'] :
#conf = config['actions'][id]
#item = eval("".join([id,"(",json.dumps(conf),")"]))
#self.actors[id.lower()] = item
"""
This function is designed to provide the orchestrator a configuration
@pre
"""
def init(self,config):
for id in config:
setup_info = config[id]
item = eval("".join([id,"(",json.dumps(setup_info),")"]))
self.actors[id.lower()] = item
def callback(self,channel,method,header,stream):
message = json.loads(stream)
if 'content' in message :
content = message['content']
print self.actors
to = message['to']
if isinstance(content,basestring) and content.lower() in ['quit'] or to=='quit':
if content.lower() == 'quit' or to == 'quit':
print '**** closing ',self.getIdentifier()
channel.close()
else:
id = to.lower()
actor = self.actors[id]
if actor is not None and actor.isValid(content) :
actor.init(self.config['actions'])
actor.process(content)
else:
content = {"status":"invalid","content":content}
#self.post(to=sender,content=content)
def run(self):
info = {}
host = self.config['api']
uid = self.config['key']
qid = self.config['id']
qlistener = QueueListener(qid=qid,uid=uid,host=host)
qlistener.callback = self.callback
qlistener.read()
r = [self.process(item) for item in self.items]
"""
This class is designed to send a message to a given AMQP enpoint
The AMQP endpoint is implemented by QueueWriter class
"""
class Alert(Actor):
def process(self,item):
pass
config = {"id":"demo","key":"[0v8]-247&7!v3","api":"localhost"}
actor = Kill(config)
actor.start()
# class Alert(Actor):
# def process(self,item):
# pass
config = {"id":"demo-100","key":"[0v8]-247&7!v3","api":"localhost"}
actor_1 = Kill(config)
actor_1.start()
if __name__ == '__main__':
thread = Orchestrator()
thread.start()

24
src/utils/agents/data-collector.py
Unescape View File

@ -27,6 +27,8 @@ class ICollector(Thread) :
self.factory = DataSourceFactory()
self.init()
self.name = 'data-collector@'+self.id
def init(self):
@ -51,7 +53,9 @@ class ICollector(Thread) :
self.quit = False
#self.DELAY = PARAMS['delay']*60
self.DELAY = self.config['delay']
#
# we need to instanciate the actor orchestrator
#
"""
This function returns an instance of a data collector class :
ProcessDetails, FileWatch, ... provided the class name
@ -70,6 +74,7 @@ class ICollector(Thread) :
write_class = self.config['store']['class']['write']
read_args = self.config['store']['args']
DELAY = self.config['delay'] * 60
while self.quit == False:
for thread in self.pool :
@ -84,7 +89,20 @@ class ICollector(Thread) :
else:
label = id
row = data
#
# At this point we should check for the status and if it prompts an action
# @TODO Use a design pattern for this ... (Aggregation?)
# - submit the row to Event for analysis
# - The event orchestrator will handle things from this point on
#
message = {}
message['to'] = thread.getName()
message['content'] = row
qwriter = QueueWriter(host=self.config['api'],uid=self.config['key'],qid=self.id)
qwriter.write(label=self.id,row = message)
qwriter.close()
self.lock.acquire()
store = self.factory.instance(type=write_class,args=read_args)
store.flush(size=200)
@ -105,4 +123,4 @@ class ICollector(Thread) :
if __name__ == '__main__':
thread = ICollector()
# thread.daemon = True
thread.start()
thread.start()

18
src/utils/transport.py
Unescape View File

@ -261,6 +261,7 @@ class MessageQueue:
self.close()
return resp
def close(self):
if self.connection.is_closed == False :
self.channel.close()
self.connection.close()
"""
@ -351,7 +352,6 @@ class QueueReader(MessageQueue,Reader):
self.durable = False
self.size = -1
self.data = {}
def init(self,qid):
properties = pika.ConnectionParameters(host=self.host)
@ -368,6 +368,7 @@ class QueueReader(MessageQueue,Reader):
"""
def callback(self,channel,method,header,stream):
r = []
if re.match("^\{|\[",stream) is not None:
r = json.loads(stream)
@ -399,9 +400,12 @@ class QueueReader(MessageQueue,Reader):
# We enabled the reader to be able to read from several queues (sequentially for now)
# The qid parameter will be an array of queues the reader will be reading from
#
if isinstance(self.qid,basestring) :
self.qid = [self.qid]
for qid in self.qid:
self.init(qid)
# r[qid] = []
if self.info.method.message_count > 0:
self.channel.basic_consume(self.callback,queue=qid,no_ack=False);
@ -420,8 +424,8 @@ class QueueListener(QueueReader):
self.channel = self.connection.channel()
self.channel.exchange_declare(exchange=self.uid,type='direct',durable=True )
self.info = self.channel.queue_declare(exclusive=True,queue=qid)
print self.info.method.queue
self.info = self.channel.queue_declare(passive=True,exclusive=True,queue=qid)
self.channel.queue_bind(exchange=self.uid,queue=self.info.method.queue,routing_key=qid)
#self.callback = callback
def read(self):
@ -583,7 +587,7 @@ class CouchdbWriter(Couchdb,Writer):
self.dbase.save_doc(document)
def flush(self,**params) :
size = params['size']
size = params['size'] if 'size' in params else 0
has_changed = False
document = self.dbase.get(self.uid)
for key in document:
@ -591,7 +595,7 @@ class CouchdbWriter(Couchdb,Writer):
content = document[key]
else:
continue
if isinstance(content,list):
if isinstance(content,list) and size > 0:
index = len(content) - size
content = content[index:]
document[key] = content
@ -599,8 +603,8 @@ class CouchdbWriter(Couchdb,Writer):
else:
document[key] = {}
has_changed = True
if has_changed:
self.dbase.save_doc(document)
self.dbase.save_doc(document)
def archive(self,params=None):
document = self.dbase.get(self.uid)

Write Preview
Loading…
Cancel
Save