Merge branch 'community' of http://dev.the-phi.com/git/steve/monitor into community

config.json

@@ -1,15 +1,18 @@
 {
-	"virtual-env":{
-		"class":"Sandbox",
-		"config":{
-			"3-launchpad":{"requirements":"/Users/steve/Documents/git/repair-file/required.txt","sandbox":"/Users/steve/Documents/git/sandbox"}
-		}
+	"id":"zulu-hacker",
+	"key":"4q-h8r5-247&!570p=[0v8]x360",
+	"api":"localhost",
+	"delay":0.5,
+	"store":{
+		"class":{"read":"CouchdbReader","write":"CouchdbWriter"},
+		"args":{"uri":"http://localhost:5984","dbname":"monitor","uid":"logs"}
 	},
-	"processes":{
-		"class":"DetailProcess",
-		"config":{
-			"system":["postgresql","couchdb","httpd"]
-			}
+	"procs":["kate","firefox"],
+	"folders":["/home/steve/tmp","/home/steve/git","/home/steve/Downloads"],
+	"actions":{
+		"folders":{"threshold":"10mb","action":"archive"},
+		"apps":{"firefox":"","kate":""	}
 	}
 	
+	
 }

init.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# This script is designed to handle various operations related to setting up, starting, stopping the python application
+# It will assume the requirements file is at the root (not with the source code)
+#
+
+export PYTHONPATH=$PWD/src
+pip_upgrade='sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o'
+install(){
+	
+	virtualenv sandbox
+	sandbox/bin/pip install -r requirements.txt
+	`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade` 
+
+
+}
+upgrade(){
+	git pull
+	count=`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|wc -l`
+	if [ ! "$count" = "0" ]; then
+	 	`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade`
+	 else
+	 	echo "No Upgrade required for sandbox"
+	 fi
+	
+}
+start(){
+	
+	if [ "$1" = "collector" ]; then
+		sandbox/bin/python src/utils/agents/data-collector.py --path $PWD/config.json
+	else
+
+		sandbox/bin/python src/api/index.py --path $PWD/config.json &
+	fi
+
+}
+stop(){
+	ps -eo pid,command|grep python|grep -E "$PWD"|grep index.py|grep -E "^ {0,}[0-9]+" -o |xargs kill -9
+	ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -o |xargs kill -9
+}
+
+status(){
+	pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep index.py|grep -E "^ {0,}[0-9]+" -m 1 -o`
+	if [ "$pid" = "" ]; then
+		echo "API IS OFFLINE"
+	else
+		echo "API IS ONLINE $pid"
+	fi
+	pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -m 1 -o`
+	if [ "$pid" = "" ]; then 
+		echo "DATA-COLLECTOR IS OFFLINE"
+	else
+		echo "DATA-COLLECTOR IS ONLINE $pid"
+	fi
+
+}
+
+if [ "$1" = "start" ]; then
+	if [ "$2" = "collector" ]; then
+		start "collector"
+	else
+		start
+	fi
+else
+	$1
+
+fi

readme.md

@@ -6,6 +6,20 @@ The program answers basic questions:
 	- Is a given program still running
 	- How much resource (memory/cpu) a program is using up
 	- The number of processes found
+	- Folder monitoring ...
+	
+#Architecture
+
+The architecture of the system is distributed with a central master node,
+    {
+        "id":"",
+        "key":"",
+        "apps":[],
+        "sandbox":[{"path":"","requirements":""}],
+        "folders":["path-1"],
+        "store":{}
+        "actions":{}
+    }
 
 The agent will perform three basic functions :
 

requirements.txt

@@ -1,13 +1,16 @@
-aniso8601==1.2.0
-click==6.6
-couchdbkit==0.6.5
 Flask==0.11.1
 Flask-Session==0.3.0
 Flask-SocketIO==2.8.2
-http-parser==0.8.3
-itsdangerous==0.24
 Jinja2==2.8
 MarkupSafe==0.23
+Werkzeug==0.11.11
+aniso8601==1.2.0
+argparse==1.2.1
+click==6.6
+couchdbkit==0.6.5
+http-parser==0.8.3
+itsdangerous==0.24
+ngram==3.3.0
 numpy==1.11.3
 pika==0.10.0
 python-dateutil==2.6.0
@@ -17,4 +20,4 @@ pytz==2016.10
 restkit==4.2.2
 six==1.10.0
 socketpool==0.5.3
-Werkzeug==0.11.11
+wsgiref==0.1.2

src/api/static/img/uml-design

@@ -0,0 +1,243 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<XMI xmlns:UML="http://schema.omg.org/spec/UML/1.3" timestamp="2017-04-07T08:29:04" xmi.version="1.2" verified="false">
+ <XMI.header>
+  <XMI.documentation>
+   <XMI.exporter>umbrello uml modeller http://umbrello.kde.org</XMI.exporter>
+   <XMI.exporterVersion>1.6.9</XMI.exporterVersion>
+   <XMI.exporterEncoding>UnicodeUTF8</XMI.exporterEncoding>
+  </XMI.documentation>
+  <XMI.metamodel href="UML.xml" xmi.name="UML" xmi.version="1.3"/>
+ </XMI.header>
+ <XMI.content>
+  <UML:Model isRoot="false" isSpecification="false" name="UML Model" isAbstract="false" xmi.id="m1" isLeaf="false">
+   <UML:Namespace.ownedElement>
+    <UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="folder" isAbstract="false" isLeaf="false" xmi.id="folder" visibility="public"/>
+    <UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="datatype" isAbstract="false" isLeaf="false" xmi.id="datatype" visibility="public"/>
+    <UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="agent" isAbstract="false" isLeaf="false" xmi.id="rSZCzDKpjhw5" visibility="public"/>
+    <UML:Stereotype isRoot="false" isSpecification="false" namespace="m1" name="interface" isAbstract="false" isLeaf="false" xmi.id="interface" visibility="public"/>
+    <UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Logical View" isAbstract="false" isLeaf="false" xmi.id="Logical View" visibility="public">
+     <UML:Namespace.ownedElement>
+      <UML:Package isRoot="false" stereotype="folder" isSpecification="false" namespace="Logical View" name="Datatypes" isAbstract="false" isLeaf="false" xmi.id="Datatypes" visibility="public">
+       <UML:Namespace.ownedElement>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="int" isAbstract="false" isLeaf="false" xmi.id="fCoa6Ljfk80G" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="char" isAbstract="false" isLeaf="false" xmi.id="XVF672bO1S1I" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="bool" isAbstract="false" isLeaf="false" xmi.id="HU1uO6O1JAZP" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="float" isAbstract="false" isLeaf="false" xmi.id="UI3ER3SwXh1e" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="double" isAbstract="false" isLeaf="false" xmi.id="hc0i20yJS0Dp" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="short" isAbstract="false" isLeaf="false" xmi.id="6aqNkOnEgoR7" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="long" isAbstract="false" isLeaf="false" xmi.id="pt2Mb3yGdzxf" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="unsigned int" isAbstract="false" isLeaf="false" xmi.id="xvyPvAC0i1NS" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="unsigned short" isAbstract="false" isLeaf="false" xmi.id="P8f5w6BmxD8W" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="unsigned long" isAbstract="false" isLeaf="false" xmi.id="E7nr4kW1fSOb" visibility="public"/>
+        <UML:DataType isRoot="false" stereotype="datatype" isSpecification="false" namespace="Datatypes" name="string" isAbstract="false" isLeaf="false" xmi.id="abbIcwjz5O41" visibility="public"/>
+       </UML:Namespace.ownedElement>
+      </UML:Package>
+      <UML:Interface isRoot="false" stereotype="interface" isSpecification="false" namespace="Logical View" name="Agent" isAbstract="true" isLeaf="false" xmi.id="RXqBshUB41L5" visibility="public">
+       <UML:Classifier.feature>
+        <UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="init" isAbstract="false" isLeaf="false" xmi.id="wViGJvmJF1zV" visibility="public"/>
+        <UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="execute" isAbstract="false" isLeaf="false" xmi.id="7FOYGyKLJkjG" visibility="public"/>
+        <UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="status" isAbstract="false" isLeaf="false" xmi.id="q8oPwCisiO61" visibility="public"/>
+        <UML:Operation isRoot="false" isQuery="false" isSpecification="false" name="instance" isAbstract="false" isLeaf="false" xmi.id="2fGeoj3czd3G" visibility="public" ownerScope="classifier"/>
+       </UML:Classifier.feature>
+      </UML:Interface>
+      <UML:Interface isRoot="false" stereotype="interface" isSpecification="false" namespace="Logical View" name="data-store" isAbstract="true" isLeaf="false" xmi.id="dqWyyHKwPkeW" visibility="public"/>
+     </UML:Namespace.ownedElement>
+     <XMI.extension xmi.extender="umbrello">
+      <diagrams>
+       <diagram usefillcolor="1" linewidth="0" showpubliconly="0" snapy="25" isopen="1" showatts="1" zoom="100" griddotcolor="#d3d3d3" name="class diagram" canvasheight="0" showpackage="1" canvaswidth="0" type="1" showops="1" xmi.id="J1rdJrVlcu5U" showgrid="0" showattsig="1" backgroundcolor="#ffffff" linecolor="#ff0000" snapcsgrid="0" fillcolor="#ffff00" showattribassocs="1" localid="-1" snapgrid="0" showscope="1" showopsig="1" snapx="25" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" documentation="">
+        <widgets/>
+        <messages/>
+        <associations/>
+       </diagram>
+      </diagrams>
+     </XMI.extension>
+    </UML:Model>
+    <UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Use Case View" isAbstract="false" isLeaf="false" xmi.id="Use Case View" visibility="public">
+     <UML:Namespace.ownedElement/>
+    </UML:Model>
+    <UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Component View" isAbstract="false" isLeaf="false" xmi.id="Component View" visibility="public">
+     <UML:Namespace.ownedElement>
+      <UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="data-collector" isAbstract="false" isLeaf="false" xmi.id="Tz06QOf0xEHX" visibility="public" executable="0"/>
+      <UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Learner" isAbstract="false" isLeaf="false" xmi.id="qVP8MjpaiUjw" visibility="public" executable="0"/>
+      <UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Kill" isAbstract="false" isLeaf="false" xmi.id="IsIXtH00UPZi" visibility="public" executable="0"/>
+      <UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="rVfXulYAdqhX" visibility="public" client="IsIXtH00UPZi"/>
+      <UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Start" isAbstract="false" isLeaf="false" xmi.id="1TAcIp7NVdvX" visibility="public" executable="0"/>
+      <UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="PvoWBjUADfkV" visibility="public" client="1TAcIp7NVdvX"/>
+      <UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Notify" isAbstract="false" isLeaf="false" xmi.id="HSy87jRVwqsY" visibility="public" executable="0"/>
+      <UML:Component isRoot="false" stereotype="rSZCzDKpjhw5" isSpecification="false" namespace="Component View" name="Mailer" isAbstract="false" isLeaf="false" xmi.id="2X1nlev04t8B" visibility="public" executable="0"/>
+      <UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="HyCj3nQ3KPr5" visibility="public" client="2X1nlev04t8B"/>
+      <UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="RXqBshUB41L5" xmi.id="jpx2Co5CY9GC" visibility="public" client="HSy87jRVwqsY"/>
+      <UML:Association isSpecification="false" namespace="Component View" name="" xmi.id="xfmTo0d7U3Co" visibility="public">
+       <UML:Association.connection>
+        <UML:AssociationEnd isNavigable="false" aggregation="none" isSpecification="false" changeability="changeable" name="" type="Tz06QOf0xEHX" xmi.id="GLVKohqCd1VT" visibility="public"/>
+        <UML:AssociationEnd isNavigable="true" aggregation="none" isSpecification="false" changeability="changeable" name="" type="RXqBshUB41L5" xmi.id="cgz2pZckFNB1" visibility="public"/>
+       </UML:Association.connection>
+      </UML:Association>
+      <UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="NoSQL" isAbstract="false" isLeaf="false" xmi.id="ZPS2o7GDkc5b" visibility="public" executable="0"/>
+      <UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="Queue" isAbstract="false" isLeaf="false" xmi.id="yAQh9XwiuMkg" visibility="public" executable="0"/>
+      <UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="dqWyyHKwPkeW" xmi.id="roUjSuB43yq5" visibility="public" client="yAQh9XwiuMkg"/>
+      <UML:Abstraction isSpecification="false" namespace="Component View" name="" supplier="dqWyyHKwPkeW" xmi.id="up5vlwzBogrF" visibility="public" client="ZPS2o7GDkc5b"/>
+      <UML:Association isSpecification="false" namespace="Component View" name="" xmi.id="EZF1F7EEwgSu" visibility="public">
+       <UML:Association.connection>
+        <UML:AssociationEnd isNavigable="false" aggregation="none" isSpecification="false" changeability="changeable" name="" type="Tz06QOf0xEHX" xmi.id="WZqWR7Y3vUHh" visibility="public"/>
+        <UML:AssociationEnd isNavigable="true" aggregation="none" isSpecification="false" changeability="changeable" name="" type="dqWyyHKwPkeW" xmi.id="2WNNqxT3Uh2j" visibility="public"/>
+       </UML:Association.connection>
+      </UML:Association>
+      <UML:Component isRoot="false" isSpecification="false" namespace="Component View" name="Dashboard" isAbstract="false" isLeaf="false" xmi.id="mGviuNaRvQvK" visibility="public" executable="0"/>
+      <UML:Association isSpecification="false" namespace="Component View" name="" xmi.id="Mm0rT2YFGUvA" visibility="public">
+       <UML:Association.connection>
+        <UML:AssociationEnd isNavigable="false" aggregation="none" isSpecification="false" changeability="changeable" name="" type="mGviuNaRvQvK" xmi.id="pTZirT2NwOlu" visibility="public"/>
+        <UML:AssociationEnd isNavigable="true" aggregation="none" isSpecification="false" changeability="changeable" name="" type="dqWyyHKwPkeW" xmi.id="0rp9GA8cw6RN" visibility="public"/>
+       </UML:Association.connection>
+      </UML:Association>
+      <UML:Dependency isSpecification="false" namespace="Component View" name="" supplier="dqWyyHKwPkeW" xmi.id="80aOSVWFy4yp" visibility="public" client="qVP8MjpaiUjw"/>
+     </UML:Namespace.ownedElement>
+     <XMI.extension xmi.extender="umbrello">
+      <diagrams>
+       <diagram usefillcolor="0" linewidth="0" showpubliconly="0" snapy="25" isopen="1" showatts="1" zoom="100" griddotcolor="#d3d3d3" name="component diagram" canvasheight="614" showpackage="1" canvaswidth="1042" type="7" showops="1" xmi.id="SFlPQuwezjV1" showgrid="0" showattsig="1" backgroundcolor="#ffffff" linecolor="#000000" snapcsgrid="0" fillcolor="#ffff00" showattribassocs="1" localid="-1" snapgrid="0" showscope="1" showopsig="1" snapx="25" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" documentation="">
+        <widgets>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-678" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="v7rsoLtFIrMg" linewidth="0" linecolor="#000000" xmi.id="Tz06QOf0xEHX" x="-1736" width="159" showstereotype="1"/>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-412" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="OPLKYyaGrclm" linewidth="0" linecolor="#000000" xmi.id="qVP8MjpaiUjw" x="-1550" width="146" showstereotype="1"/>
+         <interfacewidget usefillcolor="0" isinstance="0" linewidth="0" usesdiagramusefillcolor="0" showpubliconly="0" showoperations="1" height="102" y="-690.5" width="93" showattsigs="601" showpackage="0" xmi.id="RXqBshUB41L5" usesdiagramfillcolor="0" x="-1500.5" linecolor="#000000" fillcolor="#ffff00" localid="bSOrj87QKpFY" showscope="1" showattributes="0" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" showopsigs="601" drawascircle="0"/>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-810" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="Lr1cZksQrfnn" linewidth="0" linecolor="#000000" xmi.id="IsIXtH00UPZi" x="-1276" width="118" showstereotype="1"/>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-687" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="fCxXTyzSroIP" linewidth="0" linecolor="#000000" xmi.id="1TAcIp7NVdvX" x="-1114" width="118" showstereotype="1"/>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-579" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="65" localid="rbXRChMtT5dh" linewidth="0" linecolor="#000000" xmi.id="HSy87jRVwqsY" x="-1293" width="150" showstereotype="1"/>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-803" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="cZfZNX5E6adI" linewidth="0" linecolor="#000000" xmi.id="2X1nlev04t8B" x="-1515" width="118" showstereotype="1"/>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-426" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="nIBN2jAINAlN" linewidth="0" linecolor="#000000" xmi.id="ZPS2o7GDkc5b" x="-1891" width="145" showstereotype="1"/>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-319" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="Rzr9ZxkMFuhF" linewidth="0" linecolor="#000000" xmi.id="yAQh9XwiuMkg" x="-1888" width="147" showstereotype="1"/>
+         <interfacewidget usefillcolor="0" isinstance="0" linewidth="0" usesdiagramusefillcolor="0" showpubliconly="0" showoperations="1" height="40" y="-410" width="40" showattsigs="601" showpackage="0" xmi.id="dqWyyHKwPkeW" usesdiagramfillcolor="0" x="-1674" linecolor="#000000" fillcolor="#ffff00" localid="GYF8sxpJueQR" showscope="1" showattributes="0" showstereotype="1" textcolor="#000000" font="Sans Serif,9,-1,5,50,0,0,0,0,0" showopsigs="601" drawascircle="1">
+          <floatingtext usefillcolor="1" isinstance="0" linewidth="0" pretext="" usesdiagramusefillcolor="1" height="21" y="66" posttext="" width="78" xmi.id="pwDCepiuLv1D" usesdiagramfillcolor="1" x="-22" text="data-store" linecolor="#000000" fillcolor="none" localid="b8no2T4fJacH" showstereotype="1" textcolor="none" font="Ubuntu,11,-1,5,50,0,0,0,0,0" role="700"/>
+         </interfacewidget>
+         <componentwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-312" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="64" localid="VTNOxe5xARKz" linewidth="0" linecolor="#000000" xmi.id="mGviuNaRvQvK" x="-1546" width="152" showstereotype="1"/>
+         <boxwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-817" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="338" localid="deiZieMDdXfx" linewidth="0" linecolor="#000000" xmi.id="TyIE7mFOMqb2" x="-1945" width="998" showstereotype="1"/>
+         <boxwidget usesdiagramfillcolor="0" fillcolor="#ffff00" font="Sans Serif,9,-1,5,50,0,0,0,0,0" y="-462" usefillcolor="0" isinstance="0" textcolor="#000000" usesdiagramusefillcolor="0" height="259" localid="r9osJyLub2TH" linewidth="0" linecolor="#0000ff" xmi.id="gqVHN8Fq6V2C" x="-1945" width="1000" showstereotype="1"/>
+         <floatingtext usefillcolor="0" isinstance="0" linewidth="0" pretext="" usesdiagramusefillcolor="0" height="32" y="-537" posttext="" width="154" xmi.id="M7Vv9zCpnf7A" usesdiagramfillcolor="0" x="-1933" text=" DETECTION" linecolor="#000000" fillcolor="#ffff00" localid="GFaa3AbvJPnb" showstereotype="1" textcolor="#000000" font="Sans Serif,18,-1,5,50,0,0,0,0,0" role="700"/>
+         <floatingtext usefillcolor="0" isinstance="0" linewidth="0" pretext="" usesdiagramusefillcolor="0" height="32" y="-248" posttext="" width="163" xmi.id="mD94avfu28pB" usesdiagramfillcolor="0" x="-1116" text="PREDICTION " linecolor="#000000" fillcolor="#ffff00" localid="nKysny43DccE" showstereotype="1" textcolor="#000000" font="Sans Serif,18,-1,5,50,0,0,0,0,0" role="700"/>
+        </widgets>
+        <messages/>
+        <associations>
+         <assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="rVfXulYAdqhX" widgetaid="IsIXtH00UPZi" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-746" startx="-1223"/>
+           <endpoint endx="-1407.5" endy="-649"/>
+           <point y="-649" x="-1223"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="PvoWBjUADfkV" widgetaid="1TAcIp7NVdvX" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-650" startx="-1114"/>
+           <endpoint endx="-1407.5" endy="-650"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="HyCj3nQ3KPr5" widgetaid="2X1nlev04t8B" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-739" startx="-1467"/>
+           <endpoint endx="-1467" endy="-690.5"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="511" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="jpx2Co5CY9GC" widgetaid="HSy87jRVwqsY" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-579" startx="-1224"/>
+           <endpoint endx="-1407.5" endy="-649"/>
+           <point y="-649" x="-1224"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="1" fillcolor="none" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="512" totalcountb="2" linewidth="0" widgetbid="RXqBshUB41L5" linecolor="#000000" xmi.id="xfmTo0d7U3Co" widgetaid="Tz06QOf0xEHX" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-651.5" startx="-1577"/>
+           <endpoint endx="-1500.5" endy="-651.5"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="511" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="roUjSuB43yq5" widgetaid="yAQh9XwiuMkg" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-319" startx="-1741"/>
+           <endpoint endx="-1674" endy="-370"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="1" fillcolor="none" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="511" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="up5vlwzBogrF" widgetaid="ZPS2o7GDkc5b" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-410" startx="-1746"/>
+           <endpoint endx="-1674" endy="-410"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="1" type="512" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="EZF1F7EEwgSu" widgetaid="Tz06QOf0xEHX" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-614" startx="-1652"/>
+           <endpoint endx="-1652" endy="-410"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="1" fillcolor="none" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="512" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="Mm0rT2YFGUvA" widgetaid="mGviuNaRvQvK" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-312" startx="-1546"/>
+           <endpoint endx="-1634" endy="-370"/>
+          </linepath>
+         </assocwidget>
+         <assocwidget usesdiagramfillcolor="0" fillcolor="#ffff00" indexa="1" font="Sans Serif,9,-1,5,50,0,0,0,0,0" usefillcolor="1" indexb="1" textcolor="none" usesdiagramusefillcolor="0" type="502" totalcountb="2" linewidth="0" widgetbid="dqWyyHKwPkeW" linecolor="#000000" xmi.id="80aOSVWFy4yp" widgetaid="qVP8MjpaiUjw" seqnum="" totalcounta="2">
+          <linepath layout="Polyline">
+           <startpoint starty="-410" startx="-1550"/>
+           <endpoint endx="-1634" endy="-410"/>
+          </linepath>
+         </assocwidget>
+        </associations>
+       </diagram>
+      </diagrams>
+     </XMI.extension>
+    </UML:Model>
+    <UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Deployment View" isAbstract="false" isLeaf="false" xmi.id="Deployment View" visibility="public">
+     <UML:Namespace.ownedElement/>
+    </UML:Model>
+    <UML:Model isRoot="false" isSpecification="false" namespace="m1" name="Entity Relationship Model" isAbstract="false" isLeaf="false" xmi.id="Entity Relationship Model" visibility="public">
+     <UML:Namespace.ownedElement/>
+    </UML:Model>
+   </UML:Namespace.ownedElement>
+  </UML:Model>
+ </XMI.content>
+ <XMI.extensions xmi.extender="umbrello">
+  <docsettings uniqueid="wLA7xmMizVkY" viewid="SFlPQuwezjV1" documentation=""/>
+  <listview>
+   <listitem id="Views" open="1" type="800">
+    <listitem id="Component View" open="1" type="821">
+     <listitem id="SFlPQuwezjV1" label="component diagram" open="0" type="819"/>
+     <listitem id="mGviuNaRvQvK" open="1" type="822"/>
+     <listitem id="Tz06QOf0xEHX" open="1" type="822"/>
+     <listitem id="IsIXtH00UPZi" open="1" type="822"/>
+     <listitem id="qVP8MjpaiUjw" open="1" type="822"/>
+     <listitem id="2X1nlev04t8B" open="1" type="822"/>
+     <listitem id="ZPS2o7GDkc5b" open="1" type="822"/>
+     <listitem id="HSy87jRVwqsY" open="1" type="822"/>
+     <listitem id="yAQh9XwiuMkg" open="1" type="822"/>
+     <listitem id="1TAcIp7NVdvX" open="1" type="822"/>
+    </listitem>
+    <listitem id="Deployment View" open="1" type="827"/>
+    <listitem id="Entity Relationship Model" open="1" type="836"/>
+    <listitem id="Logical View" open="1" type="801">
+     <listitem id="RXqBshUB41L5" open="1" type="817">
+      <listitem id="7FOYGyKLJkjG" open="0" type="815"/>
+      <listitem id="wViGJvmJF1zV" open="0" type="815"/>
+      <listitem id="2fGeoj3czd3G" open="0" type="815"/>
+      <listitem id="q8oPwCisiO61" open="0" type="815"/>
+     </listitem>
+     <listitem id="J1rdJrVlcu5U" label="class diagram" open="0" type="807"/>
+     <listitem id="dqWyyHKwPkeW" open="1" type="817"/>
+     <listitem id="Datatypes" open="0" type="830">
+      <listitem id="HU1uO6O1JAZP" open="1" type="829"/>
+      <listitem id="XVF672bO1S1I" open="1" type="829"/>
+      <listitem id="hc0i20yJS0Dp" open="1" type="829"/>
+      <listitem id="UI3ER3SwXh1e" open="1" type="829"/>
+      <listitem id="fCoa6Ljfk80G" open="1" type="829"/>
+      <listitem id="pt2Mb3yGdzxf" open="1" type="829"/>
+      <listitem id="6aqNkOnEgoR7" open="1" type="829"/>
+      <listitem id="abbIcwjz5O41" open="1" type="829"/>
+      <listitem id="xvyPvAC0i1NS" open="1" type="829"/>
+      <listitem id="E7nr4kW1fSOb" open="1" type="829"/>
+      <listitem id="P8f5w6BmxD8W" open="1" type="829"/>
+     </listitem>
+    </listitem>
+    <listitem id="Use Case View" open="1" type="802"/>
+   </listitem>
+  </listview>
+  <codegeneration>
+   <codegenerator language="C++"/>
+  </codegeneration>
+ </XMI.extensions>
+</XMI>

src/api/static/js/jx

@@ -1 +1 @@
-Subproject commit 36f9f10ff44406cc0418cc5934eb475bc77ebb1e
+Subproject commit 3d00f5a126574f2277cdac25d60008ee35dc8740

src/utils/agents/actor.py

@@ -13,112 +13,291 @@
 import json
 from threading import Thread
 import os
+import shutil
 import subprocess
 from monitor import ProcessCounter
-from utils.transport import QueueListener, QueueWriter
+from utils.transport import QueueListener, QueueWriter, QueueReader
+from utils.params import PARAMS
+from ngram import NGram as ng
 class Actor(Thread):
-	def __init__(self,config):
-		Thread.__init__(self)
-		self.config = config
-		self.items = []
-		self.__id = config['id']
-        def getIdentifier(self):
-                return self.__id
-            
-	def init(self,litems):
-		self.items = litems
+    def __init__(self):
+        Thread.__init__(self)
+        pass
+    def getIdentifier(self):
+        return self.__class__.__name__.lower()
+    """
+        Initializing the class with configuration. The configuration will be specific to each subclass
+
+    """
+    def init(self,config,item=None):
+        self.config = config
+        self.item = item
 	def process(self,item):
 		pass
-	def execute(self,cmd):
-		stream = None
-		try:
-                        print self.getIdentifier()
-                        print cmd
-			handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
-			stream = handler.communicate()[0]
-		except Exception,e:
-			pass
-		return stream
-	def callback(self,channel,method,header,stream):
-                print [self.getIdentifier(),stream]
-                message = json.loads(stream)
-                content = message['content']
-                sender = message['from']
-                if content.lower() == 'quit' :
-                    channel.close()
-                    print " *** ",self.getIdentifier()
-                elif content.lower() == 'ping':
-                    self.post(to=sender,content="1")
-                else:
-                    self.process(content)
-                    self.post(to=sender,content=content)
-                
-                #message = None
-                #try:
-                    #message = json.loads(stream)
-                #except Exception, e:
-                    #pass
-                #if message is not None:
-                    #if 'id' in message :
-                        #if 'payload' in message:
-                            #self.execute(message['payload']
-        """
-            Sending a message to a queue with parameters to,from,content
-        """
-        def post(self,**args):
-                to      = args['to'] 
-                content = args['content']
-                message = {"from":self.getIdentifier(),"to":to,"content":content}
-                host	= self.config['api']
-                uid	= self.config['key']
-                qid	= to#self.config['id']
-                
-                qwriter = QueueWriter(host=host,uid=uid,qid=qid)
-                qwriter.init(qid)
-                qwriter.write(label=qid,row=content)
-                #qwriter.close()
-                pass
-	def run(self):
-		info = {}
-		host	= self.config['api']
-		uid	= self.config['key']
-		qid	= self.config['id']
-		
-		qlistener = QueueListener(qid=qid,uid=uid,host=host)		
-		qlistener.callback = self.callback
-		qlistener.read()
-		r = [self.process(item) for item in self.items]
+    def isValid(self,item):
+        return False
+
+    def execute(self,cmd):
+        stream = None
+        try:
+            subprocess.call (cmd,shell=False)
+            #stream = handler.communicate()[0]
+        except Exception,e:
+            pass
+        
+    def run(self):
+        if self.item is not None:
+            self.process(self.item)
+    """
+        Sending a message to a queue with parameters to,from,content
+    """
+    def post(self,**args):
+        pass
+"""
+    This is designed to handle folders i.e cleaning/archiving the folders
+    
+"""
+class Folders(Actor):
+    def init(self,config,item):
+        Actor.init(self,config,item)
+        self.lfolders   = config['folders']
+        self.config     = config['actions']['folders']
+        self.threshold  = self.get_size(self.config['threshold'])
+        self.item       = item
+    
+    def archive(self,item):
+    """
+        This function will archive all files in a given folder
+        @pre : isValid
+    """
+        folder = item['label']
+        signature='-'.join([str(item['date']),str(item['count']),'-files'])
+        tarball=os.sep([folder,signature])
+        shutil.make_archive(tarball,'tar',folder)
+        self.clean(item)
+        #
+        # @TODO: The archive can be uploaded to the cloud or else where
+        #   - This allows the submission of data to a processing engine if there ever were one
+        #
+        pass
+    
+    def clean(self,item):
+    """
+        This function consists in deleting files from a given folder
+    """
+        rpath = item['label']
+        lists = os.listdir(item['label'])
+        for name in list() :
+            path = os.sep([item['label'],name])
+            if os.path.isdir(path) :
+                shutil.rmtree(path)
+            else:
+                os.remove(path)
+        #
+        # 
+
+    def get_size(self,value):
+        units = {'MB':1000,'GB':1000000,'TB':1000000000} # converting to kb
+        key = set(unites) & set(re.split('(\d+)',value.upper()))
+        if len(key) == 0:
+            return -1
+        key = key.pop()
+        return float(value.upper().replace('MB','').strip()) * units[key]
+    
+    def isvalid(self,item):        
+    """
+        This function returns whether the following :
+        p : folder exists
+        q : has_reached threashold
+    """
+        
+        
+        p = os.path.exists(item['label']) and item['label'] in self.lfolders
+    
+        q = self.get_size(item['size']) >= self.threshold
+        return p and q
+    
+    def process(self,item):
+        if self.isValid(item) :
+            
+            name = self.config['action']
+            stream = "".join([name,'(',json.dumps(item),')'])
+            eval(stream)
+        
+        
 class Kill(Actor):
-	def __init__(self,config):
-		Actor.__init__(self,config)
+    
+    def isValid(self,item):                
+        return (item is not None) and (item in self.config)
 	def process(self,item):
-		cmd = "".join(["ps -eo pid,command|grep ",item,'|grep -E "^ {0,1}[0-9]+" -o|xargs kill -9'])
-		self.execute(cmd)
+		args = "".join(["-eo pid,command|grep ",item,'|grep -E "^ {0,1}[0-9]+" -o|xargs kill -9'])
+		self.execute(["ps",args])
 		#
 		# We need to make sure we can get assess the process on this server
 		#
+
 class Start(Actor):
-	def __init__(self,config):
-		Actor.__init__(self,config)
-	def process(self,item):
-		path = item['path']
-		args = item['args'] if 'args' in item else ''
-		cmd = " ".join([path,args])
-		self.execute(cmd)
+    def __init__(self):
+        Actor.__init__(self)
+        self.ng = None
+    
+    def init(self,config,item):
+        Actor.init(self,config,item)
+        self.config = config['apps']
+        self.ng = ng(self.config.keys())
+
+    def isValid(self,name):
+        items = self.ng.search(name) 
+        if len(items) == 0 :
+            return False
+        else:
+            return items[0][1] > 0.1
+                    
+    def process(self,row):
+        name    = row['label']
+        items   = self.ng.search(name)[0]
+        app = items[0]
 
+
+        args = self.config[app]
+        cmd = " ".join([app,args])
+        
+        self.execute([app,args])
 """
-    This class is designed to send a message to a given AMQP enpoint
-    The AMQP endpoint is implemented by QueueWriter class
+    This class is designed to handle applications i.e start/stopping applications
+    @TODO: Assess if a reboot is required, by looking at the variance/anomaly detection
+"""
+class Apps(Actor):
+    def __init__(self):
+        Actor.__init__(self)
+        self.crashes = []
+        self.running = []
+    
+    def isValid(self,rows):
+        status = [row['status'] for row in rows]
+        return 'crash' in status
+    
+    def classify(self,rows):
+        self.crashes = []
+        self.running = []
+        for row in rows:
+            if row['status'] == 'crash' :
+                self.crashes.append(row)
+            else:
+                self.running.append(row)
+    def reboot(self):
+        for row_run in self.running:
+            pass
+    def start(self):
+        for row_crash in self.crashes:
+            thread = Start()
+            thread.init(self.config,row_crash)
+            thread.daemon = True
+            thread.start()
+        
+    def process(self,rows):
+        self.classify(rows)
+        if self.crashes :
+            self.start()
+        if self.running:
+            self.reboot()
+        
+        
+class Event(Thread):    
+    def __init__(self,config):
+        pass
+    def run(self):
 """
-class Alert(Actor):
-        def process(self,item):
-                pass
+    The orchestrator class is designed to aggregate actions and communicate back to the caller
+    Mesage passing is structured as follows {from,to,content} The content is designed to be understood by the actor
+    
+    The orchestrator is implemented using a simple iterator design-pattern
+    @TODO: action specifications should be provided remotely
+"""
+class Orchestrator(Actor):
+    
+    def __init__(self,config=None):
+        Actor.__init__(self)
+        if config is None:
+            f = open(PARAMS['path'])
+            config = json.loads(f.read())
+            f.close()
+        self.config = config
+        Actor.__init__(self)
+        
+        self.actors = {"apps":Apps(),"folders":Folders()}
+        self.is_master_node = False
+        self.items = []
+        #
+        # If the configuration only has id,key then this is NOT the maestro
+        #
+        host = config['api']
+        qid = config['id']
+        print "Initialized ***** ",self.getIdentifier(), " as ",config['id']
+        
+            #
+            # This object will have to request for the configuration
+            #
+        #for id in config['actions'] :
+            #conf = config['actions'][id]
+            #item = eval("".join([id,"(",json.dumps(conf),")"]))
+            #self.actors[id.lower()]  = item
+        """
+            This function is designed to provide the orchestrator a configuration             
+            @pre
+        """
+    def init(self,config):
+            
+        for id in config:
             
+            setup_info  = config[id]
+            item        = eval("".join([id,"(",json.dumps(setup_info),")"]))
+            self.actors[id.lower()]  = item
                 
-config = {"id":"demo","key":"[0v8]-247&7!v3","api":"localhost"}
-actor = Kill(config)
-actor.start()
+    def callback(self,channel,method,header,stream):
+                
+        message = json.loads(stream)
+        if 'content' in message :
+            content = message['content']
+	    print self.actors        
+            to = message['to']
+            if isinstance(content,basestring) and content.lower() in ['quit'] or to=='quit':
+                if content.lower() == 'quit' or to == 'quit':
+                    print '**** closing ',self.getIdentifier()
+                    channel.close()
+            else:
+
+                id = to.lower()
+                actor = self.actors[id]
+                
+                if actor is not None and actor.isValid(content) :
+                    actor.init(self.config['actions'])                            
+                    actor.process(content)
+                else:
+                    content = {"status":"invalid","content":content}
+                
+                #self.post(to=sender,content=content)
+                
+    def run(self):
+
+        info = {}
+        host	= self.config['api']
+        uid	= self.config['key']
+        qid	= self.config['id']
+		
+        qlistener = QueueListener(qid=qid,uid=uid,host=host)		
+        qlistener.callback = self.callback
+        qlistener.read()
+        r = [self.process(item) for item in self.items]
+
+"""
+    This class is designed to send a message to a given AMQP enpoint
+    The AMQP endpoint is implemented by QueueWriter class
+"""
+# class Alert(Actor):
+#     def process(self,item):
+#         pass
 
-config = {"id":"demo-100","key":"[0v8]-247&7!v3","api":"localhost"}
-actor_1 = Kill(config)
-actor_1.start()
+if __name__ == '__main__':
+    thread = Orchestrator()
+    thread.start()

src/utils/agents/data-collector.py

@@ -27,6 +27,8 @@ class ICollector(Thread) :
 		self.factory	= DataSourceFactory()
 		self.init()
 		self.name = 'data-collector@'+self.id
+		
+		
 	def init(self):
 		
 		
@@ -51,7 +53,9 @@ class ICollector(Thread) :
 		self.quit = False
 		#self.DELAY = PARAMS['delay']*60
 		self.DELAY = self.config['delay']
-		
+		#
+		# we need to instanciate the actor orchestrator
+		#
 	"""
 		This function returns an instance of a data collector class :
 		ProcessDetails, FileWatch, ... provided the class name
@@ -70,6 +74,7 @@ class ICollector(Thread) :
 		write_class 	= self.config['store']['class']['write']
 		read_args	= self.config['store']['args']
 		DELAY	= self.config['delay'] * 60
+		
 		while self.quit == False:
 			
 			for thread in self.pool :
@@ -84,6 +89,19 @@ class ICollector(Thread) :
 				else:
 					label = id
 					row = data
+				#
+				# At this point we should check for the status and if it prompts an action
+				# @TODO Use a design pattern for this ... (Aggregation?)
+				#   - submit the row to Event for analysis
+				#   - The event orchestrator will handle things from this point on
+				#
+				message = {}
+				
+				message['to'] = thread.getName()
+				message['content'] = row
+				qwriter = QueueWriter(host=self.config['api'],uid=self.config['key'],qid=self.id)
+				qwriter.write(label=self.id,row = message)
+				qwriter.close()
 				
 				self.lock.acquire()
 				store = self.factory.instance(type=write_class,args=read_args)

src/utils/transport.py

@@ -261,6 +261,7 @@ class MessageQueue:
 		self.close()
 		return resp
 	def close(self):
+            if self.connection.is_closed == False :
 		self.channel.close()
 		self.connection.close()
 """
@@ -351,7 +352,6 @@ class QueueReader(MessageQueue,Reader):
 			self.durable = False
 		self.size = -1
 		self.data = {}
-	
 	def init(self,qid):
 		
 		properties = pika.ConnectionParameters(host=self.host)
@@ -368,6 +368,7 @@ class QueueReader(MessageQueue,Reader):
 
 	"""
 	def callback(self,channel,method,header,stream):
+                
 		r = []
 		if re.match("^\{|\[",stream) is not None:
 			r = json.loads(stream)
@@ -399,9 +400,12 @@ class QueueReader(MessageQueue,Reader):
 		# We enabled the reader to be able to read from several queues (sequentially for now)
 		# The qid parameter will be an array of queues the reader will be reading from
 		#
+		if isinstance(self.qid,basestring) :
+                    self.qid = [self.qid]
 		for qid in self.qid:
 			self.init(qid)
 			# r[qid] = []
+			
 			if self.info.method.message_count > 0:
 				
 				self.channel.basic_consume(self.callback,queue=qid,no_ack=False);
@@ -420,8 +424,8 @@ class QueueListener(QueueReader):
 		self.channel	= self.connection.channel()
 		self.channel.exchange_declare(exchange=self.uid,type='direct',durable=True )
 
-		self.info = self.channel.queue_declare(exclusive=True,queue=qid)
-		print self.info.method.queue
+		self.info = self.channel.queue_declare(passive=True,exclusive=True,queue=qid)
+		
 		self.channel.queue_bind(exchange=self.uid,queue=self.info.method.queue,routing_key=qid)
 		#self.callback = callback
 	def read(self):
@@ -583,7 +587,7 @@ class CouchdbWriter(Couchdb,Writer):
 		self.dbase.save_doc(document)
 	def flush(self,**params) :
 		
-		size = params['size']
+		size = params['size'] if 'size' in params else 0
 		has_changed = False	
 		document = self.dbase.get(self.uid)
 		for key in document:
@@ -591,7 +595,7 @@ class CouchdbWriter(Couchdb,Writer):
 				content = document[key]
 			else:
 				continue
-			if isinstance(content,list):
+			if isinstance(content,list) and size > 0:
 				index = len(content) - size
 				content = content[index:]
 				document[key] = content
@@ -599,8 +603,8 @@ class CouchdbWriter(Couchdb,Writer):
 			else:
 				document[key] = {}
 				has_changed = True
-		if has_changed:
-			self.dbase.save_doc(document)
+		
+		self.dbase.save_doc(document)
 			
 	def archive(self,params=None):
 		document = self.dbase.get(self.uid)