CO - Update transport, registration with stripe, @TODO: Need to update other modules

couchdb/monitor-config.js

@@ -0,0 +1,12 @@
+{
+	"_id": "_design/config",
+	"language": "javascript",
+	"views": {
+	    "apps": {
+		"map": "function(doc) {\n\tif(doc.config != null){\n\t\tif(doc.config.apps != null){\n\t\t\temit(doc._id,doc.config.apps)\n\t\t}\n\t}else{\n\t\t\temit(doc._id,{})\n\t\t}\n}"
+	    },
+	    "folders": {
+		"map": "function(doc) {\n\tthreshold = null\n\tif(doc.config != null){\n\t\tif(doc.config.folders != null){\n\t\t\tfolders = doc.config.folders ;\n\t\t\tif (folders.threshold != null){ \n\t\t\t\tthreshold = folders.threshold\n\t\t\t}\n\t\t}\n\t}\n\tthreshold = (threshold == null)?\"5 MB\" : threshold ;\n\tr = {threshold:threshold}\n\temit(doc._id,r)\n}"
+	    }
+	}
+     }

couchdb/monitor-stats.js

@@ -0,0 +1,38 @@
+{
+	"_id": "_design/summary",
+	"language": "javascript",
+	"views": {
+	    "app_resources": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar logs = doc[id][index]\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\temit(doc._id,logs[i])\n\t\t\t}\n\t\t\t//emit(doc._id,doc[id][index][0])//doc[id][index])\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar cpu = 0 ;\n\tvar mem_avail = 0 ;\n\tvar mem_used = 0\n\n\tfor(i in values){\n\t\trow = values[i]\n\t\tmem_avail \t= (mem_avail < row.memory_available)?row.memory_available:mem_avail\n\t\tmem_used \t+= (row.memory_available*row.memory_usage/100)\n\t\tcpu\t\t+= row.cpu_usage\n\t}\n\n\tif(mem_avail > 999 && mem_avail < 999999){\n\t\tunits=\"MB\"\n\t}else {\n\t\tunits=\"GB\"\n\t}\n\tmem_avail = parseFloat(mem_avail.toFixed(2))\n\tmem_used = parseFloat(mem_used.toFixed(2))\n\n\treturn {\"app_count\":values.length,\"memory_available\":mem_avail,\"memory_usage\":mem_used,\"cpu_usage\":cpu,\"units\":units}\n}"
+	    },
+	    "app_names": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar size   = doc[id].length\n\t\t\tvar offset = 25\n\t\t\t//\n\t\t\t// We get the last 25 rows ( a days worth of observations\n\t\t\t//\n\t\t\tvar rows = doc[id].slice(size-offset,size)\n\t\t\tfor(var i=0; i < rows.length; i++){\n\t\t\t\tfor(var j=0; j < rows[i].length; j++){\n\t\t\t\t\tvar info = {}\n\t\t\t\t\tinfo[id]=rows[i][j]\n\t\t\t\t\t\n\t\t\t\t\temit(doc._id,info)\t\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar m = {} ;\n\tvar r = {}\n\tfor (var i =0; i < values.length; i++){\n\t\tvar rows = values[i]\n\t\tfor (node in rows){\n\t\t\tvar log = rows[node]\n\t\t\tif(m[node] == null){\n\t\t\t\tm[node]={}\n\t\t\t\tr[node]=[]\n\t\t\t}\n\t\t\tvar app = log.label.toLowerCase()\n\t\t\tif(m[node][app] == null){\n\t\t\t\tm[node][app] = 1\n\t\t\t\tr[node].push(app)\n\t\t\t}\n\t\t}\n\t}\n\treturn r\n\t\n}"
+	    },
+	    "folder_size": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^folder.+/) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar logs = doc[id][index]\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\temit(doc._id,logs[i])\n\t\t\t}\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar size=0\n\tvar units = {\"MB\":1000,\"GB\":1000000,\"TB\":1000000000}\n\tfor(i in values){\n\t\trow = values[i]\n\t\tsvalue = row.size.match(/^[\\d,.]+/i)\n\t\tukey \t= row.size.match(/[a-z]+$/i)\n\t\tsize\t+= parseFloat(svalue*units[ukey])\n\t}\n\tif (size > 999 && size < 999999){\n\t\tunits=\"MB\"\n\t\tsize /=1000\n\t}else if(size > 999999 && 999999999){\n\t\tunits=\"GB\"\n\t\tsize /=1000000\n\t}else{ \n\t\tunits = \"TB\"\n\t\tsize /=1000000000\n\t}\n\tsize = parseFloat(size.toFixed(2))\n\treturn {\"folder_count\":values.length,\"size\":size,\"units\":units} //parseFloat(values[0].size.match(/^[\\d,.]+/))\n}"
+	    },
+	    "app_status": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\n\t\t\tif(doc[id][index].length > 0){\n\t\t\t\temit(doc._id,doc[id][index])\n\t\t\t}\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar status = {\"crash\":0,\"idle\":0,\"running\":0}\n\tfor(i in values){\n\t\tfor(ii in values[i]){\n\t\t\trow = values[i][ii]\n\t\t\tid = row.status\n\t\t\tif(status[id] != null){\n\t\t\t\tstatus[id] += 1\n\t\t\t}\n\n\t\t}\n\t}\n\treturn status\n}"
+	    },
+	    "resource_usage_trend": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i)){\n\t\t\tend = doc[id].length -1\n\t\t\tbeg = (doc[id].length > 25 )?doc[id].length - 25:0\n\t\t\tfor(var i = end; i > beg ; --i){\n\t\t\t\temit(doc._id,doc[id][i])//doc[id][index])\n\t\t\t}\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar cpu = [] ;\n\tvar mem_avail = [] ;\n\tvar mem_used = []\n\tvar size = 20\n\tvar beg = null\n\tvar end = null\n\tapps = {}\n\tapp_count = 0\n\tfor(i in values){\n\t\t//end = values[i].length -1\n\t\t//beg = end - size\n\t\tfor(ii in values[i]){\n\t\t\trow = values[i][ii]\n\t\t\tapps[row.label] = 1\n\t\t\tif(beg == null){\n\t\t\t\tbeg = ([row.year,row.month,row.day]).join('/')+' ' +([row.hour,row.minute]).join(':')\n\t\t\t}\n\t\t\tmem_avail.push( parseFloat(row.memory_available.toFixed(2)))\n\t\t\tmem_used.push(row.memory_available*row.memory_usage/100)\n\t\t\tcpu.push(row.cpu_usage)\n\t\t\t\n\t\t}\n\t\tend = ([row.year,row.month,row.day]).join('/')+' ' +([row.hour,row.minute]).join(':')\n\t\n\t\t\n\t}\n\t//\n\t// counting applications found in the logs\n\t// There is no requirement on consistency of the logs\n\t//\n\tfor(i in apps){\n\t\tapp_count += 1\n\t}\n\t\n\t//return row\t//return values[0].slice(beg,end)\n\treturn {\"app_count\":app_count,\"beg\":beg,\"end\":end,\"memory_available\":mem_avail,\"memory_usage\":mem_used,\"cpu_usage\":cpu}\n}"
+	    },
+	    "app_status_details": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i)){\n\t\t\tindex = doc[id].length -1 \n\t\t\tbeg = (doc[id].length > 0)?doc[id].length -13 : 0\n\t\t\tvar logs = doc[id].slice(beg,index);//[index]\n\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\tvar rec = logs[i]\n\t\t\t\tfor (ii in rec){\n\t\t\t\t\tr = {}\n\t\t\t\t\tr[id] = rec[ii]\n\t\t\t\t\temit(doc._id,r)\n\t\t\t\t}\n\t\t\t}\n\t\t\t//emit(doc._id,doc[id][index][0])//doc[id][index])\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar cpu = 0 ;\n\tvar mem_avail = 0 ;\n\tvar mem_used = 0\n\tr = {}\n\t//\n\t// retrieving the memory available\n\n\tfor(i in values){\n\t\trow = values[i]\n\t\tfor(id in row){\n\t\t\tmem_avail = (mem_avail < row[id].memory_available)?row[id].memory_available:mem_avail \n\t\t}\n\t}\n\t//\n\t// Determining appropriate units of measurement\n\t//\n\t\t\n\tif(mem_avail > 999 && mem_avail < 999999){\n\t\tunits=\"MB\"\n\t}else {\n\t\tunits=\"GB\"\n\t}\n\n\n\tfor(i in values){\n\n\t\trec = values[i]\n\t\tfor(key in rec){\n\t\t\trow = rec[key]\n\t\t\tvar id = row.label\n\t\t\tif(r[key] == null){ r[key] = {} }\n\t\t\tif (r[key][id] == null){\n\t\t\t\tr[key][id] = {\"beg\":null,\"end\":null,\"crash\":0,\"idle\":0,\"running\":0}\n\t\t\t}\n\t\t\tr[key][id][row.status] += 1\n\t\t\tif(r[key][id].beg == null){\n\t\t\t\tr[key][id].beg = ([row.year,row.month,row.day]).join('/')+' '+([row.hour,row.minute]).join(':')\n\t\t\t}\n\t\t\tr[key][id].end = ([row.year,row.month,row.day]).join('/')+' '+([row.hour,row.minute]).join(':')\n\t\t}\n\t}\n\n\n\t//mem_avail = parseFloat(mem_avail.toFixed(2))\n\t//mem_used = parseFloat(mem_used.toFixed(2))\n\n\treturn r //{\"app_count\":values.length,\"memory_available\":mem_avail,\"memory_usage\":mem_used,\"cpu_usage\":cpu,\"units\":units}\n}"
+	    },
+	    "app_resource_usage_details": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i)){\n\t\t\tindex = doc[id].length -1 \n\t\t\tbeg = (doc[id].length > 25)?doc[id].length -25 :0\n\t\t\tvar logs = doc[id].slice(beg,index);//[index]\n\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\tvar rec = logs[i]\n\t\t\t\tfor (ii in rec){\n\t\t\t\t\tvar info = {}\n\t\t\t\t\tinfo[id] = rec[ii]\n\t\t\t\t\temit(doc._id,info)//rec[ii])\n\t\t\t\t}\n\t\t\t}\n\t\t\t//emit(doc._id,doc[id][index][0])//doc[id][index])\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar cpu = 0 ;\n\tvar mem_avail = 0 ;\n\tvar mem_used = 0\n\tr = {}\n\t//\n\t// retrieving the memory available\n\tfor(i in values){\n\t\tvar rvalues = values[i]\n\t\tfor(id in rvalues){\n\t\t\trow = rvalues[id]\n\t\t\tmem_avail = parseFloat(row.memory_available) ;//mem_avail < row.memory_available)?row.memory_available:mem_avail \n\t\t\tapp_id = row.label.toLowerCase()\n\t\t\tif(r[id] == null){\n\t\t\t\t\n\t\t\t\tr[id] = {}\n\t\t\t}\n\t\t\tif(r[id][app_id] == null){\n\t\t\t\tr[id][app_id] = {memory_available:0,cpu:[],memory_used:[],dates:[]}\n\t\t\t}\n\n\t\t\tr[id][app_id].memory_available += mem_avail\n\t\t\t//app_id = row.label.toLowerCase()\n\n\t\t\tif(mem_avail > 999 && mem_avail < 999999){\n\t\t\t\tunits=\"MB\"\n\t\t\t\t\n\t\t\t}else {\n\t\t\t\tunits=\"GB\"\n\t\t\t}\t\n\t\t\tr[id][app_id].units = units\n\t\t}\n\t\tfor(var id in rvalues){\n\t\t\trow = rvalues[id]\n\t\t\tapp_id = row.label.toLowerCase()\n\t\t\tif(r[id][app_id].units == 'MB'){\t\t\t\t\n\t\t\t\tr[id][app_id].memory_available = parseFloat((r[id][app_id].memory_available/ 1000).toFixed(2))\n\t\t\t}else{\n\t\t\t\t//\n\t\t\t\t// assuming we have GB\n\t\t\t\tr[id][app_id].memory_available = parseFloat((r[id][app_id].memory_available/ 1000000).toFixed(2))\n\t\t\t}\n\t\t\tdate = new Date(row.year,row.month,row.day,row.hour,row.minute)\n\t\t\tdate = ([row.hour,row.minute]).join('H:')//{month:row.month,year:row.year,day:row.day,hour:row.hour,minute:row.minute}\n\t\t\tr[id][app_id].dates.push(date)\t\t\t\t\t\n\t\t\t//r[id][app_id].end = \t([row.year,row.month,row.day]).join('/')+' '+([row.hour,row.minute]).join(':')\n\t\t\tmem_avail = r[id][app_id].memory_available == 0?1:r[id][app_id].memory_available\n\t\t\t//r[id][app_id].memory_used.push(parseFloat((mem_avail*row.memory_usage).toFixed(2)))\n\t\t\tr[id][app_id].memory_used.push(parseFloat(row.memory_usage.toFixed(2)))\n\t\t\tr[id][app_id].cpu.push(row.cpu_usage)\n\t\t\t\n\t\t\t\n\t\t}\n\t\t\n\t}\n\treturn r\n}"
+	    },
+	    "nodes": {
+		"map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar size   = doc[id].length\n\t\t\tvar offset = 25\n\t\t\t//\n\t\t\t// We get the last 25 rows ( a days worth of observations\n\t\t\t//\n\t\t\tvar rows = doc[id].slice(size-offset,size)\n\t\t\tfor(var i=0; i < rows.length; i++){\n\t\t\t\tfor(var j=0; j < rows[i].length; j++){\n\t\t\t\t\tvar info = {}\n\t\t\t\t\tinfo[id]=rows[i][j]\n\t\t\t\t\t\n\t\t\t\t\temit(doc._id,id)\t\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t}\n\t}\n}",
+		"reduce": "function(keys,values){\n\tvar m = {}\n\tfor(var i in values){\n\t\tvar id = values[i]\n\t\tif(m[id] == null){\n\t\t\tm[id] = 0;\n\t\t}\n\t\tm[id] += 1\n\t}\n\treturn Object.keys(m)\n}"
+	    }
+	}
+     }

couchdb/monitor.js

@@ -0,0 +1,10 @@
+{
+   "_id": "_design/federation",
+   "_rev": "1-e3073f79279708acb28ee569d7ff208e",
+   "language": "javascript",
+   "views": {
+       "uid_map": {
+           "map": "function(doc) {\n\tfor (var i in doc.emails){\n\t\temit(doc.emails[i],doc._id)\n\t}\n}"
+       }
+   }
+}

init.sh

@@ -30,7 +30,7 @@ start(){
 		sandbox/bin/python src/utils/agents/data-collector.py --path $PWD/config.json
 	else
 
-		sandbox/bin/python src/api/index.py --path $PWD/config.json --context monitor &
+		sandbox/bin/python src/api/index.py --path $PWD/config.json &
 	fi
 
 }

src/api/index.py

@@ -15,7 +15,7 @@
 
 from flask import Flask, session, request, redirect, Response
 from flask.templating import render_template
-
+import requests
 from flask_session import Session
 import time
 import sys
@@ -23,6 +23,7 @@ import os
 import json
 import re
 import monitor
+import uuid
 
 import Queue
 from utils.transport import *
@@ -55,6 +56,10 @@ factory = DataSourceFactory()
 
 @app.route('/')
 def home():
+	"""
+		This is the initial function (entry point), This function will load a dashboard
+		@param key	customer's key
+	"""
 	context = PARAMS['context']
 	if 'title' in PARAMS :
 		title = PARAMS['title']
@@ -64,6 +69,10 @@ def home():
 	try:
 		#
 		# The user must pass a key in heref
+		key = request.header['key']
+		args= dict(p)
+		args['uid'] = key
+		
 		gReader = factory.instance(type=class_read,args=p)
 		apps = gReader.view('summary/nodes',key=p['uid'])
 
@@ -71,6 +80,77 @@ def home():
 		print (e)
 	return render_template('dashboard.html',context=context,title=title,app_names=apps)
 
+@app.route('/register',methods=['POST'])
+def register():
+	"""
+		This function is intended to manage users i.e : register a user given an email
+		Once the email is provided a key will be generated for the user
+		@param uid	user's email address
+		@param plan	plan identifier (monitor-{light,basic,xxx}
+
+	"""
+	global p
+	
+	body = request.get_json(silent=True)
+	
+	auid = str(uuid.uuid4())
+	headers = {"uid":body['uid'],"pid":body['plan'],"auid":json.dumps([auid])}
+	couchdb = factory.instance(type=class_read,args=p)
+	r = couchdb.view('federation/uid_map',key=body['uid'])
+	q = couchdb.view('federation/uid_map',key=auid)
+	if not r and not q:
+		
+		# The uid has not yet been assigned
+		url="https://the-phi.com/store/init/monitor"
+		r = requests.post(url,headers=headers)	
+		
+		#
+		# Now we should create a document with the generated key
+		# @TODO: Have an endpoint map uid/key
+		args = dict(p)
+		
+		args['dbname']	= body['plan']
+		args['uid']	= auid
+		
+		couchdb = factory.instance(type=class_write,args=args)
+		couchdb.write(label='emails',row=[auid,body['uid']])
+		plan = {body['plan']:json.loads(r.text)}
+		session['plan'] = [plan]
+		couchdb.write(label='plan',row=plan)
+		r = auid 
+		
+		pass
+
+	else:
+		#
+		# at this point we have the alternate identifier
+		#@TODO : Initiate a retry
+		#
+		
+		args = dict(p)
+		args['uid'] = r		#-- returned uid
+		couchdb = factory.instance(type=class_read,args=args)
+		document = couchdb.read()
+		r = list(set(document['emails']) - set(body['uid']))
+		r  = [item for item in r if len(item) == 36]
+		if len(r) > 0 :
+			r = r[len(r)-1]
+			session['plan'] = document['plan']
+		else:
+			r = ""
+		pass
+	#
+	# make sure the plan is a valid one i.e monitor-*-*
+	# This should be stored in the session so we can proceed going forward
+	if r != "":
+		session['key'] = r
+	return r
+def init():
+	"""
+		This function is designed to initialize with fundamental data i.e :
+		Which plan is valid, and currently on going.
+	"""
+	pass
 @app.route('/1/get/nodes')
 def get_nodes():
 	"""

src/utils/transport.py

@@ -586,10 +586,14 @@ class CouchdbWriter(Couchdb,Writer):
 		
 		document = self.dbase.get(self.uid)
 		label = params['label']
+
 		row	= params['row']
 		if label not in document :
-			document[label] = []
+			document[label] = [] if isinstance(row,list) else {}
-		document[label].append(row)
+		if isinstance(document[label],list) :
+			document[label].append(row)
+		else :
+			document[label] = dict(document[label],**row)
 		self.dbase.save_doc(document)
 	def flush(self,**params) :