diff --git a/couchdb/monitor-config.js b/couchdb/monitor-config.js new file mode 100644 index 0000000..c38e34b --- /dev/null +++ b/couchdb/monitor-config.js @@ -0,0 +1,12 @@ +{ + "_id": "_design/config", + "language": "javascript", + "views": { + "apps": { + "map": "function(doc) {\n\tif(doc.config != null){\n\t\tif(doc.config.apps != null){\n\t\t\temit(doc._id,doc.config.apps)\n\t\t}\n\t}else{\n\t\t\temit(doc._id,{})\n\t\t}\n}" + }, + "folders": { + "map": "function(doc) {\n\tthreshold = null\n\tif(doc.config != null){\n\t\tif(doc.config.folders != null){\n\t\t\tfolders = doc.config.folders ;\n\t\t\tif (folders.threshold != null){ \n\t\t\t\tthreshold = folders.threshold\n\t\t\t}\n\t\t}\n\t}\n\tthreshold = (threshold == null)?\"5 MB\" : threshold ;\n\tr = {threshold:threshold}\n\temit(doc._id,r)\n}" + } + } + } \ No newline at end of file diff --git a/couchdb/monitor-stats.js b/couchdb/monitor-stats.js new file mode 100644 index 0000000..b03c8de --- /dev/null +++ b/couchdb/monitor-stats.js @@ -0,0 +1,38 @@ +{ + "_id": "_design/summary", + "language": "javascript", + "views": { + "app_resources": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar logs = doc[id][index]\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\temit(doc._id,logs[i])\n\t\t\t}\n\t\t\t//emit(doc._id,doc[id][index][0])//doc[id][index])\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar cpu = 0 ;\n\tvar mem_avail = 0 ;\n\tvar mem_used = 0\n\n\tfor(i in values){\n\t\trow = values[i]\n\t\tmem_avail \t= (mem_avail < row.memory_available)?row.memory_available:mem_avail\n\t\tmem_used \t+= (row.memory_available*row.memory_usage/100)\n\t\tcpu\t\t+= row.cpu_usage\n\t}\n\n\tif(mem_avail > 999 && mem_avail < 999999){\n\t\tunits=\"MB\"\n\t}else {\n\t\tunits=\"GB\"\n\t}\n\tmem_avail = parseFloat(mem_avail.toFixed(2))\n\tmem_used = parseFloat(mem_used.toFixed(2))\n\n\treturn {\"app_count\":values.length,\"memory_available\":mem_avail,\"memory_usage\":mem_used,\"cpu_usage\":cpu,\"units\":units}\n}" + }, + "app_names": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar size = doc[id].length\n\t\t\tvar offset = 25\n\t\t\t//\n\t\t\t// We get the last 25 rows ( a days worth of observations\n\t\t\t//\n\t\t\tvar rows = doc[id].slice(size-offset,size)\n\t\t\tfor(var i=0; i < rows.length; i++){\n\t\t\t\tfor(var j=0; j < rows[i].length; j++){\n\t\t\t\t\tvar info = {}\n\t\t\t\t\tinfo[id]=rows[i][j]\n\t\t\t\t\t\n\t\t\t\t\temit(doc._id,info)\t\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar m = {} ;\n\tvar r = {}\n\tfor (var i =0; i < values.length; i++){\n\t\tvar rows = values[i]\n\t\tfor (node in rows){\n\t\t\tvar log = rows[node]\n\t\t\tif(m[node] == null){\n\t\t\t\tm[node]={}\n\t\t\t\tr[node]=[]\n\t\t\t}\n\t\t\tvar app = log.label.toLowerCase()\n\t\t\tif(m[node][app] == null){\n\t\t\t\tm[node][app] = 1\n\t\t\t\tr[node].push(app)\n\t\t\t}\n\t\t}\n\t}\n\treturn r\n\t\n}" + }, + "folder_size": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^folder.+/) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar logs = doc[id][index]\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\temit(doc._id,logs[i])\n\t\t\t}\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar size=0\n\tvar units = {\"MB\":1000,\"GB\":1000000,\"TB\":1000000000}\n\tfor(i in values){\n\t\trow = values[i]\n\t\tsvalue = row.size.match(/^[\\d,.]+/i)\n\t\tukey \t= row.size.match(/[a-z]+$/i)\n\t\tsize\t+= parseFloat(svalue*units[ukey])\n\t}\n\tif (size > 999 && size < 999999){\n\t\tunits=\"MB\"\n\t\tsize /=1000\n\t}else if(size > 999999 && 999999999){\n\t\tunits=\"GB\"\n\t\tsize /=1000000\n\t}else{ \n\t\tunits = \"TB\"\n\t\tsize /=1000000000\n\t}\n\tsize = parseFloat(size.toFixed(2))\n\treturn {\"folder_count\":values.length,\"size\":size,\"units\":units} //parseFloat(values[0].size.match(/^[\\d,.]+/))\n}" + }, + "app_status": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\n\t\t\tif(doc[id][index].length > 0){\n\t\t\t\temit(doc._id,doc[id][index])\n\t\t\t}\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar status = {\"crash\":0,\"idle\":0,\"running\":0}\n\tfor(i in values){\n\t\tfor(ii in values[i]){\n\t\t\trow = values[i][ii]\n\t\t\tid = row.status\n\t\t\tif(status[id] != null){\n\t\t\t\tstatus[id] += 1\n\t\t\t}\n\n\t\t}\n\t}\n\treturn status\n}" + }, + "resource_usage_trend": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i)){\n\t\t\tend = doc[id].length -1\n\t\t\tbeg = (doc[id].length > 25 )?doc[id].length - 25:0\n\t\t\tfor(var i = end; i > beg ; --i){\n\t\t\t\temit(doc._id,doc[id][i])//doc[id][index])\n\t\t\t}\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar cpu = [] ;\n\tvar mem_avail = [] ;\n\tvar mem_used = []\n\tvar size = 20\n\tvar beg = null\n\tvar end = null\n\tapps = {}\n\tapp_count = 0\n\tfor(i in values){\n\t\t//end = values[i].length -1\n\t\t//beg = end - size\n\t\tfor(ii in values[i]){\n\t\t\trow = values[i][ii]\n\t\t\tapps[row.label] = 1\n\t\t\tif(beg == null){\n\t\t\t\tbeg = ([row.year,row.month,row.day]).join('/')+' ' +([row.hour,row.minute]).join(':')\n\t\t\t}\n\t\t\tmem_avail.push( parseFloat(row.memory_available.toFixed(2)))\n\t\t\tmem_used.push(row.memory_available*row.memory_usage/100)\n\t\t\tcpu.push(row.cpu_usage)\n\t\t\t\n\t\t}\n\t\tend = ([row.year,row.month,row.day]).join('/')+' ' +([row.hour,row.minute]).join(':')\n\t\n\t\t\n\t}\n\t//\n\t// counting applications found in the logs\n\t// There is no requirement on consistency of the logs\n\t//\n\tfor(i in apps){\n\t\tapp_count += 1\n\t}\n\t\n\t//return row\t//return values[0].slice(beg,end)\n\treturn {\"app_count\":app_count,\"beg\":beg,\"end\":end,\"memory_available\":mem_avail,\"memory_usage\":mem_used,\"cpu_usage\":cpu}\n}" + }, + "app_status_details": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i)){\n\t\t\tindex = doc[id].length -1 \n\t\t\tbeg = (doc[id].length > 0)?doc[id].length -13 : 0\n\t\t\tvar logs = doc[id].slice(beg,index);//[index]\n\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\tvar rec = logs[i]\n\t\t\t\tfor (ii in rec){\n\t\t\t\t\tr = {}\n\t\t\t\t\tr[id] = rec[ii]\n\t\t\t\t\temit(doc._id,r)\n\t\t\t\t}\n\t\t\t}\n\t\t\t//emit(doc._id,doc[id][index][0])//doc[id][index])\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar cpu = 0 ;\n\tvar mem_avail = 0 ;\n\tvar mem_used = 0\n\tr = {}\n\t//\n\t// retrieving the memory available\n\n\tfor(i in values){\n\t\trow = values[i]\n\t\tfor(id in row){\n\t\t\tmem_avail = (mem_avail < row[id].memory_available)?row[id].memory_available:mem_avail \n\t\t}\n\t}\n\t//\n\t// Determining appropriate units of measurement\n\t//\n\t\t\n\tif(mem_avail > 999 && mem_avail < 999999){\n\t\tunits=\"MB\"\n\t}else {\n\t\tunits=\"GB\"\n\t}\n\n\n\tfor(i in values){\n\n\t\trec = values[i]\n\t\tfor(key in rec){\n\t\t\trow = rec[key]\n\t\t\tvar id = row.label\n\t\t\tif(r[key] == null){ r[key] = {} }\n\t\t\tif (r[key][id] == null){\n\t\t\t\tr[key][id] = {\"beg\":null,\"end\":null,\"crash\":0,\"idle\":0,\"running\":0}\n\t\t\t}\n\t\t\tr[key][id][row.status] += 1\n\t\t\tif(r[key][id].beg == null){\n\t\t\t\tr[key][id].beg = ([row.year,row.month,row.day]).join('/')+' '+([row.hour,row.minute]).join(':')\n\t\t\t}\n\t\t\tr[key][id].end = ([row.year,row.month,row.day]).join('/')+' '+([row.hour,row.minute]).join(':')\n\t\t}\n\t}\n\n\n\t//mem_avail = parseFloat(mem_avail.toFixed(2))\n\t//mem_used = parseFloat(mem_used.toFixed(2))\n\n\treturn r //{\"app_count\":values.length,\"memory_available\":mem_avail,\"memory_usage\":mem_used,\"cpu_usage\":cpu,\"units\":units}\n}" + }, + "app_resource_usage_details": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i)){\n\t\t\tindex = doc[id].length -1 \n\t\t\tbeg = (doc[id].length > 25)?doc[id].length -25 :0\n\t\t\tvar logs = doc[id].slice(beg,index);//[index]\n\n\t\t\tfor(var i=0; i < logs.length; i++){\n\t\t\t\tvar rec = logs[i]\n\t\t\t\tfor (ii in rec){\n\t\t\t\t\tvar info = {}\n\t\t\t\t\tinfo[id] = rec[ii]\n\t\t\t\t\temit(doc._id,info)//rec[ii])\n\t\t\t\t}\n\t\t\t}\n\t\t\t//emit(doc._id,doc[id][index][0])//doc[id][index])\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar cpu = 0 ;\n\tvar mem_avail = 0 ;\n\tvar mem_used = 0\n\tr = {}\n\t//\n\t// retrieving the memory available\n\tfor(i in values){\n\t\tvar rvalues = values[i]\n\t\tfor(id in rvalues){\n\t\t\trow = rvalues[id]\n\t\t\tmem_avail = parseFloat(row.memory_available) ;//mem_avail < row.memory_available)?row.memory_available:mem_avail \n\t\t\tapp_id = row.label.toLowerCase()\n\t\t\tif(r[id] == null){\n\t\t\t\t\n\t\t\t\tr[id] = {}\n\t\t\t}\n\t\t\tif(r[id][app_id] == null){\n\t\t\t\tr[id][app_id] = {memory_available:0,cpu:[],memory_used:[],dates:[]}\n\t\t\t}\n\n\t\t\tr[id][app_id].memory_available += mem_avail\n\t\t\t//app_id = row.label.toLowerCase()\n\n\t\t\tif(mem_avail > 999 && mem_avail < 999999){\n\t\t\t\tunits=\"MB\"\n\t\t\t\t\n\t\t\t}else {\n\t\t\t\tunits=\"GB\"\n\t\t\t}\t\n\t\t\tr[id][app_id].units = units\n\t\t}\n\t\tfor(var id in rvalues){\n\t\t\trow = rvalues[id]\n\t\t\tapp_id = row.label.toLowerCase()\n\t\t\tif(r[id][app_id].units == 'MB'){\t\t\t\t\n\t\t\t\tr[id][app_id].memory_available = parseFloat((r[id][app_id].memory_available/ 1000).toFixed(2))\n\t\t\t}else{\n\t\t\t\t//\n\t\t\t\t// assuming we have GB\n\t\t\t\tr[id][app_id].memory_available = parseFloat((r[id][app_id].memory_available/ 1000000).toFixed(2))\n\t\t\t}\n\t\t\tdate = new Date(row.year,row.month,row.day,row.hour,row.minute)\n\t\t\tdate = ([row.hour,row.minute]).join('H:')//{month:row.month,year:row.year,day:row.day,hour:row.hour,minute:row.minute}\n\t\t\tr[id][app_id].dates.push(date)\t\t\t\t\t\n\t\t\t//r[id][app_id].end = \t([row.year,row.month,row.day]).join('/')+' '+([row.hour,row.minute]).join(':')\n\t\t\tmem_avail = r[id][app_id].memory_available == 0?1:r[id][app_id].memory_available\n\t\t\t//r[id][app_id].memory_used.push(parseFloat((mem_avail*row.memory_usage).toFixed(2)))\n\t\t\tr[id][app_id].memory_used.push(parseFloat(row.memory_usage.toFixed(2)))\n\t\t\tr[id][app_id].cpu.push(row.cpu_usage)\n\t\t\t\n\t\t\t\n\t\t}\n\t\t\n\t}\n\treturn r\n}" + }, + "nodes": { + "map": "function(doc) {\n\tfor(id in doc){\n\t\tif(id.match(/^apps.+$/i) && doc[id].length > 0){\n\t\t\tindex = doc[id].length -1 \n\t\t\tvar size = doc[id].length\n\t\t\tvar offset = 25\n\t\t\t//\n\t\t\t// We get the last 25 rows ( a days worth of observations\n\t\t\t//\n\t\t\tvar rows = doc[id].slice(size-offset,size)\n\t\t\tfor(var i=0; i < rows.length; i++){\n\t\t\t\tfor(var j=0; j < rows[i].length; j++){\n\t\t\t\t\tvar info = {}\n\t\t\t\t\tinfo[id]=rows[i][j]\n\t\t\t\t\t\n\t\t\t\t\temit(doc._id,id)\t\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t}\n\t}\n}", + "reduce": "function(keys,values){\n\tvar m = {}\n\tfor(var i in values){\n\t\tvar id = values[i]\n\t\tif(m[id] == null){\n\t\t\tm[id] = 0;\n\t\t}\n\t\tm[id] += 1\n\t}\n\treturn Object.keys(m)\n}" + } + } + } \ No newline at end of file diff --git a/couchdb/monitor.js b/couchdb/monitor.js new file mode 100644 index 0000000..aec9e55 --- /dev/null +++ b/couchdb/monitor.js @@ -0,0 +1,10 @@ +{ + "_id": "_design/federation", + "_rev": "1-e3073f79279708acb28ee569d7ff208e", + "language": "javascript", + "views": { + "uid_map": { + "map": "function(doc) {\n\tfor (var i in doc.emails){\n\t\temit(doc.emails[i],doc._id)\n\t}\n}" + } + } +} \ No newline at end of file diff --git a/init.sh b/init.sh index 8f9e06f..4c1df69 100644 --- a/init.sh +++ b/init.sh @@ -30,7 +30,7 @@ start(){ sandbox/bin/python src/utils/agents/data-collector.py --path $PWD/config.json else - sandbox/bin/python src/api/index.py --path $PWD/config.json --context monitor & + sandbox/bin/python src/api/index.py --path $PWD/config.json & fi } diff --git a/src/api/index.py b/src/api/index.py index a1d456e..5999009 100644 --- a/src/api/index.py +++ b/src/api/index.py @@ -15,7 +15,7 @@ from flask import Flask, session, request, redirect, Response from flask.templating import render_template - +import requests from flask_session import Session import time import sys @@ -23,6 +23,7 @@ import os import json import re import monitor +import uuid import Queue from utils.transport import * @@ -55,6 +56,10 @@ factory = DataSourceFactory() @app.route('/') def home(): + """ + This is the initial function (entry point), This function will load a dashboard + @param key customer's key + """ context = PARAMS['context'] if 'title' in PARAMS : title = PARAMS['title'] @@ -64,6 +69,10 @@ def home(): try: # # The user must pass a key in heref + key = request.header['key'] + args= dict(p) + args['uid'] = key + gReader = factory.instance(type=class_read,args=p) apps = gReader.view('summary/nodes',key=p['uid']) @@ -71,6 +80,77 @@ def home(): print (e) return render_template('dashboard.html',context=context,title=title,app_names=apps) +@app.route('/register',methods=['POST']) +def register(): + """ + This function is intended to manage users i.e : register a user given an email + Once the email is provided a key will be generated for the user + @param uid user's email address + @param plan plan identifier (monitor-{light,basic,xxx} + + """ + global p + + body = request.get_json(silent=True) + + auid = str(uuid.uuid4()) + headers = {"uid":body['uid'],"pid":body['plan'],"auid":json.dumps([auid])} + couchdb = factory.instance(type=class_read,args=p) + r = couchdb.view('federation/uid_map',key=body['uid']) + q = couchdb.view('federation/uid_map',key=auid) + if not r and not q: + + # The uid has not yet been assigned + url="https://the-phi.com/store/init/monitor" + r = requests.post(url,headers=headers) + + # + # Now we should create a document with the generated key + # @TODO: Have an endpoint map uid/key + args = dict(p) + + args['dbname'] = body['plan'] + args['uid'] = auid + + couchdb = factory.instance(type=class_write,args=args) + couchdb.write(label='emails',row=[auid,body['uid']]) + plan = {body['plan']:json.loads(r.text)} + session['plan'] = [plan] + couchdb.write(label='plan',row=plan) + r = auid + + pass + + else: + # + # at this point we have the alternate identifier + #@TODO : Initiate a retry + # + + args = dict(p) + args['uid'] = r #-- returned uid + couchdb = factory.instance(type=class_read,args=args) + document = couchdb.read() + r = list(set(document['emails']) - set(body['uid'])) + r = [item for item in r if len(item) == 36] + if len(r) > 0 : + r = r[len(r)-1] + session['plan'] = document['plan'] + else: + r = "" + pass + # + # make sure the plan is a valid one i.e monitor-*-* + # This should be stored in the session so we can proceed going forward + if r != "": + session['key'] = r + return r +def init(): + """ + This function is designed to initialize with fundamental data i.e : + Which plan is valid, and currently on going. + """ + pass @app.route('/1/get/nodes') def get_nodes(): """ diff --git a/src/utils/transport.py b/src/utils/transport.py index 8ed1370..bace7b3 100644 --- a/src/utils/transport.py +++ b/src/utils/transport.py @@ -586,10 +586,14 @@ class CouchdbWriter(Couchdb,Writer): document = self.dbase.get(self.uid) label = params['label'] + row = params['row'] if label not in document : - document[label] = [] - document[label].append(row) + document[label] = [] if isinstance(row,list) else {} + if isinstance(document[label],list) : + document[label].append(row) + else : + document[label] = dict(document[label],**row) self.dbase.save_doc(document) def flush(self,**params) :