bug fix: ...

.gitignore

@@ -1,2 +1,3 @@
 sandbox
 *.pyc
+.DS_Store

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,7 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="useProjectProfile" value="false" />
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.10 virtualenv at ~/Documents/Programming/monitor/sandbox" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/monitor.iml" filepath="$PROJECT_DIR$/.idea/monitor.iml" />
+    </modules>
+  </component>
+</project>

.idea/monitor.iml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TemplatesService">
+    <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
+    <option name="TEMPLATE_FOLDERS">
+      <list>
+        <option value="$MODULE_DIR$/src/api/templates" />
+      </list>
+    </option>
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,583 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="834735bc-31bd-474a-9b31-c9bb58743d7d" name="Default" comment="">
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/config.json" afterPath="$PROJECT_DIR$/config.json" />
+    </list>
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="TRACKING_ENABLED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="CreatePatchCommitExecutor">
+    <option name="PATCH_PATH" value="" />
+  </component>
+  <component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
+  <component name="FileEditorManager">
+    <splitter split-orientation="horizontal" split-proportion="0.54361874">
+      <split-first>
+        <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
+          <file leaf-file-name="dashboard.html" pinned="false" current-in-tab="false">
+            <entry file="file://$PROJECT_DIR$/src/api/templates/dashboard.html">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="213">
+                  <caret line="65" column="5" lean-forward="true" selection-start-line="65" selection-start-column="5" selection-end-line="65" selection-end-column="5" />
+                  <folding>
+                    <element signature="e#2414#2713#0#HTML" expanded="false" />
+                  </folding>
+                </state>
+              </provider>
+            </entry>
+          </file>
+          <file leaf-file-name="dashboard.js" pinned="false" current-in-tab="true">
+            <entry file="file://$PROJECT_DIR$/src/api/static/js/dashboard.js">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="226">
+                  <caret line="26" column="0" lean-forward="true" selection-start-line="26" selection-start-column="0" selection-end-line="26" selection-end-column="0" />
+                  <folding>
+                    <marker date="1499136933000" expanded="true" signature="63:670" ph="{...}" />
+                    <marker date="1499136933000" expanded="true" signature="638:670" ph="{...}" />
+                  </folding>
+                </state>
+              </provider>
+            </entry>
+          </file>
+        </leaf>
+      </split-first>
+      <split-second>
+        <leaf>
+          <file leaf-file-name="index.py" pinned="false" current-in-tab="true">
+            <entry file="file://$PROJECT_DIR$/src/api/index.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="259">
+                  <caret line="81" column="15" lean-forward="false" selection-start-line="81" selection-start-column="14" selection-end-line="81" selection-end-column="15" />
+                  <folding />
+                </state>
+              </provider>
+            </entry>
+          </file>
+          <file leaf-file-name="config.json" pinned="false" current-in-tab="false">
+            <entry file="file://$PROJECT_DIR$/config.json">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="270">
+                  <caret line="18" column="0" lean-forward="true" selection-start-line="18" selection-start-column="0" selection-end-line="18" selection-end-column="0" />
+                  <folding />
+                </state>
+              </provider>
+            </entry>
+          </file>
+        </leaf>
+      </split-second>
+    </splitter>
+  </component>
+  <component name="FindInProjectRecents">
+    <findStrings>
+      <find>nodes</find>
+      <find>console.log</find>
+    </findStrings>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/src/api/index.py" />
+        <option value="$PROJECT_DIR$/config.json" />
+        <option value="$PROJECT_DIR$/src/api/templates/dashboard.html" />
+        <option value="$PROJECT_DIR$/src/api/static/js/dashboard.js" />
+      </list>
+    </option>
+  </component>
+  <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsGulpfileManager">
+    <detection-done>true</detection-done>
+    <sorting>DEFINITION_ORDER</sorting>
+  </component>
+  <component name="ProjectFrameBounds">
+    <option name="y" value="22" />
+    <option name="width" value="1280" />
+    <option name="height" value="774" />
+  </component>
+  <component name="ProjectView">
+    <navigator currentView="ProjectPane" proportions="" version="1">
+      <flattenPackages />
+      <showMembers />
+      <showModules />
+      <showLibraryContents />
+      <hideEmptyPackages />
+      <abbreviatePackageNames />
+      <autoscrollToSource />
+      <autoscrollFromSource />
+      <sortByType />
+      <manualOrder />
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="ProjectPane">
+        <subPane>
+          <PATH>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+          </PATH>
+          <PATH>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="src" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+          </PATH>
+          <PATH>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="src" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="api" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+          </PATH>
+          <PATH>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="src" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="api" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="templates" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+          </PATH>
+          <PATH>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="src" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="api" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="static" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="js" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+          </PATH>
+          <PATH>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="monitor" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="src" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="api" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+            <PATH_ELEMENT>
+              <option name="myItemId" value="static" />
+              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
+            </PATH_ELEMENT>
+          </PATH>
+        </subPane>
+      </pane>
+      <pane id="Scope" />
+      <pane id="Scratches" />
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="settings.editor.selected.configurable" value="vcs.Git" />
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="RunManager">
+    <configuration default="true" type="BashConfigurationType" factoryName="Bash">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="INTERPRETER_PATH" value="/bin/bash" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <option name="SCRIPT_NAME" value="" />
+      <option name="PARAMETERS" value="" />
+      <module name="" />
+      <envs />
+      <method />
+    </configuration>
+    <configuration default="true" type="DjangoTestsConfigurationType" factoryName="Django tests">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="monitor" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="TARGET" value="" />
+      <option name="SETTINGS_FILE" value="" />
+      <option name="CUSTOM_SETTINGS" value="false" />
+      <option name="USE_OPTIONS" value="false" />
+      <option name="OPTIONS" value="" />
+      <method />
+    </configuration>
+    <configuration default="true" type="JavaScriptTestRunnerJest" factoryName="Jest">
+      <node-interpreter value="project" />
+      <working-dir value="" />
+      <envs />
+      <scope-kind value="ALL" />
+      <method />
+    </configuration>
+    <configuration default="true" type="JavaScriptTestRunnerProtractor" factoryName="Protractor">
+      <config-file value="" />
+      <node-interpreter value="project" />
+      <envs />
+      <method />
+    </configuration>
+    <configuration default="true" type="JavascriptDebugType" factoryName="JavaScript Debug">
+      <method />
+    </configuration>
+    <configuration default="true" type="PyBehaveRunConfigurationType" factoryName="Behave">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="monitor" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="ADDITIONAL_ARGS" value="" />
+      <method />
+    </configuration>
+    <configuration default="true" type="PyLettuceRunConfigurationType" factoryName="Lettuce">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="monitor" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="ADDITIONAL_ARGS" value="" />
+      <method />
+    </configuration>
+    <configuration default="true" type="PythonConfigurationType" factoryName="Python">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="monitor" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <method />
+    </configuration>
+    <configuration default="true" type="Tox" factoryName="Tox">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <module name="monitor" />
+      <method />
+    </configuration>
+    <configuration default="true" type="js.build_tools.gulp" factoryName="Gulp.js">
+      <method />
+    </configuration>
+    <configuration default="true" type="js.build_tools.npm" factoryName="npm">
+      <command value="run" />
+      <scripts />
+      <node-interpreter value="project" />
+      <envs />
+      <method />
+    </configuration>
+    <configuration default="true" type="tests" factoryName="Doctests">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="monitor" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="" />
+      <option name="CLASS_NAME" value="" />
+      <option name="METHOD_NAME" value="" />
+      <option name="FOLDER_NAME" value="" />
+      <option name="TEST_TYPE" value="TEST_SCRIPT" />
+      <option name="PATTERN" value="" />
+      <option name="USE_PATTERN" value="false" />
+      <method />
+    </configuration>
+    <configuration default="true" type="tests" factoryName="Unittests">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="monitor" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="_new_additionalArguments" value="&quot;&quot;" />
+      <option name="_new_target" value="&quot;.&quot;" />
+      <option name="_new_targetType" value="&quot;PATH&quot;" />
+      <method />
+    </configuration>
+  </component>
+  <component name="ShelveChangesManager" show_recycled="false">
+    <option name="remove_strategy" value="false" />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="834735bc-31bd-474a-9b31-c9bb58743d7d" name="Default" comment="" />
+      <created>1499042529793</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1499042529793</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="0" y="22" width="1280" height="774" extended-state="6" />
+    <editor active="true" />
+    <layout>
+      <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
+      <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
+      <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32991204" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
+      <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32991204" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.15347335" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
+      <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32991204" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
+      <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
+      <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
+      <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
+      <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
+      <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+    </layout>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="processedProjectFiles" value="true" />
+  </component>
+  <component name="VcsContentAnnotationSettings">
+    <option name="myLimit" value="2678400000" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager />
+    <watches-manager />
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/src/api/index.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/api/templates/dashboard.html">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="855">
+          <caret line="57" column="17" lean-forward="false" selection-start-line="57" selection-start-column="16" selection-end-line="57" selection-end-column="17" />
+          <folding>
+            <element signature="e#2414#2713#0#HTML" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/config.json">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/start.sh">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="120">
+          <caret line="8" column="0" lean-forward="false" selection-start-line="8" selection-start-column="0" selection-end-line="8" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/config.json">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/start.sh">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/config.json">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/start.sh">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/config.json">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/start.sh">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/start.sh">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="120">
+          <caret line="8" column="0" lean-forward="false" selection-start-line="8" selection-start-column="0" selection-end-line="8" selection-end-column="0" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/config.json">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="270">
+          <caret line="18" column="0" lean-forward="true" selection-start-line="18" selection-start-column="0" selection-end-line="18" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/api/index.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="259">
+          <caret line="81" column="15" lean-forward="false" selection-start-line="81" selection-start-column="14" selection-end-line="81" selection-end-column="15" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/api/templates/dashboard.html">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="213">
+          <caret line="65" column="5" lean-forward="true" selection-start-line="65" selection-start-column="5" selection-end-line="65" selection-end-column="5" />
+          <folding>
+            <element signature="e#2414#2713#0#HTML" expanded="false" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/api/static/js/dashboard.js">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="226">
+          <caret line="26" column="0" lean-forward="true" selection-start-line="26" selection-start-column="0" selection-end-line="26" selection-end-column="0" />
+          <folding>
+            <marker date="1499136933000" expanded="true" signature="63:670" ph="{...}" />
+            <marker date="1499136933000" expanded="true" signature="638:670" ph="{...}" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>

README.md

@@ -0,0 +1,43 @@
+# Introduction
+
+This program is designed to monitor processes running on a linux/osx machine. It assumes logging hasn't been properly done.
+The program answers basic questions:
+
+	- Is a given program still running
+	- How much resource (memory/cpu) a program is using up
+	- The number of processes found
+	- Folder monitoring ...
+	
+#Architecture
+
+The architecture of the system is distributed with a central master node,
+    {
+        "id":"",
+        "key":"",
+        "apps":[],
+        "sandbox":[{"path":"","requirements":""}],
+        "folders":["path-1"],
+        "store":{}
+        "actions":{}
+    }
+
+The agent will perform three basic functions :
+
+	- data collection (building up the dataset)
+	- classification of status for a given process:
+
+		x	y	z
+		0	0	1	idle
+		1	1	1	running
+		0	0	0	crash
+	
+	with x : memory used
+	     y : cpu usage
+	     z : number of processes found
+
+	- Prediction of crashes using multivariate anomaly detection
+	This allows the learner to determine if something unusual is happening.
+
+
+
+

bin/smart-logger

@@ -0,0 +1 @@
+../smart/__main__.py

info.py

@@ -0,0 +1 @@
+smart/info.py

init.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-#
-# This script is designed to handle various operations related to setting up, starting, stopping the python application
-# It will assume the requirements file is at the root (not with the source code)
-#
-
-export PYTHONPATH=$PWD/src
-pip_upgrade='sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o'
-install(){
-	
-	virtualenv sandbox
-	sandbox/bin/pip install -r requirements.txt
-	`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade` 
-
-
-}
-upgrade(){
-	git pull
-	count=`$PWD/sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|wc -l`
-	if [ ! "$count" = "0" ]; then
-	 	`$PWD/sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade`
-	 else
-	 	echo "No Upgrade required for sandbox"
-	 fi
-	
-}
-start(){
-	
-		$PWD/sandbox/bin/python src/data-collector.py --path $PWD/config.json
-
-}
-stop(){
-	ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -o |xargs kill -9
-}
-
-status(){
-	pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -m 1 -o`
-	if [ "$pid" = "" ]; then 
-		echo "Data Collector is Offline"
-	else
-		echo "Data Collector is  Online $pid"
-	fi
-
-}
-
-$1
-

readme.md

@@ -6,6 +6,20 @@ The program answers basic questions:
 	- Is a given program still running
 	- How much resource (memory/cpu) a program is using up
 	- The number of processes found
+	- Folder monitoring ...
+	
+#Architecture
+
+The architecture of the system is distributed with a central master node,
+    {
+        "id":"",
+        "key":"",
+        "apps":[],
+        "sandbox":[{"path":"","requirements":""}],
+        "folders":["path-1"],
+        "store":{}
+        "actions":{}
+    }
 
 The agent will perform three basic functions :
 

requirements.txt

@@ -1,29 +0,0 @@
-aniso8601==1.2.0
-certifi==2017.7.27.1
-chardet==3.0.4
-click==6.6
-couchdbkit==0.6.5
-Flask==0.11.1
-Flask-Session==0.3.0
-Flask-SocketIO==2.8.2
-http-parser==0.8.3
-idna==2.5
-itsdangerous==0.24
-Jinja2==2.8
-MarkupSafe==0.23
-ngram==3.3.0
-numpy==1.11.3
-pika==0.10.0
-python-dateutil==2.6.0
-python-engineio==1.1.0
-python-socketio==1.6.2
-pytz==2016.10
-requests==2.18.3
-restkit==4.2.2
-scipy==1.0.0
-six==1.10.0
-socketpool==0.5.3
-urllib3==1.22
-Werkzeug==0.11.11
-xmljson
-xmltodict

setup.py

@@ -3,18 +3,21 @@
 from setuptools import setup, find_packages
 import os
 import sys
-def read(fname):
+import info
-    return open(os.path.join(os.path.dirname(__file__), fname)).read() 
+# def read(fname):
+#     return open(os.path.join(os.path.dirname(__file__), fname)).read() 
 args    = {
-    "name":"smart-top",
+    "name":info.__app_name__,
-    "version":"1.0.6",
+    "version":info.__version__,
-    "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
+    "author":info.__author__,"author_email":"info@the-phi.com",
-    "license":"MIT",
+    "license":info.__license__,
-    "packages":["smart","smart.top","smart.folder","smart.logger"]}
+    "packages":["smart","smart.top","smart.folder","smart.logger","smart.files"]
+    }
 args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite']
-args["install_requires"] = ['pandas','numpy','requests','data-transport@git+https://healthcareio.the-phi.com/git/code/transport.git']
+args["install_requires"] = ['typer','pandas','numpy','data-transport@git+https://github.com/lnyemba/data-transport.git']
+
 args["url"] =   "https://dev.the-phi.com/git/steve/smart-top.git"
-args['scripts'] = ['bin/smart-top']
+args['scripts'] = ['bin/smart-logger']
 # args['entry_point'] = {'console-scripts':['smart-top=smart-top:main']}
 #
 #@TODO:

smart/__init__.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 """
 This framework allows data to be logged to a given data store i.e :
     - disk, cloud (google, dropbox, box, sugarsync or s3) or a queue server
@@ -7,9 +8,146 @@ usage:
 dependencies :
     data-transport  pip install git+https://dev.the-phi.com/git/steve/data-transport.git
 """
+import smart
 import smart.top
 import smart.folder
+import smart.top
 import smart.logger
+import smart.files
+import uuid
+import typer
+import smart.info
+import json
+import os
+import transport
+import shutil
+from datetime import datetime
+_cli = typer.Typer()
+
+
+@_cli.command(name='log-intruder')
+def intrusion(path:str='/var/log/auth.log', year:int=datetime.now().year):
+    """
+    This function 
+    """
+    _r = smart.logger.read(path=path,year=year)
+    if _r :
+        for _id in _r :
+            if hasattr(smart.logger,_id):
+                try:
+                    _pointer  = getattr(smart.logger,_id)
+                    _df = _pointer(_r[_id])
+                    
+                    post(_df,_id)
+                except Exception as e:
+                    print (e)
+                    pass
+    else:
+        print ()
+        print ("Nothing out of the ordinary was found in")
+        print (f"{path}")
+@_cli.command(name='top')
+def apply_apps (app:str=None,user:str=None):
+    """
+    This function looks at applications/commands running on the system
+    """
+    
+    _df = smart.top.read()
+    _id = 'apps' if not app else app
+    if app  :
+        _index = _df.name == app
+        if _index.sum() :
+            _df = _df[_index]
+    post(_df,_id)
+    
+@_cli.command(name='archive')
+def _archive():
+    """
+    This function will archive the database, by renaming it into
+    """
+    
+    _suffix = datetime.now()
+    _suffix = "-".join([str(_value) for _value in [_suffix.year,_suffix.month,_suffix.day,_suffix.hour,_suffix.minute]])
+    _path = os.sep.join([smart.info.__home__,smart.info.__database__]) 
+    _src = _path + '.db3'
+    if os.path.exists(_src):
+        _target = _path +'-archived-on-'+ _suffix+'.db3'
+        shutil.move(_src,_target)
+        _msg = f"""Archive created successfully at:
+        {_target}"""
+    else:
+        _msg = """
+        Archive function is not available at this time, please try after logs have been stored
+        """
+    print(_msg)
+@_cli.command(name='folder')
+def apply_folder(path:str):
+    """
+    This function will read the content of a folder and generate a 
+    """
+    _df = smart.folder.read(path=path)
+    # print (_df)
+    post(_df,'folders')
+    pass
+@_cli.command (name='files')
+def apply_files(folder:str) :
+    _df = smart.files.read(folder)
+    post(_df,'files')
+@_cli.command(name='register')
+def apply_signup (email:str,key:str=None,provider:str='sqlite') :
+    _config = {"system":{"email":email,"uid":str(uuid.uuid4()),"version":smart.info.__version__},"store":{"provider":provider,"context":"write"}}
+    _db = smart.info.__database__
+    if provider in ['sqlite','sqlite3'] :
+        _db = os.sep.join([smart.info.__home__,_db+'.db3'])
+        _config['store']['database'] = _db
+    else:
+        _config['store']['database'] = _db
+    #
+    # Let us store this in a folder
+    _PATH = smart.info.__home__
+    _verb = "written"
+    if not os.path.exists(_PATH) :
+        os.mkdir(_PATH)
+    else:
+        _verb = "updated"
+    f = open(os.sep.join([_PATH,'config.json']),'w')
+    f.write(json.dumps(_config))
+    f.close()
+    _msg = f"""
+        The configuration file was {_verb} successfully at {smart.info.__home__}
+        data store: 
+            provider {provider}
+            database {_db}
+
+        If your database has security enabled, consider updating "{smart.info.__home__}{os.sep}config.json" For appropriate security
+        Visit https://github.com/lnyemba/data-transport for more information
+    """
+    print ()
+    print (_msg)
+    pass
+def post(_df,_table):
+    """
+    Store data in a given location 
+    """
+    _path = os.sep.join([smart.info.__home__,'config.json'])
+    f = open (_path)
+    _config = json.loads(f.read())
+    f.close()
+    _store = _config['store']
+    if _store['provider'] in ['mongodb','mongo','couch','couchdb'] :
+        _store['collection'] = _table
+    else:
+        _store['table'] = _table
+   
+        
+    writer = transport.factory.instance(**_store)
+    writer.write(_df)
+    if hasattr(writer,'close') :
+        writer.close()
+
+if __name__ == '__main__' :
+    _cli()
+
 # from transport import factory
 
 # class logger :

smart/__main__.py

@@ -0,0 +1 @@
+__init__.py

smart/alert/__init__.py

@@ -1,15 +0,0 @@
-"""
-This file will submit an alert to either a mailbox given a set of parameters, this will perform as following :
-    - as-a-service
-    - embedded
-"""
-
-check = None
-def post(**args):
-    """
-    This function will submit a report to a given target provided some input 
-    :key    will perform as-a-service
-    :data   data that will be submitted to smtp/queue server
-    :smtp   will send the file to a mailbox
-    """
-    pass

smart/files/__init__.py

@@ -0,0 +1,64 @@
+"""
+This file will submit an alert to either a mailbox given a set of parameters, this will perform as following :
+    - as-a-service
+    - embedded
+"""
+import os
+import pandas as pd
+import subprocess
+import glob
+from datetime import datetime
+
+def post(**args):
+    """
+    This function will submit a report to a given target provided some input 
+    :key    will perform as-a-service
+    :data   data that will be submitted to smtp/queue server
+    :smtp   will send the file to a mailbox
+    """
+    pass
+def parse(_stream):
+    """
+    :stream     single from the output command that has been executed
+    """
+    _blocks = _stream.replace('  ',' ').split(' ')
+    if len(_blocks) > 6 :
+        _user = _blocks[1]
+        _group= _blocks[2]
+        _size = _blocks[3]  # if units are not specified please interpet this as bytes
+        _date = "-".join(_blocks[4:6])
+        _time = _blocks[6]
+        _name = _blocks[-1]
+        if ':' not in _time :
+            _date = _date+' '+_time
+            _time = '00:00'
+        else:
+            _date = _date+'-'+str(datetime.now().year)
+        _name = _blocks[-1]
+        return {'user':_user,'date':_date,'time':_time,'size':_size,'content':None,'name':_name}
+def apply(_cmd, parser=None):
+    handler = subprocess.Popen(_cmd,shell=True,stdout=subprocess.PIPE,encoding='utf-8')	    		
+    stream = handler.communicate()[0].split('\n')
+    stream =  [line.strip() for line in stream]
+    if not parser :
+        # print (dict(zip(['hash','names'],stream[0].split())))
+        
+        stream = [ line.strip().replace('  ',' ').split(' ') for line in stream if len(line.strip().split()) == 2]
+        return pd.DataFrame([dict(zip(['content','name'],line)) for line in stream])
+        # return pd.DataFrame([ line.split() for line in stream ])
+        
+        # return pd.DataFrame( dict(zip(['checksum','name'],[line.strip().split(' '))) for line in stream if line.strip() != '']) )
+    else:
+        return pd.DataFrame([ parser(line.strip()) for line in stream if line.strip() != ''])
+def read (path):
+    """
+    This function will read files in a folder and provide has expressions of the files
+    """
+    _cmd = ["""find :path -type f -exec md5sum "{}" + """ , """find :path -type f -exec ls -lh "{}" + |grep -E " .*$" -o """]
+    _df = apply(_cmd[0].replace(":path",path))
+    _data= apply(_cmd[1].replace(":path",path),parse)
+    if _data.shape[0] == _df.shape[0] :
+        _data['content'] = _df.content
+    return _data
+    
+

smart/folder/__init__.py

@@ -9,6 +9,8 @@ import os
 import pandas as pd
 import io
 import datetime
+import glob
+
 class Util :
     def size(self,stream):
         
@@ -17,10 +19,15 @@ class Util :
         value,units =  re.match('^(.+)([A-Z]+$)',stream).groups()
         value = float(value)
         if 'G' == units :
-            value *= 1000
+            units = 'GB'
+        #     value *= 1000
+            
         elif 'K' == units:
-            value /= 1000
+            units = 'KB'
+        #     value /= 1000
+        else :
             units = 'MB'
+        # units = 'MB'
         return {"size":value,"units":units}
     def content(self,stream):
         return {"content":stream.split(' ')[0].strip()}
@@ -53,8 +60,9 @@ def read(**args):
 
         r = dict(r, **rows)
     N = 0 if not os.path.exists(args['path']) else len( os.listdir(args['path']))
+    path = args['path'] if args['path'].endswith('/')else args['path']+os.sep
     r['path'] = args['path']
-    r['files']= N
+    r['files']= len([filename for filename in glob.iglob(path+'**/**', recursive=True)])
     r['name'] = args['path'].split(os.sep)[-1:][0]
     r['node'] = os.uname()[1]
     r['date'] = datetime.datetime.now().strftime('%m-%d-%Y')

smart/folder/__main__.py

@@ -1,2 +1,2 @@
 import smart.folder
-print (smart.folder.read(path='/home/steve/dev/data/vumc/aou'))
+print (smart.folder.read(path='/home/steve/tmp/logs'))

smart/info.py

@@ -0,0 +1,15 @@
+import os
+__app_name__= "smart-logger"
+__version__ = "1.0-RC"
+__author__ = "Steve L. Nyemba, info@the-phi.com"
+__home__ = os.sep.join([os.environ['HOME'],'.smart-logger'])
+__database__='smart_logs'
+__license__= """
+Copyright 2017 - 2023, The Phi Technology
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""

smart/logger/__init__.py

@@ -1,52 +1,77 @@
+"""
+This file looks into the logs to determine if there is any intrusion or provides means to assess logs
+"""
+
 import pandas as pd
 import numpy as np
 import transport
 import datetime
 import io
 import json
-import requests
+import re
+from datetime import datetime
 
-def subscribe (self,**args) :
+_date  = "(^[A-Z][a-z]{2}) ([0-9]{2}) ([0-9]{2})\:([0-9]){2}\:([0-9]{2})"
-    """
+_ip = "\d+\.\d+\.\d+\.\d+"
-    This function will subscribe an email to a given service (report,notification). If already susbcribed no further action will be performed
+_regex = {
-    :email  provide a valid email for the free plan. Upgrades will be done via the website
+    'login':{'pattern':f'{_date} .*Accepted password for ([a-z]+) from ({_ip})', 'columns':['month','day','hour','minute','second','user','ip']},
-    :id     service identifier accepted values are GOOGLE_DRIVE,DROPBOX,BOX,ONE_DRIVE    
+    'attacks':{'pattern':f'{_date} .*Invalid user ([a-z,0-9]+) from ({_ip})','columns':['month','day','hour','minute','second','user','ip']},
+    'risk':{'pattern':f'{_date} .*Failed password for ([a-z,0-9]+) from ({_ip})','columns':['month','day','hour','minute','second','user','ip']} #-- accounts at risk
     
+}
+_map = {'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,'Jul':7,'Aug':8,'Sep':9,'Oct':10,'Nov':11,'Dec':12}
+def risk (_content,_id='user'):
     """
-    url = "https://the-phi.com/store/smart-top/subscribe"    
+    compute the risk associated with accounts given the counts, this should be indicated by the number of failed password attempts in a given time frame
-    SERVICES=['GOOGLE','DROPBOX','BOX','ONE_DRIVE']
-    if args['id'].upper() in SERVICES :
-        data = {"email":args['email']}
-        requests.post(url,data=data)
-    pass
-
-def log(**args) :
     """
-    This function will write to a designated location provided a set of inputs
+    _df  = pd.DataFrame(_content)
-    :store  mongo,file,couch,api
+    _g = _df.groupby([_id]).apply(lambda row: {'start_date':row.date.min(),'end_date':row.date.max() ,'count':row[_id].size} )
+    _df = pd.DataFrame(_g.tolist())
+    _df[_id] = _g.index
+    _df.start_date = _df.start_date.astype(str)
+    _df.end_date = _df.end_date.astype(str)
+    return _df
+def attacks (_content):
     """
-    #
+    This function will compute counts associated with a given set of ip addresses. If behind a load balancer IP can be ignored and counts will reflect break-in attempts
-    # @TODO: Provide facility to write to a given cloud store (google,one-drive ...)
-    #   This will have to be supported by some sort of subscription service
-    #
-    STORE_MAP = {"mongo":"MongoWriter","disk":"DiskWriter","couch":"CouchWriter",'sqlite':'SQLiteWriter'}
-    if 'store' not in args :
-        _id = 'console'
-    else:
-        _id = 'disk' if args['store'] == 'file' else args['store']
-        _id = 'disk' if _id == 'sqlite' else _id
-    if _id == 'console' :
     """
-        We are going to print whatever we have to the console ... using the tool in cli mode
+    return risk(_content,'ip')
+def login(_content):
+    return risk(_content,'user')
+def read (**_args):
     """
-        print()
+    :path path of the auth.log files to load
-        print (args['data'])
+    """
-        print ()
+    _year = _args['year'] if 'year' in _args else datetime.now().year
-        # stream = args['memory']
+    _path = _args['path']
-        # stream.write(json.dumps(args['row']) if isinstance(args['row'],dict) else args['row'])
+    f = open(_path)
-        # stream.write("\n")
+    _content = f.read().split('\n')
-    else:
+    f.close()
-        store_type  = ".".join([args['store'],STORE_MAP[_id]])
+    r = {}
-        store_args  = args['params']
+    for line in  _content :
-        store       = transport.factory.instance(type=store_type,args=store_args)    
+        for _id in _regex :
-        store.write( args['row'])
+            _pattern = _regex[_id]['pattern']
+            _columns = _regex[_id]['columns']
+            
+            _out = re.search(_pattern,line)
+            if _out :
+                try:
+                    _object = dict(zip(_columns,_out.groups()[:]))
+                    if _id not in r :
+                        r[_id] = []
+                    _month = _object['month']
+                    if _month in _map :
+                        _object['month'] = _map[ _month ]
+                    for field in ['day','month','hour','minute','second'] :
+                        _object[field] = int (_object[field])
+                    _object['date'] = datetime ( year=_year,month=_object['month'], day=_object['day'], hour=_object['hour'],minute=_object['minute'],second=_object['second'])#'-'.join([str(_object['month']),str(_object['day'])]) + ' '+_object['time']
+                    # _object['date'] = np.datetime64(_object['date'])
+                    r[_id].append(_object)
+                except Exception as e:
+                    print(e)
+                    pass
+    #
+    # At this point we have essential information formatted
+    #   Summarizing this information will serve as a means to compress it
+    #
+    return r

smart/top/__init__.py

@@ -29,10 +29,10 @@ class Util:
             name = p[0]
             args = " ".join(p[1:])
         else:
-            name = cmd.split('/')[len(cmd.split(os.sep))-1]
+            name = cmd.split(os.sep)[len(cmd.split(os.sep))-1]
             args = " ".join(stream[index:]) if index > 0 else ""
         
-        return [name,cmd,args]        
+        return [name,cmd.replace('"',"\\'"),args.replace('"',"\\'")]        
     def parse(self,rows,xchar=';'):
         """
         This function parses the document returned by the execution of the command returns a document that will have to be parsed and formatted
@@ -42,7 +42,6 @@ class Util:
         ARGS_INDEX = 6
         
         for item in rows :
-            
             if rows.index(item) != 0 :
                 parts = item.split(xchar)
                 row = parts[:TIME_INDEX]
@@ -64,7 +63,6 @@ def read(**args) :
     cmd     = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'"
     xchar   = ";"
     try:
-        
         handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)			
         stream = handler.communicate()[0]        
         if sys.version_info[0] > 2 :
@@ -86,22 +84,22 @@ def read(**args) :
         df['node'] = np.repeat(os.uname()[1],df.shape[0])
         df.columns =['pid','user','mem','cpu','status','started','name','cmd','args','date','time','node']
         
-        
         #
         # We should filter the name of the apps we are interested in here (returning the full logs )
         # @TODO: Add filter here to handle filter on different columns
         #
 
-        if 'name' in args :
+        if 'name' in args and args['name']:
             names = args['name'].split(',')
             r = pd.DataFrame()
             for name in names :
                 # tmp = df[df.name == name.strip() ]
-                ii = df.apply(lambda row: row['name'] == name.strip() or (name.strip() in str(row['name'])),axis=1).tolist()
+                # ii = df.apply(lambda row: row['name'] == name.strip() or (name.strip() in str(row['name'])),axis=1).tolist()
+                ii = df.apply(lambda row: type(row['cmd']) ==str and name.strip() in row['cmd'],axis=1).tolist()
                 tmp= df[ii]
                 # tmp.index = np.arange(tmp.shape[0])
                 if tmp.empty:
-                    tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":_name,"cmd":None,"args":None,"date":d,"time":t,"node":n}
+                    tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":name,"cmd":None,"args":None,"date":d,"time":t,"node":n}
                     
                 else:
                     r = r.append(tmp,ignore_index=False)
@@ -112,8 +110,8 @@ def read(**args) :
         # For security reasons lets has the args columns with an MD5 or sha256
         #
         
-        if not df.empty and 'args' in df : 
+        # if not df.empty and 'args' in df : 
-            df.args = [hashlib.md5(str(value).encode('utf-8')).hexdigest() for value in df.args.tolist()]
+        #     df.args = [hashlib.md5(str(value).encode('utf-8')).hexdigest() for value in df.args.tolist()]
         STATUS = {'R':'RUNNING','Z':'DEAD','D':'STASIS','S':'SLEEP','Sl':'SLEEP','Ss':'SLEEP','W':'PAGING','T':'DEAD'}
         df.status = df.status.apply(lambda value: STATUS.get(value,'UNKNOWN'))
         if 'cols' in args :
@@ -127,7 +125,7 @@ def read(**args) :
             logger(data=df)
         df.index = np.arange(df.shape[0])
         
-        return df.to_dict(orient='records')
+        return df #.to_dict(orient='records')
 
     except Exception as e:
         print (e)

smart/top/__main__.py

@@ -13,7 +13,9 @@ import sys
 import os
 import re
 import time
+import numpy as np
 # df = pd.DataFrame (smart.top.read(name='firefox,code'))
+HOME_FOLDER = os.environ['HOME']
 SYS_ARGS = {}
 if len(sys.argv) > 1:
     
@@ -30,9 +32,10 @@ if len(sys.argv) > 1:
                 
         
         i += 2
+
 if __name__ == '__main__' :
     try:
-        if 'help' in SYS_ARGS:
+        if 'help' in SYS_ARGS :
             print (help_me)
             sys.exit(0)
         if 'watch' in SYS_ARGS :
@@ -46,15 +49,37 @@ if __name__ == '__main__' :
             print ("================================= SMART TOP ================================= ")
             print ()
             df = pd.DataFrame()
-            if 'name' in SYS_ARGS :
+            if 'folder' in SYS_ARGS :
-                df = df.append(pd.DataFrame(smart.top.read(name=SYS_ARGS['name'])))
+                for path in SYS_ARGS['folder'].split(',') :
+                    #df = df.concat(smart.folder.read(path=path))
+                    _df = smart.folder.read(path=path)
+                    df = _df if df.shape[0] == 0 else pd.concat(df,_df)
+                cols = df.columns.tolist()
             else:
-                df = pd.DataFrame(smart.top.read())
+                df =  smart.top.read()
-            # df = pd.DataFrame(smart.top.read(name='fire'))    
+                cols = ['pid','name','user','cpu','mem','started','date','time','status']
+            
+                
+            
+            
             
-            log = log.append(df)
             
-            print (df[['pid','name','user','cpu','mem','started','date','time','status']])
+            for key in SYS_ARGS :
+                value = SYS_ARGS[key]
+                
+                if not value or key not in df.columns:
+                    continue 
+                ii = df.apply(lambda row:re.match(value,str(row[key])) is not None,axis=1)
+                df = df[ii].copy()
+            
+            df.index = np.arange(df.shape[0])
+                
+            # df = pd.DataFrame(smart.top.read(name='fire'))    
+            
+            #log = log.append(df)
+            log = df if log.shape[0] ==0 else pd.concat(log,df)
+            if not df.empty :
+                print (df[cols])
             if 'watch' in SYS_ARGS :                
                 time.sleep(SYS_ARGS['watch'])
                 

start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+#script_dir=`dirname $0`
+#cd $script_dir
+#/bin/bash -c ". activate sandbox; exec /bin/bash -i"
+`source activate sandbox`
+export PYTHONPATH=$PWD/src
+python src/utils/agents/data-collector.py --path /Users/michaelmead/Documents/Programming/monitor/config.json --title "Seekers Dashboard"