Compare commits

..

No commits in common. 'master' and 'data-collector' have entirely different histories.

1
.gitignore vendored

@ -1,3 +1,2 @@
sandbox sandbox
*.pyc *.pyc
.DS_Store

@ -1,7 +0,0 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="useProjectProfile" value="false" />
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

@ -1,4 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.10 virtualenv at ~/Documents/Programming/monitor/sandbox" project-jdk-type="Python SDK" />
</project>

@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/monitor.iml" filepath="$PROJECT_DIR$/.idea/monitor.iml" />
</modules>
</component>
</project>

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/src/api/templates" />
</list>
</option>
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

@ -1,583 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="834735bc-31bd-474a-9b31-c9bb58743d7d" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/config.json" afterPath="$PROJECT_DIR$/config.json" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="CreatePatchCommitExecutor">
<option name="PATCH_PATH" value="" />
</component>
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="FileEditorManager">
<splitter split-orientation="horizontal" split-proportion="0.54361874">
<split-first>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="dashboard.html" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/api/templates/dashboard.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="213">
<caret line="65" column="5" lean-forward="true" selection-start-line="65" selection-start-column="5" selection-end-line="65" selection-end-column="5" />
<folding>
<element signature="e#2414#2713#0#HTML" expanded="false" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="dashboard.js" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/api/static/js/dashboard.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="226">
<caret line="26" column="0" lean-forward="true" selection-start-line="26" selection-start-column="0" selection-end-line="26" selection-end-column="0" />
<folding>
<marker date="1499136933000" expanded="true" signature="63:670" ph="{...}" />
<marker date="1499136933000" expanded="true" signature="638:670" ph="{...}" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</split-first>
<split-second>
<leaf>
<file leaf-file-name="index.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/api/index.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="259">
<caret line="81" column="15" lean-forward="false" selection-start-line="81" selection-start-column="14" selection-end-line="81" selection-end-column="15" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="config.json" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270">
<caret line="18" column="0" lean-forward="true" selection-start-line="18" selection-start-column="0" selection-end-line="18" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
</leaf>
</split-second>
</splitter>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>nodes</find>
<find>console.log</find>
</findStrings>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/api/index.py" />
<option value="$PROJECT_DIR$/config.json" />
<option value="$PROJECT_DIR$/src/api/templates/dashboard.html" />
<option value="$PROJECT_DIR$/src/api/static/js/dashboard.js" />
</list>
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="ProjectFrameBounds">
<option name="y" value="22" />
<option name="width" value="1280" />
<option name="height" value="774" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="api" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="api" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="templates" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="api" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="static" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="js" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="monitor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="api" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="static" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane>
</pane>
<pane id="Scope" />
<pane id="Scratches" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="settings.editor.selected.configurable" value="vcs.Git" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager">
<configuration default="true" type="BashConfigurationType" factoryName="Bash">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="INTERPRETER_PATH" value="/bin/bash" />
<option name="WORKING_DIRECTORY" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SCRIPT_NAME" value="" />
<option name="PARAMETERS" value="" />
<module name="" />
<envs />
<method />
</configuration>
<configuration default="true" type="DjangoTestsConfigurationType" factoryName="Django tests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="monitor" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
<option name="TARGET" value="" />
<option name="SETTINGS_FILE" value="" />
<option name="CUSTOM_SETTINGS" value="false" />
<option name="USE_OPTIONS" value="false" />
<option name="OPTIONS" value="" />
<method />
</configuration>
<configuration default="true" type="JavaScriptTestRunnerJest" factoryName="Jest">
<node-interpreter value="project" />
<working-dir value="" />
<envs />
<scope-kind value="ALL" />
<method />
</configuration>
<configuration default="true" type="JavaScriptTestRunnerProtractor" factoryName="Protractor">
<config-file value="" />
<node-interpreter value="project" />
<envs />
<method />
</configuration>
<configuration default="true" type="JavascriptDebugType" factoryName="JavaScript Debug">
<method />
</configuration>
<configuration default="true" type="PyBehaveRunConfigurationType" factoryName="Behave">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="monitor" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
<option name="ADDITIONAL_ARGS" value="" />
<method />
</configuration>
<configuration default="true" type="PyLettuceRunConfigurationType" factoryName="Lettuce">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="monitor" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
<option name="ADDITIONAL_ARGS" value="" />
<method />
</configuration>
<configuration default="true" type="PythonConfigurationType" factoryName="Python">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="monitor" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
<option name="SCRIPT_NAME" value="" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<method />
</configuration>
<configuration default="true" type="Tox" factoryName="Tox">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
<module name="monitor" />
<method />
</configuration>
<configuration default="true" type="js.build_tools.gulp" factoryName="Gulp.js">
<method />
</configuration>
<configuration default="true" type="js.build_tools.npm" factoryName="npm">
<command value="run" />
<scripts />
<node-interpreter value="project" />
<envs />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="Doctests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="monitor" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
<option name="SCRIPT_NAME" value="" />
<option name="CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="FOLDER_NAME" value="" />
<option name="TEST_TYPE" value="TEST_SCRIPT" />
<option name="PATTERN" value="" />
<option name="USE_PATTERN" value="false" />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="Unittests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="monitor" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;.&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method />
</configuration>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="834735bc-31bd-474a-9b31-c9bb58743d7d" name="Default" comment="" />
<created>1499042529793</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1499042529793</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="0" y="22" width="1280" height="774" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32991204" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32991204" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.15347335" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32991204" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="processedProjectFiles" value="true" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/src/api/index.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/api/templates/dashboard.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="855">
<caret line="57" column="17" lean-forward="false" selection-start-line="57" selection-start-column="16" selection-end-line="57" selection-end-column="17" />
<folding>
<element signature="e#2414#2713#0#HTML" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/start.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="8" column="0" lean-forward="false" selection-start-line="8" selection-start-column="0" selection-end-line="8" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/start.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/start.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/start.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/start.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="8" column="0" lean-forward="false" selection-start-line="8" selection-start-column="0" selection-end-line="8" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270">
<caret line="18" column="0" lean-forward="true" selection-start-line="18" selection-start-column="0" selection-end-line="18" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/api/index.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="259">
<caret line="81" column="15" lean-forward="false" selection-start-line="81" selection-start-column="14" selection-end-line="81" selection-end-column="15" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/api/templates/dashboard.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="213">
<caret line="65" column="5" lean-forward="true" selection-start-line="65" selection-start-column="5" selection-end-line="65" selection-end-column="5" />
<folding>
<element signature="e#2414#2713#0#HTML" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/api/static/js/dashboard.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="226">
<caret line="26" column="0" lean-forward="true" selection-start-line="26" selection-start-column="0" selection-end-line="26" selection-end-column="0" />
<folding>
<marker date="1499136933000" expanded="true" signature="63:670" ph="{...}" />
<marker date="1499136933000" expanded="true" signature="638:670" ph="{...}" />
</folding>
</state>
</provider>
</entry>
</component>
</project>

@ -1,43 +0,0 @@
# Introduction
This program is designed to monitor processes running on a linux/osx machine. It assumes logging hasn't been properly done.
The program answers basic questions:
- Is a given program still running
- How much resource (memory/cpu) a program is using up
- The number of processes found
- Folder monitoring ...
#Architecture
The architecture of the system is distributed with a central master node,
{
"id":"",
"key":"",
"apps":[],
"sandbox":[{"path":"","requirements":""}],
"folders":["path-1"],
"store":{}
"actions":{}
}
The agent will perform three basic functions :
- data collection (building up the dataset)
- classification of status for a given process:
x y z
0 0 1 idle
1 1 1 running
0 0 0 crash
with x : memory used
y : cpu usage
z : number of processes found
- Prediction of crashes using multivariate anomaly detection
This allows the learner to determine if something unusual is happening.

@ -1 +0,0 @@
../smart/__main__.py

@ -1 +0,0 @@
smart/info.py

@ -0,0 +1,47 @@
#!/bin/bash
#
# This script is designed to handle various operations related to setting up, starting, stopping the python application
# It will assume the requirements file is at the root (not with the source code)
#
export PYTHONPATH=$PWD/src
pip_upgrade='sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o'
install(){
virtualenv sandbox
sandbox/bin/pip install -r requirements.txt
`sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade`
}
upgrade(){
git pull
count=`$PWD/sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|wc -l`
if [ ! "$count" = "0" ]; then
`$PWD/sandbox/bin/pip freeze|sort |diff requirements.txt -|grep \<|grep -E " .+$" -o|sandbox/bin/pip install --upgrade`
else
echo "No Upgrade required for sandbox"
fi
}
start(){
$PWD/sandbox/bin/python src/data-collector.py --path $PWD/config.json
}
stop(){
ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -o |xargs kill -9
}
status(){
pid=`ps -eo pid,command|grep python|grep -E "$PWD"|grep data-collector|grep -E "^ {0,}[0-9]+" -m 1 -o`
if [ "$pid" = "" ]; then
echo "Data Collector is Offline"
else
echo "Data Collector is Online $pid"
fi
}
$1

@ -6,20 +6,6 @@ The program answers basic questions:
- Is a given program still running - Is a given program still running
- How much resource (memory/cpu) a program is using up - How much resource (memory/cpu) a program is using up
- The number of processes found - The number of processes found
- Folder monitoring ...
#Architecture
The architecture of the system is distributed with a central master node,
{
"id":"",
"key":"",
"apps":[],
"sandbox":[{"path":"","requirements":""}],
"folders":["path-1"],
"store":{}
"actions":{}
}
The agent will perform three basic functions : The agent will perform three basic functions :

@ -0,0 +1,29 @@
aniso8601==1.2.0
certifi==2017.7.27.1
chardet==3.0.4
click==6.6
couchdbkit==0.6.5
Flask==0.11.1
Flask-Session==0.3.0
Flask-SocketIO==2.8.2
http-parser==0.8.3
idna==2.5
itsdangerous==0.24
Jinja2==2.8
MarkupSafe==0.23
ngram==3.3.0
numpy==1.11.3
pika==0.10.0
python-dateutil==2.6.0
python-engineio==1.1.0
python-socketio==1.6.2
pytz==2016.10
requests==2.18.3
restkit==4.2.2
scipy==1.0.0
six==1.10.0
socketpool==0.5.3
urllib3==1.22
Werkzeug==0.11.11
xmljson
xmltodict

@ -3,21 +3,18 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
import os import os
import sys import sys
import info def read(fname):
# def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read()
# return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = { args = {
"name":info.__app_name__, "name":"smart-top",
"version":info.__version__, "version":"1.0.6",
"author":info.__author__,"author_email":"info@the-phi.com", "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":info.__license__, "license":"MIT",
"packages":["smart","smart.top","smart.folder","smart.logger","smart.files"] "packages":["smart","smart.top","smart.folder","smart.logger"]}
}
args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite'] args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite']
args["install_requires"] = ['typer','pandas','numpy','data-transport@git+https://github.com/lnyemba/data-transport.git'] args["install_requires"] = ['pandas','numpy','requests','data-transport@git+https://healthcareio.the-phi.com/git/code/transport.git']
args["url"] = "https://dev.the-phi.com/git/steve/smart-top.git" args["url"] = "https://dev.the-phi.com/git/steve/smart-top.git"
args['scripts'] = ['bin/smart-logger'] args['scripts'] = ['bin/smart-top']
# args['entry_point'] = {'console-scripts':['smart-top=smart-top:main']} # args['entry_point'] = {'console-scripts':['smart-top=smart-top:main']}
# #
#@TODO: #@TODO:

@ -1,4 +1,3 @@
#!/usr/bin/env python
""" """
This framework allows data to be logged to a given data store i.e : This framework allows data to be logged to a given data store i.e :
- disk, cloud (google, dropbox, box, sugarsync or s3) or a queue server - disk, cloud (google, dropbox, box, sugarsync or s3) or a queue server
@ -8,146 +7,9 @@ usage:
dependencies : dependencies :
data-transport pip install git+https://dev.the-phi.com/git/steve/data-transport.git data-transport pip install git+https://dev.the-phi.com/git/steve/data-transport.git
""" """
import smart
import smart.top import smart.top
import smart.folder import smart.folder
import smart.top
import smart.logger import smart.logger
import smart.files
import uuid
import typer
import smart.info
import json
import os
import transport
import shutil
from datetime import datetime
_cli = typer.Typer()
@_cli.command(name='log-intruder')
def intrusion(path:str='/var/log/auth.log', year:int=datetime.now().year):
"""
This function
"""
_r = smart.logger.read(path=path,year=year)
if _r :
for _id in _r :
if hasattr(smart.logger,_id):
try:
_pointer = getattr(smart.logger,_id)
_df = _pointer(_r[_id])
post(_df,_id)
except Exception as e:
print (e)
pass
else:
print ()
print ("Nothing out of the ordinary was found in")
print (f"{path}")
@_cli.command(name='top')
def apply_apps (app:str=None,user:str=None):
"""
This function looks at applications/commands running on the system
"""
_df = smart.top.read()
_id = 'apps' if not app else app
if app :
_index = _df.name == app
if _index.sum() :
_df = _df[_index]
post(_df,_id)
@_cli.command(name='archive')
def _archive():
"""
This function will archive the database, by renaming it into
"""
_suffix = datetime.now()
_suffix = "-".join([str(_value) for _value in [_suffix.year,_suffix.month,_suffix.day,_suffix.hour,_suffix.minute]])
_path = os.sep.join([smart.info.__home__,smart.info.__database__])
_src = _path + '.db3'
if os.path.exists(_src):
_target = _path +'-archived-on-'+ _suffix+'.db3'
shutil.move(_src,_target)
_msg = f"""Archive created successfully at:
{_target}"""
else:
_msg = """
Archive function is not available at this time, please try after logs have been stored
"""
print(_msg)
@_cli.command(name='folder')
def apply_folder(path:str):
"""
This function will read the content of a folder and generate a
"""
_df = smart.folder.read(path=path)
# print (_df)
post(_df,'folders')
pass
@_cli.command (name='files')
def apply_files(folder:str) :
_df = smart.files.read(folder)
post(_df,'files')
@_cli.command(name='register')
def apply_signup (email:str,key:str=None,provider:str='sqlite') :
_config = {"system":{"email":email,"uid":str(uuid.uuid4()),"version":smart.info.__version__},"store":{"provider":provider,"context":"write"}}
_db = smart.info.__database__
if provider in ['sqlite','sqlite3'] :
_db = os.sep.join([smart.info.__home__,_db+'.db3'])
_config['store']['database'] = _db
else:
_config['store']['database'] = _db
#
# Let us store this in a folder
_PATH = smart.info.__home__
_verb = "written"
if not os.path.exists(_PATH) :
os.mkdir(_PATH)
else:
_verb = "updated"
f = open(os.sep.join([_PATH,'config.json']),'w')
f.write(json.dumps(_config))
f.close()
_msg = f"""
The configuration file was {_verb} successfully at {smart.info.__home__}
data store:
provider {provider}
database {_db}
If your database has security enabled, consider updating "{smart.info.__home__}{os.sep}config.json" For appropriate security
Visit https://github.com/lnyemba/data-transport for more information
"""
print ()
print (_msg)
pass
def post(_df,_table):
"""
Store data in a given location
"""
_path = os.sep.join([smart.info.__home__,'config.json'])
f = open (_path)
_config = json.loads(f.read())
f.close()
_store = _config['store']
if _store['provider'] in ['mongodb','mongo','couch','couchdb'] :
_store['collection'] = _table
else:
_store['table'] = _table
writer = transport.factory.instance(**_store)
writer.write(_df)
if hasattr(writer,'close') :
writer.close()
if __name__ == '__main__' :
_cli()
# from transport import factory # from transport import factory
# class logger : # class logger :

@ -1 +0,0 @@
__init__.py

@ -0,0 +1,15 @@
"""
This file will submit an alert to either a mailbox given a set of parameters, this will perform as following :
- as-a-service
- embedded
"""
check = None
def post(**args):
"""
This function will submit a report to a given target provided some input
:key will perform as-a-service
:data data that will be submitted to smtp/queue server
:smtp will send the file to a mailbox
"""
pass

@ -1,64 +0,0 @@
"""
This file will submit an alert to either a mailbox given a set of parameters, this will perform as following :
- as-a-service
- embedded
"""
import os
import pandas as pd
import subprocess
import glob
from datetime import datetime
def post(**args):
"""
This function will submit a report to a given target provided some input
:key will perform as-a-service
:data data that will be submitted to smtp/queue server
:smtp will send the file to a mailbox
"""
pass
def parse(_stream):
"""
:stream single from the output command that has been executed
"""
_blocks = _stream.replace(' ',' ').split(' ')
if len(_blocks) > 6 :
_user = _blocks[1]
_group= _blocks[2]
_size = _blocks[3] # if units are not specified please interpet this as bytes
_date = "-".join(_blocks[4:6])
_time = _blocks[6]
_name = _blocks[-1]
if ':' not in _time :
_date = _date+' '+_time
_time = '00:00'
else:
_date = _date+'-'+str(datetime.now().year)
_name = _blocks[-1]
return {'user':_user,'date':_date,'time':_time,'size':_size,'content':None,'name':_name}
def apply(_cmd, parser=None):
handler = subprocess.Popen(_cmd,shell=True,stdout=subprocess.PIPE,encoding='utf-8')
stream = handler.communicate()[0].split('\n')
stream = [line.strip() for line in stream]
if not parser :
# print (dict(zip(['hash','names'],stream[0].split())))
stream = [ line.strip().replace(' ',' ').split(' ') for line in stream if len(line.strip().split()) == 2]
return pd.DataFrame([dict(zip(['content','name'],line)) for line in stream])
# return pd.DataFrame([ line.split() for line in stream ])
# return pd.DataFrame( dict(zip(['checksum','name'],[line.strip().split(' '))) for line in stream if line.strip() != '']) )
else:
return pd.DataFrame([ parser(line.strip()) for line in stream if line.strip() != ''])
def read (path):
"""
This function will read files in a folder and provide has expressions of the files
"""
_cmd = ["""find :path -type f -exec md5sum "{}" + """ , """find :path -type f -exec ls -lh "{}" + |grep -E " .*$" -o """]
_df = apply(_cmd[0].replace(":path",path))
_data= apply(_cmd[1].replace(":path",path),parse)
if _data.shape[0] == _df.shape[0] :
_data['content'] = _df.content
return _data

@ -9,8 +9,6 @@ import os
import pandas as pd import pandas as pd
import io import io
import datetime import datetime
import glob
class Util : class Util :
def size(self,stream): def size(self,stream):
@ -19,15 +17,10 @@ class Util :
value,units = re.match('^(.+)([A-Z]+$)',stream).groups() value,units = re.match('^(.+)([A-Z]+$)',stream).groups()
value = float(value) value = float(value)
if 'G' == units : if 'G' == units :
units = 'GB' value *= 1000
# value *= 1000
elif 'K' == units: elif 'K' == units:
units = 'KB' value /= 1000
# value /= 1000 units = 'MB'
else :
units = 'MB'
# units = 'MB'
return {"size":value,"units":units} return {"size":value,"units":units}
def content(self,stream): def content(self,stream):
return {"content":stream.split(' ')[0].strip()} return {"content":stream.split(' ')[0].strip()}
@ -60,14 +53,13 @@ def read(**args):
r = dict(r, **rows) r = dict(r, **rows)
N = 0 if not os.path.exists(args['path']) else len( os.listdir(args['path'])) N = 0 if not os.path.exists(args['path']) else len( os.listdir(args['path']))
path = args['path'] if args['path'].endswith('/')else args['path']+os.sep
r['path'] = args['path'] r['path'] = args['path']
r['files']= len([filename for filename in glob.iglob(path+'**/**', recursive=True)]) r['files']= N
r['name'] = args['path'].split(os.sep)[-1:][0] r['name'] = args['path'].split(os.sep)[-1:][0]
r['node'] = os.uname()[1] r['node'] = os.uname()[1]
r['date'] = datetime.datetime.now().strftime('%m-%d-%Y') r['date'] = datetime.datetime.now().strftime('%m-%d-%Y')
r['time'] = datetime.datetime.now().strftime('%H:%M:%S') r['time'] = datetime.datetime.now().strftime('%H:%M:%S')
return pd.DataFrame([r]) return pd.DataFrame([r])
pass pass

@ -1,2 +1,2 @@
import smart.folder import smart.folder
print (smart.folder.read(path='/home/steve/tmp/logs')) print (smart.folder.read(path='/home/steve/dev/data/vumc/aou'))

@ -1,15 +0,0 @@
import os
__app_name__= "smart-logger"
__version__ = "1.0-RC"
__author__ = "Steve L. Nyemba, info@the-phi.com"
__home__ = os.sep.join([os.environ['HOME'],'.smart-logger'])
__database__='smart_logs'
__license__= """
Copyright 2017 - 2023, The Phi Technology
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the Software), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""

@ -1,77 +1,52 @@
"""
This file looks into the logs to determine if there is any intrusion or provides means to assess logs
"""
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import transport import transport
import datetime import datetime
import io import io
import json import json
import re import requests
from datetime import datetime
_date = "(^[A-Z][a-z]{2}) ([0-9]{2}) ([0-9]{2})\:([0-9]){2}\:([0-9]{2})" def subscribe (self,**args) :
_ip = "\d+\.\d+\.\d+\.\d+"
_regex = {
'login':{'pattern':f'{_date} .*Accepted password for ([a-z]+) from ({_ip})', 'columns':['month','day','hour','minute','second','user','ip']},
'attacks':{'pattern':f'{_date} .*Invalid user ([a-z,0-9]+) from ({_ip})','columns':['month','day','hour','minute','second','user','ip']},
'risk':{'pattern':f'{_date} .*Failed password for ([a-z,0-9]+) from ({_ip})','columns':['month','day','hour','minute','second','user','ip']} #-- accounts at risk
}
_map = {'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,'Jul':7,'Aug':8,'Sep':9,'Oct':10,'Nov':11,'Dec':12}
def risk (_content,_id='user'):
"""
compute the risk associated with accounts given the counts, this should be indicated by the number of failed password attempts in a given time frame
"""
_df = pd.DataFrame(_content)
_g = _df.groupby([_id]).apply(lambda row: {'start_date':row.date.min(),'end_date':row.date.max() ,'count':row[_id].size} )
_df = pd.DataFrame(_g.tolist())
_df[_id] = _g.index
_df.start_date = _df.start_date.astype(str)
_df.end_date = _df.end_date.astype(str)
return _df
def attacks (_content):
""" """
This function will compute counts associated with a given set of ip addresses. If behind a load balancer IP can be ignored and counts will reflect break-in attempts This function will subscribe an email to a given service (report,notification). If already susbcribed no further action will be performed
:email provide a valid email for the free plan. Upgrades will be done via the website
:id service identifier accepted values are GOOGLE_DRIVE,DROPBOX,BOX,ONE_DRIVE
""" """
return risk(_content,'ip') url = "https://the-phi.com/store/smart-top/subscribe"
def login(_content): SERVICES=['GOOGLE','DROPBOX','BOX','ONE_DRIVE']
return risk(_content,'user') if args['id'].upper() in SERVICES :
def read (**_args): data = {"email":args['email']}
requests.post(url,data=data)
pass
def log(**args) :
""" """
:path path of the auth.log files to load This function will write to a designated location provided a set of inputs
:store mongo,file,couch,api
""" """
_year = _args['year'] if 'year' in _args else datetime.now().year
_path = _args['path']
f = open(_path)
_content = f.read().split('\n')
f.close()
r = {}
for line in _content :
for _id in _regex :
_pattern = _regex[_id]['pattern']
_columns = _regex[_id]['columns']
_out = re.search(_pattern,line)
if _out :
try:
_object = dict(zip(_columns,_out.groups()[:]))
if _id not in r :
r[_id] = []
_month = _object['month']
if _month in _map :
_object['month'] = _map[ _month ]
for field in ['day','month','hour','minute','second'] :
_object[field] = int (_object[field])
_object['date'] = datetime ( year=_year,month=_object['month'], day=_object['day'], hour=_object['hour'],minute=_object['minute'],second=_object['second'])#'-'.join([str(_object['month']),str(_object['day'])]) + ' '+_object['time']
# _object['date'] = np.datetime64(_object['date'])
r[_id].append(_object)
except Exception as e:
print(e)
pass
# #
# At this point we have essential information formatted # @TODO: Provide facility to write to a given cloud store (google,one-drive ...)
# Summarizing this information will serve as a means to compress it # This will have to be supported by some sort of subscription service
# #
return r STORE_MAP = {"mongo":"MongoWriter","disk":"DiskWriter","couch":"CouchWriter",'sqlite':'SQLiteWriter'}
if 'store' not in args :
_id = 'console'
else:
_id = 'disk' if args['store'] == 'file' else args['store']
_id = 'disk' if _id == 'sqlite' else _id
if _id == 'console' :
"""
We are going to print whatever we have to the console ... using the tool in cli mode
"""
print()
print (args['data'])
print ()
# stream = args['memory']
# stream.write(json.dumps(args['row']) if isinstance(args['row'],dict) else args['row'])
# stream.write("\n")
else:
store_type = ".".join([args['store'],STORE_MAP[_id]])
store_args = args['params']
store = transport.factory.instance(type=store_type,args=store_args)
store.write( args['row'])

@ -29,10 +29,10 @@ class Util:
name = p[0] name = p[0]
args = " ".join(p[1:]) args = " ".join(p[1:])
else: else:
name = cmd.split(os.sep)[len(cmd.split(os.sep))-1] name = cmd.split('/')[len(cmd.split(os.sep))-1]
args = " ".join(stream[index:]) if index > 0 else "" args = " ".join(stream[index:]) if index > 0 else ""
return [name,cmd.replace('"',"\\'"),args.replace('"',"\\'")] return [name,cmd,args]
def parse(self,rows,xchar=';'): def parse(self,rows,xchar=';'):
""" """
This function parses the document returned by the execution of the command returns a document that will have to be parsed and formatted This function parses the document returned by the execution of the command returns a document that will have to be parsed and formatted
@ -42,6 +42,7 @@ class Util:
ARGS_INDEX = 6 ARGS_INDEX = 6
for item in rows : for item in rows :
if rows.index(item) != 0 : if rows.index(item) != 0 :
parts = item.split(xchar) parts = item.split(xchar)
row = parts[:TIME_INDEX] row = parts[:TIME_INDEX]
@ -54,7 +55,7 @@ class Util:
m.append(row) m.append(row)
return m return m
def read(**args) : def read(**args) :
""" """
This function will perform the actual reads of process informations. This function will perform the actual reads of process informations.
@ -63,13 +64,14 @@ def read(**args) :
cmd = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'" cmd = "ps -eo pid,user,pmem,pcpu,stat,etime,args|awk 'OFS=\";\" {$1=$1; if($5 > 9) print }'"
xchar = ";" xchar = ";"
try: try:
handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE) handler = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE)
stream = handler.communicate()[0] stream = handler.communicate()[0]
if sys.version_info[0] > 2 : if sys.version_info[0] > 2 :
rows = str(stream).split('\\n') rows = str(stream).split('\\n')
else: else:
rows = stream.split('\n') rows = stream.split('\n')
formatter = Util() formatter = Util()
m = formatter.parse(rows) m = formatter.parse(rows)
@ -84,22 +86,22 @@ def read(**args) :
df['node'] = np.repeat(os.uname()[1],df.shape[0]) df['node'] = np.repeat(os.uname()[1],df.shape[0])
df.columns =['pid','user','mem','cpu','status','started','name','cmd','args','date','time','node'] df.columns =['pid','user','mem','cpu','status','started','name','cmd','args','date','time','node']
# #
# We should filter the name of the apps we are interested in here (returning the full logs ) # We should filter the name of the apps we are interested in here (returning the full logs )
# @TODO: Add filter here to handle filter on different columns # @TODO: Add filter here to handle filter on different columns
# #
if 'name' in args and args['name']: if 'name' in args :
names = args['name'].split(',') names = args['name'].split(',')
r = pd.DataFrame() r = pd.DataFrame()
for name in names : for name in names :
# tmp = df[df.name == name.strip() ] # tmp = df[df.name == name.strip() ]
# ii = df.apply(lambda row: row['name'] == name.strip() or (name.strip() in str(row['name'])),axis=1).tolist() ii = df.apply(lambda row: row['name'] == name.strip() or (name.strip() in str(row['name'])),axis=1).tolist()
ii = df.apply(lambda row: type(row['cmd']) ==str and name.strip() in row['cmd'],axis=1).tolist()
tmp= df[ii] tmp= df[ii]
# tmp.index = np.arange(tmp.shape[0]) # tmp.index = np.arange(tmp.shape[0])
if tmp.empty: if tmp.empty:
tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":name,"cmd":None,"args":None,"date":d,"time":t,"node":n} tmp = {"pid":None,"user":None,"mem":0,"cpu":0,"status":"-100","started":None,"name":_name,"cmd":None,"args":None,"date":d,"time":t,"node":n}
else: else:
r = r.append(tmp,ignore_index=False) r = r.append(tmp,ignore_index=False)
@ -110,8 +112,8 @@ def read(**args) :
# For security reasons lets has the args columns with an MD5 or sha256 # For security reasons lets has the args columns with an MD5 or sha256
# #
# if not df.empty and 'args' in df : if not df.empty and 'args' in df :
# df.args = [hashlib.md5(str(value).encode('utf-8')).hexdigest() for value in df.args.tolist()] df.args = [hashlib.md5(str(value).encode('utf-8')).hexdigest() for value in df.args.tolist()]
STATUS = {'R':'RUNNING','Z':'DEAD','D':'STASIS','S':'SLEEP','Sl':'SLEEP','Ss':'SLEEP','W':'PAGING','T':'DEAD'} STATUS = {'R':'RUNNING','Z':'DEAD','D':'STASIS','S':'SLEEP','Sl':'SLEEP','Ss':'SLEEP','W':'PAGING','T':'DEAD'}
df.status = df.status.apply(lambda value: STATUS.get(value,'UNKNOWN')) df.status = df.status.apply(lambda value: STATUS.get(value,'UNKNOWN'))
if 'cols' in args : if 'cols' in args :
@ -124,8 +126,8 @@ def read(**args) :
logger = args['logger'] logger = args['logger']
logger(data=df) logger(data=df)
df.index = np.arange(df.shape[0]) df.index = np.arange(df.shape[0])
return df #.to_dict(orient='records') return df.to_dict(orient='records')
except Exception as e: except Exception as e:
print (e) print (e)

@ -13,9 +13,7 @@ import sys
import os import os
import re import re
import time import time
import numpy as np
# df = pd.DataFrame (smart.top.read(name='firefox,code')) # df = pd.DataFrame (smart.top.read(name='firefox,code'))
HOME_FOLDER = os.environ['HOME']
SYS_ARGS = {} SYS_ARGS = {}
if len(sys.argv) > 1: if len(sys.argv) > 1:
@ -32,10 +30,9 @@ if len(sys.argv) > 1:
i += 2 i += 2
if __name__ == '__main__' : if __name__ == '__main__' :
try: try:
if 'help' in SYS_ARGS : if 'help' in SYS_ARGS:
print (help_me) print (help_me)
sys.exit(0) sys.exit(0)
if 'watch' in SYS_ARGS : if 'watch' in SYS_ARGS :
@ -49,37 +46,15 @@ if __name__ == '__main__' :
print ("================================= SMART TOP ================================= ") print ("================================= SMART TOP ================================= ")
print () print ()
df = pd.DataFrame() df = pd.DataFrame()
if 'folder' in SYS_ARGS : if 'name' in SYS_ARGS :
for path in SYS_ARGS['folder'].split(',') : df = df.append(pd.DataFrame(smart.top.read(name=SYS_ARGS['name'])))
#df = df.concat(smart.folder.read(path=path))
_df = smart.folder.read(path=path)
df = _df if df.shape[0] == 0 else pd.concat(df,_df)
cols = df.columns.tolist()
else: else:
df = smart.top.read() df = pd.DataFrame(smart.top.read())
cols = ['pid','name','user','cpu','mem','started','date','time','status']
for key in SYS_ARGS :
value = SYS_ARGS[key]
if not value or key not in df.columns:
continue
ii = df.apply(lambda row:re.match(value,str(row[key])) is not None,axis=1)
df = df[ii].copy()
df.index = np.arange(df.shape[0])
# df = pd.DataFrame(smart.top.read(name='fire')) # df = pd.DataFrame(smart.top.read(name='fire'))
#log = log.append(df) log = log.append(df)
log = df if log.shape[0] ==0 else pd.concat(log,df)
if not df.empty : print (df[['pid','name','user','cpu','mem','started','date','time','status']])
print (df[cols])
if 'watch' in SYS_ARGS : if 'watch' in SYS_ARGS :
time.sleep(SYS_ARGS['watch']) time.sleep(SYS_ARGS['watch'])
@ -93,4 +68,4 @@ if __name__ == '__main__' :
print ("... Exiting, Thanks for using smart-top") print ("... Exiting, Thanks for using smart-top")
# pass # pass
# print (df.groupby(['user'])['cpu','mem'].sum()) # print (df.groupby(['user'])['cpu','mem'].sum())

@ -1,8 +0,0 @@
#!/bin/bash
#script_dir=`dirname $0`
#cd $script_dir
#/bin/bash -c ". activate sandbox; exec /bin/bash -i"
`source activate sandbox`
export PYTHONPATH=$PWD/src
python src/utils/agents/data-collector.py --path /Users/michaelmead/Documents/Programming/monitor/config.json --title "Seekers Dashboard"
Loading…
Cancel
Save