{ "cells": [ { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", "This notebook is designed to run experiments around demographics on registered tier\n", "The \n", "\"\"\"\n", "import pandas as pd\n", "import numpy as np\n", "from pandas_risk import *\n", "\n", "ATTRIBUTES = ['race','ethnicity','birth_date','state','city','zip','marital_status','education','language','home_owner','income','employment_status','living_situation','active_duty_status','gender_identity','birth_place','death_date','death_cause','orientation']\n", "dfs = pd.read_csv('scenario-settings.csv')\n", "dfc = pd.read_gbq(\"SELECT * FROM deid_risk.registered_dec_01\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "cols_o = dfs.loc[(dfs.fo & dfs.fi) ==1].feature.tolist()\n", "cols_i = dfs.loc[(dfs.fo + dfs.fi )>=1 ].feature.tolist()\n", "cols_a = dfs.feature.tolist()\n", "\n", "cols_v = ['birth_date','gender_identity','race','state','city','birth_place'] #-- voter registration\n", "#remove the dates fields because dates are shifted\n", "cols_o = [i for i in cols_o if i not in ['birth_date','death_date']]\n", "cols_i = [i for i in cols_i if i not in ['birth_date','death_date']]\n", "cols_a = [i for i in cols_a if i not in ['birth_date','death_date']]\n", "cols_v = [i for i in cols_v if i not in ['birth_date', 'death_date']]" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "# print(dfs)\n", "# print(cols_o)\n", "# print(cols_i)\n", "\n" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
field_countflaggroup_countmarketerprosecutorunique_row_ratio
08high-conj65320.0562341.00.021368
111high-disj474470.4084731.00.278554
216all607180.5227241.00.408189
35voter-reg13160.0113291.00.002944
\n", "
" ], "text/plain": [ " field_count flag group_count marketer prosecutor unique_row_ratio\n", "0 8 high-conj 6532 0.056234 1.0 0.021368\n", "1 11 high-disj 47447 0.408473 1.0 0.278554\n", "2 16 all 60718 0.522724 1.0 0.408189\n", "3 5 voter-reg 1316 0.011329 1.0 0.002944" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "r = pd.concat([dfc[cols_o].deid.evaluate(),dfc[cols_i].deid.evaluate(),dfc[cols_a].deid.evaluate(),dfc[cols_v].deid.evaluate() ])\n", "r.index = np.arange(r.shape[0]).astype(np.int64)\n", "r['flag']=['high-conj','high-disj','all','voter-reg']\n", "\n", "r\n" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEuCAYAAAB1QVLBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFxVJREFUeJzt3X+U3XV95/HnK+FHtohSIcsiAZO6UYGFEjdEUY/yU0E0bP3RwuourFa2q1A89lSBFqzUY6ly3HqUrlKbo9a1ILCWGKIgVVyRU8gAWWzMcow0QmArETGy7okQfe8f906cjHcydyZ3cplPno9z5uR+P9/PzH3nnpnXfOf9/d7vJ1WFJKktc4ZdgCRp8Ax3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoP2GtYTH3TQQbVw4cJhPb0kzUp33333D6tq/mTzhhbuCxcuZGRkZFhPL0mzUpLv9zPPtowkNchwl6QGGe6S1KCh9dwlteupp55i06ZNbN26ddilzFrz5s1jwYIF7L333tP6fMNd0sBt2rSJ/fffn4ULF5Jk2OXMOlXFY489xqZNm1i0aNG0voZtGUkDt3XrVg488ECDfZqScOCBB+7SXz6Gu6QZYbDvml19/Qx3SWqQPXdpN1h40U3DLqEvG684Y0a+7qD//zNV50Ruu+02rrzySlatWtXX/LVr1/LII4/wmte8ZoYrm5hH7pK0E9u2bZvy56xdu5bVq1fP+PPsjOEuqUkbN27khS98Ieeeey7Pf/7zefOb38ytt97Ky172MhYvXsxdd93FXXfdxfHHH8+SJUt46Utfyv333w/Apz/9aZYvX85JJ53EySefvMPXXbNmDUuWLOF73/seP/3pT3nrW9/KsmXLWLJkCTfeeCNPPvkkl112Gddeey3HHnss1157bc95kz3PrrItI6lZGzZs4LrrrmPFihUcd9xxfP7zn+f2229n5cqVfPCDH+Szn/0s3/zmN9lrr7249dZbueSSS7jhhhsAuOeee7jvvvt49rOfzW233QbAHXfcwQUXXMCNN97I4YcfziWXXMJJJ53EihUr+PGPf8yyZcs45ZRTuPzyyxkZGeHjH/84wITzxj/PIBnukpq1aNEijj76aACOOuooTj75ZJJw9NFHs3HjRrZs2cI555zDd7/7XZLw1FNPbf/cU089dYfAXb9+Peeddx633HILz3nOcwC45ZZbWLlyJVdeeSXQuQT0wQcf/JU6djZv/PMMiuEuqVn77rvv9sdz5szZvj1nzhy2bdvGpZdeyoknnsgXv/hFNm7cyAknnLB9/n777bfD1zrkkEPYunUr99577/ZwrypuuOEGXvCCF+ww984779xhe2fzxj/PoPTVc09yWpL7k2xIclGP/ecm2ZxkbffjdwdfqiQN1pYtWzj00EOBTv97Zw444ABuuukmLr744u1tmle/+tV87GMfo6oAuPfeewHYf//9eeKJJ7Z/7kTzZtKkR+5J5gJXAacCm4A1SVZW1XfGTb22qs6fgRolzXK7+9LFfr3nPe/hnHPO4QMf+ABnnDF5jQcffDCrVq3i9NNPZ8WKFVx66aW8613v4phjjuEXv/gFixYtYtWqVZx44olcccUVHHvssVx88cUTzptJGf1NMuGE5HjgT6rq1d3tiwGq6s/GzDkXWDqVcF+6dGm5WIf2FHvade7r16/niCOOGMjX2pP1eh2T3F1VSyf73H7aMocCD43Z3tQdG+8NSe5Lcn2Sw3p9oSTnJRlJMrJ58+Y+nlqSNB2Dus79S8DCqjoG+CrwmV6TqurqqlpaVUvnz590CUBJ0jT1E+4PA2OPxBd0x7arqseq6mfdzU8B/3Yw5UmarSZr+WrndvX16yfc1wCLkyxKsg9wFrBy7IQkh4zZXA6s36WqJM1q8+bN47HHHjPgp2n0fu7z5s2b9teY9GqZqtqW5HzgZmAusKKq1iW5HBipqpXA7ydZDmwDfgScO+2KJM16CxYsYNOmTXhubfpGV2Karr7exFRVq4HV48YuG/P4YuDiaVchqSl77733tFcQ0mB44zBJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhq0Vz+TkpwGfBSYC3yqqq6YYN4bgOuB46pqZGBVaigWXnTTsEvoy8Yrzhh2CdLTzqRH7knmAlcBpwNHAmcnObLHvP2BC4E7B12kJGlq+mnLLAM2VNUDVfUkcA1wZo95fwr8ObB1gPVJkqahn3A/FHhozPam7th2SV4EHFZVO/07Psl5SUaSjGzevHnKxUqS+rPLJ1STzAE+AvzBZHOr6uqqWlpVS+fPn7+rTy1JmkA/4f4wcNiY7QXdsVH7A/8GuC3JRuAlwMokSwdVpCRpavoJ9zXA4iSLkuwDnAWsHN1ZVVuq6qCqWlhVC4F/AJZ7tYwkDc+k4V5V24DzgZuB9cAXqmpdksuTLJ/pAiVJU9fXde5VtRpYPW7ssgnmnrDrZUmSdoXvUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN6ivck5yW5P4kG5Jc1GP/7yX5dpK1SW5PcuTgS5Uk9WvScE8yF7gKOB04Eji7R3h/vqqOrqpjgQ8BHxl4pZKkvvVz5L4M2FBVD1TVk8A1wJljJ1TVT8Zs7gfU4EqUJE3VXn3MORR4aMz2JuDF4ycleSfwbmAf4KSBVCdJmpaBnVCtqquq6nnAe4E/7jUnyXlJRpKMbN68eVBPLUkap59wfxg4bMz2gu7YRK4B/l2vHVV1dVUtraql8+fP779KSdKU9BPua4DFSRYl2Qc4C1g5dkKSxWM2zwC+O7gSJUlTNWnPvaq2JTkfuBmYC6yoqnVJLgdGqmolcH6SU4CngMeBc2ayaEnSzvVzQpWqWg2sHjd22ZjHFw64LknSLvAdqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNaivcE9yWpL7k2xIclGP/e9O8p0k9yX5+yTPHXypkqR+TRruSeYCVwGnA0cCZyc5cty0e4GlVXUMcD3woUEXKknqXz9H7suADVX1QFU9CVwDnDl2QlV9var+X3fzH4AFgy1TkjQV/YT7ocBDY7Y3dccm8jbgy712JDkvyUiSkc2bN/dfpSRpSgZ6QjXJW4ClwId77a+qq6tqaVUtnT9//iCfWpI0xl59zHkYOGzM9oLu2A6SnAL8EfDKqvrZYMqTJE1HP0fua4DFSRYl2Qc4C1g5dkKSJcAngeVV9ejgy5QkTcWk4V5V24DzgZuB9cAXqmpdksuTLO9O+zDwDOC6JGuTrJzgy0mSdoN+2jJU1Wpg9bixy8Y8PmXAdUmSdoHvUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUF9hXuS05Lcn2RDkot67H9FknuSbEvyxsGXKUmaiknDPclc4CrgdOBI4OwkR46b9iBwLvD5QRcoSZq6vfqYswzYUFUPACS5BjgT+M7ohKra2N33ixmoUZI0Rf20ZQ4FHhqzvak7NmVJzksykmRk8+bN0/kSkqQ+7NYTqlV1dVUtraql8+fP351PLUl7lH7C/WHgsDHbC7pjkqSnqX7CfQ2wOMmiJPsAZwErZ7YsSdKumDTcq2obcD5wM7Ae+EJVrUtyeZLlAEmOS7IJeBPwySTrZrJoSdLO9XO1DFW1Glg9buyyMY/X0GnXSJKeBnyHqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KC+7i0zWyy86KZhl9CXjVecMewSJDXOI3dJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqUFMrMUnaM7jq2uT6OnJPclqS+5NsSHJRj/37Jrm2u//OJAsHXagkqX+ThnuSucBVwOnAkcDZSY4cN+1twONV9a+B/wr8+aALlST1r58j92XAhqp6oKqeBK4Bzhw350zgM93H1wMnJ8ngypQkTUU/PfdDgYfGbG8CXjzRnKralmQLcCDww7GTkpwHnNfd/L9J7p9O0bvZQYz7f+yq7Nl/1/h6Do6v5WDNltfzuf1M2q0nVKvqauDq3fmcuyrJSFUtHXYdrfD1HBxfy8Fq7fXspy3zMHDYmO0F3bGec5LsBTwLeGwQBUqSpq6fcF8DLE6yKMk+wFnAynFzVgLndB+/EfhaVdXgypQkTcWkbZluD/184GZgLrCiqtYluRwYqaqVwF8Df5NkA/AjOr8AWjGr2kizgK/n4PhaDlZTr2c8wJak9nj7AUlqkOEuSQ0y3CWpQYa7JDXIu0JqxiR5ZlX9JMmze+wu4CdV9fPdXZcEkOT1PYa3AN+uqkd3dz2D5tUyYyR5S1V9Lsm7e+wuOpd5rqyqx3dzabNSklVV9dok/0Tn9Rt/v6FnAH9VVZfs/upmlyRP0HkNf2UXUFX1zN1c0qyX5CbgeODr3aETgLuBRcDlVfU3QyptIDxy39F+3X/3n2D/IuC/AC/ZPeXMblX12u6/i3rt795x9B8Bw30SVTXR96Smby/giKr6AUCSg4HP0rl31v8EDPdWVNUnu/++f6I53TdvaQqSvAxYW1U/TfIW4EXAX1TVg8ARw61udpigtbVdVf1od9XSkMNGg73r0e7Yj5I8NayiBsW2TA9J5gNvBxYy5hdgVb11WDXNZknuA34TOAb4NPAp4Ler6pXDrGs2maC1NbpdVfUbQylsFkvyl8DhwHXdoTfSubvtHwKrqurEYdU2CIZ7D0nuAL5Jp/+2/YRfVd0wtKJmsST3VNWLklwGPFxVfz06NuzaZqPuUfxiYN7oWFV9Y3gVzU7dNSdeD7y8O/Qt4IZW7otlW6a3X6uq9w67iIY8keRi4C3AK5LMAfYeck2zUpLfBS6kc3fWtXTO/9wBnDzMumajqqokI8CWqro1ya/ROcn/xJBLGwivc+9tVZLXDLuIhvwO8DPgbVX1z3SC6cPDLWnWuhA4Dvh+t22whM7le5qiJG+ns3LcJ7tDhwJ/N7yKBsu2TA/dy872A54ERk+seLmZhi7Jmqo6Lsla4MVV9bMk66rqqGHXNtt0X8NlwJ1VtaQ79u2qOnq4lQ2GbZkevOxsMJLcXlUv73GNttdmT9+mJAfQOcL8apLHge8PuabZ6mdV9eTocs/dhYaaOdr1yH0CSZYDr+hu3lZVq4ZZjzReklfSWfXsK93F6zUFST4E/Bj4j8AFwDuA71TVHw21sAEx3HtIcgWdvuZ/7w6dTWdhkouHV9Xs47XZejrrnth/G/AqOn9N3gx8qpWrZQz3HrrXZR9bVb/obs8F7q2qY4Zb2ewy7trsw4HHu48PAB6c6J2r0kzr/kx/tqrePOxaZopXy0zsgDGPnzW0KmaxqlrUfXPNrcDrquqgqjoQeC1wy3Cr056se8O653bXhW6SJ1R7+zPg3iRfp3Ok+QrgouGWNKu9pKrePrpRVV/u9julYXoA+FaSlcBPRwer6iPDK2lwDPcequpvk9xGp+8O8N7u9dmankeS/DHwue72m4FHhliPBPC97sccJr5Z4Kxlz72HJL8FfK2qtnS3DwBOqKpm3uCwO3VPrL6PX1599A06t1T1hKqeFpL8q9YO4Az3HpKsrapjx43dO/pGB01fkkOq6v8Muw5prBbvdeQJ1d56vS62sAbjpmEXIPUwfiGZWc9w720kyUeSPK/78RE6d4jUrmvuh0hN+KthFzBohntvF9C5r8y1wDXAVuCdQ62oHc39EGn2STI3yf8e3a6qvxxmPTPBnvsk7BEPRvdNIwez4+InDw6vIu3pktwIXNDq96F95MndRGdZOE1TkgvoXC3zAzqLn4TOO1d9x6+G6deBdUnuYsfr3JcPr6TBMdwnZ494110IvKCqHht2IdIYlw67gJlkuE/OHvGuewgXlNDTTFV9I8lzgcVjVmKaO+y6BsWe+wTsEe+6JO/uPjwKeAGdFtfPRve38jZvzU7dlZjOA55dVc9Lshj4RFU1sWShR+492CMemNG3dD/Y/din+yE9HbyT7kpMAFX13ST/crglDY7h3ps94gGoqvcPuwZpJ5peiclw780e8QAl+RK/+kOzBRgBPllVW3d/VRLfSHIJ8C+SnEpnJaYvDbmmgbHnPoY94pmR5KPAfOBvu0O/A/yETuA/s6r+w7Bq056r10pMVdXMBRQeue/IHvHMeGlVHTdm+0tJ1lTVcUnWDa0q7ekuqKqPMuaKuCQXdsdmPY/cNeOSrAdePXq1UZLD6RwlHeHdNjUsve4E2dL3o0fuPdgjHrg/AG5P8j06f/4uAt6RZD/gM0OtTHucJGcD/x5Y1F2FadQzgWbWGPDIvQd7xIOXZF/ghd3N+/0FqWHpvnFpEZ3lNMcun/kEcF9VbRtKYQNmuPcw2g/uNZZkXVUdNazaZpMkJ1XV15K8vtf+qvofu7smaawkB/PL5TTvqqpHh1nPINmW6e0ZSQ4f1yN+Rnffk8Mra9Z5JfA14HXd7dEjidE3hRnuGpokbwKuBG6j8z35sSR/WFXXD7WwAfHIvYckrwE+QWfx3O09YjrfBG+vqr8YXnWzT5J5wBuAhfzygKKq6vKhFaU9XpL/BZw6erSeZD5wa1X95nArGwyP3HuoqtXd+0z06hEb7FP3d8CPgXvoLHwCDb0TULPWnHFtmMdoaAEjw32MnfSIn5fEHvH0Laiq04ZdhDTOl5PczI4XTqweYj0DZbjvyB7xzLgjydFV9e1hFyKN8SjwOeDY7vbVVfXFIdYzUPbce7BHPBhJvk3nl+JewGLgATq3cwid19O7bGpokrwP+G0617ZfC1xXVT8YblWDY7j3kOQr/LJH/PPucHlvmanpXk88oar6/u6qRZpIkmPotGTeAGyqqlOGXNJA2JbpzR7xABjemiUeBf6ZzgnVZu7n3syZ4QG7I8nRwy5C0sxJ8o4ktwF/DxxI5zLnZlqFHrmPMa5H/J+S2COW2nUY8K6qWjvsQmaCPfcx7BFLaoXhLkkNsucuSQ0y3CWpQYa79jhJfj/J+iQPJ/n4sOuRZoJXy2hP9A7glO7H0iHXIs0Ij9y1R0nyCeA3gC8Dvz5m/HVJ7kxyb5Jbu4s4kGR+kq8mWZfkU0m+n+SgIZUv9c1w1x6lqn4PeAQ4EXh8zK7bgZd0F0e+BnhPd/x9wNe6q29dDxy+G8uVps22jNSxALg2ySHAPsA/dcdfDvwWQFV9JcnjE3y+9LTikbvU8THg41V1NPCfgXlDrkfaJYa71PEs4OHu43PGjH+Lzm1hSfIqxvTppaczw13q+BPguiR3Az8cM/5+4FVJ/hF4E527Bz6x+8uTpsbbD0g7kWRf4OdVtS3J8cB/q6pjJ/s8adg8oSrt3OHAF5LMAZ4E3j7keqS+eOQuSQ2y5y5JDTLcJalBhrskNchwl6QGGe6S1KD/DzwQEyuRP8dtAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "fig_o = r.plot(kind='bar',x='flag',y=['marketer']).get_figure()\n" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "writer = pd.ExcelWriter('out-116kpatients-phase-1.xlsx',engine='xlsxwriter')\n", "r.to_excel(writer,'phase-1')\n", "writer.save()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
featurefifo
0race11
1ethnicity11
2birth_date11
3city11
4state11
5marital_status11
6education10
7language00
8home_owner11
9income01
10employment_status10
11living_situation00
12active_duty_status00
13gender_identity11
14birth_place00
15death_date11
16death_cause11
17orientation00
\n", "
" ], "text/plain": [ " feature fi fo\n", "0 race 1 1\n", "1 ethnicity 1 1\n", "2 birth_date 1 1\n", "3 city 1 1\n", "4 state 1 1\n", "5 marital_status 1 1\n", "6 education 1 0\n", "7 language 0 0\n", "8 home_owner 1 1\n", "9 income 0 1\n", "10 employment_status 1 0\n", "11 living_situation 0 0\n", "12 active_duty_status 0 0\n", "13 gender_identity 1 1\n", "14 birth_place 0 0\n", "15 death_date 1 1\n", "16 death_cause 1 1\n", "17 orientation 0 0" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dfs\n" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "names = pd.read_csv('family-history.csv').name.tolist()\n", "path ='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json'\n", "sql = \"\"\"\n", "SELECT * FROM deid_risk.registered_medical_history_dec_001\n", "\"\"\"\n", "dfm = pd.read_gbq(\"SELECT * FROM deid_risk.registered_medical_history_dec_001\",private_key=path,dialect='standard')" ] }, { "cell_type": "code", "execution_count": 69, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(0.9343780009344719, 1.269831148073964)" ] }, "execution_count": 69, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cols = list( set(dfm.columns.tolist()) - set(['person_id']))\n", "r = pd.DataFrame(dfm[cols].count(),columns=['counts'])\n", "r['attributes'] = r.index\n", "r['rate'] = 100*(r.counts / dfm.shape[0])\n", "r.rate.mean(),np.sqrt(r.rate.var())" ] }, { "cell_type": "code", "execution_count": 81, "metadata": {}, "outputs": [], "source": [ "writer = pd.ExcelWriter('/home/steve/tmp/simple.xlsx', engine='xlsxwriter')\n", "r.to_excel(writer,sheet_name='p1')\n", "workbook = writer.book\n", "worksheet = workbook.add_worksheet()\n", "b = pd.DataFrame({\"id\":np.random.choice(10,30)})" ] }, { "cell_type": "code", "execution_count": 80, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['__class__',\n", " '__delattr__',\n", " '__dict__',\n", " '__doc__',\n", " '__format__',\n", " '__getattribute__',\n", " '__hash__',\n", " '__init__',\n", " '__module__',\n", " '__new__',\n", " '__reduce__',\n", " '__reduce_ex__',\n", " '__repr__',\n", " '__setattr__',\n", " '__sizeof__',\n", " '__str__',\n", " '__subclasshook__',\n", " '__weakref__',\n", " '_assemble_xml_file',\n", " '_button_params',\n", " '_calculate_spans',\n", " '_calculate_x_split_width',\n", " '_check_dimensions',\n", " '_comment_params',\n", " '_convert_date_time',\n", " '_convert_name_area',\n", " '_csv_join',\n", " '_encode_password',\n", " '_escape_attributes',\n", " '_escape_data',\n", " '_escape_url',\n", " '_extract_filter_tokens',\n", " '_get_palette_color',\n", " '_get_range_data',\n", " '_initialize',\n", " '_isinf',\n", " '_isnan',\n", " '_opt_close',\n", " '_opt_reopen',\n", " '_parse_filter_expression',\n", " '_parse_filter_tokens',\n", " '_position_object_emus',\n", " '_position_object_pixels',\n", " '_prepare_chart',\n", " '_prepare_header_image',\n", " '_prepare_header_vml_objects',\n", " '_prepare_image',\n", " '_prepare_shape',\n", " '_prepare_tables',\n", " '_prepare_vml_objects',\n", " '_set_filehandle',\n", " '_set_icon_props',\n", " '_set_spark_color',\n", " '_set_xml_writer',\n", " '_size_col',\n", " '_size_row',\n", " '_sort_pagebreaks',\n", " '_table_function_to_formula',\n", " '_write',\n", " '_write_array_formula',\n", " '_write_auto_filter',\n", " '_write_autofilters',\n", " '_write_blank',\n", " '_write_boolean',\n", " '_write_brk',\n", " '_write_cell',\n", " '_write_cell_array_formula',\n", " '_write_cell_value',\n", " '_write_cf_rule',\n", " '_write_cfvo',\n", " '_write_col_breaks',\n", " '_write_col_info',\n", " '_write_color',\n", " '_write_color_axis',\n", " '_write_color_first',\n", " '_write_color_high',\n", " '_write_color_last',\n", " '_write_color_low',\n", " '_write_color_markers',\n", " '_write_color_negative',\n", " '_write_color_scale',\n", " '_write_color_series',\n", " '_write_cols',\n", " '_write_conditional_formats',\n", " '_write_conditional_formatting',\n", " '_write_conditional_formatting_2010',\n", " '_write_custom_filter',\n", " '_write_custom_filters',\n", " '_write_data_bar',\n", " '_write_data_bar_ext',\n", " '_write_data_validation',\n", " '_write_data_validations',\n", " '_write_datetime',\n", " '_write_dimension',\n", " '_write_drawing',\n", " '_write_drawings',\n", " '_write_empty_row',\n", " '_write_ext',\n", " '_write_ext_list',\n", " '_write_ext_list_data_bars',\n", " '_write_ext_list_sparklines',\n", " '_write_filter',\n", " '_write_filter_column',\n", " '_write_filters',\n", " '_write_font',\n", " '_write_formula',\n", " '_write_formula_1',\n", " '_write_formula_2',\n", " '_write_formula_element',\n", " '_write_freeze_panes',\n", " '_write_header_footer',\n", " '_write_hyperlink_external',\n", " '_write_hyperlink_internal',\n", " '_write_hyperlinks',\n", " '_write_icon_set',\n", " '_write_legacy_drawing',\n", " '_write_legacy_drawing_hf',\n", " '_write_merge_cell',\n", " '_write_merge_cells',\n", " '_write_number',\n", " '_write_odd_footer',\n", " '_write_odd_header',\n", " '_write_optimized_sheet_data',\n", " '_write_outline_pr',\n", " '_write_page_margins',\n", " '_write_page_set_up_pr',\n", " '_write_page_setup',\n", " '_write_panes',\n", " '_write_phonetic_pr',\n", " '_write_print_options',\n", " '_write_rich_string',\n", " '_write_row',\n", " '_write_row_breaks',\n", " '_write_rows',\n", " '_write_rstring_color',\n", " '_write_selection',\n", " '_write_selections',\n", " '_write_sheet_data',\n", " '_write_sheet_format_pr',\n", " '_write_sheet_pr',\n", " '_write_sheet_protection',\n", " '_write_sheet_view',\n", " '_write_sheet_views',\n", " '_write_single_row',\n", " '_write_spark_color',\n", " '_write_sparkline_group',\n", " '_write_sparkline_groups',\n", " '_write_sparklines',\n", " '_write_split_panes',\n", " '_write_string',\n", " '_write_tab_color',\n", " '_write_table_part',\n", " '_write_table_parts',\n", " '_write_token_as_string',\n", " '_write_underline',\n", " '_write_url',\n", " '_write_vert_align',\n", " '_write_worksheet',\n", " '_write_x14_axis_color',\n", " '_write_x14_border_color',\n", " '_write_x14_cf_rule',\n", " '_write_x14_cfvo',\n", " '_write_x14_data_bar',\n", " '_write_x14_negative_border_color',\n", " '_write_x14_negative_fill_color',\n", " '_xml_close',\n", " '_xml_data_element',\n", " '_xml_declaration',\n", " '_xml_empty_tag',\n", " '_xml_empty_tag_unencoded',\n", " '_xml_end_tag',\n", " '_xml_formula_element',\n", " '_xml_inline_string',\n", " '_xml_number_element',\n", " '_xml_rich_inline_string',\n", " '_xml_rich_si_element',\n", " '_xml_si_element',\n", " '_xml_start_tag',\n", " '_xml_start_tag_unencoded',\n", " '_xml_string_element',\n", " 'activate',\n", " 'active',\n", " 'active_pane',\n", " 'add_sparkline',\n", " 'add_table',\n", " 'autofilter',\n", " 'autofilter_area',\n", " 'autofilter_ref',\n", " 'black_white',\n", " 'buttons_list',\n", " 'center_horizontally',\n", " 'center_vertically',\n", " 'charts',\n", " 'col_formats',\n", " 'col_size_changed',\n", " 'col_sizes',\n", " 'colinfo',\n", " 'comments',\n", " 'comments_author',\n", " 'comments_list',\n", " 'comments_visible',\n", " 'cond_formats',\n", " 'conditional_format',\n", " 'constant_memory',\n", " 'data_bars_2010',\n", " 'data_validation',\n", " 'date_1904',\n", " 'default_col_pixels',\n", " 'default_date_format',\n", " 'default_row_height',\n", " 'default_row_pixels',\n", " 'default_row_zeroed',\n", " 'default_url_format',\n", " 'dim_colmax',\n", " 'dim_colmin',\n", " 'dim_rowmax',\n", " 'dim_rowmin',\n", " 'draft_quality',\n", " 'drawing',\n", " 'drawing_links',\n", " 'dxf_priority',\n", " 'escapes',\n", " 'excel2003_style',\n", " 'excel_version',\n", " 'ext_sheets',\n", " 'external_comment_links',\n", " 'external_drawing_links',\n", " 'external_hyper_links',\n", " 'external_table_links',\n", " 'external_vml_links',\n", " 'fh',\n", " 'fileclosed',\n", " 'filter_cols',\n", " 'filter_column',\n", " 'filter_column_list',\n", " 'filter_on',\n", " 'filter_range',\n", " 'filter_type',\n", " 'fit_height',\n", " 'fit_page',\n", " 'fit_to_pages',\n", " 'fit_width',\n", " 'footer',\n", " 'footer_images',\n", " 'freeze_panes',\n", " 'get_name',\n", " 'has_comments',\n", " 'has_header_vml',\n", " 'has_vml',\n", " 'hbreaks',\n", " 'hcenter',\n", " 'header',\n", " 'header_footer_aligns',\n", " 'header_footer_changed',\n", " 'header_footer_scales',\n", " 'header_images',\n", " 'header_images_list',\n", " 'hidden',\n", " 'hide',\n", " 'hide_gridlines',\n", " 'hide_row_col_headers',\n", " 'hide_zero',\n", " 'hlink_count',\n", " 'hlink_refs',\n", " 'horizontal_dpi',\n", " 'hyperlinks',\n", " 'images',\n", " 'index',\n", " 'insert_button',\n", " 'insert_chart',\n", " 'insert_image',\n", " 'insert_textbox',\n", " 'internal_fh',\n", " 'is_chartsheet',\n", " 'is_right_to_left',\n", " 'last_shape_id',\n", " 'leading_zeros',\n", " 'margin_bottom',\n", " 'margin_footer',\n", " 'margin_header',\n", " 'margin_left',\n", " 'margin_right',\n", " 'margin_top',\n", " 'merge',\n", " 'merge_range',\n", " 'name',\n", " 'names',\n", " 'nan_inf_to_errors',\n", " 'orientation',\n", " 'original_row_height',\n", " 'outline_below',\n", " 'outline_changed',\n", " 'outline_col_level',\n", " 'outline_on',\n", " 'outline_right',\n", " 'outline_row_level',\n", " 'outline_settings',\n", " 'outline_style',\n", " 'page_order',\n", " 'page_setup_changed',\n", " 'page_start',\n", " 'page_view',\n", " 'palette',\n", " 'panes',\n", " 'paper_size',\n", " 'previous_row',\n", " 'print_across',\n", " 'print_area',\n", " 'print_area_range',\n", " 'print_comments',\n", " 'print_gridlines',\n", " 'print_headers',\n", " 'print_options_changed',\n", " 'print_row_col_headers',\n", " 'print_scale',\n", " 'protect',\n", " 'protect_options',\n", " 'rel_count',\n", " 'remove_timezone',\n", " 'repeat_col_range',\n", " 'repeat_columns',\n", " 'repeat_row_range',\n", " 'repeat_rows',\n", " 'right_to_left',\n", " 'row_col_headers',\n", " 'row_data_fh',\n", " 'row_data_fh_closed',\n", " 'row_data_filename',\n", " 'row_size_changed',\n", " 'row_sizes',\n", " 'row_spans',\n", " 'rstring',\n", " 'screen_gridlines',\n", " 'select',\n", " 'selected',\n", " 'selections',\n", " 'set_cols',\n", " 'set_column',\n", " 'set_comments_author',\n", " 'set_default_row',\n", " 'set_first_sheet',\n", " 'set_footer',\n", " 'set_h_pagebreaks',\n", " 'set_header',\n", " 'set_landscape',\n", " 'set_margins',\n", " 'set_page_view',\n", " 'set_paper',\n", " 'set_portrait',\n", " 'set_print_scale',\n", " 'set_row',\n", " 'set_rows',\n", " 'set_selection',\n", " 'set_start_page',\n", " 'set_tab_color',\n", " 'set_v_pagebreaks',\n", " 'set_vba_name',\n", " 'set_zoom',\n", " 'shape_hash',\n", " 'shapes',\n", " 'show_comments',\n", " 'show_zeros',\n", " 'sparklines',\n", " 'split_panes',\n", " 'str_table',\n", " 'strings_to_formulas',\n", " 'strings_to_numbers',\n", " 'strings_to_urls',\n", " 'tab_color',\n", " 'table',\n", " 'tables',\n", " 'tmpdir',\n", " 'use_data_bars_2010',\n", " 'validations',\n", " 'vba_codename',\n", " 'vbreaks',\n", " 'vcenter',\n", " 'vertical_dpi',\n", " 'vml_data_id',\n", " 'vml_drawing_links',\n", " 'vml_header_id',\n", " 'vml_shape_id',\n", " 'worksheet_meta',\n", " 'write',\n", " 'write_array_formula',\n", " 'write_blank',\n", " 'write_boolean',\n", " 'write_column',\n", " 'write_comment',\n", " 'write_datetime',\n", " 'write_formula',\n", " 'write_match',\n", " 'write_number',\n", " 'write_rich_string',\n", " 'write_row',\n", " 'write_string',\n", " 'write_url',\n", " 'xls_colmax',\n", " 'xls_rowmax',\n", " 'xls_strmax',\n", " 'zoom',\n", " 'zoom_scale_normal']" ] }, "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dir(worksheet)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.15rc1" } }, "nbformat": 4, "nbformat_minor": 2 }