You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
privacykit/notebooks/registered-tier.ipynb

860 lines
33 KiB
Plaintext

{
"cells": [
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"\"\"\"\n",
"This notebook is designed to run experiments around demographics on registered tier\n",
"The \n",
"\"\"\"\n",
"import pandas as pd\n",
"import numpy as np\n",
"from pandas_risk import *\n",
"\n",
"ATTRIBUTES = ['race','ethnicity','birth_date','state','city','zip','marital_status','education','language','home_owner','income','employment_status','living_situation','active_duty_status','gender_identity','birth_place','death_date','death_cause','orientation']\n",
"dfs = pd.read_csv('scenario-settings.csv')\n",
"dfc = pd.read_gbq(\"SELECT * FROM deid_risk.registered_dec_01\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"cols_o = dfs.loc[(dfs.fo & dfs.fi) ==1].feature.tolist()\n",
"cols_i = dfs.loc[(dfs.fo + dfs.fi )>=1 ].feature.tolist()\n",
"cols_a = dfs.feature.tolist()\n",
"\n",
"cols_v = ['birth_date','gender_identity','race','state','city','birth_place'] #-- voter registration\n",
"#remove the dates fields because dates are shifted\n",
"cols_o = [i for i in cols_o if i not in ['birth_date','death_date']]\n",
"cols_i = [i for i in cols_i if i not in ['birth_date','death_date']]\n",
"cols_a = [i for i in cols_a if i not in ['birth_date','death_date']]\n",
"cols_v = [i for i in cols_v if i not in ['birth_date', 'death_date']]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# print(dfs)\n",
"# print(cols_o)\n",
"# print(cols_i)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>field_count</th>\n",
" <th>flag</th>\n",
" <th>group_count</th>\n",
" <th>marketer</th>\n",
" <th>prosecutor</th>\n",
" <th>unique_row_ratio</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>8</td>\n",
" <td>high-conj</td>\n",
" <td>6532</td>\n",
" <td>0.056234</td>\n",
" <td>1.0</td>\n",
" <td>0.021368</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>11</td>\n",
" <td>high-disj</td>\n",
" <td>47447</td>\n",
" <td>0.408473</td>\n",
" <td>1.0</td>\n",
" <td>0.278554</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>16</td>\n",
" <td>all</td>\n",
" <td>60718</td>\n",
" <td>0.522724</td>\n",
" <td>1.0</td>\n",
" <td>0.408189</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>5</td>\n",
" <td>voter-reg</td>\n",
" <td>1316</td>\n",
" <td>0.011329</td>\n",
" <td>1.0</td>\n",
" <td>0.002944</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" field_count flag group_count marketer prosecutor unique_row_ratio\n",
"0 8 high-conj 6532 0.056234 1.0 0.021368\n",
"1 11 high-disj 47447 0.408473 1.0 0.278554\n",
"2 16 all 60718 0.522724 1.0 0.408189\n",
"3 5 voter-reg 1316 0.011329 1.0 0.002944"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"r = pd.concat([dfc[cols_o].deid.evaluate(),dfc[cols_i].deid.evaluate(),dfc[cols_a].deid.evaluate(),dfc[cols_v].deid.evaluate() ])\n",
"r.index = np.arange(r.shape[0]).astype(np.int64)\n",
"r['flag']=['high-conj','high-disj','all','voter-reg']\n",
"\n",
"r\n"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEuCAYAAAB1QVLBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFxVJREFUeJzt3X+U3XV95/HnK+FHtohSIcsiAZO6UYGFEjdEUY/yU0E0bP3RwuourFa2q1A89lSBFqzUY6ly3HqUrlKbo9a1ILCWGKIgVVyRU8gAWWzMcow0QmArETGy7okQfe8f906cjHcydyZ3cplPno9z5uR+P9/PzH3nnpnXfOf9/d7vJ1WFJKktc4ZdgCRp8Ax3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoP2GtYTH3TQQbVw4cJhPb0kzUp33333D6tq/mTzhhbuCxcuZGRkZFhPL0mzUpLv9zPPtowkNchwl6QGGe6S1KCh9dwlteupp55i06ZNbN26ddilzFrz5s1jwYIF7L333tP6fMNd0sBt2rSJ/fffn4ULF5Jk2OXMOlXFY489xqZNm1i0aNG0voZtGUkDt3XrVg488ECDfZqScOCBB+7SXz6Gu6QZYbDvml19/Qx3SWqQPXdpN1h40U3DLqEvG684Y0a+7qD//zNV50Ruu+02rrzySlatWtXX/LVr1/LII4/wmte8ZoYrm5hH7pK0E9u2bZvy56xdu5bVq1fP+PPsjOEuqUkbN27khS98Ieeeey7Pf/7zefOb38ytt97Ky172MhYvXsxdd93FXXfdxfHHH8+SJUt46Utfyv333w/Apz/9aZYvX85JJ53EySefvMPXXbNmDUuWLOF73/seP/3pT3nrW9/KsmXLWLJkCTfeeCNPPvkkl112Gddeey3HHnss1157bc95kz3PrrItI6lZGzZs4LrrrmPFihUcd9xxfP7zn+f2229n5cqVfPCDH+Szn/0s3/zmN9lrr7249dZbueSSS7jhhhsAuOeee7jvvvt49rOfzW233QbAHXfcwQUXXMCNN97I4YcfziWXXMJJJ53EihUr+PGPf8yyZcs45ZRTuPzyyxkZGeHjH/84wITzxj/PIBnukpq1aNEijj76aACOOuooTj75ZJJw9NFHs3HjRrZs2cI555zDd7/7XZLw1FNPbf/cU089dYfAXb9+Peeddx633HILz3nOcwC45ZZbWLlyJVdeeSXQuQT0wQcf/JU6djZv/PMMiuEuqVn77rvv9sdz5szZvj1nzhy2bdvGpZdeyoknnsgXv/hFNm7cyAknnLB9/n777bfD1zrkkEPYunUr99577/ZwrypuuOEGXvCCF+ww984779xhe2fzxj/PoPTVc09yWpL7k2xIclGP/ecm2ZxkbffjdwdfqiQN1pYtWzj00EOBTv97Zw444ABuuukmLr744u1tmle/+tV87GMfo6oAuPfeewHYf//9eeKJJ7Z/7kTzZtKkR+5J5gJXAacCm4A1SVZW1XfGTb22qs6fgRolzXK7+9LFfr3nPe/hnHPO4QMf+ABnnDF5jQcffDCrVq3i9NNPZ8WKFVx66aW8613v4phjjuEXv/gFixYtYtWqVZx44olcccUVHHvssVx88cUTzptJGf1NMuGE5HjgT6rq1d3tiwGq6s/GzDkXWDqVcF+6dGm5WIf2FHvade7r16/niCOOGMjX2pP1eh2T3F1VSyf73H7aMocCD43Z3tQdG+8NSe5Lcn2Sw3p9oSTnJRlJMrJ58+Y+nlqSNB2Dus79S8DCqjoG+CrwmV6TqurqqlpaVUvnz590CUBJ0jT1E+4PA2OPxBd0x7arqseq6mfdzU8B/3Yw5UmarSZr+WrndvX16yfc1wCLkyxKsg9wFrBy7IQkh4zZXA6s36WqJM1q8+bN47HHHjPgp2n0fu7z5s2b9teY9GqZqtqW5HzgZmAusKKq1iW5HBipqpXA7ydZDmwDfgScO+2KJM16CxYsYNOmTXhubfpGV2Karr7exFRVq4HV48YuG/P4YuDiaVchqSl77733tFcQ0mB44zBJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhq0Vz+TkpwGfBSYC3yqqq6YYN4bgOuB46pqZGBVaigWXnTTsEvoy8Yrzhh2CdLTzqRH7knmAlcBpwNHAmcnObLHvP2BC4E7B12kJGlq+mnLLAM2VNUDVfUkcA1wZo95fwr8ObB1gPVJkqahn3A/FHhozPam7th2SV4EHFZVO/07Psl5SUaSjGzevHnKxUqS+rPLJ1STzAE+AvzBZHOr6uqqWlpVS+fPn7+rTy1JmkA/4f4wcNiY7QXdsVH7A/8GuC3JRuAlwMokSwdVpCRpavoJ9zXA4iSLkuwDnAWsHN1ZVVuq6qCqWlhVC4F/AJZ7tYwkDc+k4V5V24DzgZuB9cAXqmpdksuTLJ/pAiVJU9fXde5VtRpYPW7ssgnmnrDrZUmSdoXvUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN6ivck5yW5P4kG5Jc1GP/7yX5dpK1SW5PcuTgS5Uk9WvScE8yF7gKOB04Eji7R3h/vqqOrqpjgQ8BHxl4pZKkvvVz5L4M2FBVD1TVk8A1wJljJ1TVT8Zs7gfU4EqUJE3VXn3MORR4aMz2JuDF4ycleSfwbmAf4KSBVCdJmpaBnVCtqquq6nnAe4E/7jUnyXlJRpKMbN68eVBPLUkap59wfxg4bMz2gu7YRK4B/l2vHVV1dVUtraql8+fP779KSdKU9BPua4DFSRYl2Qc4C1g5dkKSxWM2zwC+O7gSJUlTNWnPvaq2JTkfuBmYC6yoqnVJLgdGqmolcH6SU4CngMeBc2ayaEnSzvVzQpWqWg2sHjd22ZjHFw64LknSLvAdqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNaivcE9yWpL7k2xIclGP/e9O8p0k9yX5+yTPHXypkqR+TRruSeYCVwGnA0cCZyc5cty0e4GlVXUMcD3woUEXKknqXz9H7suADVX1QFU9CVwDnDl2QlV9var+X3fzH4AFgy1TkjQV/YT7ocBDY7Y3dccm8jbgy712JDkvyUiSkc2bN/dfpSRpSgZ6QjXJW4ClwId77a+qq6tqaVUtnT9//iCfWpI0xl59zHkYOGzM9oLu2A6SnAL8EfDKqvrZYMqTJE1HP0fua4DFSRYl2Qc4C1g5dkKSJcAngeVV9ejgy5QkTcWk4V5V24DzgZuB9cAXqmpdksuTLO9O+zDwDOC6JGuTrJzgy0mSdoN+2jJU1Wpg9bixy8Y8PmXAdUmSdoHvUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUF9hXuS05Lcn2RDkot67H9FknuSbEvyxsGXKUmaiknDPclc4CrgdOBI4OwkR46b9iBwLvD5QRcoSZq6vfqYswzYUFUPACS5BjgT+M7ohKra2N33ixmoUZI0Rf20ZQ4FHhqzvak7NmVJzksykmRk8+bN0/kSkqQ+7NYTqlV1dVUtraql8+fP351PLUl7lH7C/WHgsDHbC7pjkqSnqX7CfQ2wOMmiJPsAZwErZ7YsSdKumDTcq2obcD5wM7Ae+EJVrUtyeZLlAEmOS7IJeBPwySTrZrJoSdLO9XO1DFW1Glg9buyyMY/X0GnXSJKeBnyHqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KC+7i0zWyy86KZhl9CXjVecMewSJDXOI3dJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqUFMrMUnaM7jq2uT6OnJPclqS+5NsSHJRj/37Jrm2u//OJAsHXagkqX+ThnuSucBVwOnAkcDZSY4cN+1twONV9a+B/wr8+aALlST1r58j92XAhqp6oKqeBK4Bzhw350zgM93H1wMnJ8ngypQkTUU/PfdDgYfGbG8CXjzRnKralmQLcCDww7GTkpwHnNfd/L9J7p9O0bvZQYz7f+yq7Nl/1/h6Do6v5WDNltfzuf1M2q0nVKvqauDq3fmcuyrJSFUtHXYdrfD1HBxfy8Fq7fXspy3zMHDYmO0F3bGec5LsBTwLeGwQBUqSpq6fcF8DLE6yKMk+wFnAynFzVgLndB+/EfhaVdXgypQkTcWkbZluD/184GZgLrCiqtYluRwYqaqVwF8Df5NkA/AjOr8AWjGr2kizgK/n4PhaDlZTr2c8wJak9nj7AUlqkOEuSQ0y3CWpQYa7JDXIu0JqxiR5ZlX9JMmze+wu4CdV9fPdXZcEkOT1PYa3AN+uqkd3dz2D5tUyYyR5S1V9Lsm7e+wuOpd5rqyqx3dzabNSklVV9dok/0Tn9Rt/v6FnAH9VVZfs/upmlyRP0HkNf2UXUFX1zN1c0qyX5CbgeODr3aETgLuBRcDlVfU3QyptIDxy39F+3X/3n2D/IuC/AC/ZPeXMblX12u6/i3rt795x9B8Bw30SVTXR96Smby/giKr6AUCSg4HP0rl31v8EDPdWVNUnu/++f6I53TdvaQqSvAxYW1U/TfIW4EXAX1TVg8ARw61udpigtbVdVf1od9XSkMNGg73r0e7Yj5I8NayiBsW2TA9J5gNvBxYy5hdgVb11WDXNZknuA34TOAb4NPAp4Ler6pXDrGs2maC1NbpdVfUbQylsFkvyl8DhwHXdoTfSubvtHwKrqurEYdU2CIZ7D0nuAL5Jp/+2/YRfVd0wtKJmsST3VNWLklwGPFxVfz06NuzaZqPuUfxiYN7oWFV9Y3gVzU7dNSdeD7y8O/Qt4IZW7otlW6a3X6uq9w67iIY8keRi4C3AK5LMAfYeck2zUpLfBS6kc3fWtXTO/9wBnDzMumajqqokI8CWqro1ya/ROcn/xJBLGwivc+9tVZLXDLuIhvwO8DPgbVX1z3SC6cPDLWnWuhA4Dvh+t22whM7le5qiJG+ns3LcJ7tDhwJ/N7yKBsu2TA/dy872A54ERk+seLmZhi7Jmqo6Lsla4MVV9bMk66rqqGHXNtt0X8NlwJ1VtaQ79u2qOnq4lQ2GbZkevOxsMJLcXlUv73GNttdmT9+mJAfQOcL8apLHge8PuabZ6mdV9eTocs/dhYaaOdr1yH0CSZYDr+hu3lZVq4ZZjzReklfSWfXsK93F6zUFST4E/Bj4j8AFwDuA71TVHw21sAEx3HtIcgWdvuZ/7w6dTWdhkouHV9Xs47XZejrrnth/G/AqOn9N3gx8qpWrZQz3HrrXZR9bVb/obs8F7q2qY4Zb2ewy7trsw4HHu48PAB6c6J2r0kzr/kx/tqrePOxaZopXy0zsgDGPnzW0KmaxqlrUfXPNrcDrquqgqjoQeC1wy3Cr056se8O653bXhW6SJ1R7+zPg3iRfp3Ok+QrgouGWNKu9pKrePrpRVV/u9julYXoA+FaSlcBPRwer6iPDK2lwDPcequpvk9xGp+8O8N7u9dmankeS/DHwue72m4FHhliPBPC97sccJr5Z4Kxlz72HJL8FfK2qtnS3DwBOqKpm3uCwO3VPrL6PX1599A06t1T1hKqeFpL8q9YO4Az3HpKsrapjx43dO/pGB01fkkOq6v8Muw5prBbvdeQJ1d56vS62sAbjpmEXIPUwfiGZWc9w720kyUeSPK/78RE6d4jUrmvuh0hN+KthFzBohntvF9C5r8y1wDXAVuCdQ62oHc39EGn2STI3yf8e3a6qvxxmPTPBnvsk7BEPRvdNIwez4+InDw6vIu3pktwIXNDq96F95MndRGdZOE1TkgvoXC3zAzqLn4TOO1d9x6+G6deBdUnuYsfr3JcPr6TBMdwnZ494110IvKCqHht2IdIYlw67gJlkuE/OHvGuewgXlNDTTFV9I8lzgcVjVmKaO+y6BsWe+wTsEe+6JO/uPjwKeAGdFtfPRve38jZvzU7dlZjOA55dVc9Lshj4RFU1sWShR+492CMemNG3dD/Y/din+yE9HbyT7kpMAFX13ST/crglDY7h3ps94gGoqvcPuwZpJ5peiclw780e8QAl+RK/+kOzBRgBPllVW3d/VRLfSHIJ8C+SnEpnJaYvDbmmgbHnPoY94pmR5KPAfOBvu0O/A/yETuA/s6r+w7Bq056r10pMVdXMBRQeue/IHvHMeGlVHTdm+0tJ1lTVcUnWDa0q7ekuqKqPMuaKuCQXdsdmPY/cNeOSrAdePXq1UZLD6RwlHeHdNjUsve4E2dL3o0fuPdgjHrg/AG5P8j06f/4uAt6RZD/gM0OtTHucJGcD/x5Y1F2FadQzgWbWGPDIvQd7xIOXZF/ghd3N+/0FqWHpvnFpEZ3lNMcun/kEcF9VbRtKYQNmuPcw2g/uNZZkXVUdNazaZpMkJ1XV15K8vtf+qvofu7smaawkB/PL5TTvqqpHh1nPINmW6e0ZSQ4f1yN+Rnffk8Mra9Z5JfA14HXd7dEjidE3hRnuGpokbwKuBG6j8z35sSR/WFXXD7WwAfHIvYckrwE+QWfx3O09YjrfBG+vqr8YXnWzT5J5wBuAhfzygKKq6vKhFaU9XpL/BZw6erSeZD5wa1X95nArGwyP3HuoqtXd+0z06hEb7FP3d8CPgXvoLHwCDb0TULPWnHFtmMdoaAEjw32MnfSIn5fEHvH0Laiq04ZdhDTOl5PczI4XTqweYj0DZbjvyB7xzLgjydFV9e1hFyKN8SjwOeDY7vbVVfXFIdYzUPbce7BHPBhJvk3nl+JewGLgATq3cwid19O7bGpokrwP+G0617ZfC1xXVT8YblWDY7j3kOQr/LJH/PPucHlvmanpXk88oar6/u6qRZpIkmPotGTeAGyqqlOGXNJA2JbpzR7xABjemiUeBf6ZzgnVZu7n3syZ4QG7I8nRwy5C0sxJ8o4ktwF/DxxI5zLnZlqFHrmPMa5H/J+S2COW2nUY8K6qWjvsQmaCPfcx7BFLaoXhLkkNsucuSQ0y3CWpQYa79jhJfj/J+iQPJ/n4sOuRZoJXy2hP9A7glO7H0iHXIs0Ij9y1R0nyCeA3gC8Dvz5m/HVJ7kxyb5Jbu4s4kGR+kq8mWZfkU0m+n+SgIZUv9c1w1x6lqn4PeAQ4EXh8zK7bgZd0F0e+BnhPd/x9wNe6q29dDxy+G8uVps22jNSxALg2ySHAPsA/dcdfDvwWQFV9JcnjE3y+9LTikbvU8THg41V1NPCfgXlDrkfaJYa71PEs4OHu43PGjH+Lzm1hSfIqxvTppaczw13q+BPguiR3Az8cM/5+4FVJ/hF4E527Bz6x+8uTpsbbD0g7kWRf4OdVtS3J8cB/q6pjJ/s8adg8oSrt3OHAF5LMAZ4E3j7keqS+eOQuSQ2y5y5JDTLcJalBhrskNchwl6QGGe6S1KD/DzwQEyuRP8dtAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"fig_o = r.plot(kind='bar',x='flag',y=['marketer']).get_figure()\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"writer = pd.ExcelWriter('out-116kpatients-phase-1.xlsx',engine='xlsxwriter')\n",
"r.to_excel(writer,'phase-1')\n",
"writer.save()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>feature</th>\n",
" <th>fi</th>\n",
" <th>fo</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>race</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>ethnicity</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>birth_date</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>city</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>state</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>marital_status</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>education</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>language</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>home_owner</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>income</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>employment_status</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>living_situation</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>active_duty_status</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>gender_identity</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>birth_place</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>death_date</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>death_cause</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>orientation</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" feature fi fo\n",
"0 race 1 1\n",
"1 ethnicity 1 1\n",
"2 birth_date 1 1\n",
"3 city 1 1\n",
"4 state 1 1\n",
"5 marital_status 1 1\n",
"6 education 1 0\n",
"7 language 0 0\n",
"8 home_owner 1 1\n",
"9 income 0 1\n",
"10 employment_status 1 0\n",
"11 living_situation 0 0\n",
"12 active_duty_status 0 0\n",
"13 gender_identity 1 1\n",
"14 birth_place 0 0\n",
"15 death_date 1 1\n",
"16 death_cause 1 1\n",
"17 orientation 0 0"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfs\n"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"names = pd.read_csv('family-history.csv').name.tolist()\n",
"path ='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json'\n",
"sql = \"\"\"\n",
"SELECT * FROM deid_risk.registered_medical_history_dec_001\n",
"\"\"\"\n",
"dfm = pd.read_gbq(\"SELECT * FROM deid_risk.registered_medical_history_dec_001\",private_key=path,dialect='standard')"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(0.9343780009344719, 1.269831148073964)"
]
},
"execution_count": 69,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cols = list( set(dfm.columns.tolist()) - set(['person_id']))\n",
"r = pd.DataFrame(dfm[cols].count(),columns=['counts'])\n",
"r['attributes'] = r.index\n",
"r['rate'] = 100*(r.counts / dfm.shape[0])\n",
"r.rate.mean(),np.sqrt(r.rate.var())"
]
},
{
"cell_type": "code",
"execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
"writer = pd.ExcelWriter('/home/steve/tmp/simple.xlsx', engine='xlsxwriter')\n",
"r.to_excel(writer,sheet_name='p1')\n",
"workbook = writer.book\n",
"worksheet = workbook.add_worksheet()\n",
"b = pd.DataFrame({\"id\":np.random.choice(10,30)})"
]
},
{
"cell_type": "code",
"execution_count": 80,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['__class__',\n",
" '__delattr__',\n",
" '__dict__',\n",
" '__doc__',\n",
" '__format__',\n",
" '__getattribute__',\n",
" '__hash__',\n",
" '__init__',\n",
" '__module__',\n",
" '__new__',\n",
" '__reduce__',\n",
" '__reduce_ex__',\n",
" '__repr__',\n",
" '__setattr__',\n",
" '__sizeof__',\n",
" '__str__',\n",
" '__subclasshook__',\n",
" '__weakref__',\n",
" '_assemble_xml_file',\n",
" '_button_params',\n",
" '_calculate_spans',\n",
" '_calculate_x_split_width',\n",
" '_check_dimensions',\n",
" '_comment_params',\n",
" '_convert_date_time',\n",
" '_convert_name_area',\n",
" '_csv_join',\n",
" '_encode_password',\n",
" '_escape_attributes',\n",
" '_escape_data',\n",
" '_escape_url',\n",
" '_extract_filter_tokens',\n",
" '_get_palette_color',\n",
" '_get_range_data',\n",
" '_initialize',\n",
" '_isinf',\n",
" '_isnan',\n",
" '_opt_close',\n",
" '_opt_reopen',\n",
" '_parse_filter_expression',\n",
" '_parse_filter_tokens',\n",
" '_position_object_emus',\n",
" '_position_object_pixels',\n",
" '_prepare_chart',\n",
" '_prepare_header_image',\n",
" '_prepare_header_vml_objects',\n",
" '_prepare_image',\n",
" '_prepare_shape',\n",
" '_prepare_tables',\n",
" '_prepare_vml_objects',\n",
" '_set_filehandle',\n",
" '_set_icon_props',\n",
" '_set_spark_color',\n",
" '_set_xml_writer',\n",
" '_size_col',\n",
" '_size_row',\n",
" '_sort_pagebreaks',\n",
" '_table_function_to_formula',\n",
" '_write',\n",
" '_write_array_formula',\n",
" '_write_auto_filter',\n",
" '_write_autofilters',\n",
" '_write_blank',\n",
" '_write_boolean',\n",
" '_write_brk',\n",
" '_write_cell',\n",
" '_write_cell_array_formula',\n",
" '_write_cell_value',\n",
" '_write_cf_rule',\n",
" '_write_cfvo',\n",
" '_write_col_breaks',\n",
" '_write_col_info',\n",
" '_write_color',\n",
" '_write_color_axis',\n",
" '_write_color_first',\n",
" '_write_color_high',\n",
" '_write_color_last',\n",
" '_write_color_low',\n",
" '_write_color_markers',\n",
" '_write_color_negative',\n",
" '_write_color_scale',\n",
" '_write_color_series',\n",
" '_write_cols',\n",
" '_write_conditional_formats',\n",
" '_write_conditional_formatting',\n",
" '_write_conditional_formatting_2010',\n",
" '_write_custom_filter',\n",
" '_write_custom_filters',\n",
" '_write_data_bar',\n",
" '_write_data_bar_ext',\n",
" '_write_data_validation',\n",
" '_write_data_validations',\n",
" '_write_datetime',\n",
" '_write_dimension',\n",
" '_write_drawing',\n",
" '_write_drawings',\n",
" '_write_empty_row',\n",
" '_write_ext',\n",
" '_write_ext_list',\n",
" '_write_ext_list_data_bars',\n",
" '_write_ext_list_sparklines',\n",
" '_write_filter',\n",
" '_write_filter_column',\n",
" '_write_filters',\n",
" '_write_font',\n",
" '_write_formula',\n",
" '_write_formula_1',\n",
" '_write_formula_2',\n",
" '_write_formula_element',\n",
" '_write_freeze_panes',\n",
" '_write_header_footer',\n",
" '_write_hyperlink_external',\n",
" '_write_hyperlink_internal',\n",
" '_write_hyperlinks',\n",
" '_write_icon_set',\n",
" '_write_legacy_drawing',\n",
" '_write_legacy_drawing_hf',\n",
" '_write_merge_cell',\n",
" '_write_merge_cells',\n",
" '_write_number',\n",
" '_write_odd_footer',\n",
" '_write_odd_header',\n",
" '_write_optimized_sheet_data',\n",
" '_write_outline_pr',\n",
" '_write_page_margins',\n",
" '_write_page_set_up_pr',\n",
" '_write_page_setup',\n",
" '_write_panes',\n",
" '_write_phonetic_pr',\n",
" '_write_print_options',\n",
" '_write_rich_string',\n",
" '_write_row',\n",
" '_write_row_breaks',\n",
" '_write_rows',\n",
" '_write_rstring_color',\n",
" '_write_selection',\n",
" '_write_selections',\n",
" '_write_sheet_data',\n",
" '_write_sheet_format_pr',\n",
" '_write_sheet_pr',\n",
" '_write_sheet_protection',\n",
" '_write_sheet_view',\n",
" '_write_sheet_views',\n",
" '_write_single_row',\n",
" '_write_spark_color',\n",
" '_write_sparkline_group',\n",
" '_write_sparkline_groups',\n",
" '_write_sparklines',\n",
" '_write_split_panes',\n",
" '_write_string',\n",
" '_write_tab_color',\n",
" '_write_table_part',\n",
" '_write_table_parts',\n",
" '_write_token_as_string',\n",
" '_write_underline',\n",
" '_write_url',\n",
" '_write_vert_align',\n",
" '_write_worksheet',\n",
" '_write_x14_axis_color',\n",
" '_write_x14_border_color',\n",
" '_write_x14_cf_rule',\n",
" '_write_x14_cfvo',\n",
" '_write_x14_data_bar',\n",
" '_write_x14_negative_border_color',\n",
" '_write_x14_negative_fill_color',\n",
" '_xml_close',\n",
" '_xml_data_element',\n",
" '_xml_declaration',\n",
" '_xml_empty_tag',\n",
" '_xml_empty_tag_unencoded',\n",
" '_xml_end_tag',\n",
" '_xml_formula_element',\n",
" '_xml_inline_string',\n",
" '_xml_number_element',\n",
" '_xml_rich_inline_string',\n",
" '_xml_rich_si_element',\n",
" '_xml_si_element',\n",
" '_xml_start_tag',\n",
" '_xml_start_tag_unencoded',\n",
" '_xml_string_element',\n",
" 'activate',\n",
" 'active',\n",
" 'active_pane',\n",
" 'add_sparkline',\n",
" 'add_table',\n",
" 'autofilter',\n",
" 'autofilter_area',\n",
" 'autofilter_ref',\n",
" 'black_white',\n",
" 'buttons_list',\n",
" 'center_horizontally',\n",
" 'center_vertically',\n",
" 'charts',\n",
" 'col_formats',\n",
" 'col_size_changed',\n",
" 'col_sizes',\n",
" 'colinfo',\n",
" 'comments',\n",
" 'comments_author',\n",
" 'comments_list',\n",
" 'comments_visible',\n",
" 'cond_formats',\n",
" 'conditional_format',\n",
" 'constant_memory',\n",
" 'data_bars_2010',\n",
" 'data_validation',\n",
" 'date_1904',\n",
" 'default_col_pixels',\n",
" 'default_date_format',\n",
" 'default_row_height',\n",
" 'default_row_pixels',\n",
" 'default_row_zeroed',\n",
" 'default_url_format',\n",
" 'dim_colmax',\n",
" 'dim_colmin',\n",
" 'dim_rowmax',\n",
" 'dim_rowmin',\n",
" 'draft_quality',\n",
" 'drawing',\n",
" 'drawing_links',\n",
" 'dxf_priority',\n",
" 'escapes',\n",
" 'excel2003_style',\n",
" 'excel_version',\n",
" 'ext_sheets',\n",
" 'external_comment_links',\n",
" 'external_drawing_links',\n",
" 'external_hyper_links',\n",
" 'external_table_links',\n",
" 'external_vml_links',\n",
" 'fh',\n",
" 'fileclosed',\n",
" 'filter_cols',\n",
" 'filter_column',\n",
" 'filter_column_list',\n",
" 'filter_on',\n",
" 'filter_range',\n",
" 'filter_type',\n",
" 'fit_height',\n",
" 'fit_page',\n",
" 'fit_to_pages',\n",
" 'fit_width',\n",
" 'footer',\n",
" 'footer_images',\n",
" 'freeze_panes',\n",
" 'get_name',\n",
" 'has_comments',\n",
" 'has_header_vml',\n",
" 'has_vml',\n",
" 'hbreaks',\n",
" 'hcenter',\n",
" 'header',\n",
" 'header_footer_aligns',\n",
" 'header_footer_changed',\n",
" 'header_footer_scales',\n",
" 'header_images',\n",
" 'header_images_list',\n",
" 'hidden',\n",
" 'hide',\n",
" 'hide_gridlines',\n",
" 'hide_row_col_headers',\n",
" 'hide_zero',\n",
" 'hlink_count',\n",
" 'hlink_refs',\n",
" 'horizontal_dpi',\n",
" 'hyperlinks',\n",
" 'images',\n",
" 'index',\n",
" 'insert_button',\n",
" 'insert_chart',\n",
" 'insert_image',\n",
" 'insert_textbox',\n",
" 'internal_fh',\n",
" 'is_chartsheet',\n",
" 'is_right_to_left',\n",
" 'last_shape_id',\n",
" 'leading_zeros',\n",
" 'margin_bottom',\n",
" 'margin_footer',\n",
" 'margin_header',\n",
" 'margin_left',\n",
" 'margin_right',\n",
" 'margin_top',\n",
" 'merge',\n",
" 'merge_range',\n",
" 'name',\n",
" 'names',\n",
" 'nan_inf_to_errors',\n",
" 'orientation',\n",
" 'original_row_height',\n",
" 'outline_below',\n",
" 'outline_changed',\n",
" 'outline_col_level',\n",
" 'outline_on',\n",
" 'outline_right',\n",
" 'outline_row_level',\n",
" 'outline_settings',\n",
" 'outline_style',\n",
" 'page_order',\n",
" 'page_setup_changed',\n",
" 'page_start',\n",
" 'page_view',\n",
" 'palette',\n",
" 'panes',\n",
" 'paper_size',\n",
" 'previous_row',\n",
" 'print_across',\n",
" 'print_area',\n",
" 'print_area_range',\n",
" 'print_comments',\n",
" 'print_gridlines',\n",
" 'print_headers',\n",
" 'print_options_changed',\n",
" 'print_row_col_headers',\n",
" 'print_scale',\n",
" 'protect',\n",
" 'protect_options',\n",
" 'rel_count',\n",
" 'remove_timezone',\n",
" 'repeat_col_range',\n",
" 'repeat_columns',\n",
" 'repeat_row_range',\n",
" 'repeat_rows',\n",
" 'right_to_left',\n",
" 'row_col_headers',\n",
" 'row_data_fh',\n",
" 'row_data_fh_closed',\n",
" 'row_data_filename',\n",
" 'row_size_changed',\n",
" 'row_sizes',\n",
" 'row_spans',\n",
" 'rstring',\n",
" 'screen_gridlines',\n",
" 'select',\n",
" 'selected',\n",
" 'selections',\n",
" 'set_cols',\n",
" 'set_column',\n",
" 'set_comments_author',\n",
" 'set_default_row',\n",
" 'set_first_sheet',\n",
" 'set_footer',\n",
" 'set_h_pagebreaks',\n",
" 'set_header',\n",
" 'set_landscape',\n",
" 'set_margins',\n",
" 'set_page_view',\n",
" 'set_paper',\n",
" 'set_portrait',\n",
" 'set_print_scale',\n",
" 'set_row',\n",
" 'set_rows',\n",
" 'set_selection',\n",
" 'set_start_page',\n",
" 'set_tab_color',\n",
" 'set_v_pagebreaks',\n",
" 'set_vba_name',\n",
" 'set_zoom',\n",
" 'shape_hash',\n",
" 'shapes',\n",
" 'show_comments',\n",
" 'show_zeros',\n",
" 'sparklines',\n",
" 'split_panes',\n",
" 'str_table',\n",
" 'strings_to_formulas',\n",
" 'strings_to_numbers',\n",
" 'strings_to_urls',\n",
" 'tab_color',\n",
" 'table',\n",
" 'tables',\n",
" 'tmpdir',\n",
" 'use_data_bars_2010',\n",
" 'validations',\n",
" 'vba_codename',\n",
" 'vbreaks',\n",
" 'vcenter',\n",
" 'vertical_dpi',\n",
" 'vml_data_id',\n",
" 'vml_drawing_links',\n",
" 'vml_header_id',\n",
" 'vml_shape_id',\n",
" 'worksheet_meta',\n",
" 'write',\n",
" 'write_array_formula',\n",
" 'write_blank',\n",
" 'write_boolean',\n",
" 'write_column',\n",
" 'write_comment',\n",
" 'write_datetime',\n",
" 'write_formula',\n",
" 'write_match',\n",
" 'write_number',\n",
" 'write_rich_string',\n",
" 'write_row',\n",
" 'write_string',\n",
" 'write_url',\n",
" 'xls_colmax',\n",
" 'xls_rowmax',\n",
" 'xls_strmax',\n",
" 'zoom',\n",
" 'zoom_scale_normal']"
]
},
"execution_count": 80,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dir(worksheet)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.15rc1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}