{ "cells": [ { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", "The experiments here describe medical/family history as they associate with risk measures\n", "Additionally we will have fractional risk assessments\n", "\"\"\"\n", "import pandas as pd\n", "import numpy as np\n", "from pandas_risk import *\n", "dfm = pd.read_gbq(\"SELECT * FROM deid_risk.registered_medical_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n", "dff = pd.read_gbq(\"SELECT * FROM deid_risk.registered_family_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n", "df = pd.read_gbq(\"SELECT person_id, birth_date,city,state,home_owner,race,ethnicity,gender,birth_place,marital_status,orientation,education,employment_status,income,travel_abroad_6_months,active_duty_status FROM deid_risk.registered_dec_01\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "med_cols = np.random.choice(list(set(dfm.columns.tolist()) - set(['person_id'])),3).tolist()\n", "fam_cols = np.random.choice(list(set(dff.columns.tolist()) - set(['person_id'])),3).tolist()\n", "medical = pd.merge(df,dfm[med_cols+['person_id']],on='person_id')\n", "family = pd.merge(df,dff[fam_cols + ['person_id']],on='person_id')\n", "_tmp = pd.merge(dfm[med_cols +['person_id']],dff[fam_cols+['person_id']])\n", "data = pd.merge(df,_tmp,on='person_id')" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
field_countflaggroup_countmarketerprosecutorunique_row_ratio
021full history1153080.9926911.00.987663
118medical1153060.9926741.00.987629
218family1153040.9926561.00.987594
315no-history1153000.9926221.00.987526
43medical-only270.0002320.50.000000
53family-only1460.0012571.00.000551
\n", "
" ], "text/plain": [ " field_count flag group_count marketer prosecutor \\\n", "0 21 full history 115308 0.992691 1.0 \n", "1 18 medical 115306 0.992674 1.0 \n", "2 18 family 115304 0.992656 1.0 \n", "3 15 no-history 115300 0.992622 1.0 \n", "4 3 medical-only 27 0.000232 0.5 \n", "5 3 family-only 146 0.001257 1.0 \n", "\n", " unique_row_ratio \n", "0 0.987663 \n", "1 0.987629 \n", "2 0.987594 \n", "3 0.987526 \n", "4 0.000000 \n", "5 0.000551 " ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.concat([data.deid.evaluate(flag='full history',cols= list(set(data.columns.tolist()) - set(['person_id'])) )\n", " ,medical.deid.evaluate(flag='medical',cols=list( set(medical.columns.tolist() ) - set(['person_id']) ) )\n", " ,family.deid.evaluate(flag='family',cols=list( set(family.columns.tolist() ) - set(['person_id']) ) )\n", " ,df.deid.evaluate(flag='no-history',cols=list( set(df.columns.tolist() ) - set(['person_id']) ) )\n", " , dfm.deid.evaluate(flag='medical-only',cols=med_cols )\n", " , dff.deid.evaluate(flag='family-only',cols=fam_cols )\n", " ],ignore_index=True)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from __future__ import division\n", "def evaluate(df) :\n", " cols = list(set(df.columns.tolist()) - set(['person_id']))\n", " \n", " portions = np.round(np.random.random_sample(4),3).tolist() + np.arange(5,105,5).tolist()\n", " \n", " N = df.shape[0] - 1\n", " portions = np.divide(np.multiply(portions,N),100).astype(np.int64)\n", " portions = np.unique([n for n in portions if n > 1])\n", " \n", " r = pd.DataFrame()\n", " for num_rows in portions :\n", " \n", " indices = np.random.choice(N,num_rows,replace=False)\n", "# print (indices.size / N)\n", " flag = \" \".join([str( np.round(100*indices.size/ N,2)),'%'])\n", " r = r.append(df.loc[indices].deid.evaluate(cols=cols,flag=flag,min_group_size=2))\n", " return r" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
field_countflaggroup_countmarketerprosecutorunique_row_ratio
011UNFLAGGED1148860.9890581.00.980535
\n", "
" ], "text/plain": [ " field_count flag group_count marketer prosecutor unique_row_ratio\n", "0 11 UNFLAGGED 114886 0.989058 1.0 0.980535" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cols = list(set (df.columns.tolist()) - set(['person_id']))\n", "df[['race','state','gender_identity','ethnicity','marital_status','education','orientation','sex_at_birth','birth_date','travel_abroad_6_months','active_duty_status']].deid.evaluate()" ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['person_id',\n", " 'HearingVision_FarSightedness',\n", " 'HearingVision_Glaucoma',\n", " 'Digestive_Pancreatitis']" ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#\n", "# This is the merge with medical history\n", "\n", "cols = ['person_id'] + np.random.choice(dfm.columns[1:],3,replace=False).tolist()\n", "p = pd.merge(df,dfm[cols],on='person_id')\n", "cols\n", "# # cols = list(set(p.columns.tolist()) - set(['person_id']))\n", "# evaluate(p) #p.deid.explore(cols=cols,num_runs=100)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "cols = list( set(dfm.columns.tolist()) - set(['person_id']))\n", "cols = np.random.choice(cols,3,replace=False).tolist()\n", "p = pd.merge(dfm[['person_id']+cols],df)\n", "fcols = list(set(p.columns.tolist()) - set(['person_id']))\n", "# dfm[cols].deid.evaluate(cols=list( set(cols) - set(['person_id'])))" ] }, { "cell_type": "markdown", "metadata": { "variables": { " \" ; \".join(cols)": "InfectiousDiseases_HepatitisC ; Cancer_StomachCancer ; Circulatory_Hypertension", " p.shape[0] ": "116157", " p[fcols].deid.evaluate() ": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
field_countflaggroup_countmarketerprosecutorunique_row_ratio
037UNFLAGGED1153970.9934571.00.98886
\n
" } }, "source": [ "### Medical History\n", "\n", " We randomly select three a tributes {{ \" ; \".join(cols)}} . \n", " The dataset associated risk evaluation contains {{ p.shape[0] }} records\n", "{{ p[fcols].deid.evaluate() }}\n", "\n", " \n" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['person_id',\n", " 'InfectiousDiseases_Tuberculosis',\n", " 'SkeletalMuscular_Fibromyalgia',\n", " 'Cancer_ProstateCancer']" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cols" ] }, { "cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3" ] }, "execution_count": 67, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# dfm[cols[1:]].head()\n", "np.sum(dfm.fillna(' ').groupby(cols[1:],as_index=False).size().values <= 1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.15rc1" } }, "nbformat": 4, "nbformat_minor": 2 }