{
"cells": [
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"\"\"\"\n",
"The experiments here describe medical/family history as they associate with risk measures\n",
"Additionally we will have fractional risk assessments\n",
"\"\"\"\n",
"import pandas as pd\n",
"import numpy as np\n",
"from pandas_risk import *\n",
"dfm = pd.read_gbq(\"SELECT * FROM deid_risk.registered_medical_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n",
"dff = pd.read_gbq(\"SELECT * FROM deid_risk.registered_family_history_dec_001\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')\n",
"df = pd.read_gbq(\"SELECT person_id, birth_date,city,state,home_owner,race,ethnicity,gender,birth_place,marital_status,orientation,education,employment_status,income,travel_abroad_6_months,active_duty_status FROM deid_risk.registered_dec_01\",private_key='/home/steve/dev/google-cloud-sdk/accounts/curation-test.json')"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"med_cols = np.random.choice(list(set(dfm.columns.tolist()) - set(['person_id'])),3).tolist()\n",
"fam_cols = np.random.choice(list(set(dff.columns.tolist()) - set(['person_id'])),3).tolist()\n",
"medical = pd.merge(df,dfm[med_cols+['person_id']],on='person_id')\n",
"family = pd.merge(df,dff[fam_cols + ['person_id']],on='person_id')\n",
"_tmp = pd.merge(dfm[med_cols +['person_id']],dff[fam_cols+['person_id']])\n",
"data = pd.merge(df,_tmp,on='person_id')"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" field_count | \n",
" flag | \n",
" group_count | \n",
" marketer | \n",
" prosecutor | \n",
" unique_row_ratio | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 21 | \n",
" full history | \n",
" 115308 | \n",
" 0.992691 | \n",
" 1.0 | \n",
" 0.987663 | \n",
"
\n",
" \n",
" 1 | \n",
" 18 | \n",
" medical | \n",
" 115306 | \n",
" 0.992674 | \n",
" 1.0 | \n",
" 0.987629 | \n",
"
\n",
" \n",
" 2 | \n",
" 18 | \n",
" family | \n",
" 115304 | \n",
" 0.992656 | \n",
" 1.0 | \n",
" 0.987594 | \n",
"
\n",
" \n",
" 3 | \n",
" 15 | \n",
" no-history | \n",
" 115300 | \n",
" 0.992622 | \n",
" 1.0 | \n",
" 0.987526 | \n",
"
\n",
" \n",
" 4 | \n",
" 3 | \n",
" medical-only | \n",
" 27 | \n",
" 0.000232 | \n",
" 0.5 | \n",
" 0.000000 | \n",
"
\n",
" \n",
" 5 | \n",
" 3 | \n",
" family-only | \n",
" 146 | \n",
" 0.001257 | \n",
" 1.0 | \n",
" 0.000551 | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" field_count flag group_count marketer prosecutor \\\n",
"0 21 full history 115308 0.992691 1.0 \n",
"1 18 medical 115306 0.992674 1.0 \n",
"2 18 family 115304 0.992656 1.0 \n",
"3 15 no-history 115300 0.992622 1.0 \n",
"4 3 medical-only 27 0.000232 0.5 \n",
"5 3 family-only 146 0.001257 1.0 \n",
"\n",
" unique_row_ratio \n",
"0 0.987663 \n",
"1 0.987629 \n",
"2 0.987594 \n",
"3 0.987526 \n",
"4 0.000000 \n",
"5 0.000551 "
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pd.concat([data.deid.evaluate(flag='full history',cols= list(set(data.columns.tolist()) - set(['person_id'])) )\n",
" ,medical.deid.evaluate(flag='medical',cols=list( set(medical.columns.tolist() ) - set(['person_id']) ) )\n",
" ,family.deid.evaluate(flag='family',cols=list( set(family.columns.tolist() ) - set(['person_id']) ) )\n",
" ,df.deid.evaluate(flag='no-history',cols=list( set(df.columns.tolist() ) - set(['person_id']) ) )\n",
" , dfm.deid.evaluate(flag='medical-only',cols=med_cols )\n",
" , dff.deid.evaluate(flag='family-only',cols=fam_cols )\n",
" ],ignore_index=True)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from __future__ import division\n",
"def evaluate(df) :\n",
" cols = list(set(df.columns.tolist()) - set(['person_id']))\n",
" \n",
" portions = np.round(np.random.random_sample(4),3).tolist() + np.arange(5,105,5).tolist()\n",
" \n",
" N = df.shape[0] - 1\n",
" portions = np.divide(np.multiply(portions,N),100).astype(np.int64)\n",
" portions = np.unique([n for n in portions if n > 1])\n",
" \n",
" r = pd.DataFrame()\n",
" for num_rows in portions :\n",
" \n",
" indices = np.random.choice(N,num_rows,replace=False)\n",
"# print (indices.size / N)\n",
" flag = \" \".join([str( np.round(100*indices.size/ N,2)),'%'])\n",
" r = r.append(df.loc[indices].deid.evaluate(cols=cols,flag=flag,min_group_size=2))\n",
" return r"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" field_count | \n",
" flag | \n",
" group_count | \n",
" marketer | \n",
" prosecutor | \n",
" unique_row_ratio | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 11 | \n",
" UNFLAGGED | \n",
" 114886 | \n",
" 0.989058 | \n",
" 1.0 | \n",
" 0.980535 | \n",
"
\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" field_count flag group_count marketer prosecutor unique_row_ratio\n",
"0 11 UNFLAGGED 114886 0.989058 1.0 0.980535"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cols = list(set (df.columns.tolist()) - set(['person_id']))\n",
"df[['race','state','gender_identity','ethnicity','marital_status','education','orientation','sex_at_birth','birth_date','travel_abroad_6_months','active_duty_status']].deid.evaluate()"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['person_id',\n",
" 'HearingVision_FarSightedness',\n",
" 'HearingVision_Glaucoma',\n",
" 'Digestive_Pancreatitis']"
]
},
"execution_count": 68,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#\n",
"# This is the merge with medical history\n",
"\n",
"cols = ['person_id'] + np.random.choice(dfm.columns[1:],3,replace=False).tolist()\n",
"p = pd.merge(df,dfm[cols],on='person_id')\n",
"cols\n",
"# # cols = list(set(p.columns.tolist()) - set(['person_id']))\n",
"# evaluate(p) #p.deid.explore(cols=cols,num_runs=100)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"cols = list( set(dfm.columns.tolist()) - set(['person_id']))\n",
"cols = np.random.choice(cols,3,replace=False).tolist()\n",
"p = pd.merge(dfm[['person_id']+cols],df)\n",
"fcols = list(set(p.columns.tolist()) - set(['person_id']))\n",
"# dfm[cols].deid.evaluate(cols=list( set(cols) - set(['person_id'])))"
]
},
{
"cell_type": "markdown",
"metadata": {
"variables": {
" \" ; \".join(cols)": "InfectiousDiseases_HepatitisC ; Cancer_StomachCancer ; Circulatory_Hypertension",
" p.shape[0] ": "116157",
" p[fcols].deid.evaluate() ": "\n\n
\n \n \n | \n field_count | \n flag | \n group_count | \n marketer | \n prosecutor | \n unique_row_ratio | \n
\n \n \n \n 0 | \n 37 | \n UNFLAGGED | \n 115397 | \n 0.993457 | \n 1.0 | \n 0.98886 | \n
\n \n
\n
"
}
},
"source": [
"### Medical History\n",
"\n",
" We randomly select three a tributes {{ \" ; \".join(cols)}} . \n",
" The dataset associated risk evaluation contains {{ p.shape[0] }} records\n",
"{{ p[fcols].deid.evaluate() }}\n",
"\n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['person_id',\n",
" 'InfectiousDiseases_Tuberculosis',\n",
" 'SkeletalMuscular_Fibromyalgia',\n",
" 'Cancer_ProstateCancer']"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cols"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"3"
]
},
"execution_count": 67,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# dfm[cols[1:]].head()\n",
"np.sum(dfm.fillna(' ').groupby(cols[1:],as_index=False).size().values <= 1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.15rc1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}