From 886230e647cdc84fd1631356066e032edfebe58d Mon Sep 17 00:00:00 2001 From: "Steve L. Nyemba -- The Architect" Date: Mon, 1 Oct 2018 11:31:15 -0500 Subject: [PATCH] pandas extension for risk --- src/pandas_risk.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/pandas_risk.py diff --git a/src/pandas_risk.py b/src/pandas_risk.py new file mode 100644 index 0000000..e631cae --- /dev/null +++ b/src/pandas_risk.py @@ -0,0 +1,91 @@ +""" + Health Information Privacy Lab + Steve L. Nyemba & Brad. Malin + + + This is an extension to the pandas data-frame that will perform a risk assessment on a variety of attributes + This implementation puts the responsibility on the user of the framework to join datasets and load the final results into a pandas data-frame. + + The code will randomly select fields and compute the risk (marketer and prosecutor) and perform a given number of runs. + + Usage: + from pandas_risk import * + + mydataframe = pd.DataFrame('/myfile.csv') + risk = mydataframe.deid.risk(id=,num_runs=) + + + @TODO: + - Provide a selected number of fields and risk will be computed for those fields. + - include journalist risk + +""" +import pandas as pd +import numpy as np + +@pd.api.extensions.register_dataframe_accessor("deid") +class deid : + """ + This class is a deidentification class that will compute risk (marketer, prosecutor) given a pandas dataframe + """ + def __init__(self,df): + self._df = df + + def risk(self,**args): + """ + @param id name of patient field + @params num_runs number of runs (default will be 100) + """ + + id = args['id'] + + num_runs = args['num_runs'] if 'num_runs' in args else 100 + r = pd.DataFrame() + + columns = list(set(self._df.columns) - set([id])) + k = len(columns) + for i in range(0,num_runs) : + # + # let's chose a random number of columns and compute marketer and prosecutor risk + # Once the fields are selected we run a groupby clause + # + + n = np.random.randint(2,k) #-- number of random fields we are picking + ii = np.random.choice(k,n,replace=False) + cols = np.array(columns)[ii].tolist() + x_ = self._df.groupby(cols).count()[id].values + r = r.append( + pd.DataFrame( + [ + { + "selected":n, + "marketer": x_.size / np.float64(np.sum(x_)), + "prosecutor":1 / np.float64(np.min(x_)) + + } + ] + ) + ) + g_size = x_.size + n_ids = np.float64(np.sum(x_)) + + return r + + +import pandas as pd +import numpy as np +from io import StringIO +csv = """ +id,sex,age,profession,drug_test +1,M,37,doctor,- +2,F,28,doctor,+ +3,M,37,doctor,- +4,M,28,doctor,+ +5,M,28,doctor,- +6,M,37,doctor,- +""" +f = StringIO() +f.write(unicode(csv)) +f.seek(0) +df = pd.read_csv(f) +print df.deid.risk(id='id',num_runs=1) \ No newline at end of file