commit
5960caa7f8
@ -0,0 +1,251 @@
|
||||
"""
|
||||
Create pseudonyms map as follows :
|
||||
table, field,value,enc,filter
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from google.oauth2 import service_account
|
||||
from google.cloud import bigquery as bq
|
||||
import json
|
||||
import threading
|
||||
import sys
|
||||
import os
|
||||
import itertools
|
||||
|
||||
DATASET_SUFFIX = '_pseudo'
|
||||
PSEUDO_TABLENAME = 'map'
|
||||
|
||||
SYS_ARGS = {'context':''}
|
||||
if len(sys.argv) > 1:
|
||||
|
||||
N = len(sys.argv)
|
||||
for i in range(1,N):
|
||||
value = None
|
||||
if sys.argv[i].startswith('--'):
|
||||
key = sys.argv[i].replace('-','')
|
||||
|
||||
if i + 1 < N:
|
||||
value = sys.argv[i + 1] = sys.argv[i+1].strip()
|
||||
if key and value:
|
||||
SYS_ARGS[key] = value
|
||||
if key == 'context':
|
||||
SYS_ARGS[key] = ('/'+value).replace('//','/')
|
||||
|
||||
i += 2
|
||||
|
||||
|
||||
class void :
|
||||
pass
|
||||
|
||||
class pseudonym :
|
||||
@staticmethod
|
||||
def meta(**args) :
|
||||
"""
|
||||
:key Bigquery private key (service account)
|
||||
:dataset dataset of the input table
|
||||
:table table name
|
||||
:filter optional filter (SQL statement)
|
||||
"""
|
||||
credentials = service_account.Credentials.from_service_account_file(args['key'])
|
||||
SQL = ["SELECT * FROM :dataset.:table"]
|
||||
if 'filter' in args :
|
||||
SQL += ['WHERE',args['filter']]
|
||||
dataset = args['dataset']
|
||||
table = args['table']
|
||||
SQL = " ".join(SQL+["LIMIT 1"]).replace(":dataset",dataset).replace(":table",table)
|
||||
|
||||
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
|
||||
return df.columns
|
||||
@staticmethod
|
||||
def apply(**args):
|
||||
"""
|
||||
This function applies the
|
||||
"""
|
||||
columns = pseudonym.meta(**args)
|
||||
#
|
||||
# we need to make the schema here
|
||||
client = bq.Client.from_service_account_json(args['key'])
|
||||
datasets = list(client.list_datasets())
|
||||
dataset_name = args['dataset']+DATASET_SUFFIX
|
||||
if np.sum( [ 1*(item.dataset_id == dataset_name) for item in datasets]) == 0:
|
||||
#-- make the target dataset
|
||||
|
||||
dataset = bq.Dataset(client.dataset(dataset_name))
|
||||
client.create_dataset(dataset)
|
||||
|
||||
for name in columns :
|
||||
p = dict(args,**{"field":name})
|
||||
p['filter'] = '' if 'filter' not in args else args['filter']
|
||||
# thread = threading.Thread(target=pseudonym.post, args=(p,))
|
||||
# thread.start()
|
||||
# if columns.tolist().index(name) == 0 :
|
||||
# thread.join()
|
||||
pseudonym.post(**p)
|
||||
|
||||
|
||||
#
|
||||
# let us submit the query
|
||||
pass
|
||||
@staticmethod
|
||||
def post(**args) :
|
||||
"""
|
||||
This function will submit a query to bigquery for insertion
|
||||
"""
|
||||
SQL = " ".join(['SELECT DISTINCT CAST(',args['field']," AS STRING) AS values, COUNT(*) as counts FROM :dataset.:table :filter"]).replace(':dataset',args['dataset'])
|
||||
SQL = SQL.replace(':table',args['table'])
|
||||
if args['filter'].strip() != '' :
|
||||
SQL = SQL.replace(":filter", "WHERE "+args['filter'])
|
||||
SQL += " ".join(['GROUP BY ',args['field'],'ORDER BY 1 '])
|
||||
TABLE_NAME = ".".join([args['dataset'],args['table']])
|
||||
credentials = service_account.Credentials.from_service_account_file(args['key'])
|
||||
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
|
||||
df['table'] = args['table']
|
||||
df['field'] = args['field']
|
||||
# df['filter']= args['filter']
|
||||
N = df.shape[0] + 10000
|
||||
beg = np.random.randint(11,200)
|
||||
df['encoded'] = np.random.choice(np.arange(beg,N),df.shape[0],replace=False)
|
||||
df = df[['table','field','values','counts','encoded']]
|
||||
# print (df.head()[:5])
|
||||
# sys.stdout.flush()
|
||||
TABLE_NAME = ".".join([args['dataset']+DATASET_SUFFIX,PSEUDO_TABLENAME])
|
||||
df.to_gbq(TABLE_NAME,credentials=credentials,if_exists='append')
|
||||
# df.to_gbq(TABLE_NAME.replace('.','_pseudo.'),credentials=credentials,if_exists='append')
|
||||
|
||||
class Builder :
|
||||
"""
|
||||
This class will build a dataset from encoded values
|
||||
"""
|
||||
def encode(self,**args):
|
||||
"""
|
||||
This function will create pseudonyms for a given table from the mapping tables
|
||||
"""
|
||||
SQL = "SELECT * FROM :dataset.:table limit 1".replace(':dataset',args['dataset']).replace(":table",args['table'])
|
||||
credentials = service_account.Credentials.from_service_account_file(args['key'])
|
||||
columns = pd.read_gbq(SQL,credentials=credentials,dialect='standard').columns.tolist()
|
||||
TEMPLATE = ['(SELECT encoded FROM :dataset'+DATASET_SUFFIX+'.'+PSEUDO_TABLENAME,"WHERE table=':table' AND field = ':name' AND CAST(values AS STRING)=CAST(:table.:name AS STRING ) ) as :name"]
|
||||
SQL = ["SELECT"]
|
||||
FIELDS = []
|
||||
for field in columns :
|
||||
FIELDS += [" ".join(TEMPLATE).replace(":name",field)]
|
||||
SQL += [",\n\t".join(FIELDS)]
|
||||
SQL += ['FROM :dataset.:table']
|
||||
return ("\n".join(SQL).replace(":dataset",args['dataset']).replace(':table',args['table']) )
|
||||
|
||||
def process(self,**args):
|
||||
"""
|
||||
:dataset
|
||||
:table
|
||||
:key
|
||||
"""
|
||||
pseudonym.apply(**args)
|
||||
def decode(self,**args):
|
||||
"""
|
||||
This function should be able to take a pseudonymized data frame and convert it to original values
|
||||
...
|
||||
"""
|
||||
|
||||
pass
|
||||
class Binary :
|
||||
"""
|
||||
This is a utility class to import and export a data to/from a binary matrix
|
||||
"""
|
||||
def __stream(self,column) :
|
||||
"""
|
||||
This function will convert a column into a binary matrix with the value-space representing each column of the resulting matrix
|
||||
:column a column vector i.e every item is a row
|
||||
"""
|
||||
values = np.unique(column)
|
||||
values.sort()
|
||||
|
||||
row_count,col_count = column.size,values.size
|
||||
matrix = [ np.zeros(col_count) for i in np.arange(row_count)]
|
||||
#
|
||||
# let's create a binary matrix of the feature that was passed in
|
||||
# The indices of the matrix are inspired by classical x,y axis
|
||||
for yi in np.arange(row_count) :
|
||||
value = column[yi]
|
||||
xi = np.where(values == value)[0][0] #-- column index
|
||||
matrix[yi][xi] = 1
|
||||
|
||||
return matrix
|
||||
def Export(self,df) :
|
||||
"""
|
||||
This function will convert a data-frame to a binary matrix
|
||||
:return _map,matrix
|
||||
"""
|
||||
#
|
||||
# This will give us a map of how each column was mapped to a bitstream
|
||||
_map = df.apply(lambda column: self.__stream(column.values),axis=0)
|
||||
#
|
||||
# We will merge this to have a healthy matrix
|
||||
_matrix = _map.apply(lambda row: list(list(itertools.chain(*row.values.tolist()))),axis=1)
|
||||
_matrix = np.matrix([list(item) for item in _matrix])
|
||||
#
|
||||
# let's format the map so we don't have an unreasonable amount of data
|
||||
#
|
||||
columns = _map.columns.tolist()
|
||||
beg = 0
|
||||
end = 0
|
||||
_map = _map.loc[0]
|
||||
_m = {}
|
||||
for name in columns :
|
||||
end += _map[name].size
|
||||
_m[name] = {"start":beg,"end":end}
|
||||
beg = end
|
||||
|
||||
return _m,_matrix
|
||||
|
||||
def Import(self,df,values,_map):
|
||||
"""
|
||||
This function will convert a binary stream into a
|
||||
:values original/pseudonymed values
|
||||
:_map field map of the binary matrix
|
||||
"""
|
||||
r = pd.DataFrame(None,columns=_map.keys())
|
||||
for key in _map:
|
||||
i = np.arange(_map[key]['start'],_map[key]['end'])
|
||||
columns = values[key]
|
||||
r[key] = df[i].apply(lambda row: np.array( columns)[row==1][0], axis=1 )
|
||||
return r
|
||||
pass
|
||||
|
||||
# has_basic = 'dataset' in SYS_ARGS.keys() and 'table' in SYS_ARGS.keys() and 'key' in SYS_ARGS.keys()
|
||||
# has_action= 'export' in SYS_ARGS.keys() or 'pseudo' in SYS_ARGS.keys()
|
||||
df = pd.DataFrame({"fname":['james','james','steve','kevin','kevin'],"lname":["bond","dean","nyemba",'james','johnson']})
|
||||
df['age'] = (np.random.sample(df.shape[0]) * 100).astype(np.int32)
|
||||
if __name__ == '__main__' :
|
||||
"""
|
||||
Run the program from the command line passing the following mandatory arguments
|
||||
python bridge.py <[--pseudo|--export <PATH>]> --dataset <dataset> --table <tablename> [--filter <table filter>]
|
||||
--pseudo will create pseudonyms for a given
|
||||
--export will export data to a specified location
|
||||
"""
|
||||
has_basic = 'dataset' in SYS_ARGS.keys() and 'table' in SYS_ARGS.keys() and 'key' in SYS_ARGS.keys()
|
||||
has_action= 'export' in SYS_ARGS.keys() or 'pseudo' in SYS_ARGS.keys()
|
||||
if has_basic and has_action :
|
||||
builder = Builder()
|
||||
if 'export' in SYS_ARGS :
|
||||
print ()
|
||||
print ("exporting ....")
|
||||
if not os.path.exists(SYS_ARGS['export']) :
|
||||
os.mkdir(SYS_ARGS['export'])
|
||||
SQL = builder.encode(**SYS_ARGS)
|
||||
credentials = service_account.Credentials.from_service_account_file(SYS_ARGS['key'])
|
||||
df = pd.read_gbq(SQL,credentials =credentials,dialect='standard')
|
||||
FILENAME = os.sep.join([SYS_ARGS['export'],SYS_ARGS['table']+'.csv'])
|
||||
#
|
||||
# This would allow us to export it to wherever we see fit
|
||||
print (FILENAME)
|
||||
df.to_csv(FILENAME,index=False)
|
||||
elif 'pseudo' in SYS_ARGS :
|
||||
builder.process(**SYS_ARGS)
|
||||
else:
|
||||
print ("")
|
||||
print ("has basic ",has_basic)
|
||||
print ("has action ",has_action)
|
||||
# pseudonym.apply(table='person',dataset='wgan_original',key='./curation-test-2.json')
|
||||
# args = {"dataset":"wgan_original","table":"observation","key":"./curation-test-2.json"}
|
||||
# builder = Builder()
|
||||
# # builder.encode(dataset='wgan_original',table='person',key='./curation-test-2.json')
|
||||
# builder.process(**args)
|
||||
Loading…
Reference in new issue