You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
"""
|
|
|
|
|
Create pseudonyms map as follows :
|
|
|
|
|
table, field,value,enc,filter
|
|
|
|
|
"""
|
|
|
|
|
import pandas as pd
|
|
|
|
|
import numpy as np
|
|
|
|
|
from google.oauth2 import service_account
|
|
|
|
|
from google.cloud import bigquery as bq
|
|
|
|
|
import json
|
|
|
|
|
import threading
|
|
|
|
|
import sys
|
|
|
|
|
import os
|
|
|
|
|
import itertools
|
|
|
|
|
|
|
|
|
|
DATASET_SUFFIX = '_pseudo'
|
|
|
|
|
PSEUDO_TABLENAME = 'map'
|
|
|
|
|
|
|
|
|
|
SYS_ARGS = {'context':''}
|
|
|
|
|
if len(sys.argv) > 1:
|
|
|
|
|
|
|
|
|
|
N = len(sys.argv)
|
|
|
|
|
for i in range(1,N):
|
|
|
|
|
value = None
|
|
|
|
|
if sys.argv[i].startswith('--'):
|
|
|
|
|
key = sys.argv[i].replace('-','')
|
|
|
|
|
# if 'filter' in args :
|
|
|
|
|
else:
|
|
|
|
|
# FILTER = FILTER.replace(field,'values')
|
|
|
|
|
# This will give us a map of how each column was mapped to a bitstream
|
|
|
|
|
# df = pd.DataFrame({"fname":['james','james','steve','kevin','kevin'],"lname":["bond","dean","nyemba",'james','johnson']})
|
|
|
|
|
# df['age'] = (np.random.sample(df.shape[0]) * 100).astype(np.int32)
|
