|
|
@ -143,7 +143,7 @@ class Components :
|
|
|
|
# columns = args['columns']
|
|
|
|
# columns = args['columns']
|
|
|
|
# df = np.array_split(df[columns].values,PART_SIZE)
|
|
|
|
# df = np.array_split(df[columns].values,PART_SIZE)
|
|
|
|
# df = pd.DataFrame(df[ int (partition) ],columns = columns)
|
|
|
|
# df = pd.DataFrame(df[ int (partition) ],columns = columns)
|
|
|
|
info = {"parition":int(partition),"gpu":_args["gpu"],"rows":df.shape[0],"cols":df.shape[1],"part_size":PART_SIZE}
|
|
|
|
info = {"parition":int(partition),"gpu":_args["gpu"],"rows":str(df.shape[0]),"cols":str(df.shape[1]),"part_size":int(PART_SIZE)}
|
|
|
|
logger.write({"module":"generate","action":"partition","input":info})
|
|
|
|
logger.write({"module":"generate","action":"partition","input":info})
|
|
|
|
_args['partition'] = int(partition)
|
|
|
|
_args['partition'] = int(partition)
|
|
|
|
_args['continuous']= args['continuous'] if 'continuous' in args else []
|
|
|
|
_args['continuous']= args['continuous'] if 'continuous' in args else []
|
|
|
@ -352,7 +352,7 @@ if __name__ == '__main__' :
|
|
|
|
args['partition'] = index
|
|
|
|
args['partition'] = index
|
|
|
|
# _df = pd.DataFrame(DATA[index],columns=args['columns'])
|
|
|
|
# _df = pd.DataFrame(DATA[index],columns=args['columns'])
|
|
|
|
args['data'] = DATA[index]
|
|
|
|
args['data'] = DATA[index]
|
|
|
|
args['data'].to_csv('aou-'+str(index)+'csv',index=False)
|
|
|
|
# args['data'].to_csv('aou-'+str(index)+'csv',index=False)
|
|
|
|
# args['reader'] = lambda: _df
|
|
|
|
# args['reader'] = lambda: _df
|
|
|
|
if int(args['num_gpu']) > 1 :
|
|
|
|
if int(args['num_gpu']) > 1 :
|
|
|
|
args['gpu'] = index
|
|
|
|
args['gpu'] = index
|
|
|
|