data-maker/pipeline.py

#!/usr/bin/env python3
	@staticmethod
		df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
			else:

		
				if set(df.columns) & set(_df.columns) :
					_columns = set(df.columns) - set(_df.columns)
					df = df[_columns]

				#
				# Let us merge the dataset here and and have a comprehensive dataset

				_df = pd.DataFrame.join(df,_df)
		# base_cols = list(set(_args['data'].columns) - set(args['columns']))	#-- rebuilt the dataset (and store it)
	f = open (filename)
	# @TODO:
		# 	time.sleep(2)