bug fix: date conversions

data/maker/__init__.py

@@ -296,35 +296,35 @@ class Generator (Learner):
             name = _item['name']
             
             if _item['type'].upper() in ['DATE','DATETIME','TIMESTAMP'] :
-                FORMAT = '%m-%d-%Y'
+                FORMAT = '%Y-%m-%d'
-                
+                
-                # try:
+                try:
-                #     #
+                    #
-                #     #-- Sometimes data isn't all it's meant to be
+                    #-- Sometimes data isn't all it's meant to be
-                #     SIZE = -1
+                    SIZE = -1
-                #     if 'format' in self.info and name in self.info['format'] :
+                    if 'format' in self.info and name in self.info['format'] :
-                #         FORMAT = self.info['format'][name]
+                        FORMAT = self.info['format'][name]
-                #         SIZE = 10
+                        SIZE = 10
-                #     elif _item['type'] in ['DATETIME','TIMESTAMP'] :
+                    elif _item['type'] in ['DATETIME','TIMESTAMP'] :
-                #             FORMAT = '%m-%d-%Y %H:%M:%S'
+                            FORMAT = '%Y-%m-%-d %H:%M:%S'
-                #             SIZE = 19
+                            SIZE = 19
-                    
+                    
-                #     if SIZE > 0 :
+                    if SIZE > 0 :
-                        
+                        
-                #         values = pd.to_datetime(_df[name], format=FORMAT).astype(str)
+                        values = pd.to_datetime(_df[name], format=FORMAT).astype(np.datetime64)
                         # _df[name] = [_date[:SIZE].strip() for _date in values]
                         
                        
-                #     # _df[name] = _df[name].astype(str)
+                    # _df[name] = _df[name].astype(str)
-                #     r[name] = FORMAT
+                    r[name] = FORMAT
-                #     # _df[name] = pd.to_datetime(_df[name], format=FORMAT) #.astype('datetime64[ns]')
+                    # _df[name] = pd.to_datetime(_df[name], format=FORMAT) #.astype('datetime64[ns]')
-                #     if _item['type'] in ['DATETIME','TIMESTAMP']:                   
+                    if _item['type'] in ['DATETIME','TIMESTAMP']:                   
-                #         pass #;_df[name] = _df[name].fillna('').astype('datetime64[ns]')
+                        pass #;_df[name] = _df[name].fillna('').astype('datetime64[ns]')
                     
-                # except Exception as e:
+                except Exception as e:
-                #     pass
+                    pass
-                # finally:
+                finally:
-                #     pass
+                    pass
             else:
                 
                 #
@@ -387,10 +387,11 @@ class Generator (Learner):
 
             # w = transport.factory.instance(doc='observation',provider='mongodb',context='write',db='IOV01_LOGS',auth_file='/home/steve/dev/transport/mongo.json')
             # w.write(_df)
+            cols = [name for name in _df.columns if name.endswith('datetime')]
             # print (_df[cols])
             
             writer = transport.factory.instance(**_store)
-            writer.write(_df,schema=_schema)
+            writer.write(_df[cols],schema=[_item for _item in _schema if _item['name'] in cols])