20 add logging for all scripts

blech_clust.py

@@ -102,9 +102,10 @@ def initialize_groups(self):
                 found_list.append(this_group)
 
         if len(found_list) > 0 and not self.force_run:
-            print(f'Data already present: {found_list}')
+            reload_msg = f'Data already present: {found_list}' + '\n' +\
+                        'Reload data? (yes/y/n/no) ::: '
             reload_data_str, continue_bool = entry_checker(
-                    msg='Reload data? (yes/y/n/no) ::: ',
+                    msg= reload_msg,
                     check_func=lambda x: x in ['y', 'yes', 'n', 'no'],
                     fail_response='Please enter (yes/y/n/no)')
         else:

utils/blech_utils.py

@@ -6,8 +6,32 @@
 import glob
 import json
 import pandas as pd
+import sys
 from datetime import datetime
 
+class Tee:
+    """Tee output to both stdout/stderr and a log file"""
+    def __init__(self, data_dir, name='output.log'):
+        self.log_path = os.path.join(data_dir, name)
+        self.file = open(self.log_path, 'a')
+        self.stdout = sys.stdout
+        self.stderr = sys.stderr
+        sys.stdout = self
+        sys.stderr = self
+
+    def write(self, data):
+        self.file.write(data)
+        self.stdout.write(data)
+        self.file.flush()
+
+    def flush(self):
+        self.file.flush()
+
+    def close(self):
+        sys.stdout = self.stdout
+        sys.stderr = self.stderr
+        self.file.close()
+
 class path_handler():
 
     def __init__(self):
@@ -28,9 +52,10 @@ class pipeline_graph_check():
     """
 
     def __init__(self, data_dir):
+        self.data_dir = data_dir
+        self.tee = Tee(data_dir)
         self.load_graph()
         self.check_graph()
-        self.data_dir = data_dir
 
     def load_graph(self):
         """
@@ -129,10 +154,19 @@ def write_to_log(self, script_path, type = 'attempted'):
             if 'attempted' not in log_dict.keys():
                 log_dict['attempted'] = {}
             log_dict['attempted'][script_path] = current_datetime
+            print('============================================================')
+            print(f'Attempting {os.path.basename(script_path)}, started at {current_datetime}')
+            print('============================================================')
         elif type == 'completed':
             if 'completed' not in log_dict.keys():
                 log_dict['completed'] = {}
             log_dict['completed'][script_path] = current_datetime
+            print('============================================================')
+            print(f'Completed {os.path.basename(script_path)}, ended at {current_datetime}')
+            print('============================================================')
+            # Close tee when completing
+            if hasattr(self, 'tee'):
+                self.tee.close()
         # log_dict[script_path] = current_datetime
         with open(self.log_path, 'w') as log_file_connect:
             json.dump(log_dict, log_file_connect, indent = 4)