monitor.py to restart ingester periodically hopefully avoiding memory leaks.

management/commands/ingester.py

@@ -42,7 +42,7 @@ class Command(BaseCommand):
 
 
         #fetching the file list
-        directory = input_with_timeout(f"[{current_time()}] Please enter the path to the directory containing the csv files [{previous_choice}]: ", 30)
+        directory = input_with_timeout(f"[{current_time()}] Please enter the path to the directory containing the csv files, timeout 60 seconds [{previous_choice}]: ", 60)
         if not directory:
             directory = previous_choice
             
@@ -170,6 +170,6 @@ class Command(BaseCommand):
             # Create a completion flag file
             with open("ingester_done.flag", "w") as f:
                 f.write("done")
-            print("Ingester done")
+            print(f"[{current_time()}] Ingester done.")
 
         asyncio.run(ingest_files())

monitor.py

@@ -0,0 +1,24 @@
+import subprocess
+import time
+import os
+
+def restart_script():
+
+    while True:
+        process = subprocess.Popen(["python", "manage.py", "ingester"])
+        time.sleep(7200)
+        if os.path.exists("ingester_done.flag"):
+            # Remove the flag file
+            os.remove("ingester_done.flag")
+            print("Ingester done, exiting monitor script.")
+            break
+        
+        process.terminate()
+        process.wait()
+        print("Restarting ingester...")
+
+
+if __name__ == "__main__":
+    restart_script()
+
+