Indexer: switched to user input (with previous value support) for the NSIDE value. Testing indexer_multithread version added. Ingester: minor code tweaks

management/commands/indexer.py

@@ -4,6 +4,9 @@ from astropy.coordinates import SkyCoord
 import astropy_healpix as ah
 import numpy as np
 from datetime import datetime, timedelta
+import gc
+import os
+import json
 
 
 
@@ -18,13 +21,41 @@ def update_catalog_file_status(catalog_file, status):              #catalog file
 
 
 
-def healpix(nside): #NSIDE 2048, ORDER ring or nested
+def healpix(): #NSIDE 2048, ORDER ring or nested
+
+    interrupted_files = CatalogFile.objects.filter(status='INDEX_IN_PROGRESS')
+    total_interrupted = interrupted_files.count()
+    print(f'[{current_time()}] Found {total_interrupted} files interrupted mid-index. Cleaning up.')
+
+    for interrupted_file in interrupted_files:
+        GaiaSource.objects.filter(catalog_file=interrupted_file).update(
+                healpix_ring_index=None, 
+                healpix_nested_index=None
+            )
+        update_catalog_file_status(interrupted_file, 'INGESTED')
 
     ingested_files = CatalogFile.objects.filter(status='INGESTED')
     total_files = ingested_files.count()
     print(f'[{current_time()}] There are {total_files} ingested files.')
     current_file_index = 0
 
+    if os.path.exists('config.json'):
+            with open('config.json', 'r') as config_file:
+                config = json.load(config_file)
+                previous_choice = config.get('nside', '')
+    else:
+        previous_choice = ''
+
+    nside = input(f'[{current_time()}] Please enter the NSIDE HEALPix parameter [{previous_choice}]:')
+
+    if not nside:
+        nside = previous_choice
+
+    with open('config.json', 'w') as config_file:
+            json.dump({'nside': nside}, config_file)
+
+    print(f"[{current_time()}] Selected nside: {nside}")
+
     for catalog_file in ingested_files:
         
         current_file_index += 1
@@ -33,14 +64,7 @@ def healpix(nside): #NSIDE 2048, ORDER ring or nested
         sources = list(catalog_file.sources.all())
         print(f'[{current_time()}] Sources loaded, calculating indices.')
 
-        if catalog_file.status == 'INDEX_IN_PROGRESS':
-            #sources.update(healpix_ring_index=None, healpix_nested_index=None)
-            GaiaSource.objects.filter(catalog_file=catalog_file).update(
-                healpix_ring_index=None, 
-                healpix_nested_index=None
-            )
-
-
+        
         ra_list = np.array([source.ra for source in sources])
         dec_list = np.array([source.dec for source in sources])
         
@@ -83,7 +107,13 @@ def healpix(nside): #NSIDE 2048, ORDER ring or nested
 
         update_catalog_file_status(catalog_file, 'INDEXED')
 
-        print(f'[{current_time()}] Database updated, sources indexed successfully.')
+        print(f'[{current_time()}] Database updated, sources indexed successfully. Initializing garbage collection.')
+
+        del sources, ra_list, dec_list, skycoord, ring_healpix, ring_healpix_indices, nested_healpix, nested_healpix_indices
+
+        gc.collect()
+
+        print(f'[{current_time()}] Garbage collection complete.')
         print(f'[{current_time()}] Progress: {current_file_index}/{total_files} files indexed.')
 
 
@@ -91,18 +121,6 @@ def healpix(nside): #NSIDE 2048, ORDER ring or nested
 
 class Command(BaseCommand):
     help = 'Index sources using healpix.'
-
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            'nside', 
-            type=int, 
-            help='NSIDE parameter for HEALPix'
-        )
-
-
     def handle(self, *args, **options):
-        nside = options['nside']
-
-        healpix(nside)
+        healpix()