returned to pyarrow backend in pandas, added timestamped outputs to the indexer implementation

management/commands/indexer.py

@@ -4,6 +4,8 @@ from astropy.coordinates import SkyCoord
 import astropy_healpix as ah
 import numpy as np
 
+def current_time():
+    return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S")
 
 def healpix():
 
@@ -13,19 +15,25 @@ def healpix():
     for catalog_file in CatalogFile.objects.all():
 
         sources = list(catalog_file.sources.all())
-        
+        print(f'[{current_time()}] Sources ready. Forming ra & dec arrays...')
+
         ra_list = np.array([source.ra for source in sources])
         dec_list = np.array([source.dec for source in sources])
-        
+        print(f'[{current_time()}] ra & dec arrays ready. Creating SkyCoord objects...')
+
         skycoord = SkyCoord(ra=ra_list, dec=dec_list, unit='deg', frame='fk5')
-        
+        print(f'[{current_time()}] SkyCoord objects ready. Calculating healpix indices...')
+
         healpix = ah.HEALPix(nside=NSIDE, order=ORDER, frame='fk5')
         healpix_indices = healpix.skycoord_to_healpix(skycoord)
-        
+        print(f'[{current_time()}] Indices ready. Assigning indices to source instances...')
+
         for source, healpix_index in zip(sources, healpix_indices):
             source.healpix_ring_index = healpix_index
-        
+        print(f'[{current_time()}] Instances ready. Bulk updating the database...')
+
         GaiaSource.objects.bulk_update(sources, ['healpix_ring_index'])
+        print(f'[{current_time()}] Database updated, sources indexed successfully.')
 
 
 class Command(BaseCommand):

management/commands/ingester.py

@@ -126,7 +126,7 @@ class Command(BaseCommand):
                     file_path, 
                     #comment='#', 
                     header=1000,
-                    #engine="pyarrow"
+                    engine="pyarrow"
                 )                    
 
                 gaiasource_fields = [field.name for field in GaiaSource._meta.get_fields()]   #get fields from the model