From 2ea452cf8168e452b058f9e0d253ab6deaf507f2 Mon Sep 17 00:00:00 2001
From: tyrin <nytyrin@stud.kpfu.ru>
Date: Thu, 12 Sep 2024 10:43:43 +0300
Subject: [PATCH] returned to pyarrow backend in pandas, added timestamped
 outputs to the indexer implementation

---
 management/commands/indexer.py  | 18 +++++++++++++-----
 management/commands/ingester.py |  2 +-
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/management/commands/indexer.py b/management/commands/indexer.py
index c318cc9..5f260ae 100644
--- a/management/commands/indexer.py
+++ b/management/commands/indexer.py
@@ -4,6 +4,8 @@ from astropy.coordinates import SkyCoord
 import astropy_healpix as ah
 import numpy as np
 
+def current_time():
+    return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S")
 
 def healpix():
 
@@ -13,19 +15,25 @@ def healpix():
     for catalog_file in CatalogFile.objects.all():
 
         sources = list(catalog_file.sources.all())
-        
+        print(f'[{current_time()}] Sources ready. Forming ra & dec arrays...')
+
         ra_list = np.array([source.ra for source in sources])
         dec_list = np.array([source.dec for source in sources])
-        
+        print(f'[{current_time()}] ra & dec arrays ready. Creating SkyCoord objects...')
+
         skycoord = SkyCoord(ra=ra_list, dec=dec_list, unit='deg', frame='fk5')
-        
+        print(f'[{current_time()}] SkyCoord objects ready. Calculating healpix indices...')
+
         healpix = ah.HEALPix(nside=NSIDE, order=ORDER, frame='fk5')
         healpix_indices = healpix.skycoord_to_healpix(skycoord)
-        
+        print(f'[{current_time()}] Indices ready. Assigning indices to source instances...')
+
         for source, healpix_index in zip(sources, healpix_indices):
             source.healpix_ring_index = healpix_index
-        
+        print(f'[{current_time()}] Instances ready. Bulk updating the database...')
+
         GaiaSource.objects.bulk_update(sources, ['healpix_ring_index'])
+        print(f'[{current_time()}] Database updated, sources indexed successfully.')
 
 
 class Command(BaseCommand):
diff --git a/management/commands/ingester.py b/management/commands/ingester.py
index 0253cee..6bcde2f 100644
--- a/management/commands/ingester.py
+++ b/management/commands/ingester.py
@@ -126,7 +126,7 @@ class Command(BaseCommand):
                     file_path, 
                     #comment='#', 
                     header=1000,
-                    #engine="pyarrow"
+                    engine="pyarrow"
                 )                    
 
                 gaiasource_fields = [field.name for field in GaiaSource._meta.get_fields()]   #get fields from the model