tyrin/GaiaDBInterface
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Indexer: improved outputs, added progress reporting.

Browse Source
This commit is contained in:
Никита Тырин 2024-09-13 13:09:39 +03:00
parent ec27a495b1
commit e83b23a35c
1 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
management/commands/indexer.py
View File

@ -7,7 +7,6 @@ from datetime import datetime, timedelta
def current_time(): def current_time():
return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S") return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S")
@ -21,12 +20,18 @@ def update_catalog_file_status(catalog_file, status): #catalog file
def healpix(nside): #NSIDE 2048, ORDER ring or nested def healpix(nside): #NSIDE 2048, ORDER ring or nested
for catalog_file in CatalogFile.objects.filter(status='INGESTED'): ingested_files = CatalogFile.objects.filter(status='INGESTED')
total_files = ingested_files.count()
print(f'[{current_time()}] There are {total_files} ingested files.')
current_file_index = 0
for catalog_file in ingested_files:
print(f'[{current_time()}] Loading sources from {catalog_file.name}...') current_file_index += 1
print(f'[{current_time()}] Processing sources from {catalog_file.name}')
sources = list(catalog_file.sources.all()) sources = list(catalog_file.sources.all())
print(f'[{current_time()}] Sources loaded, calculating indices.')
if catalog_file.status == 'INDEX_IN_PROGRESS': if catalog_file.status == 'INDEX_IN_PROGRESS':
#sources.update(healpix_ring_index=None, healpix_nested_index=None) #sources.update(healpix_ring_index=None, healpix_nested_index=None)
@ -50,18 +55,18 @@ def healpix(nside): #NSIDE 2048, ORDER ring or nested
ring_healpix = ah.HEALPix( ring_healpix = ah.HEALPix(
nside=nside, nside=nside,
order=ring, order='ring',
frame='fk5' frame='fk5'
) )
ring_healpix_indices = healpix.skycoord_to_healpix(skycoord) ring_healpix_indices = ring_healpix.skycoord_to_healpix(skycoord)
nested_healpix = ah.HEALPix( nested_healpix = ah.HEALPix(
nside=nside, nside=nside,
order=nested, order='nested',
frame='fk5' frame='fk5'
) )
nested_healpix_indices = healpix.skycoord_to_healpix(skycoord) nested_healpix_indices = nested_healpix.skycoord_to_healpix(skycoord)
for source, healpix_index in zip(sources, ring_healpix_indices): for source, healpix_index in zip(sources, ring_healpix_indices):
@ -70,6 +75,7 @@ def healpix(nside): #NSIDE 2048, ORDER ring or nested
for source, healpix_index in zip(sources, nested_healpix_indices): for source, healpix_index in zip(sources, nested_healpix_indices):
source.healpix_nested_index = healpix_index source.healpix_nested_index = healpix_index
print(f'[{current_time()}] Calculations done, updating database.')
update_catalog_file_status(catalog_file, 'INDEX_IN_PROGRESS') update_catalog_file_status(catalog_file, 'INDEX_IN_PROGRESS')
@ -77,8 +83,8 @@ def healpix(nside): #NSIDE 2048, ORDER ring or nested
update_catalog_file_status(catalog_file, 'INDEXED') update_catalog_file_status(catalog_file, 'INDEXED')
print(f'[{current_time()}] Database updated, sources indexed successfully.') print(f'[{current_time()}] Database updated, sources indexed successfully.')
print(f'[{current_time()}] Progress: {current_file_index}/{total_files} files indexed.')