tyrin/GaiaDBInterface
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Additional work on the Indexer

Browse Source
This commit is contained in:
Никита Тырин 2024-09-13 12:19:04 +03:00
parent c27ac6ce97
commit ec27a495b1
2 changed files with 50 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -1,2 +1,4 @@
# GaiaDBInterface # GaiaDBInterface
A Django application for interacting with Gaia DR3 via ORM interface A Django application for interacting with Gaia DR3 via ORM interface.

68
management/commands/indexer.py
View File

@ -5,68 +5,94 @@ import astropy_healpix as ah
import numpy as np import numpy as np
from datetime import datetime, timedelta from datetime import datetime, timedelta
def current_time(): def current_time():
return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S") return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S")
def update_catalog_file_status(catalog_file, status): #catalog file status updater def update_catalog_file_status(catalog_file, status): #catalog file status updater
catalog_file.status = status catalog_file.status = status
catalog_file.save() catalog_file.save()
def healpix(nside): #NSIDE 2048, ORDER ring or nested def healpix(nside): #NSIDE 2048, ORDER ring or nested
for catalog_file in CatalogFile.objects.filter(status='INGESTED'): for catalog_file in CatalogFile.objects.filter(status='INGESTED'):
print(f'[{current_time()}] Loading sources from {catalog_file.name}...') print(f'[{current_time()}] Loading sources from {catalog_file.name}...')
sources = list(catalog_file.sources.all()) sources = list(catalog_file.sources.all())
print(f'[{current_time()}] Sources ready. Forming ra & dec arrays...')
if catalog_file.status == 'INDEX_IN_PROGRESS': if catalog_file.status == 'INDEX_IN_PROGRESS':
sources.update(healpix_ring_index=None, healpix_nested_index=None) #sources.update(healpix_ring_index=None, healpix_nested_index=None)
GaiaSource.objects.filter(catalog_file=catalog_file).update(
healpix_ring_index=None,
healpix_nested_index=None
)
ra_list = np.array([source.ra for source in sources]) ra_list = np.array([source.ra for source in sources])
dec_list = np.array([source.dec for source in sources]) dec_list = np.array([source.dec for source in sources])
print(f'[{current_time()}] ra & dec arrays ready. Creating SkyCoord objects...')
skycoord = SkyCoord(ra=ra_list, dec=dec_list, unit='deg', frame='fk5') skycoord = SkyCoord(
print(f'[{current_time()}] SkyCoord objects ready. Calculating healpix indices...') ra=ra_list,
dec=dec_list,
unit='deg',
frame='fk5'
)
ring_healpix = ah.HEALPix(
ring_healpix = ah.HEALPix(nside=nside, order=ring, frame='fk5') nside=nside,
order=ring,
frame='fk5'
)
ring_healpix_indices = healpix.skycoord_to_healpix(skycoord) ring_healpix_indices = healpix.skycoord_to_healpix(skycoord)
print(f'[{current_time()}] Ring indices ready. Assigning indices to source instances...')
nested_healpix = ah.HEALPix(
nside=nside,
order=nested,
frame='fk5'
)
nested_healpix_indices = healpix.skycoord_to_healpix(skycoord)
for source, healpix_index in zip(sources, ring_healpix_indices): for source, healpix_index in zip(sources, ring_healpix_indices):
source.healpix_ring_index = healpix_index source.healpix_ring_index = healpix_index
print(f'[{current_time()}] Instances ready.')
for source, healpix_index in zip(sources, nested_healpix_indices):
nested_healpix = ah.HEALPix(nside=nside, order=nested, frame='fk5')
nested_healpix_indices = healpix.skycoord_to_healpix(skycoord)
print(f'[{current_time()}] Nested indices ready. Assigning indices to source instances...')
for source, healpix_index in zip(sources, healpix_indices):
source.healpix_nested_index = healpix_index source.healpix_nested_index = healpix_index
print(f'[{current_time()}] Instances ready. Bulk updating the database...')
update_catalog_file_status(catalog_file, 'INDEX_IN_PROGRESS') update_catalog_file_status(catalog_file, 'INDEX_IN_PROGRESS')
GaiaSource.objects.bulk_update(sources, ['healpix_ring_index', 'healpix_nested_index']) GaiaSource.objects.bulk_update(sources, ['healpix_ring_index', 'healpix_nested_index'])
print(f'[{current_time()}] Database updated, sources nested-indexed successfully.')
update_catalog_file_status(catalog_file, 'INDEXED') update_catalog_file_status(catalog_file, 'INDEXED')
print(f'[{current_time()}] Database updated, sources indexed successfully.')
class Command(BaseCommand): class Command(BaseCommand):
help = 'Index sources using healpix.' help = 'Index sources using healpix.'
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('nside', type=int, help='NSIDE parameter for HEALPix') parser.add_argument(
'nside',
type=int,
help='NSIDE parameter for HEALPix'
)
def handle(self, *args, **options): def handle(self, *args, **options):