tyrin/GaiaDBInterface
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

made ingester functional without inputs in preparation for a monitor script restarting it

Browse Source
This commit is contained in:
Никита Тырин 2024-09-12 11:45:02 +03:00
parent 2ea452cf81
commit e37387b68f
2 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
management/commands/indexer.py
View File

@ -3,6 +3,7 @@ from GaiaDBInterface.models import GaiaSource, CatalogFile
from astropy.coordinates import SkyCoord from astropy.coordinates import SkyCoord
import astropy_healpix as ah import astropy_healpix as ah
import numpy as np import numpy as np
from datetime import datetime, timedelta
def current_time(): def current_time():
return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S") return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S")
@ -13,7 +14,7 @@ def healpix():
ORDER='ring' ORDER='ring'
for catalog_file in CatalogFile.objects.all(): for catalog_file in CatalogFile.objects.all():
print(f'[{current_time()}] Loading sources from the database...')
sources = list(catalog_file.sources.all()) sources = list(catalog_file.sources.all())
print(f'[{current_time()}] Sources ready. Forming ra & dec arrays...') print(f'[{current_time()}] Sources ready. Forming ra & dec arrays...')

19
management/commands/ingester.py
View File

@ -6,6 +6,7 @@ import json
import glob import glob
import uuid import uuid
import asyncio import asyncio
import select
import tracemalloc import tracemalloc
from datetime import datetime, timedelta from datetime import datetime, timedelta
import pandas as pd import pandas as pd
@ -14,7 +15,13 @@ from asgiref.sync import sync_to_async
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from GaiaDBInterface.models import GaiaSource, CatalogFile from GaiaDBInterface.models import GaiaSource, CatalogFile
def input_with_timeout(prompt, timeout=30):
print(prompt, end='', flush=True)
ready, _, _ = select.select([sys.stdin], [], [], timeout)
if ready:
return sys.stdin.readline().strip()
else:
return ''
class Command(BaseCommand): class Command(BaseCommand):
help = 'Ingest CSV files into the database' help = 'Ingest CSV files into the database'
@ -32,7 +39,7 @@ class Command(BaseCommand):
#fetching the file list #fetching the file list
directory = input(f"Please enter the path to the directory containing the csv files [{previous_choice}]: ") directory = input_with_timeout(f"Please enter the path to the directory containing the csv files [{previous_choice}]: ", 30)
if not directory: if not directory:
directory = previous_choice directory = previous_choice
@ -66,7 +73,7 @@ class Command(BaseCommand):
input("Press Enter to continue...") #input("Press Enter to continue...")
@ -157,6 +164,10 @@ class Command(BaseCommand):
current_time = (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S") #Timestamp and progress self.stdout.write statement current_time = (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S") #Timestamp and progress self.stdout.write statement
self.stdout.write(f"[{current_time}] {ingested_files_count}/{len(catalog_files)}") self.stdout.write(f"[{current_time}] {ingested_files_count}/{len(catalog_files)}")
# Create a completion flag file
with open("ingester_done.flag", "w") as f:
f.write("done")
print("Ingester done")
asyncio.run(ingest_files()) asyncio.run(ingest_files())