diff --git a/management/commands/indexer.py b/management/commands/indexer.py index 678ec27..6fa3b53 100644 --- a/management/commands/indexer.py +++ b/management/commands/indexer.py @@ -16,6 +16,8 @@ def healpix(nside): #NSIDE 2048, ORDER ring or nested for catalog_file in CatalogFile.objects.filter(status='INGESTED'): + + print(f'[{current_time()}] Loading sources from {catalog_file.name}...') sources = list(catalog_file.sources.all()) print(f'[{current_time()}] Sources ready. Forming ra & dec arrays...') diff --git a/management/commands/ingester.py b/management/commands/ingester.py index c88c07e..f8b8528 100644 --- a/management/commands/ingester.py +++ b/management/commands/ingester.py @@ -7,7 +7,6 @@ import glob import uuid import asyncio import select -import tracemalloc from datetime import datetime, timedelta import pandas as pd import django @@ -146,19 +145,19 @@ class Command(BaseCommand): data_dict = df_filtered.to_dict(orient='records') #translate the df into a dict - #df = None #free up memory + gaia_source_instances = [ GaiaSource(**data, catalog_file=catalog_file) for data in data_dict #create gaiasource instances, set the foreignkey ] - #data_dict = None #free up memory + await update_catalog_file_status(catalog_file, 'IN_PROGRESS') await bulk_create_gaia_sources(gaia_source_instances) #bulk-create instances from the dict - #gaia_source_instances = None #free up memory + await update_catalog_file_status(catalog_file,'INGESTED') #update the catalogfile instance status field to 'INGESTED' @@ -166,6 +165,14 @@ class Command(BaseCommand): #Timestamp and progress self.stdout.write statement self.stdout.write(f"[{current_time()}] {ingested_files_count}/{len(catalog_files)}") + + del df + del gaiasource_fields + del common_fields + del df_filtered + del data_dict + gc.collect() + # Create a completion flag file with open("ingester_done.flag", "w") as f: