tyrin/GaiaDBInterface
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added the current_time function to ingester too

Browse Source
This commit is contained in:
Никита Тырин 2024-09-12 11:52:56 +03:00
parent 4eaad008e1
commit 2d45a653fc
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
management/commands/ingester.py
View File

@ -23,6 +23,9 @@ def input_with_timeout(prompt, timeout=30):
else: else:
return '' return ''
def current_time():
return (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S")
class Command(BaseCommand): class Command(BaseCommand):
help = 'Ingest CSV files into the database' help = 'Ingest CSV files into the database'
@ -101,8 +104,7 @@ class Command(BaseCommand):
current_time = (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S") self.stdout.write(f"[{current_time()}] Starting the data ingestion.")
self.stdout.write(f"[{current_time}] Starting the data ingestion.")
#function that iterates over all catalog files and ingests sources from them #function that iterates over all catalog files and ingests sources from them
async def ingest_files(): async def ingest_files():
@ -112,7 +114,7 @@ class Command(BaseCommand):
for catalog_file in catalog_files: for catalog_file in catalog_files:
if catalog_file.status == 'INGESTED': if catalog_file.status == 'INGESTED':
self.stdout.write(f"Skipping {catalog_file.name} as it is already ingested.") self.stdout.write(f"[{current_time()}]Skipping {catalog_file.name} as it is already ingested.")
continue continue
file_path = os.path.join(directory, catalog_file.name) file_path = os.path.join(directory, catalog_file.name)
@ -162,8 +164,8 @@ class Command(BaseCommand):
ingested_files_count = await count_ingested_files() ingested_files_count = await count_ingested_files()
current_time = (datetime.now() + timedelta(hours=3)).strftime("%H:%M:%S") #Timestamp and progress self.stdout.write statement #Timestamp and progress self.stdout.write statement
self.stdout.write(f"[{current_time}] {ingested_files_count}/{len(catalog_files)}") self.stdout.write(f"[{current_time()}] {ingested_files_count}/{len(catalog_files)}")
# Create a completion flag file # Create a completion flag file
with open("ingester_done.flag", "w") as f: with open("ingester_done.flag", "w") as f: