from django.core.management.base import BaseCommand, CommandError from datetime import date import datetime from django.utils import timezone import astropy from astropy.io import ascii import pandas as pd import pymysql from sqlalchemy import create_engine import logging import os from datetime import datetime from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch from astropy_healpix import HEALPix from astropy.coordinates import SkyCoord # High-level coordinates from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames from astropy.coordinates import Angle, Latitude, Longitude # Angles import astropy.units as u from astropy_healpix import HEALPix, neighbours from heasarc.models import HeasarcBase from artsurvey.models import ArtSurvey, ArtSurveySource, eRositaMatch from srglib.utils import find_counterparts import numpy.ma as ma from django.core.exceptions import ObjectDoesNotExist MJDREF = 51543.875 TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) def load_skymap_sources_erosita(filename, survey): """ Loads sources cross-matched with eRosita Parameters ---------- filename : str Absolute path to the file :Author: Roman Krivonos """ logger = logging.getLogger('django') if not os.path.isfile(filename): logger.error("File {} does not exist".format(filename)) return logger.info("Loading sources from {}".format(filename)) name = os.path.basename(filename) catalog_name = os.path.splitext(name)[0] logger.info("Filename {} catalog {}".format(name, catalog_name)) format_string='srg_%Y%m%d_%H%M%S' dtime = datetime.strptime(catalog_name[:-4], format_string) tm = Time(dtime, format='datetime', scale='utc') dt = tm.to_datetime(timezone=TZ_MSK) data = astropy.table.Table.read(filename,format='ascii.csv') print(data.info) """ hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER_PLATES, frame=FK5()) """ for item in data: try: #srcs = ArtSurveySource.objects.filter(name_orig=item['NAME']) src = survey.artsurveysource_set.get(name_orig=item['NAME']) except Exception as e: print("{} failed to load {}".format(item['NAME'],e)) print("Missed {} cnts {}, flux {}, exptime {}".format(item['NAME'],item['CNTS'],item['FLUX'],item['EXPTIME'],)) continue match_soft = True if item['match_soft'] == 'True' else False match_med = True if item['match_med'] == 'True' else False match_hard = True if item['match_hard'] == 'True' else False #print("{}".format(src)) src.erosita_data=True src.save() if not (match_soft or match_med or match_hard): print("3 match failed for {}".format(item['NAME'])) continue hpidx_soft = item['hpidx_soft'] if not ma.is_masked(item['hpidx_soft']) else None hpidx_med = item['hpidx_med'] if not ma.is_masked(item['hpidx_med']) else None hpidx_hard = item['hpidx_hard'] if not ma.is_masked(item['hpidx_hard']) else None ero = eRositaMatch(source=src, ero_exp = item['ero_exp'], lkh = item['lkh'], sep_soft = item['sep_soft'], match_soft = item['match_soft'], srcname_soft = item['srcname_soft'], hpidx_soft = hpidx_soft, RA_fin_soft = item['RA_fin_soft'], DEC_fin_soft = item['DEC_fin_soft'], RA_soft = item['RA_soft'], DEC_soft = item['DEC_soft'], RADEC_ERR_soft = item['RADEC_ERR_soft'], DET_LIKE_0_soft = item['DET_LIKE_0_soft'], ML_FLUX_0_soft = item['ML_FLUX_0_soft'], ML_FLUX_ERR_0_soft = item['ML_FLUX_ERR_0_soft'], ML_CTS_0_soft = item['ML_CTS_0_soft'], ML_CTS_ERR_0_soft = item['ML_CTS_ERR_0_soft'], ML_BKG_0_soft = item['ML_BKG_0_soft'], EXT_soft = item['EXT_soft'], EXT_LIKE_soft = item['EXT_LIKE_soft'], EXT_ERR_soft = item['EXT_ERR_soft'], ML_EXP_1_soft = item['ML_EXP_1_soft'], sep_med = item['sep_med'], match_med = item['match_med'], srcname_med = item['srcname_med'], hpidx_med = hpidx_med, RA_fin_med = item['RA_fin_med'], DEC_fin_med = item['DEC_fin_med'], RA_med = item['RA_med'], DEC_med = item['DEC_med'], RADEC_ERR_med = item['RADEC_ERR_med'], DET_LIKE_0_med = item['DET_LIKE_0_med'], ML_FLUX_0_med = item['ML_FLUX_0_med'], ML_FLUX_ERR_0_med = item['ML_FLUX_ERR_0_med'], ML_CTS_0_med = item['ML_CTS_0_med'], ML_CTS_ERR_0_med = item['ML_CTS_ERR_0_med'], ML_BKG_0_med = item['ML_BKG_0_med'], EXT_med = item['EXT_med'], EXT_LIKE_med = item['EXT_LIKE_med'], EXT_ERR_med = item['EXT_ERR_med'], ML_EXP_1_med = item['ML_EXP_1_med'], sep_hard = item['sep_hard'], match_hard = item['match_hard'], srcname_hard = item['srcname_hard'], hpidx_hard = hpidx_hard, RA_fin_hard = item['RA_fin_hard'], DEC_fin_hard = item['DEC_fin_hard'], RA_hard = item['RA_hard'], DEC_hard = item['DEC_hard'], RADEC_ERR_hard = item['RADEC_ERR_hard'], DET_LIKE_0_hard = item['DET_LIKE_0_hard'], ML_FLUX_0_hard = item['ML_FLUX_0_hard'], ML_FLUX_ERR_0_hard = item['ML_FLUX_ERR_0_hard'], ML_CTS_0_hard = item['ML_CTS_0_hard'], ML_CTS_ERR_0_hard = item['ML_CTS_ERR_0_hard'], ML_BKG_0_hard = item['ML_BKG_0_hard'], EXT_hard = item['EXT_hard'], EXT_LIKE_hard = item['EXT_LIKE_hard'], EXT_ERR_hard = item['EXT_ERR_hard'], ML_EXP_1_hard = item['ML_EXP_1_hard'], sep_simb = item['sep_simb'], match_simb = item['match_simb'], ra_match = item['ra_match'], dec_match = item['dec_match'], RAJ2000_simb = item['RAJ2000_simb'], DEJ2000_simb = item['DEJ2000_simb'], z_simb = item['z_simb'], otype_simb = item['otype_simb']) ero.save() class Command(BaseCommand): help = 'Initiates data dase' def handle(self, *args, **options): survey_version=1.1 try: survey = ArtSurvey.objects.get(version=survey_version) except ArtSurvey.DoesNotExist: print("ArtSurvey {} not found".format(survey_version)) return print("use ArtSurvey {}".format(survey)) srcs = survey.artsurveysource_set.all() for src in srcs: try: src.erositamatch except ObjectDoesNotExist: continue print(src.erositamatch) src.erositamatch.delete() src.erosita_data=False src.save() load_skymap_sources_erosita("/data/artxc/erosita/srg_20200610_000000_000.csv", survey) self.stdout.write(self.style.SUCCESS('Done'))