from astropy.coordinates import SkyCoord # High-level coordinates from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames from astropy.coordinates import Angle, Latitude, Longitude # Angles import astropy.units as u import re import json import pandas as pd from astropy_healpix import HEALPix from erotrans.models import EroTransSource, EroTrans from erosurvey.models import NSIDE_PLATES, ORDER_PLATES from heasarc.models import NSIDE_SOURCES, ORDER, HeasarcBase from astrobasis.models import GAIADR2 from astrobasis.models import ZTFAlert from astrobasis.models import ALeRCE from astrobasis.models import alerce_early_class from monthplan.models import Survey #import datetime from datetime import datetime from django.utils.timezone import localdate, localtime, now import astropy.units as u from astropy.time.formats import erfa from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch from astropy_healpix import HEALPix, neighbours from django_pandas.io import read_frame from srglib import correlate_utils, check_transient import numpy as np from math import pi, cos, sin from Quaternion import Quat from scipy.spatial.transform import Rotation, Slerp import logging from pprint import pprint, pformat import requests import json from srglib.utils import TZ_MSK def find_ztf_in_survey(ztf): """ Find ZTF alert in ALL surveys by HEALPIX plate""" surveys = Survey.objects.all() for survey in surveys: #total = survey.surveyhealpixplate_set.count() #print(survey.experiment, ztf, total) if survey.surveyhealpixplate_set.filter(healpix=ztf.healpix_plate).exists(): #print('add survey') ztf.survey.add(survey) ztf.save() """ if ztf.survey.exists(): return True else: return False """ def load_ztf_alerts(): """ Loads ZTF alerts https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample: rb >= 0.65 and nbad = 0 and fwhm <= 5 and elong <= 1.2 and abs(magdiff) <= 0.1 Parameters ---------- time : int Request alerts since this time in seconds :Author: Roman Krivonos """ format_string='%a, %d %b %Y %H:%M:%S %Z' #date_string='Tue, 28 Jan 2020 09:30:30 GMT' #dt = datetime.strptime(date_string, format_string) hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) hp_plates = HEALPix(nside=NSIDE_PLATES, order=ORDER_PLATES, frame=FK5()) URL = "https://mars.lco.global/" PARAMS={'format':'json', 'rb__gte':0.8,'nbad__lte':0,'fwhm__lte':5,'elong__lte':1.2} r = requests.get(url = URL, params = PARAMS) data = r.json() #print(data['total']) #pprint.pprint(data) for item in data['results']: #print(item['lco_id'],item['objectId'],item['candidate']['wall_time']) try: ztf = ZTFAlert.objects.get(lco_id=item['lco_id']) logging.info("ZTF alert ID %d is already loaded, skip" % item['lco_id']) continue except ZTFAlert.DoesNotExist: logging.info("ZTF alert ID %d is not loaded" % item['lco_id']) pass date_string = item['candidate']['wall_time'] dt = datetime.strptime(date_string, format_string) ra = item['candidate']['ra'] dec = item['candidate']['dec'] crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") healpix = hp.skycoord_to_healpix(crd) healpix_plate = hp_plates.skycoord_to_healpix(crd) ztf = ZTFAlert( objectId = item['objectId'], fid = item['candidate']['fid'], lco_id = item['lco_id'], healpix = healpix, healpix_plate = healpix_plate, ra = ra, dec = dec, programid = item['candidate']['programid'], magpsf = item['candidate']['magpsf'], sigmapsf = item['candidate']['sigmapsf'], magap = item['candidate']['magap'], magdiff = item['candidate']['magdiff'], nbad = item['candidate']['nbad'], fwhm = item['candidate']['fwhm'], elong = item['candidate']['elong'], sigmagap = item['candidate']['sigmagap'], wall_time = datetime.strptime(date_string, format_string), diffmaglim = item['candidate']['diffmaglim'], deltamagref = item['candidate']['deltamagref'], deltamaglatest = item['candidate']['deltamaglatest'], rb = item['candidate']['rb']) ztf.save() find_ztf_in_survey(ztf) def load_ztf_alerce_to_django(item): """ Loads ALeRCE ZTF alert to Django classearly=None, VS, Asteroids and Bogus objects are not loaded. """ nobs = int(item['nobs']) classearly = item['classearly'] if (classearly == None): print("ALeRCE ZTF alert %s: skip classearly=None" % item['oid']) return """ if (classearly == 20 or classearly == 21 or classearly == 22): logging.debug("ALeRCE ZTF alert '{0}': skip '{1}'".format(item['oid'],alerce_early_class(classearly))) return """ try: ztf = ALeRCE.objects.get(oid=item['oid']) if(nobs > ztf.nobs): """ if nobs is changed, reload this alert, else skip """ logging.info("ALeRCE ZTF alert '{0}': reload, new nobs '{1}' > '{2}'".format(item['oid'],ztf.nobs,nobs)) ztf.delete() pass else: logging.debug("ALeRCE ZTF alert %s is found, skip" % item['oid']) return except ALeRCE.DoesNotExist: logging.info("ALeRCE ZTF alert %s is not found, loading" % item['oid']) pass hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) hp_plates = HEALPix(nside=NSIDE_PLATES, order=ORDER_PLATES, frame=FK5()) ra = float(item['meanra']) dec = float(item['meandec']) crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") healpix = hp.skycoord_to_healpix(crd) healpix_plate = hp_plates.skycoord_to_healpix(crd) first_tm = Time(item['firstmjd'], format='mjd', scale='utc') last_tm = Time(item['lastmjd'], format='mjd', scale='utc') first_dt = first_tm.to_datetime(timezone=TZ_MSK) last_dt = last_tm.to_datetime(timezone=TZ_MSK) ztf = ALeRCE( healpix = healpix, healpix_plate = healpix_plate, classearly = item['classearly'], classrf = item['classrf'], oid = item['oid'], firstdate = first_dt, lastdate = last_dt, firstmjd = item['firstmjd'], lastmjd = item['lastmjd'], mean_magap_g = item['mean_magap_g'], mean_magap_r = item['mean_magap_r'], mean_magpsf_g = item['mean_magpsf_g'], mean_magpsf_r = item['mean_magpsf_r'], dec = dec, ra = ra, nobs = item['nobs'], pclassearly = item['pclassearly'], pclassrf = item['pclassrf'], sigma_magap_g = item['sigma_magap_g'], sigma_magap_r = item['sigma_magap_r'], sigma_magpsf_g = item['sigma_magpsf_g'], sigma_magpsf_r = item['sigma_magpsf_r'], sigmadec = item['sigmadec'], sigmara = item['sigmara']) ztf.save() find_ztf_in_survey(ztf) def load_ztf_alerce(days, extra=None): classrf = "sn ia" pclassrf = 0.1 pclassearly = 0.1 sortBy = "firstmjd" nobsmin = 2 nobsmax = 40 url = "https://ztf.alerce.online/query" current_utc = now() current_mjd = Time(current_utc).mjd firstmjd_min = current_mjd - days post={ "sortBy": sortBy, "records_per_pages": 100, "query_parameters":{ "filters": { "nobs": { "min": nobsmin, "max": nobsmax }, #"classrf": classrf, #"pclassrf": pclassrf, 'pclassearly':pclassearly, }, "dates":{ "firstmjd":{ "min": firstmjd_min } } } } if(extra): post['query_parameters']['filters'].update(extra) #pprint.pprint(post) r = requests.post(url = url, json = post) data = r.json() for item in data['result']: """ loads first page """ packet=data['result'][item] #pprint.pprint(packet['oid']) load_ztf_alerce_to_django(packet) total=int(data['total']) num_pages=int(data['num_pages']) page=int(data['page']) logging.info("ALeRCE ZTF alerts total '{0}', num_pages '{1}'".format(int(total),int(num_pages))) #return pages = list(range(2,num_pages+1)) for page in pages: """ loads all other pages, staring from 2 """ post.update( {'page':page,} ) logging.debug(pformat(post)) r = requests.post(url = url, json = post) data = r.json() for item in data['result']: packet=data['result'][item] #pprint.pprint(packet['oid']) load_ztf_alerce_to_django(packet)