292 lines
9.2 KiB
Python
292 lines
9.2 KiB
Python
from astropy.coordinates import SkyCoord # High-level coordinates
|
|
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
|
|
from astropy.coordinates import Angle, Latitude, Longitude # Angles
|
|
import astropy.units as u
|
|
import re
|
|
import json
|
|
import pandas as pd
|
|
from astropy_healpix import HEALPix
|
|
from erotrans.models import EroTransSource, EroTrans
|
|
from erosurvey.models import NSIDE_PLATES, ORDER_PLATES
|
|
from heasarc.models import NSIDE_SOURCES, ORDER, HeasarcBase
|
|
|
|
from astrobasis.models import GAIADR2
|
|
from astrobasis.models import ZTFAlert
|
|
from astrobasis.models import ALeRCE
|
|
from astrobasis.models import alerce_early_class
|
|
from monthplan.models import Survey
|
|
#import datetime
|
|
|
|
from datetime import datetime
|
|
from django.utils.timezone import localdate, localtime, now
|
|
|
|
import astropy.units as u
|
|
from astropy.time.formats import erfa
|
|
from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch
|
|
|
|
from astropy_healpix import HEALPix, neighbours
|
|
from django_pandas.io import read_frame
|
|
from srglib import correlate_utils, check_transient
|
|
import numpy as np
|
|
|
|
from math import pi, cos, sin
|
|
from Quaternion import Quat
|
|
from scipy.spatial.transform import Rotation, Slerp
|
|
|
|
|
|
import logging
|
|
from pprint import pprint, pformat
|
|
|
|
import requests
|
|
import json
|
|
|
|
from srglib.utils import TZ_MSK
|
|
|
|
|
|
def find_ztf_in_survey(ztf):
|
|
""" Find ZTF alert in ALL surveys by HEALPIX plate"""
|
|
surveys = Survey.objects.all()
|
|
for survey in surveys:
|
|
#total = survey.surveyhealpixplate_set.count()
|
|
#print(survey.experiment, ztf, total)
|
|
if survey.surveyhealpixplate_set.filter(healpix=ztf.healpix_plate).exists():
|
|
#print('add survey')
|
|
ztf.survey.add(survey)
|
|
ztf.save()
|
|
|
|
|
|
"""
|
|
if ztf.survey.exists():
|
|
return True
|
|
else:
|
|
return False
|
|
"""
|
|
def load_ztf_alerts():
|
|
""" Loads ZTF alerts
|
|
|
|
https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html
|
|
|
|
Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample:
|
|
|
|
rb >= 0.65 and
|
|
nbad = 0 and
|
|
fwhm <= 5 and
|
|
elong <= 1.2 and
|
|
abs(magdiff) <= 0.1
|
|
|
|
|
|
Parameters
|
|
----------
|
|
time : int
|
|
Request alerts since this time in seconds
|
|
|
|
|
|
:Author:
|
|
Roman Krivonos <krivonos@cosmos.ru>
|
|
|
|
"""
|
|
|
|
format_string='%a, %d %b %Y %H:%M:%S %Z'
|
|
|
|
#date_string='Tue, 28 Jan 2020 09:30:30 GMT'
|
|
#dt = datetime.strptime(date_string, format_string)
|
|
|
|
hp = HEALPix(nside=NSIDE_SOURCES,
|
|
order=ORDER, frame=FK5())
|
|
|
|
hp_plates = HEALPix(nside=NSIDE_PLATES,
|
|
order=ORDER_PLATES, frame=FK5())
|
|
|
|
URL = "https://mars.lco.global/"
|
|
PARAMS={'format':'json', 'rb__gte':0.8,'nbad__lte':0,'fwhm__lte':5,'elong__lte':1.2}
|
|
r = requests.get(url = URL, params = PARAMS)
|
|
data = r.json()
|
|
#print(data['total'])
|
|
#pprint.pprint(data)
|
|
|
|
for item in data['results']:
|
|
#print(item['lco_id'],item['objectId'],item['candidate']['wall_time'])
|
|
try:
|
|
ztf = ZTFAlert.objects.get(lco_id=item['lco_id'])
|
|
logging.info("ZTF alert ID %d is already loaded, skip" % item['lco_id'])
|
|
continue
|
|
except ZTFAlert.DoesNotExist:
|
|
logging.info("ZTF alert ID %d is not loaded" % item['lco_id'])
|
|
pass
|
|
date_string = item['candidate']['wall_time']
|
|
dt = datetime.strptime(date_string, format_string)
|
|
ra = item['candidate']['ra']
|
|
dec = item['candidate']['dec']
|
|
crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
|
|
healpix = hp.skycoord_to_healpix(crd)
|
|
healpix_plate = hp_plates.skycoord_to_healpix(crd)
|
|
ztf = ZTFAlert(
|
|
objectId = item['objectId'],
|
|
fid = item['candidate']['fid'],
|
|
lco_id = item['lco_id'],
|
|
healpix = healpix, healpix_plate = healpix_plate,
|
|
ra = ra, dec = dec,
|
|
programid = item['candidate']['programid'],
|
|
magpsf = item['candidate']['magpsf'],
|
|
sigmapsf = item['candidate']['sigmapsf'],
|
|
magap = item['candidate']['magap'],
|
|
magdiff = item['candidate']['magdiff'],
|
|
nbad = item['candidate']['nbad'],
|
|
fwhm = item['candidate']['fwhm'],
|
|
elong = item['candidate']['elong'],
|
|
sigmagap = item['candidate']['sigmagap'],
|
|
wall_time = datetime.strptime(date_string, format_string),
|
|
diffmaglim = item['candidate']['diffmaglim'],
|
|
deltamagref = item['candidate']['deltamagref'],
|
|
deltamaglatest = item['candidate']['deltamaglatest'],
|
|
rb = item['candidate']['rb'])
|
|
ztf.save()
|
|
|
|
find_ztf_in_survey(ztf)
|
|
|
|
|
|
def load_ztf_alerce_to_django(item):
|
|
""" Loads ALeRCE ZTF alert to Django
|
|
|
|
classearly=None, VS, Asteroids and Bogus objects are not loaded.
|
|
"""
|
|
|
|
nobs = int(item['nobs'])
|
|
|
|
classearly = item['classearly']
|
|
|
|
if (classearly == None):
|
|
print("ALeRCE ZTF alert %s: skip classearly=None" % item['oid'])
|
|
return
|
|
|
|
"""
|
|
if (classearly == 20 or classearly == 21 or classearly == 22):
|
|
logging.debug("ALeRCE ZTF alert '{0}': skip '{1}'".format(item['oid'],alerce_early_class(classearly)))
|
|
return
|
|
"""
|
|
|
|
try:
|
|
ztf = ALeRCE.objects.get(oid=item['oid'])
|
|
if(nobs > ztf.nobs):
|
|
""" if nobs is changed, reload this alert, else skip """
|
|
logging.info("ALeRCE ZTF alert '{0}': reload, new nobs '{1}' > '{2}'".format(item['oid'],ztf.nobs,nobs))
|
|
ztf.delete()
|
|
pass
|
|
else:
|
|
logging.debug("ALeRCE ZTF alert %s is found, skip" % item['oid'])
|
|
return
|
|
except ALeRCE.DoesNotExist:
|
|
logging.info("ALeRCE ZTF alert %s is not found, loading" % item['oid'])
|
|
pass
|
|
|
|
hp = HEALPix(nside=NSIDE_SOURCES,
|
|
order=ORDER, frame=FK5())
|
|
|
|
hp_plates = HEALPix(nside=NSIDE_PLATES,
|
|
order=ORDER_PLATES, frame=FK5())
|
|
|
|
ra = float(item['meanra'])
|
|
dec = float(item['meandec'])
|
|
crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
|
|
healpix = hp.skycoord_to_healpix(crd)
|
|
healpix_plate = hp_plates.skycoord_to_healpix(crd)
|
|
|
|
first_tm = Time(item['firstmjd'], format='mjd', scale='utc')
|
|
last_tm = Time(item['lastmjd'], format='mjd', scale='utc')
|
|
first_dt = first_tm.to_datetime(timezone=TZ_MSK)
|
|
last_dt = last_tm.to_datetime(timezone=TZ_MSK)
|
|
|
|
ztf = ALeRCE(
|
|
healpix = healpix,
|
|
healpix_plate = healpix_plate,
|
|
classearly = item['classearly'],
|
|
classrf = item['classrf'],
|
|
oid = item['oid'],
|
|
firstdate = first_dt,
|
|
lastdate = last_dt,
|
|
firstmjd = item['firstmjd'],
|
|
lastmjd = item['lastmjd'],
|
|
mean_magap_g = item['mean_magap_g'],
|
|
mean_magap_r = item['mean_magap_r'],
|
|
mean_magpsf_g = item['mean_magpsf_g'],
|
|
mean_magpsf_r = item['mean_magpsf_r'],
|
|
dec = dec,
|
|
ra = ra,
|
|
nobs = item['nobs'],
|
|
pclassearly = item['pclassearly'],
|
|
pclassrf = item['pclassrf'],
|
|
sigma_magap_g = item['sigma_magap_g'],
|
|
sigma_magap_r = item['sigma_magap_r'],
|
|
sigma_magpsf_g = item['sigma_magpsf_g'],
|
|
sigma_magpsf_r = item['sigma_magpsf_r'],
|
|
sigmadec = item['sigmadec'],
|
|
sigmara = item['sigmara'])
|
|
|
|
ztf.save()
|
|
find_ztf_in_survey(ztf)
|
|
|
|
def load_ztf_alerce(days, extra=None):
|
|
classrf = "sn ia"
|
|
pclassrf = 0.1
|
|
pclassearly = 0.1
|
|
sortBy = "firstmjd"
|
|
nobsmin = 2
|
|
nobsmax = 40
|
|
url = "https://ztf.alerce.online/query"
|
|
|
|
current_utc = now()
|
|
current_mjd = Time(current_utc).mjd
|
|
firstmjd_min = current_mjd - days
|
|
|
|
post={
|
|
"sortBy": sortBy,
|
|
"records_per_pages": 100,
|
|
"query_parameters":{
|
|
"filters": {
|
|
"nobs": {
|
|
"min": nobsmin,
|
|
"max": nobsmax
|
|
},
|
|
#"classrf": classrf,
|
|
#"pclassrf": pclassrf,
|
|
'pclassearly':pclassearly,
|
|
},
|
|
"dates":{
|
|
"firstmjd":{
|
|
"min": firstmjd_min
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(extra):
|
|
post['query_parameters']['filters'].update(extra)
|
|
|
|
#pprint.pprint(post)
|
|
|
|
r = requests.post(url = url, json = post)
|
|
data = r.json()
|
|
for item in data['result']:
|
|
""" loads first page """
|
|
packet=data['result'][item]
|
|
#pprint.pprint(packet['oid'])
|
|
load_ztf_alerce_to_django(packet)
|
|
|
|
total=int(data['total'])
|
|
num_pages=int(data['num_pages'])
|
|
page=int(data['page'])
|
|
|
|
logging.info("ALeRCE ZTF alerts total '{0}', num_pages '{1}'".format(int(total),int(num_pages)))
|
|
#return
|
|
pages = list(range(2,num_pages+1))
|
|
for page in pages:
|
|
""" loads all other pages, staring from 2 """
|
|
post.update( {'page':page,} )
|
|
logging.debug(pformat(post))
|
|
r = requests.post(url = url, json = post)
|
|
data = r.json()
|
|
for item in data['result']:
|
|
packet=data['result'][item]
|
|
#pprint.pprint(packet['oid'])
|
|
load_ztf_alerce_to_django(packet)
|