romakrivonos/srg
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more apps

Browse Source
This commit is contained in:
Roman Krivonos
2024-04-26 12:43:00 +03:00
parent 52b209b176
commit 69a2160eb7
536 changed files with 33118 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
artsurvey/management/commands/00_artsurvey_addsource.py Normal file
View File

@@ -0,0 +1,55 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
from artsurvey.utils import setup_metasource
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
def addsource(survey_version, band):
try:
survey = ArtSurvey.objects.get(version=survey_version)
print("use ArtSurvey {}".format(survey))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
try:
band = EnergyBand.objects.get(slug='E0')
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
addsource(12.2,'E0')
self.stdout.write(self.style.SUCCESS('Not completed!'))

117
artsurvey/management/commands/00_artsurvey_all.py Normal file
View File

@@ -0,0 +1,117 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
import sys
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_vlass
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
from srglib.utils import load_simbad_for_skymap_sources
from srglib.utils import load_vizier_bj
from srglib.utils import load_vizier_gaia_dr3
#from srglib.utils import select_turkpossible
def do_all(survey_version,survey_name,slug="E0"):
minrad=40
maxdist=600
#bad_category = Category.objects.get(slug='false')
#minrad=180
#maxdist=640
try:
band = EnergyBand.objects.get(slug=slug)
print("use EnergyBand {}".format(band))
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return
try:
survey = ArtSurvey.objects.get(Q(version=survey_version) & Q(name=survey_name))
print("use ArtSurvey {}".format(survey))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
srcs = survey.artsurveysource_set.all().filter(band=band).filter(log_ml_nfalse__lt=-2.3206)
print("Selected {} from {} {}".format(len(srcs),survey,band))
#sys.exit()
load_simbad_for_skymap_sources(srcs,minrad=minrad,maxdist=maxdist)
load_vizier_allwise(srcs, minrad=minrad, maxdist=maxdist)
load_vizier_bj(srcs, minrad=minrad, maxdist=60)
load_vizier_gaia_dr3(srcs, minrad=minrad, maxdist=60)
load_vizier_nvss(srcs, minrad=minrad, maxdist=maxdist)
load_vizier_first(srcs, minrad=minrad, maxdist=maxdist)
load_vizier_sumss(srcs, minrad=minrad, maxdist=maxdist)
load_vizier_vlass(srcs, minrad=minrad, maxdist=maxdist)
find_counterparts(srcs, GAIADR3.objects.all(), "gaia3", maxdist=maxdist, minrad=minrad)
"""
for src in srcs:
src.heasarc.clear()
find_counterparts(srcs, HeasarcBase.objects.all(),'heasarc', maxdist=maxdist, minrad=minrad)
"""
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
#do_all(12.5)
#do_all(1.7)
#do_all(2.7)
#do_all(1234.0)
#do_all(3.0)
#do_all(4.0)
#do_all(34.0,"34")
#do_all(5.0,"5")
#do_all(12345.0,"1-5")
#do_all(0.12,"L20")
#do_all(0.12,"L20p1")
#do_all(0.12,"L20p2")
#do_all(0.12,"L20p3")
#do_all(0.12,"L20p4")
#do_all(0.13,"L20rb")
#do_all(0.15,"L20rb_ml")
#do_all(12345.12,"1-5",slug="E5")
#do_all(12345.13,"1-5",slug="E5")
do_all(12345.14,"1-5",slug="E5")
#names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345']
#names=['GP055','GP060','GP065']
#names=['GP000','GP005','GP010','GP015','GP020','GP025','GP030','GP035','GP040','GP045','GP050','GP055','GP060','GP065','GP070','GP085','GP090',
# 'GP270','GP275','GP280','GP285','GP290','GP295','GP300','GP305','GP310','GP315','GP320','GP325']
#names=['GP315']
names=['GP330','GP335','GP340']
"""
for name in names:
do_all(0.11,name)
"""
self.stdout.write(self.style.SUCCESS('Done'))

178
artsurvey/management/commands/00_artsurvey_catalog_nfalse.py Normal file
View File

@@ -0,0 +1,178 @@
from django.core.management.base import BaseCommand, CommandError
import sys
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from django.db.models import Q
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, Catalog, MetaSource, Category
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
def get_survey(version):
try:
survey = ArtSurvey.objects.get(version=version)
print("use ArtSurvey {}".format(survey))
return survey
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(version))
return None
def get_band(slug):
try:
band = EnergyBand.objects.get(slug=slug)
print("use EnergyBand {}".format(band))
return band
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return None
def select(catalog,survey,band,log_nfalse,threshold):
srcs = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(Q(log_nfalse__lte=log_nfalse) | Q(ext__gte=1))
for src in srcs:
src.catalog.add(catalog)
ms = src.metasource
ms.catalog=catalog
ms.save()
src.save()
count=srcs.count()
print("{} {}, log_nfalse {}, selected {}, {:.1f}% false count: {:.1f}".format(survey,band.slug,log_nfalse,count,threshold*100,count*threshold))
def select_ml(catalog,survey,band,log_ml_nfalse, threshold):
srcs = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(log_ml_nfalse__lte=log_ml_nfalse) # no extended sources in 2023
bad_category = Category.objects.get(slug='false')
count=0
for src in srcs:
if(src.category == bad_category):
print(src," --bad")
continue
print(src)
src.catalog.add(catalog)
if not (src.metasource):
raise ValueError("{} has no MetaSource".format(src))
ms = src.metasource
ms.catalog=catalog
ms.save()
src.save()
count=count+1
#count=srcs.count()
print("{} {}, log_ml_nfalse {}, selected {}, {:.1f}% false count: {:.1f}".format(survey,band.slug,log_ml_nfalse,count,threshold*100,count*threshold))
def select_catalog():
minrad=40
maxdist=120
#s1s2=get_survey(12.5)
#s1=get_survey(1.7)
#s2=get_survey(2.7)
#s15=get_survey(12345.12)
s15=get_survey(12345.14)
#e0=get_band('E0')
#e1=get_band('E1')
#e2=get_band('E2')
#e3=get_band('E3')
#e4=get_band('E4')
e5=get_band('E5')
cat1=Catalog.objects.get(year=2021)
cat2=Catalog.objects.get(year=2023)
"""
RUN METASOURCE FIRST
"""
""" clean links in MetaSource """
msrcs = MetaSource.objects.all()
for ms in msrcs:
""" clean Pape II only, if exists """
if(ms.catalog == cat2):
print("{} Clean catalog {}".format(ms,cat2))
ms.catalog = None
ms.save()
srcs = ArtSurveySource.objects.all()
for src in srcs:
for cat in src.catalog.all():
if(cat == cat1):
print("{} {} -- keep".format(src,cat))
continue
if(cat == cat2):
print("{} {} -- clean".format(src,cat))
#continue
src.catalog.remove(cat2)
src.save()
continue
#sys.exit()
#select_ml(cat2,s15,e5,-3.1222,0.02)# ML appeared in second catalog
select_ml(cat2,s15,e5,-3.1264,0.02)# ML appeared in second catalog
sys.exit()
#select(cat,s1s2,e0,-2.6751,0.10)
#select(cat,s1s2,e1,-3.2786)
#select(cat,s1s2,e2,-3.3014)
#select(cat,s1s2,e3,-3.9955)
# false 1%
#select(cat,s1s2,e1,-4.0384,0.01)
#select(cat,s1s2,e2,-4.0799,0.01)
#select(cat,s1s2,e3,-4.5918,0.01)
# false 5%
#select(cat,s1,e0,-3.2900)
#select(cat,s1,e1,-3.4620 )
#select(cat,s1,e2,-3.5190 )
#select(cat,s1,e3,-4.0747 )
# false 5%
#select(cat,s2,e0,-3.3065)
#select(cat,s2,e1,-3.5003 )
#select(cat,s2,e2,-3.5262 )
#select(cat,s2,e3,-4.1576 )
# false 1%
#select(cat,s1,e0,-4.0519,0.01)
# false 1%
#select(cat,s2,e0,-4.0747,0.01)
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
select_catalog()
self.stdout.write(self.style.SUCCESS('Done'))

93
artsurvey/management/commands/00_artsurvey_catalog_sigma.py Normal file
View File

@@ -0,0 +1,93 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, Catalog, MetaSource
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
def get_survey(version):
try:
survey = ArtSurvey.objects.get(version=version)
print("use ArtSurvey {}".format(survey))
return survey
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(version))
return None
def get_band(slug):
try:
band = EnergyBand.objects.get(slug=slug)
print("use EnergyBand {}".format(band))
return band
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return None
def select(catalog,survey,band,threshold):
srcs = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(Q(sig__gte=threshold) | Q(ext__gte=1))
for src in srcs:
src.catalog.add(catalog)
ms = src.metasource
ms.catalog=catalog
ms.save()
src.save()
count=srcs.count()
print("{} {}, sign. threshold {}, selected {}".format(survey,band.slug,threshold,count))
def select_catalog():
minrad=40
maxdist=120
s1s2=get_survey(12.5)
s1=get_survey(1.7)
s2=get_survey(2.7)
e0=get_band('E0')
e1=get_band('E1')
e2=get_band('E2')
e3=get_band('E3')
cat=Catalog.objects.get(year=2021)
"""
RUN METASOURCE FIRST
"""
""" clean links in MetaSource """
msrcs = MetaSource.objects.all()
for ms in msrcs:
ms.catalog = None
ms.save()
srcs = ArtSurveySource.objects.all()
for src in srcs:
src.catalog.clear()
select(cat,s1s2,e0,4.818)
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
select_catalog()
self.stdout.write(self.style.SUCCESS('Done'))

21
artsurvey/management/commands/00_artsurvey_cds.py Normal file
View File

@@ -0,0 +1,21 @@
from django.core.management.base import BaseCommand, CommandError
import astropy
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
filename='/export/django/srg/data/artsurvey/catalog.csv'
data = astropy.table.Table.read(filename,format='ascii.csv',delimiter=',')
print(data.info)
self.stdout.write(self.style.SUCCESS('Done'))

80
artsurvey/management/commands/00_artsurvey_crossmatch.py Normal file
View File

@@ -0,0 +1,80 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import F, Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
import numpy as np
from sqlalchemy import create_engine
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, Galactic, FK5
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
from astrobasis.models import GAIADR2
from srglib.utils import find_counterparts
from srglib.utils import crossmatch
from srglib.utils import transfer
from heasarc.models import NSIDE_SOURCES, ORDER
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
""" arcsecond """
minrad=40.0
maxdist=120.0
survey1=1.1
survey2=12.1
sig_min=0.0
sig_max=1000.0
print('***')
print("*** Filter: sig_min={} sig_max={} minrad={:.1f}'' maxdist={}'' ***".format(sig_min,sig_max,minrad,maxdist))
print('***')
try:
band = EnergyBand.objects.get(slug='E0')
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
print("use EnergyBand {}".format(band))
try:
s1 = ArtSurvey.objects.get(version=survey1)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey1))
return
try:
s2 = ArtSurvey.objects.get(version=survey2)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey2))
return
srcs1=s1.artsurveysource_set.all().filter(band=band).filter(Q(sig__gt=sig_min) & Q(sig__lt=sig_max))
srcs2=s2.artsurveysource_set.all().filter(band=band).filter(Q(sig__gt=sig_min) & Q(sig__lt=sig_max))
print("use ArtSurvey {} nrows={}".format(s1, srcs1.count()))
print("use ArtSurvey {} nrows={}".format(s2, srcs2.count()))
match_count=transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist)
print("Total matched {}".format(match_count))
self.stdout.write(self.style.SUCCESS('Done'))

70
artsurvey/management/commands/00_artsurvey_find_orig.py Normal file
View File

@@ -0,0 +1,70 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
from srglib.utils import find_counterparts
from srgcat.models import SrgDataDump
def find_orig_survey(init_version,survey_name,dump_name,slug):
name=dump_name+'.'+slug
try:
survey = ArtSurvey.objects.get(version=init_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey not found")
#survey = ArtSurvey(version=init_version, name=survey_name)
#survey.save()
return
print("use ArtSurvey {}".format(survey))
try:
band = EnergyBand.objects.get(slug=slug)
print("use EnergyBand {}".format(band))
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return
""" copy catalog from this dump """
dump = SrgDataDump.objects.get(name=name)
srcs = dump.skymapsource_set.all()
for src in srcs:
#if not (src.artsurvey_source):
try:
source = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(name=src.name)
except Exception as e:
print("{} {}".format(src,e))
continue
if (source.count() > 1):
for s in source:
print("{} --> {}".format(src,s))
continue
if(source.count() == 0):
print("{} --> not found".format(src))
continue
print("{} {} {}".format(slug,src.name,src.artsurvey_source))
# artsource = ArtSurveySource.objects.create_source(src,survey,band)
# artsource.save()
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
find_orig_survey(1234.0,'1-4','S1-4','E0')
self.stdout.write(self.style.SUCCESS('Done'))

33
artsurvey/management/commands/00_artsurvey_find_skymaps.py Normal file
View File

@@ -0,0 +1,33 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from django.db.models import Q
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from artsurvey.models import EnergyBand
from artsurvey.utils import find_skymaps
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
find_skymaps()
self.stdout.write(self.style.SUCCESS('Done'))

133
artsurvey/management/commands/00_artsurvey_heasarc.py Normal file
View File

@@ -0,0 +1,133 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime, os, sys
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from math import isfinite, sqrt
from heasarc.models import HeasarcBase
from heasarc.models import Heasarc4XMMDR12
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, Category
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
from srglib.utils import load_simbad_for_skymap_sources
from artsurvey.utils import find_heasarc
from heasarc.models import HeasarcBase
from heasarc.models import HeasarcSwiftBAT105m
from heasarc.models import HeasarcIntRefCat
from heasarc.models import HeasarcXTEASSCAT
from heasarc.models import HeasarcXMMSL2
from heasarc.models import Heasarc4FGL
from heasarc.models import HeasarcIntegral2020
from heasarc.models import HeasarcMAXI7YR
from heasarc.models import HeasarcRASS2RXS
def do_heasarc(survey_version,slug,log_ml_nfalse=None):
bad_category = Category.objects.get(slug='false')
minrad=40
maxdist=600
try:
band = EnergyBand.objects.get(slug=slug)
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
try:
survey = ArtSurvey.objects.get(version=survey_version)
print("use ArtSurvey {}".format(survey))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
srcs = survey.artsurveysource_set.all().filter(band=band).filter(log_ml_nfalse__lte=log_ml_nfalse).exclude(category__exact=bad_category)
print("Selected {}".format(srcs.count()))
# normal run
find_heasarc(srcs, maxdist=maxdist, minrad=minrad, offset=False, syserr=10.0)
return
# make offset
"""
# script to read this file:
import pandas as pd
df=pd.read_csv('offset.csv')
for k in df.keys():
print("{} {:.1f} +/- {:.1f} std={:.1f}".format(k,df[k].mean(),df[k].sem(),df[k].std()))
"""
fout='offset.csv'
try:
os.remove(fout)
except OSError:
pass
with open(fout, "a") as myfile:
myfile.write("RASS2RXS,XMMSL2,MAXI7YR,SwiftBAT105m,Integral2020,4FGL\n")
for x in range(100):
find_heasarc(srcs, maxdist=maxdist, minrad=minrad, offset=True, syserr=10.0)
SwiftBAT105m=0
XMMSL2=0
FGL=0
Integral2020=0
MAXI7YR=0
RASS2RXS=0
for src in srcs:
heasarcs = src.heasarc.all()
if not (heasarcs.count()):
continue
if (heasarcs.filter(instance_of=HeasarcSwiftBAT105m).count()):
SwiftBAT105m+=1
if (heasarcs.filter(instance_of=HeasarcXMMSL2).count()):
XMMSL2+=1
if (heasarcs.filter(instance_of=Heasarc4FGL).count()):
FGL+=1
if (heasarcs.filter(instance_of=HeasarcIntegral2020).count()):
Integral2020+=1
if (heasarcs.filter(instance_of=HeasarcMAXI7YR).count()):
MAXI7YR+=1
if (heasarcs.filter(instance_of=HeasarcRASS2RXS).count()):
RASS2RXS+=1
with open(fout, "a") as myfile:
myfile.write("{}, {}, {}, {}, {}, {}\n".format(RASS2RXS,XMMSL2,MAXI7YR,SwiftBAT105m,Integral2020,FGL))
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
#all=HeasarcBase.objects.all()
#print(all.count())
#do_heasarc(12345.12,'E5',log_ml_nfalse=-3.1222)
#do_heasarc(12345.13,'E5',log_ml_nfalse=-3.1264)
do_heasarc(12345.14,'E5',log_ml_nfalse=-3.1264)
"""
srcs = ArtSurveySource.objects.all().filter(heasarc__heasarc4xmmdr12__isnull=False)[:10]
print("Selected {} in total".format(srcs.count()))
for src in srcs:
print(src.get_absolute_url())
"""
self.stdout.write(self.style.SUCCESS('Done'))

105
artsurvey/management/commands/00_artsurvey_init.py Normal file
View File

@@ -0,0 +1,105 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
from srglib.utils import find_counterparts
from srgcat.models import SrgDataDump
def init_survey(init_version,survey_name,dump_name,slug):
name=dump_name+'.'+slug
try:
survey = ArtSurvey.objects.get(Q(version=init_version) & Q(name=survey_name))
except ArtSurvey.DoesNotExist:
print("create ArtSurvey")
survey = ArtSurvey(version=init_version, name=survey_name)
survey.save()
print("use ArtSurvey {}".format(survey))
try:
band = EnergyBand.objects.get(slug=slug)
print("use EnergyBand {}".format(band))
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return
""" copy catalog from this dump """
dump = SrgDataDump.objects.get(name=name)
print(dump)
srcs = dump.skymapsource_set.all()
for src in srcs:
if not (src.artsurvey_source):
print("{} {}".format(slug,src))
artsource = ArtSurveySource.objects.create_source(src,survey,band)
artsource.save()
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
"""
bands = ['E0','E1','E2','E3','E4']
for slug in bands:
init_survey(12.5,'S12',slug)
"""
#init_survey(123.0,'S123','E0')
#init_survey(1234.0,'1-4','S1-4','E0')
#init_survey(3.0,'3','S3','E0')
#version=0.1
#init_survey(version,'L20','L20','E0')
#init_survey(0.13,'L20rb','L20rb','E0')
#init_survey(0.15,'L20rb_ml','L20rb_ml','E0')
#init_survey(12345.12,'1-5', '22y','E0')
#init_survey(12345.12,'1-5', '22y','E5')
#init_survey(12345.13,'1-5', '22y','E5')
init_survey(12345.14,'1-5', '22y','E5')
#init_survey(version,'GC2019','GC','E0')
#init_survey(34.0,'34','S34','E0')
#init_survey(5.0,'5','S5','E0')
#init_survey(12345.0,'1-5','S1-5','E0')
#init_survey(0.12,'L20','L20','E0')
#init_survey(0.12,'L20p1','L20p1','E0')
#init_survey(0.12,'L20p2','L20p2','E0')
#init_survey(0.12,'L20p3','L20p3','E0')
#init_survey(0.12,'L20p4','L20p4','E0')
version=0.11
#fields=[50,45,40,35,30,'025','020','015','010','005','000','355','350','345',]
#fields=['070','075','080','085','090']
#fields=['270','275','280','285','290','295','300','305','310','215','320','325']
#fields=['315']
fields=['330','335','340']
"""
for field in fields:
init_survey(version,"GP{}".format(field),"GP{}".format(field),'E0')
init_survey(version,"GP{}".format(field),"GP{}".format(field),'E1')
init_survey(version,"GP{}".format(field),"GP{}".format(field),'E2')
init_survey(version,"GP{}".format(field),"GP{}".format(field),'E3')
init_survey(version,"GP{}".format(field),"GP{}".format(field),'E4')
"""
self.stdout.write(self.style.SUCCESS('Done'))

33
artsurvey/management/commands/00_artsurvey_mark_turkpossible.py Normal file
View File

@@ -0,0 +1,33 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from django.db.models import Q
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from artsurvey.models import EnergyBand
from artsurvey.utils import select_turkpossible
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
select_turkpossible(12345.11,'E0')
self.stdout.write(self.style.SUCCESS('Done'))

176
artsurvey/management/commands/00_artsurvey_mark_xrays.py Normal file
View File

@@ -0,0 +1,176 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from django.db.models import Q
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from artsurvey.models import EnergyBand
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
from srglib.utils import load_simbad_for_skymap_sources
from heasarc.models import HeasarcSwiftBAT105m
from heasarc.models import HeasarcXMMSL2
from heasarc.models import Heasarc4FGL
from heasarc.models import HeasarcIntegral2020, HeasarcMAXI7YR, HeasarcRASS2RXS
def do_all(survey_version,slug, log_ml_nfalse=None):
minrad=40
maxdist=600
#minrad=180
#maxdist=640
try:
band = EnergyBand.objects.get(slug=slug)
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
try:
survey = ArtSurvey.objects.get(version=survey_version)
print("use ArtSurvey {}".format(survey))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
srcs = survey.artsurveysource_set.all().filter(band=band).filter(log_ml_nfalse__lte=log_ml_nfalse)
for src in srcs:
src.xrays=False
src.save()
SwiftBAT105m=0
XMMSL2=0
FGL=0
Integral2020=0
MAXI7YR=0
RASS2RXS=0
for src in srcs:
heasarcs = src.heasarc.all()
#if not (heasarcs.count()):
# continue
if (heasarcs.filter(instance_of=HeasarcSwiftBAT105m).count()):
SwiftBAT105m+=1
src.xrays=True
src.save()
if (heasarcs.filter(instance_of=HeasarcXMMSL2).count()):
XMMSL2+=1
src.xrays=True
src.save()
if (heasarcs.filter(instance_of=Heasarc4FGL).count()):
FGL+=1
src.xrays=True
src.save()
if (heasarcs.filter(instance_of=HeasarcIntegral2020).count()):
Integral2020+=1
src.xrays=True
src.save()
if (heasarcs.filter(instance_of=HeasarcMAXI7YR).count()):
MAXI7YR+=1
src.xrays=True
src.save()
if (heasarcs.filter(instance_of=HeasarcRASS2RXS).count()):
RASS2RXS+=1
src.xrays=True
src.save()
print("{} {}".format(src,heasarcs.count()))
selection=srcs.filter(xrays__exact=True)
print("Xrays=True for {} from {}".format(selection.count(),srcs.count()))
print("SwiftBAT105m={}".format(SwiftBAT105m))
print("XMMSL2={}".format(XMMSL2))
print("4FGL={}".format(FGL))
print("Integral2020={}".format(Integral2020))
print("MAXI7YR={}".format(MAXI7YR))
print("RASS2RXS={}".format(RASS2RXS))
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
#do_all(12.5,'E0')
#do_all(1.7)
#do_all(2.7)
#do_all(12345.13,'E5', log_ml_nfalse=-3.1264)
do_all(12345.14,'E5', log_ml_nfalse=-3.1264)
self.stdout.write(self.style.SUCCESS('Done'))
"""
SwiftBAT105m=7
XMMSL2=2
4FGL=26
Integral2020=8
MAXI7YR=7
RASS2RXS=6
SwiftBAT105m=10
XMMSL2=1
4FGL=17
Integral2020=7
MAXI7YR=1
RASS2RXS=4
Xrays=True for 41 from 1541
SwiftBAT105m=5
XMMSL2=2
4FGL=20
Integral2020=6
MAXI7YR=3
RASS2RXS=6
Xrays=True for 30 from 1541
SwiftBAT105m=7
XMMSL2=1
4FGL=13
Integral2020=3
MAXI7YR=7
RASS2RXS=4
Xrays=True for 34 from 1541
SwiftBAT105m=5
XMMSL2=1
4FGL=18
Integral2020=2
MAXI7YR=5
RASS2RXS=4
Xrays=True for 44 from 1541
SwiftBAT105m=5
XMMSL2=7
4FGL=17
Integral2020=9
MAXI7YR=9
RASS2RXS=4
"""

60
artsurvey/management/commands/00_artsurvey_metasource_check.py Normal file
View File

@@ -0,0 +1,60 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import F, Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
import numpy as np
from sqlalchemy import create_engine
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, Galactic, FK5
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource, Catalog
from astrobasis.models import GAIADR2
from srglib.utils import find_counterparts
from srglib.utils import crossmatch
from srglib.utils import transfer
from heasarc.models import NSIDE_SOURCES, ORDER
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
srcs = MetaSource.objects.all()
print("Check for empty MetaSource")
for src in srcs:
count = src.artsurveysource_set.all().count()
if (count==0):
print("Empty MetaSource {} paper={}".format(src,src.catalog))
self.stdout.write(self.style.SUCCESS('Done'))
print("Check for empty Paper reference")
for src in srcs:
if not src.catalog:
continue
asrcs = src.artsurveysource_set.all()
count=0
for a in asrcs:
if not (a.catalog.all().count()):
continue
if(src.catalog in a.catalog.all()):
count=count+1
if (count==0):
print("Empty MetaSource {} paper={}".format(src,src.catalog))
self.stdout.write(self.style.SUCCESS('Done'))

98
artsurvey/management/commands/00_artsurvey_metasource_manual_setup.py Normal file
View File

@@ -0,0 +1,98 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from artsurvey.utils import setup_metasource
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
def do_all(survey_version):
minrad=40
maxdist=120
try:
survey = ArtSurvey.objects.get(version=survey_version)
print("use ArtSurvey {}".format(survey))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
srcs = survey.artsurveysource_set.all()
print("Selected {} sources".format(srcs.count()))
for src in srcs:
if(src.skymapsource_set.all().count()):
orig=src.skymapsource_set.all()[0]
"""
src.nfalse=orig.nfalse
src.save()
print("{} <-- {}".format(src.nfalse,orig.nfalse))
"""
if not orig.ext_id == '':
#src.ext_id=orig.ext_id
#src.save()
print("{} {} <-- {}".format(src,src.ext_id,orig.ext_id))
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
"""
setup_metasource('J041324.8+102752','J041326.3+102725')
setup_metasource('J151003.9+333057','J151014.3+332942')
setup_metasource('J210336.2+454507','J210331.8+454453')
setup_metasource('J142557.4+374954','J142604.6+374930')
setup_metasource('J171226.3-232138','J171223.5-232048')
setup_metasource('J151350.7-590814','J151355.6-590806')
setup_metasource('J151350.7-590814','J151355.7-590806')
setup_metasource('J151350.7-590814','J151355.6-590802')
setup_metasource('J151350.7-590814','J151354.2-590805')
setup_metasource('J151350.7-590814','J151354.1-590805')
setup_metasource('J151350.7-590814','J151354.3-590805')
setup_metasource('J151350.7-590814','J151355.4-590802')
setup_metasource('J151350.7-590814','J151354.5-590806')
# "Empty" MetaSource SRGA J151354.5-590806 has been deleted by hand
"""
"""
setup_metasource('J151054.2+054422','J151054.9+054357')
setup_metasource('J151054.2+054422','J151053.6+054345')
setup_metasource('J151054.2+054422','J151053.6+054344')
"""
setup_metasource('J210336.2+454507','J210331.8+454453')
setup_metasource('J151054.2+054422','J151054.9+054357')
setup_metasource('J151054.2+054422','J151053.6+054344')
setup_metasource('J151054.2+054422','J151053.6+054345')
setup_metasource('J142557.4+374954','J142604.6+374930')
setup_metasource('J121033.1+392421','J121030.2+392422')
setup_metasource('J123855.1-271830','J123823.9-253148')
setup_metasource('J174155.6-121207','J174154.0-121231')
setup_metasource('J184817.8-031020','J184816.1-030956')
setup_metasource('J092234.6-631738','J092239.7-631738')
#setup_metasource('','')
self.stdout.write(self.style.SUCCESS('Done'))

41
artsurvey/management/commands/00_artsurvey_radio.py Normal file
View File

@@ -0,0 +1,41 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=2.3
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
load_vizier_nvss(srcs, minrad=40, maxdist=120)
load_vizier_first(srcs, minrad=40, maxdist=120)
load_vizier_sumss(srcs, minrad=40, maxdist=120)
self.stdout.write(self.style.SUCCESS('Done'))

125
artsurvey/management/commands/00_artsurvey_replica.py Normal file
View File

@@ -0,0 +1,125 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from django.db.models import Q
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
from srglib.utils import find_counterparts
from srgcat.models import SrgDataDump
def make_survey_replica(version,slug):
""" note that we don't copy catalogs, i.e. papers reletions """
""" delete all replicas of this survey """
try:
replicas = ArtSurvey.objects.filter(Q(version=version) & Q(parent__isnull=False))
for replica in replicas:
print("Delete Replica ArtSurvey {} --> {}".format(replica,replica.parent))
replicas.delete()
except ArtSurvey.DoesNotExist:
pass
""" For cloning select only original surveys, i.e. without parents """
try:
survey = ArtSurvey.objects.get(Q(version=version) & Q(parent=None))
except ArtSurvey.DoesNotExist:
print("For cloning select only original surveys, i.e. without parents")
print("Survey {} with parent=None not found".format(version))
return
print("use ArtSurvey {}".format(survey))
try:
band = EnergyBand.objects.get(slug=slug)
print("use EnergyBand {}".format(band))
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return
child = ArtSurvey(version=version,parent=survey)
child.save()
srcs = survey.artsurveysource_set.all().filter(band=band)
print("Selected {}".format(srcs.count()))
for src in srcs:
print("Cloning {}".format(src))
self_pk = src.pk
onames = src.othername_set.all()
for o in onames:
print("{} OtherNames {}".format(src,o))
ads = src.ads.all()
simbad = src.simbad.all()
nvss = src.nvss.all()
first = src.first.all()
sumss = src.sumss.all()
heasarc = src.heasarc.all()
allwise = src.allwise.all()
gaia3 = src.gaia3.all()
gaia3_primary = src.gaia3_primary
clone = src
clone.pk = None
clone.id = None
clone._state.adding = True
clone.survey = child
clone.save()
try:
parent = ArtSurveySource.objects.get(pk=self_pk)
except ArtSurveySource.DoesNotExist:
print("Parent source not found for {}".format(clone))
return
clone.parent = parent
clone.ads.set(ads)
clone.simbad.set(simbad)
clone.nvss.set(nvss)
clone.first.set(first)
clone.sumss.set(sumss)
clone.heasarc.set(heasarc)
clone.allwise.set(allwise)
clone.gaia3.set(gaia3)
""" clone OtherName """
for o in onames:
print("clone OtherName {}".format(o))
o.source=clone
o.pk=None
o.id=None
o._state.adding = True
o.save()
clone.save()
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
"""
bands = ['E0','E1','E2','E3','E4']
for slug in bands:
init_survey(12.5,'S12',slug)
"""
make_survey_replica(12.5,'E0')
self.stdout.write(self.style.SUCCESS('Done'))

77
artsurvey/management/commands/00_artsurvey_subquery.py Normal file
View File

@@ -0,0 +1,77 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Count
from django.db.models import OuterRef, Subquery, Exists
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource
from srglib.utils import find_counterparts
from srgcat.models import SrgDataDump
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
#srcs = ArtSurveySource.objects.all()
""" Find first survey """
version=12.2
try:
survey = ArtSurvey.objects.get(version=version)
except ArtSurvey.DoesNotExist:
return
print("use ArtSurvey {}".format(survey))
try:
exclude_survey = ArtSurvey.objects.get(version=2.4)
except ArtSurvey.DoesNotExist:
return
print("exclude ArtSurvey {}".format(exclude_survey))
try:
band = EnergyBand.objects.get(slug='E0')
print("use EnergyBand {}".format(band))
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
try:
exclude_band = EnergyBand.objects.get(slug='E0')
print("use exclude EnergyBand {}".format(exclude_band))
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
srcs = ArtSurveySource.objects.all().filter(survey=survey)
srcs=srcs.filter(band=band)
subquery=MetaSource.objects.filter(artsurveysource=OuterRef("pk")).filter(artsurveysource__survey=exclude_survey).filter(artsurveysource__band=exclude_band)
#srcs=srcs.annotate(exclude=Subquery(subquery.values("name")[:1]))
srcs=srcs.annotate(exclude=Exists(subquery))
for s in srcs:
print(s,s.exclude)
print(srcs.count())
"""
msrcs=MetaSource.objects.annotate(num_sources=Count("artsurveysource"))
for m in msrcs:
print(m, m.num_sources)
"""
self.stdout.write(self.style.SUCCESS('Done'))

57
artsurvey/management/commands/00_artsurvey_sync_notes.py Normal file
View File

@@ -0,0 +1,57 @@
from django.core.management.base import BaseCommand, CommandError
import astropy
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
try:
band = EnergyBand.objects.get(slug='E0')
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
try:
survey = ArtSurvey.objects.get(version=12.5)
print("use ArtSurvey {}".format(survey))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
filename='/export/django/srg/data/artsurvey_notes_revised_paper1.tex'
data = astropy.table.Table.read(filename,format='ascii.csv',delimiter='|')
print(data.info)
for item in data:
name=item[0]
notes=item[1]
print()
print("[new] {} --> {}".format(item[0],item[1]))
try:
srcs=ArtSurveySource.objects.filter(name__exact=name).filter(survey=survey).filter(band=band)
except Exception as e:
print(e)
continue
if(srcs.count()>1):
print("More than one source found!")
return
for src in srcs:
print("[old] {} --> {}".format(src.name,src.notes_paper))
#src.notes_paper=notes
#src.save()
self.stdout.write(self.style.SUCCESS('Done'))

175
artsurvey/management/commands/00_artsurvey_transfer.old Normal file
View File

@@ -0,0 +1,175 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, Galactic, FK5
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from astrobasis.models import GAIADR2
from srglib.utils import find_counterparts
from heasarc.models import NSIDE_SOURCES, ORDER
def transfer_data(psrc,child):
print("--> Transfer data from {} (S{} E{}) to {} (S{} E{})".format(psrc,psrc.survey, psrc.band, child, child.survey, child.band))
child.owner=psrc.owner
child.notes=psrc.notes
child.redshift=psrc.redshift
child.class_id=psrc.class_id
child.object_class=psrc.object_class
child.cname=psrc.cname
child.refid=psrc.refid
child.erosita_data=psrc.erosita_data
child.gaia_primary=psrc.gaia_primary
child.class_tentative=psrc.class_tentative
child.category=psrc.category
for a in psrc.ads.all():
child.ads.add(a)
""" clone OtherName """
for o in psrc.othername_set.all():
o.source=child
o.pk=None
o.save()
child.save()
class Command(BaseCommand):
help = 'DEPRECATED'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
""" arcsecond """
tolerance=10
parent_version=1.1
children_version=1.2
try:
parent = ArtSurvey.objects.get(version=parent_version)
except ArtSurvey.DoesNotExist:
print("parent ArtSurvey {} not found".format(parent_version))
return
try:
children = ArtSurvey.objects.get(version=children_version)
except ArtSurvey.DoesNotExist:
print("children ArtSurvey {} not found".format(children_version))
return
print("use parent ArtSurvey {}".format(parent))
print("use children ArtSurvey {}".format(children))
""" Reset parent in children's relations """
for child in children.artsurveysource_set.all():
child.parent=None
child.owner=None
child.notes=''
child.redshift=0.0
child.class_id=0
child.object_class=None
child.cname=''
child.refid=0
child.erosita_data=False
child.gaia_primary=None
child.class_tentative=False
child.category=None
child.ads.clear()
for o in child.othername_set.all():
print("Delete {} in {}".format(o,child))
o.delete()
child.save()
for psrc in parent.artsurveysource_set.all():
c = SkyCoord(psrc.ra, psrc.dec, frame="fk5", unit="deg")
#print("{} {}".format(psrc,psrc.healpix))
try:
csrcs=children.artsurveysource_set.all().filter(healpix=psrc.healpix)
except:
#self.stdout.write(self.style.ERROR("Missed childrens for {}".format(psrc)))
print("[cent] Missed childrens for {}".format(psrc))
continue
#print("{} {} --> {} {}".format(psrc,psrc.healpix,csrc, csrc.healpix))
for csrc in csrcs:
c0 = SkyCoord(csrc.ra, csrc.dec, frame="fk5", unit="deg")
sep=c.separation(c0).arcsecond
if(sep <= tolerance):
print("[cent] Found children {} (S{} E{}) for parent {} (S{} E{}) at {:.1f}''".format(csrc,csrc.survey,csrc.band,psrc,psrc.survey,psrc.band,sep))
csrc.parent=psrc
csrc.save()
transfer_data(psrc,csrc)
else:
#print("[cent] Possible children {} for parent {} is too far away {:.1f}'' (tol {}'')".format(csrc,psrc,sep,tolerance))
pass
neib = neighbours(psrc.healpix, NSIDE_SOURCES, order=ORDER)
for heal in neib:
try:
csrcs=children.artsurveysource_set.all().filter(healpix=heal)
except:
#print("neib: Missed children for {}".format(psrc))
continue
for csrc in csrcs:
c0 = SkyCoord(csrc.ra, csrc.dec, frame="fk5", unit="deg")
sep=c.separation(c0).arcsecond
if(sep <= tolerance):
self.stdout.write(self.style.SUCCESS("[neib] Found children {} (S{} E{}) for parent {} (S{} E{}) at {:1f}''".format(csrc,csrc.survey,csrc.band,psrc,psrc.survey,psrc.band,sep)))
csrc.parent=psrc
csrc.save()
transfer_data(psrc,csrc)
else:
print("[neib] Possible children {} for parent {} is too far away {:.1f}'' (tol {}'')".format(csrc,psrc,sep,tolerance))
self.stdout.write(self.style.SUCCESS('Done'))
return
""" dalshe ne rabotaet """
""" print warning for missed sources """
for psrc in parent.artsurveysource_set.all():
if (psrc.children.all().count() == 0):
self.stdout.write(self.style.ERROR("Missed children for {}".format(psrc)))
if (psrc.children.all().count() > 1):
self.stdout.write(self.style.ERROR("Too many ({}) children for {}".format(psrc.children.all().count(), psrc)))
if (psrc.children.all().count() == 1):
""" transfer catalog information """
child = psrc.children.all()[0]
print("Transfer data from {} (S{} E{}) to {} (S{} E{})".format(psrc,psrc.survey, psrc.band, child, child.survey, child.band))
child.owner=psrc.owner
child.notes=psrc.notes
child.redshift=psrc.redshift
child.class_id=psrc.class_id
child.object_class=psrc.object_class
child.cname=psrc.cname
child.refid=psrc.refid
child.erosita_data=psrc.erosita_data
child.gaia_primary=psrc.gaia_primary
child.class_tentative=psrc.class_tentative
child.category=psrc.category
for a in psrc.ads.all():
child.ads.add(a)
""" clone OtherName """
for o in psrc.othername_set.all():
o.source=child
o.pk=None
o.save()
child.save()
self.stdout.write(self.style.SUCCESS('Done'))

188
artsurvey/management/commands/00_artsurvey_transfer.py Normal file
View File

@@ -0,0 +1,188 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import F, Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
import numpy as np
from sqlalchemy import create_engine
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, Galactic, FK5
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand
from astrobasis.models import GAIADR2
from srglib.utils import find_counterparts
from srglib.utils import crossmatch
from srglib.utils import transfer
from heasarc.models import NSIDE_SOURCES, ORDER
def do_transfer(parent, child_version, child_name, slug, minrad=30, maxdist=500):
""" the same band assumed for parent and child """
try:
band = EnergyBand.objects.get(slug=slug)
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
print("use EnergyBand {}".format(band))
try:
s1 = ArtSurvey.objects.get(Q(version=parent)) # & Q(archived=True))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(parent))
return
try:
s2 = ArtSurvey.objects.get(Q(version=child_version) & Q(name=child_name))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(child_version))
return
srcs1=s1.artsurveysource_set.all().filter(band=band)
srcs2=s2.artsurveysource_set.all().filter(band=band)
print("[{}] use parent ArtSurvey {} nrows={}".format(slug, s1, srcs1.count()))
print("[{}] use child ArtSurvey {} nrows={}".format(slug, s2, srcs2.count()))
match_count = transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist)
print("Total matched {}".format(match_count))
def do_transfer_band(parent, parent_band, child_version, child_name, child_band, minrad=30, maxdist=500):
""" the same band assumed for parent and child """
try:
band1 = EnergyBand.objects.get(slug=parent_band)
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return
try:
band2 = EnergyBand.objects.get(slug=child_band)
except EnergyBand.DoesNotExist:
print("EnergyBand not found")
return
print("use EnergyBand {} --> {}".format(band1, band2))
try:
s1 = ArtSurvey.objects.get(Q(version=parent)) # & Q(archived=True))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(parent))
return
try:
s2 = ArtSurvey.objects.get(Q(version=child_version) & Q(name=child_name))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(child_version))
return
srcs1=s1.artsurveysource_set.all().filter(band=band1)
srcs2=s2.artsurveysource_set.all().filter(band=band2)
print("[{}] use parent ArtSurvey {} nrows={}".format(child_band, s1, srcs1.count()))
print("[{}] use child ArtSurvey {} nrows={}".format(child_band, s2, srcs2.count()))
match_count = transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist)
print("Total matched {}".format(match_count))
def do_transfer_by_name(parent_version, parent_name, child_version, child_name, slug, minrad=30, maxdist=500):
try:
band = EnergyBand.objects.get(slug=slug)
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
print("use EnergyBand {}".format(band))
try:
s1 = ArtSurvey.objects.get(Q(version=parent_version) & Q(name=parent_name))
except ArtSurvey.DoesNotExist:
print("*** parent ArtSurvey {} {} not found".format(parent_name,parent_version))
return
try:
s2 = ArtSurvey.objects.get(Q(version=child_version) & Q(name=child_name))
except ArtSurvey.DoesNotExist:
print("*** child ArtSurvey {} not found".format(child_version))
return
srcs1=s1.artsurveysource_set.all().filter(band=band)
srcs2=s2.artsurveysource_set.all().filter(band=band)
print("[{}] use parent ArtSurvey {} nrows={}".format(slug, s1, srcs1.count()))
print("[{}] use child ArtSurvey {} nrows={}".format(slug, s2, srcs2.count()))
match_count = transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist)
print("Total matched {}".format(match_count))
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
""" arcsecond """
minrad=30.0
maxdist=500.0
"""
bands = ['E0','E1','E2','E3']
for slug in bands:
print (slug)
do_transfer(12.4, 12.5, slug, minrad=minrad, maxdist=maxdist)
do_transfer(1.6, 1.7, 'E0', minrad=minrad, maxdist=maxdist)
do_transfer(2.6, 2.7, 'E0', minrad=minrad, maxdist=maxdist)
"""
#do_transfer(12.5, 1234.0, 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer(12.5, 3.0, 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer(12.5, 34.0,"34", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer(12.5, 5.0,"5", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer(1234.0, 12345.0,"1-5", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer(12345.0, 12345.1,"1-5", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer(12345.0, 12345.11,"1-5", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer(12345.11, 12345.12,"1-5", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer_band(12345.11, "E0", 12345.12,"1-5", 'E5', minrad=minrad, maxdist=maxdist)
#do_transfer_band(12345.12, "E5", 12345.13,"1-5", 'E5', minrad=minrad, maxdist=maxdist)
do_transfer_band(12345.13, "E5", 12345.14,"1-5", 'E5', minrad=minrad, maxdist=maxdist)
#names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345',]
#names=['GP055','GP060','GP065']
names=['GP070','GP075','GP080','GP085','GP090', 'GP270','GP275','GP280','GP285','GP290','GP295','GP300','GP305','GP310','GP215','GP320','GP325']
"""
for name in names:
do_transfer(12.5, 0.11, name, 'E0', minrad=minrad, maxdist=maxdist)
"""
"""
names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345',]
for name in names:
do_transfer_by_name(0.10, name, 0.11, name, 'E0', minrad=minrad, maxdist=maxdist)
"""
#do_transfer_by_name(0.10, "L20", 0.12, "L20", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer_by_name(0.10, "L20", 0.12, "L20p1", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer_by_name(0.10, "L20", 0.12, "L20p2", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer_by_name(0.10, "L20", 0.12, "L20p3", 'E0', minrad=minrad, maxdist=maxdist)
#do_transfer_by_name(0.10, "L20", 0.12, "L20p4", 'E0', minrad=minrad, maxdist=maxdist)
self.stdout.write(self.style.SUCCESS('Done'))

76
artsurvey/management/commands/00_artsurvey_update.py Normal file
View File

@@ -0,0 +1,76 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import load_vizier_nvss
from srglib.utils import load_vizier_first
from srglib.utils import load_vizier_sumss
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
from srglib.utils import load_vizier_allwise
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, Galactic, FK5
from astropy.coordinates import BarycentricTrueEcliptic
from astropy.coordinates import HeliocentricTrueEcliptic
from astropy.coordinates import GeocentricTrueEcliptic
def do_all(survey_version):
minrad=40
maxdist=120
try:
survey = ArtSurvey.objects.get(version=survey_version)
print("use ArtSurvey {}".format(survey))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
srcs = survey.artsurveysource_set.all()
print("Selected {} sources".format(srcs.count()))
for src in srcs:
crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg")
src.ecl_l=crd.transform_to(BarycentricTrueEcliptic).lon.value
src.ecl_b=crd.transform_to(BarycentricTrueEcliptic).lat.value
src.save()
return
for src in srcs:
if(src.skymapsource_set.all().count()):
orig=src.skymapsource_set.all()[0]
"""
src.nfalse=orig.nfalse
src.save()
print("{} <-- {}".format(src.nfalse,orig.nfalse))
"""
if not orig.ext_id == '':
src.ext_id=orig.ext_id
src.save()
print("{} {} <-- {}".format(src,src.ext_id,orig.ext_id))
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
do_all(12345.0)
self.stdout.write(self.style.SUCCESS('Done'))

67
artsurvey/management/commands/00_artsurvey_update_marshall.py Normal file
View File

@@ -0,0 +1,67 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import F, Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
import numpy as np
from sqlalchemy import create_engine
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, Galactic, FK5
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource
from artsurvey.utils import update_marshall
from astrobasis.models import GAIADR2
from srglib.utils import find_counterparts
from srglib.utils import crossmatch
from srglib.utils import transfer
from heasarc.models import NSIDE_SOURCES, ORDER
def update_local(survey_version,survey_name,en='E0',minrad=15,maxdist=120):
try:
s = ArtSurvey.objects.get(Q(version=survey_version) & Q(name=survey_name))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} {} not found".format(survey_name,survey_version))
return
try:
band = EnergyBand.objects.get(slug=en)
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
srcs=s.artsurveysource_set.all().filter(band=band)
count = update_marshall(srcs)
print("use ArtSurvey {} band={} nrows={} , marshall={}".format(s, band, srcs.count(), count))
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
""" arcsecond """
minrad=30.0
maxdist=500.0
""" Detele MetaSource """
#msrcs=MetaSource.objects.all()
#print("Delete MetaSource nrows={}".format(msrcs.count()))
#msrcs.delete()
update_local(12345.14, "1-5", en='E5')
self.stdout.write(self.style.SUCCESS('Done'))

137
artsurvey/management/commands/00_artsurvey_update_metasource.py Normal file
View File

@@ -0,0 +1,137 @@
from django.core.management.base import BaseCommand, CommandError
from django.db.models import F, Q
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
import numpy as np
from sqlalchemy import create_engine
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, Galactic, FK5
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource
from artsurvey.utils import update_metasource
from astrobasis.models import GAIADR2
from srglib.utils import find_counterparts
from srglib.utils import crossmatch
from srglib.utils import transfer
from heasarc.models import NSIDE_SOURCES, ORDER
def update_meta(survey_version,survey_name,en='E0',minrad=15,maxdist=120):
try:
s = ArtSurvey.objects.get(Q(version=survey_version) & Q(name=survey_name))
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} {} not found".format(survey_name,survey_version))
return
try:
band = EnergyBand.objects.get(slug=en)
except EnergyBand.DoesNotExist:
print("EnergyBand bot found")
return
srcs=s.artsurveysource_set.all().filter(band=band)
count = update_metasource(srcs, minrad=minrad, maxdist=maxdist)
print("use ArtSurvey {} band={} nrows={} , meta={}".format(s, band, srcs.count(), count))
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
""" arcsecond """
minrad=30.0
maxdist=500.0
""" Detele MetaSource """
#msrcs=MetaSource.objects.all()
#print("Delete MetaSource nrows={}".format(msrcs.count()))
#msrcs.delete()
"""
Start filling MetaSource with the most deep survey
Attention! Start with S12.E0 because only it contains extended objects
"""
"""
update_meta(survey=12.5, en='E0', minrad=minrad, maxdist=maxdist)
update_meta(survey=12.5, en='E1', minrad=minrad, maxdist=maxdist)
update_meta(survey=12.5, en='E2', minrad=minrad, maxdist=maxdist)
update_meta(survey=12.5, en='E3', minrad=minrad, maxdist=maxdist)
update_meta(survey=12.5, en='E4', minrad=minrad, maxdist=maxdist)
update_meta(survey=1.7, en='E0', minrad=minrad, maxdist=maxdist)
update_meta(survey=1.7, en='E1', minrad=minrad, maxdist=maxdist)
update_meta(survey=1.7, en='E2', minrad=minrad, maxdist=maxdist)
update_meta(survey=1.7, en='E3', minrad=minrad, maxdist=maxdist)
update_meta(survey=1.7, en='E4', minrad=minrad, maxdist=maxdist)
update_meta(survey=2.7, en='E0', minrad=minrad, maxdist=maxdist)
update_meta(survey=2.7, en='E1', minrad=minrad, maxdist=maxdist)
update_meta(survey=2.7, en='E2', minrad=minrad, maxdist=maxdist)
update_meta(survey=2.7, en='E3', minrad=minrad, maxdist=maxdist)
update_meta(survey=2.7, en='E4', minrad=minrad, maxdist=maxdist)
"""
#update_meta(survey=0.0, en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(survey=0.0, en='E1', minrad=minrad, maxdist=maxdist)
#update_meta(survey=0.0, en='E2', minrad=minrad, maxdist=maxdist)
#update_meta(survey=123.0, en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(survey=1234.0, en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(survey=3.0, en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(survey=4.0, en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(34.0, "34", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(5.0, "5", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(12345.0, "1-5", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(12345.1, "1-5", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(12345.11, "1-5", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(12345.12, "1-5", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(12345.12, "1-5", en='E5', minrad=minrad, maxdist=maxdist)
#update_meta(12345.13, "1-5", en='E5', minrad=minrad, maxdist=maxdist)
update_meta(12345.14, "1-5", en='E5', minrad=minrad, maxdist=maxdist)
#update_meta(0.12, "L20", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(0.12, "L20p1", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(0.12, "L20p2", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(0.12, "L20p3", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(0.12, "L20p4", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(0.13, "L20rb", en='E0', minrad=minrad, maxdist=maxdist)
#update_meta(0.15, "L20rb_ml", en='E0', minrad=minrad, maxdist=maxdist)
version=0.11
#names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345',]
#names=['GP055','GP060','GP065']
#names=['GP070','GP075','GP080','GP085','GP090', 'GP270','GP275','GP280','GP285','GP290','GP295','GP300','GP305','GP310','GP215','GP320','GP325']
#names=['GP315']
names=['GP330','GP335','GP340']
"""
for name in names:
update_meta(version, name, en='E0', minrad=minrad, maxdist=maxdist)
update_meta(version, name, en='E1', minrad=minrad, maxdist=maxdist)
update_meta(version, name, en='E2', minrad=minrad, maxdist=maxdist)
update_meta(version, name, en='E3', minrad=minrad, maxdist=maxdist)
update_meta(version, name, en='E4', minrad=minrad, maxdist=maxdist)
"""
self.stdout.write(self.style.SUCCESS('Done'))

49
artsurvey/management/commands/00_init_energy_bands.py Normal file
View File

@@ -0,0 +1,49 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import EnergyBand, ArtSurveySource, ArtSurvey
from srglib.utils import find_counterparts
from srgcat.models import SrgDataDump
"""
0 4.0 12.0
1 4.0 7.0
2 7.0 12.0
3 12.0 20.0
4 20.0 30.0
"""
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
bands = EnergyBand.objects.all()
bands.delete()
e0=EnergyBand(e1=4.0, e2=12.0, slug='E0')
e0.save()
e1=EnergyBand(e1=4.0, e2=7.0, slug='E1')
e1.save()
e2=EnergyBand(e1=7.0, e2=12.0, slug='E2')
e2.save()
e3=EnergyBand(e1=12.0, e2=20.0, slug='E3')
e3.save()
e4=EnergyBand(e1=20.0, e2=30.0, slug='E4')
e4.save()
self.stdout.write(self.style.SUCCESS('Done'))

36
artsurvey/management/commands/01_artsurvey_heasarc.py Normal file
View File

@@ -0,0 +1,36 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import find_counterparts
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=2.3
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
find_counterparts(srcs, HeasarcBase.objects.all(),'heasarc',minrad=40,verbose=True)
self.stdout.write(self.style.SUCCESS('Done'))

199
artsurvey/management/commands/01_artsurvey_load_erositamatch.py Normal file
View File

@@ -0,0 +1,199 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
import logging
import os
from datetime import datetime
from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurvey, ArtSurveySource, eRositaMatch
from srglib.utils import find_counterparts
import numpy.ma as ma
from django.core.exceptions import ObjectDoesNotExist
MJDREF = 51543.875
TZ_UTC = TimezoneInfo(utc_offset=0*u.hour)
TZ_MSK = TimezoneInfo(utc_offset=3*u.hour)
def load_skymap_sources_erosita(filename, survey):
""" Loads sources cross-matched with eRosita
Parameters
----------
filename : str
Absolute path to the file
:Author:
Roman Krivonos <krivonos@cosmos.ru>
"""
logger = logging.getLogger('django')
if not os.path.isfile(filename):
logger.error("File {} does not exist".format(filename))
return
logger.info("Loading sources from {}".format(filename))
name = os.path.basename(filename)
catalog_name = os.path.splitext(name)[0]
logger.info("Filename {} catalog {}".format(name, catalog_name))
format_string='srg_%Y%m%d_%H%M%S'
dtime = datetime.strptime(catalog_name[:-4], format_string)
tm = Time(dtime, format='datetime', scale='utc')
dt = tm.to_datetime(timezone=TZ_MSK)
data = astropy.table.Table.read(filename,format='ascii.csv')
print(data.info)
"""
hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
hp_plate = HEALPix(nside=NSIDE_PLATES,
order=ORDER_PLATES,
frame=FK5())
"""
for item in data:
try:
#srcs = ArtSurveySource.objects.filter(name_orig=item['NAME'])
src = survey.artsurveysource_set.get(name_orig=item['NAME'])
except Exception as e:
print("{} failed to load {}".format(item['NAME'],e))
print("Missed {} cnts {}, flux {}, exptime {}".format(item['NAME'],item['CNTS'],item['FLUX'],item['EXPTIME'],))
continue
match_soft = True if item['match_soft'] == 'True' else False
match_med = True if item['match_med'] == 'True' else False
match_hard = True if item['match_hard'] == 'True' else False
#print("{}".format(src))
src.erosita_data=True
src.save()
if not (match_soft or match_med or match_hard):
print("3 match failed for {}".format(item['NAME']))
continue
hpidx_soft = item['hpidx_soft'] if not ma.is_masked(item['hpidx_soft']) else None
hpidx_med = item['hpidx_med'] if not ma.is_masked(item['hpidx_med']) else None
hpidx_hard = item['hpidx_hard'] if not ma.is_masked(item['hpidx_hard']) else None
ero = eRositaMatch(source=src,
ero_exp = item['ero_exp'],
lkh = item['lkh'],
sep_soft = item['sep_soft'],
match_soft = item['match_soft'],
srcname_soft = item['srcname_soft'],
hpidx_soft = hpidx_soft,
RA_fin_soft = item['RA_fin_soft'],
DEC_fin_soft = item['DEC_fin_soft'],
RA_soft = item['RA_soft'],
DEC_soft = item['DEC_soft'],
RADEC_ERR_soft = item['RADEC_ERR_soft'],
DET_LIKE_0_soft = item['DET_LIKE_0_soft'],
ML_FLUX_0_soft = item['ML_FLUX_0_soft'],
ML_FLUX_ERR_0_soft = item['ML_FLUX_ERR_0_soft'],
ML_CTS_0_soft = item['ML_CTS_0_soft'],
ML_CTS_ERR_0_soft = item['ML_CTS_ERR_0_soft'],
ML_BKG_0_soft = item['ML_BKG_0_soft'],
EXT_soft = item['EXT_soft'],
EXT_LIKE_soft = item['EXT_LIKE_soft'],
EXT_ERR_soft = item['EXT_ERR_soft'],
ML_EXP_1_soft = item['ML_EXP_1_soft'],
sep_med = item['sep_med'],
match_med = item['match_med'],
srcname_med = item['srcname_med'],
hpidx_med = hpidx_med,
RA_fin_med = item['RA_fin_med'],
DEC_fin_med = item['DEC_fin_med'],
RA_med = item['RA_med'],
DEC_med = item['DEC_med'],
RADEC_ERR_med = item['RADEC_ERR_med'],
DET_LIKE_0_med = item['DET_LIKE_0_med'],
ML_FLUX_0_med = item['ML_FLUX_0_med'],
ML_FLUX_ERR_0_med = item['ML_FLUX_ERR_0_med'],
ML_CTS_0_med = item['ML_CTS_0_med'],
ML_CTS_ERR_0_med = item['ML_CTS_ERR_0_med'],
ML_BKG_0_med = item['ML_BKG_0_med'],
EXT_med = item['EXT_med'],
EXT_LIKE_med = item['EXT_LIKE_med'],
EXT_ERR_med = item['EXT_ERR_med'],
ML_EXP_1_med = item['ML_EXP_1_med'],
sep_hard = item['sep_hard'],
match_hard = item['match_hard'],
srcname_hard = item['srcname_hard'],
hpidx_hard = hpidx_hard,
RA_fin_hard = item['RA_fin_hard'],
DEC_fin_hard = item['DEC_fin_hard'],
RA_hard = item['RA_hard'],
DEC_hard = item['DEC_hard'],
RADEC_ERR_hard = item['RADEC_ERR_hard'],
DET_LIKE_0_hard = item['DET_LIKE_0_hard'],
ML_FLUX_0_hard = item['ML_FLUX_0_hard'],
ML_FLUX_ERR_0_hard = item['ML_FLUX_ERR_0_hard'],
ML_CTS_0_hard = item['ML_CTS_0_hard'],
ML_CTS_ERR_0_hard = item['ML_CTS_ERR_0_hard'],
ML_BKG_0_hard = item['ML_BKG_0_hard'],
EXT_hard = item['EXT_hard'],
EXT_LIKE_hard = item['EXT_LIKE_hard'],
EXT_ERR_hard = item['EXT_ERR_hard'],
ML_EXP_1_hard = item['ML_EXP_1_hard'],
sep_simb = item['sep_simb'],
match_simb = item['match_simb'],
ra_match = item['ra_match'],
dec_match = item['dec_match'],
RAJ2000_simb = item['RAJ2000_simb'],
DEJ2000_simb = item['DEJ2000_simb'],
z_simb = item['z_simb'],
otype_simb = item['otype_simb'])
ero.save()
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
survey_version=1.1
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
for src in srcs:
try:
src.erositamatch
except ObjectDoesNotExist:
continue
print(src.erositamatch)
src.erositamatch.delete()
src.erosita_data=False
src.save()
load_skymap_sources_erosita("/data/artxc/erosita/srg_20200610_000000_000.csv", survey)
self.stdout.write(self.style.SUCCESS('Done'))

196
artsurvey/management/commands/02_artsurvey_load_wise.py Normal file
View File

@@ -0,0 +1,196 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
import logging
import os
from datetime import datetime
from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch
from astropy.io import fits
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from astropy_healpix import HEALPix, neighbours
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurvey, ArtSurveySource, WiseCounterpart
from srglib.utils import find_counterparts
import numpy.ma as ma
from django.core.exceptions import ObjectDoesNotExist
MJDREF = 51543.875
TZ_UTC = TimezoneInfo(utc_offset=0*u.hour)
TZ_MSK = TimezoneInfo(utc_offset=3*u.hour)
def load_skymap_sources_wise(filename, survey):
""" Loads sources cross-matched with eRosita
Parameters
----------
filename : str
Absolute path to the file
:Author:
Roman Krivonos <krivonos@cosmos.ru>
"""
logger = logging.getLogger('django')
if not os.path.isfile(filename):
logger.error("File {} does not exist".format(filename))
return
logger.info("Loading sources from {}".format(filename))
name = os.path.basename(filename)
catalog_name = os.path.splitext(name)[0]
logger.info("Filename {} catalog {}".format(name, catalog_name))
format_string='srg_%Y%m%d_%H%M%S_000_ps2wf'
dtime = datetime.strptime(catalog_name[:-4], format_string)
tm = Time(dtime, format='datetime', scale='utc')
dt = tm.to_datetime(timezone=TZ_MSK)
hdul = fits.open(filename)
data = hdul[1].data
hdul.info()
print(hdul[1].columns)
for item in data:
try:
#srcs = ArtSurveySource.objects.filter(name_orig=item['NAME'])
src = survey.artsurveysource_set.get(name_orig=item['NAME_cl'])
except Exception as e:
print("{} failed to load {}".format(item['NAME_cl'],e))
print("Missed {} cnts {}, flux {}, exptime {}".format(item['NAME_cl'],item['CNTS_cl'],item['FLUX_cl'],item['EXPTIME_cl'],))
continue
src.wise_data=True
src.save()
w1mag = item['w1mag'] if not (ma.is_masked(item['w1mag']) or item['w1mag'] < 0.0) else None
w2mag = item['w2mag'] if not (ma.is_masked(item['w2mag']) or item['w2mag'] < 0.0) else None
dw1mag = item['dw1mag'] if not (ma.is_masked(item['dw1mag']) or item['dw1mag'] < 0.0) else None
dw2mag = item['dw2mag'] if not (ma.is_masked(item['dw2mag']) or item['dw2mag'] < 0.0) else None
w1flux = item['w1flux'] if not (ma.is_masked(item['w1flux']) or item['w1flux'] < 0.0) else None
w2flux = item['w2flux'] if not (ma.is_masked(item['w2flux']) or item['w2flux'] < 0.0) else None
dw1flux = item['dw1flux'] if not (ma.is_masked(item['dw1flux']) or item['dw1flux'] < 0.0) else None
dw2flux = item['dw2flux'] if not (ma.is_masked(item['dw2flux']) or item['dw2flux'] < 0.0) else None
gPSFMag = item['gPSFMag'] if not (ma.is_masked(item['gPSFMag']) or item['gPSFMag'] < 0.0) else None
iPSFMag = item['iPSFMag'] if not (ma.is_masked(item['iPSFMag']) or item['iPSFMag'] < 0.0) else None
rPSFMag = item['rPSFMag'] if not (ma.is_masked(item['rPSFMag']) or item['rPSFMag'] < 0.0) else None
yPSFMag = item['yPSFMag'] if not (ma.is_masked(item['yPSFMag']) or item['yPSFMag'] < 0.0) else None
zPSFMag = item['zPSFMag'] if not (ma.is_masked(item['zPSFMag']) or item['zPSFMag'] < 0.0) else None
primaryDetection = True if not (ma.is_masked(item['primaryDetection']) or int(item['primaryDetection']) == 0) else False
bestDetection = True if not (ma.is_masked(item['bestDetection']) or int(item['bestDetection']) == 0) else False
wise = WiseCounterpart(source=src,
sep=item['sep_cl'],
objID=item['objID'],
raStack=item['raStack'],
decStack=item['decStack'],
raStackErr=item['raStackErr'],
decStackErr=item['decStackErr'],
raMean=item['raMean'],
decMean=item['decMean'],
raMeanErr=item['raMeanErr'],
decMeanErr=item['decMeanErr'],
objInfoFlag=item['objInfoFlag'],
qualityFlag=item['qualityFlag'],
primaryDetection=primaryDetection,
bestDetection=bestDetection,
gPSFMag=gPSFMag,
gPSFMagErr=item['gPSFMagErr'],
gApMag=item['gApMag'],
gApMagErr=item['gApMagErr'],
gKronMag=item['gKronMag'],
gKronMagErr=item['gKronMagErr'],
rPSFMag=rPSFMag,
rPSFMagErr=item['rPSFMagErr'],
rApMag=item['rApMag'],
rApMagErr=item['rApMagErr'],
rKronMag=item['rKronMag'],
rKronMagErr=item['rKronMagErr'],
iPSFMag=iPSFMag,
iPSFMagErr=item['iPSFMagErr'],
iApMag=item['iApMag'],
iApMagErr=item['iApMagErr'],
iKronMag=item['iKronMag'],
iKronMagErr=item['iKronMagErr'],
zPSFMag=zPSFMag,
zPSFMagErr=item['zPSFMagErr'],
zApMag=item['zApMag'],
zApMagErr=item['zApMagErr'],
zKronMag=item['zKronMag'],
zKronMagErr=item['zKronMagErr'],
yPSFMag=yPSFMag,
yPSFMagErr=item['yPSFMagErr'],
yApMag=item['yApMag'],
yApMagErr=item['yApMagErr'],
yKronMag=item['yKronMag'],
yKronMagErr=item['yKronMagErr'],
wisefield=item['wisefield'],
fitext=item['fitext'],
devaucou=item['devaucou'],
star=item['star'],
w1flux=w1flux,
dw1flux=dw1flux,
w1mag=w1mag,
dw1mag=dw1mag,
w1reff=item['w1reff'],
w2flux=w2flux,
dw2flux=dw2flux,
w2mag=w2mag,
dw2mag=dw2mag,
w2reff=item['w2reff'])
wise.save()
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
survey_version=1.1
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
"""
srcs = survey.artsurveysource_set.all()
for src in srcs:
try:
src.erositamatch
except ObjectDoesNotExist:
continue
print(src.erositamatch)
src.erositamatch.delete()
src.erosita_data=False
src.save()
"""
w = WiseCounterpart.objects.all()
w.delete()
load_skymap_sources_wise("/data/artxc/wise/srg_20200610_191805_000_ps2wf_r40.fits", survey)
self.stdout.write(self.style.SUCCESS('Done'))

37
artsurvey/management/commands/02_artsurvey_simbad.py Normal file
View File

@@ -0,0 +1,37 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import load_simbad_for_skymap_sources
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=12.1
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
#srcs = ArtSurveySource.objects.all()
load_simbad_for_skymap_sources(srcs,minrad=40,maxdist=120)
self.stdout.write(self.style.SUCCESS('Done'))

37
artsurvey/management/commands/03_arsurvey_allwise.py Normal file
View File

@@ -0,0 +1,37 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import load_vizier_allwise
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=12.1
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
load_vizier_allwise(srcs, minrad=40, maxdist=120)
self.stdout.write(self.style.SUCCESS('Done'))

42
artsurvey/management/commands/03_artsurvey_gaia.py Normal file
View File

@@ -0,0 +1,42 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from astrobasis.models import GAIADR3
from srglib.utils import find_counterparts
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=1.3
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
#srcs = ArtSurveySource.objects.all()
find_counterparts(srcs, GAIADR3.objects.all(), "gaia3", maxdist=60., minrad=40.)

37
artsurvey/management/commands/04_artsurvey_nvss.py Normal file
View File

@@ -0,0 +1,37 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import load_vizier_nvss
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=12.1
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
load_vizier_nvss(srcs, minrad=40, maxdist=120)
self.stdout.write(self.style.SUCCESS('Done'))

37
artsurvey/management/commands/05_artsurvey_first.py Normal file
View File

@@ -0,0 +1,37 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import load_vizier_first
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=12.0
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
load_vizier_first(srcs, minrad=40, maxdist=120)
self.stdout.write(self.style.SUCCESS('Done'))

37
artsurvey/management/commands/06_artsurvey_sumss.py Normal file
View File

@@ -0,0 +1,37 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.models import HeasarcBase
from artsurvey.models import ArtSurveySource, ArtSurvey
from srglib.utils import load_vizier_sumss
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
survey_version=12.0
try:
survey = ArtSurvey.objects.get(version=survey_version)
except ArtSurvey.DoesNotExist:
print("ArtSurvey {} not found".format(survey_version))
return
print("use ArtSurvey {}".format(survey))
srcs = survey.artsurveysource_set.all()
load_vizier_sumss(srcs, minrad=40, maxdist=120)
self.stdout.write(self.style.SUCCESS('Done'))

1
artsurvey/management/commands/` Normal file
View File

@@ -0,0 +1 @@