romakrivonos/srg
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more apps

Browse Source
This commit is contained in:
Roman Krivonos
2024-04-26 12:43:00 +03:00
parent 52b209b176
commit 69a2160eb7
536 changed files with 33118 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
srgcat/management/commands/00_init_skymaps.py Normal file
View File

@@ -0,0 +1,183 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix, neighbours
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from astropy.io import fits
from django.db.models import Q
from srgcat.models import SkyMaps
from monthplan.models import SurveyHealpixPlate, NSIDE_PLATES, ORDER
# XMMU JHHMMSS.s+/-DDMMSS.
# www.cosmos.esa.int/web/xmm-newton/source-naming-convention
def make_source_name(key, ra, dec):
c = SkyCoord(ra, dec, frame=FK5(), unit="deg")
str1 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=1).split()
str2 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=0).split()
name = key+" J%s%s" % (str1[0].replace('h','').replace('m','').replace('s',''),
str2[1].replace('d','').replace('m','').replace('s',''))
return name
def load_skymaps(filename):
#hdul = fits.open(filename)
#hdul.info()
table = Table.read(filename)
df = table.to_pandas()
for i in df.index:
skymap = SkyMaps(
SMAPNR = int(df['SMAPNR'][i]),
RA_MIN = float(df['RA_MIN'][i]),
RA_MAX = float(df['RA_MAX'][i]),
DE_MIN = float(df['DE_MIN'][i]),
DE_MAX = float(df['DE_MAX'][i]),
RA_CEN = float(df['RA_CEN'][i]),
DE_CEN = float(df['DE_CEN'][i]),
ELON_CEN = float(df['ELON_CEN'][i]),
ELAT_CEN = float(df['ELAT_CEN'][i]),
GLON_CEN = float(df['GLON_CEN'][i]),
GLAT_CEN = float(df['GLAT_CEN'][i]),
X_MIN = float(df['X_MIN'][i]),
Y_MIN = float(df['Y_MIN'][i]),
OWNER = int(df['OWNER'][i]),
N_NBRS = int(df['N_NBRS'][i]),
FIELD1 = int(df['FIELD1'][i]),
FIELD2 = int(df['FIELD2'][i]),
FIELD3 = int(df['FIELD3'][i]),
FIELD4 = int(df['FIELD4'][i]),
FIELD5 = int(df['FIELD5'][i]),
FIELD6 = int(df['FIELD6'][i]),
FIELD7 = int(df['FIELD7'][i]),
FIELD8 = int(df['FIELD8'][i]),
FIELD9 = int(df['FIELD9'][i]))
skymap.save()
pass
def find_neighbours():
skymaps = SkyMaps.objects.all()
for skymap in skymaps:
skymap.neighbours.clear()
try:
if(skymap.FIELD1 > 0):
n1 = SkyMaps.objects.get(SMAPNR=skymap.FIELD1)
skymap.neighbours.add(n1)
if(skymap.FIELD2 > 0):
n2 = SkyMaps.objects.get(SMAPNR=skymap.FIELD2)
skymap.neighbours.add(n2)
if(skymap.FIELD3 > 0):
n3 = SkyMaps.objects.get(SMAPNR=skymap.FIELD3)
skymap.neighbours.add(n3)
if(skymap.FIELD4 > 0):
n4 = SkyMaps.objects.get(SMAPNR=skymap.FIELD4)
skymap.neighbours.add(n4)
if(skymap.FIELD5 > 0):
n5 = SkyMaps.objects.get(SMAPNR=skymap.FIELD5)
skymap.neighbours.add(n5)
if(skymap.FIELD6 > 0):
n6 = SkyMaps.objects.get(SMAPNR=skymap.FIELD6)
skymap.neighbours.add(n6)
if(skymap.FIELD7 > 0):
n7 = SkyMaps.objects.get(SMAPNR=skymap.FIELD7)
skymap.neighbours.add(n7)
if(skymap.FIELD8 > 0):
n8 = SkyMaps.objects.get(SMAPNR=skymap.FIELD8)
skymap.neighbours.add(n8)
if(skymap.FIELD9 > 0):
n9 = SkyMaps.objects.get(SMAPNR=skymap.FIELD9)
skymap.neighbours.add(n9)
except:
print(skymap," neighbour not found")
skymap.save()
print(skymap)
def find_plates():
skymaps = SkyMaps.objects.all()
for skymap in skymaps:
skymap.survey_healpix_plate.clear()
plates = SurveyHealpixPlate.objects.all()
for plate in plates:
ra = plate.ra
dec = plate.dec
try:
skymap = SkyMaps.objects.get(Q(RA_MIN__lte=ra) & Q(RA_MAX__gt=ra) & Q(DE_MIN__lte=dec) & Q(DE_MAX__gt=dec))
except Exception as e:
print("%s (%s)" % (e.message, type(e)))
return
skymap.survey_healpix_plate.add(plate)
skymap.save()
around = neighbours(plate.healpix, NSIDE_PLATES, order=ORDER)
for index in around:
if(index<0):
continue
try:
pl = SurveyHealpixPlate.objects.get(healpix=index)
except:
print("Healpix index {} not found, exit".format(index))
return
skymap.survey_healpix_plate.add(pl)
skymap.save()
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
#skymaps = SkyMaps.objects.all()
#skymaps.delete()
#load_skymaps('/data/erosita/SKYMAPS.fits')
#find_neighbours()
find_plates()
return
"""
some checks:
"""
skymaps = SkyMaps.objects.all()[:10]
for skymap in skymaps:
print(skymap, skymap.FIELD1,
skymap.FIELD2,
skymap.FIELD3,
skymap.FIELD4,
skymap.FIELD5,
skymap.FIELD6,
skymap.FIELD7,
skymap.FIELD8,
skymap.FIELD9)
n = skymap.survey_healpix_plate.all()
for nn in n:
print(nn)
self.stdout.write(self.style.SUCCESS('Done'))

90
srgcat/management/commands/00_load_art_gc.py Normal file
View File

@@ -0,0 +1,90 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
# XMMU JHHMMSS.s+/-DDMMSS.
# www.cosmos.esa.int/web/xmm-newton/source-naming-convention
def make_source_name(key, ra, dec):
c = SkyCoord(ra, dec, frame=FK5(), unit="deg")
str1 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=1).split()
str2 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=0).split()
name = key+" J%s%s" % (str1[0].replace('h','').replace('m','').replace('s',''),
str2[1].replace('d','').replace('m','').replace('s',''))
return name
def load_gc_table(filename, name, e1, e2):
datafile='/export/django/srg/data/Rodion_GC/'+filename
cat = ArtCat(name=name, e1=e1, e2=e2)
cat.save()
table = Table.read(datafile, format='ascii')
# ra dec sig cnts/s erg/s/cm^2
for item in table:
ra=item['ra']
dec=item['dec']
src_name = make_source_name('IKIa', ra, dec)
crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
print(ra,dec,crd.galactic.l,crd.galactic.b)
src = ArtSource(catalog=cat,
name=src_name,
radec_error=30.0,
ra=item['ra'],
dec=item['dec'],
rate=item['cnts/s'],
flux=item['erg/s/cm^2'],
sign=item['sig'])
src.save()
print('--> Successfully loaded "%s"' % name)
pass
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
# create sqlalchemy engine
# engine = create_engine("mysql+pymysql://{user}:{pw}@localhost/{db}"
# .format(user="heauser",
# pw="srg2019@L2_heasarc",
# db="heasarc_db"))
cats = ArtCat.objects.all()
cats.delete()
load_gc_table('src.list', "ArtCat GC e04to12keV v1.0", 4.0, 12.0)
load_gc_table('src_10_20.list', "ArtCat GC e10to20keV v1.0", 10.0, 20.0)
load_gc_table('src_20_30.list', "ArtCat GC e20to30keV v1.0", 20.0, 30.0)
hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
# update healpix indexes
srcs = ArtSource.objects.all()
for src in srcs:
crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg")
src.healpix = hp.skycoord_to_healpix(crd)
src.save()
self.stdout.write(self.style.SUCCESS('Done'))

144
srgcat/management/commands/00_load_gyro.py Normal file
View File

@@ -0,0 +1,144 @@
from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
import numpy as np
from srglib.utils import slerp, quat_to_pol_and_roll
from Quaternion import Quat
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from astropy.io import fits
from django.db.models import Q
from datetime import datetime
from astropy.time import Time
from srgcat.models import SkyMaps, SrgDataDump, Gyro
from monthplan.models import SurveyHealpixPlate, Survey
from srglib.utils import TZ_UTC, TZ_MSK, load_gyro_file, mission2date
import glob
import os
import logging
def find_closest(now, closest_greater_qs, closest_less_qs):
try:
try:
closest_greater = closest_greater_qs[0]
except IndexError:
closest = closest_less_qs[0]
try:
closest_less = closest_less_qs[0]
except IndexError:
closest = closest_greater_qs[0]
except IndexError:
raise self.model.DoesNotExist("There is no closest object because there are no objects.")
print("closest_less:",closest_less.ra,closest_less.dec)
print("closest_greater:",closest_greater.ra,closest_greater.dec)
delta = (closest_greater.dtime-now).total_seconds()/(closest_greater.dtime-closest_less.dtime).total_seconds()
#print((now-closest_greater.dtime).total_seconds())
#print((closest_greater.dtime-closest_less.dtime).total_seconds())
print('delta ',delta)
ra = np.interp(delta, [0.0, 1.0], [closest_less.ra, closest_greater.ra])
dec = np.interp(delta, [0.0, 1.0], [closest_less.dec, closest_greater.dec])
#quat = slerp([1,0,0,0], [0,0,0,1], np.arange(0, 1, 0.001))
quat = slerp([closest_less.q2, closest_less.q3, closest_less.q4, closest_less.q1],
[closest_greater.q2, closest_greater.q3, closest_greater.q4, closest_greater.q1],
[delta])
qfin=Quat(attitude=quat[0])
ra = qfin.ra
dec=qfin.dec
roll=qfin.roll
return ra, dec
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
logging.getLogger().setLevel(logging.INFO)
gyros = Gyro.objects.all()
#print(gyros.count())
#filename='/srg/a1/work/oper/data/2020/srg_20201125_182700/L0/srg_20201125_182700_000_gyro.fits'
#load_gyro_file(filename)
#now_utc = timezone.now()
#survey = Survey.objects.filter(Q(start__lte=now_utc) & Q(stop__gte=now_utc))[0]
#closest_greater_qs = survey.surveypath_set.filter(dtime__gt=now_utc).order_by('dtime')
#closest_less_qs = survey.surveypath_set.filter(dtime__lt=now_utc).order_by('-dtime')
#closest_ra, closest_dec = find_closest(now_utc, closest_greater_qs, closest_less_qs)
#print(closest_ra, closest_dec)
#path = survey.surveypath_set.all()
#print(path.count())
#for p in path:
# mjd=Time(p.dtime, format='datetime')
# #print(p.dtime,mjd)
# break
for g in gyros:
dtime=Time(mission2date(g.obt), format='datetime').datetime
mjd=Time(g.mjd, format='mjd') - 3*u.hour
msk=mjd.to_datetime(timezone=TZ_MSK)
utc=mjd.to_datetime(timezone=TZ_UTC)
try:
survey = Survey.objects.filter(Q(start__lte=utc) & Q(stop__gte=utc))[0]
except:
continue
closest_greater_qs = survey.surveypath_set.filter(dtime__gt=utc).order_by('dtime')
closest_less_qs = survey.surveypath_set.filter(dtime__lt=utc).order_by('-dtime')
try:
try:
closest_greater = closest_greater_qs[0]
except IndexError:
closest = closest_less_qs[0]
try:
closest_less = closest_less_qs[0]
except IndexError:
closest = closest_greater_qs[0]
except IndexError:
raise self.model.DoesNotExist("There is no closest object because there are no objects.")
closest_ra, closest_dec = find_closest(utc, closest_greater_qs, closest_less_qs)
crd_path = SkyCoord(closest_ra, closest_dec, frame="fk5", unit="deg")
crd_gyro = SkyCoord(g.ra, g.dec, frame="fk5", unit="deg")
print(utc)
print(g.ra)
print('1 ',closest_greater.dtime)
print('1 ',closest_greater.ra)
print('2 ',closest_less.dtime)
print('2 ',closest_less.ra)
print('interp -->',closest_ra)
print(crd_gyro.separation(crd_path).arcmin)
break
self.stdout.write(self.style.SUCCESS('Done'))

90
srgcat/management/commands/01_load_galcenter_for_artsurvey.py Normal file
View File

@@ -0,0 +1,90 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
import logging
from astropy.io import fits
from django.db.models import Q
from datetime import datetime
from astropy.time import Time
from astropy.io import fits
from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource
from monthplan.models import SurveyHealpixPlate
from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources
from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir
from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed
from srglib.utils import load_skymap_sources_for_artsurvey
import glob
import os
import logging
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
logging.getLogger().setLevel(logging.DEBUG)
#path='/export/django/srg/data/Rodion_GC/18Apr2022'
#load_skymap_sources_for_artsurvey("{}/src_list_0.fits".format(path),'GC.E0',force=True)
#return
#path='/export/django/srg/data/Andy_L20'
#load_skymap_sources_for_artsurvey("{}/cat_lp20.fits".format(path),'L20.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/lp20_20220307.fits".format(path),'L20.E0',force=True)
#path='/export/django/srg/data/Galplane_l20'
#load_skymap_sources_for_artsurvey("{}/src_list_0.fits".format(path),'L20rb.E0',force=True)
#path='/export/django/srg/data/Galplane_l20'
#load_skymap_sources_for_artsurvey("{}/src_list_0.fits".format(path),'L20rb_ml.E0',force=True)
#path='/srg/a1/work/oper/staff/kate/skymap/catalog_artdays/'
#load_skymap_sources_for_artsurvey("{}/22y_cat_0.fits".format(path),'22y.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/22y_cat_5.fits".format(path),'22y.E5',force=True)
path='/srg/a1/work/kate/skymap_artdays/catalog/'
load_skymap_sources_for_artsurvey("{}/22y_cat_5.fits".format(path),'22y.E5',force=True)
#load_skymap_sources_for_artsurvey("{}/scan_tot_cat.fits".format(path),'L20.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/scan_0_cat.fits".format(path),'L20p1.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/scan_1_cat.fits".format(path),'L20p2.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/scan_2_cat.fits".format(path),'L20p3.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/scan_3_cat.fits".format(path),'L20p4.E0',force=True)
force=True
ebands=['0','1','2','3','4']
#fields=['040','025','020','015','010','005','000','355','350','345']
#fields=['070','075','080','085','090',]
#fields=['270','275','280','285','290','295','300','305','310','215','320','325']
#fields=['315']
fields=['330','335','340']
path='/srg/a1/work/rodion/SRG/art-xc/data/galmap/field_data/'
"""
for field in fields:
for eband in ebands:
load_skymap_sources_for_artsurvey("{}/{}/1g/cat_1g_{}.fits".format(path,field,eband),
"GP{}.E{}".format(field,eband),
force=force)
"""
self.stdout.write(self.style.SUCCESS('Done'))

50
srgcat/management/commands/01_load_skymap_erosita.py Normal file
View File

@@ -0,0 +1,50 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
import logging
from astropy.io import fits
from django.db.models import Q
from datetime import datetime
from astropy.time import Time
from astropy.io import fits
from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource
from monthplan.models import SurveyHealpixPlate
from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources
from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir
from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed
from srglib.utils import load_skymap_sources_erosita
import glob
import os
import logging
import sys
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
logger = logging.getLogger('django')
logger.setLevel(logging.INFO)
load_skymap_sources_erosita('/data/artxc/erosita/srg_20200322_000000_000.csv')
self.stdout.write(self.style.SUCCESS('Done'))

100
srgcat/management/commands/01_load_skymap_for_artsurvey.py Normal file
View File

@@ -0,0 +1,100 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
import logging
from astropy.io import fits
from django.db.models import Q
from datetime import datetime
from astropy.time import Time
from astropy.io import fits
from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource
from monthplan.models import SurveyHealpixPlate
from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources
from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir
from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed
from srglib.utils import load_skymap_sources_for_artsurvey
from srglib.utils import update_skymap_sources_for_artsurvey
import glob
import os
import logging
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
logging.getLogger().setLevel(logging.DEBUG)
path='/srg/work/oper/staff/kate/skymap/catalog'
"""
load_skymap_sources_for_artsurvey("{}/1y_cat_0.fits".format(path),'S12.E0',force=True)
load_skymap_sources_for_artsurvey("{}/1y_cat_1.fits".format(path),'S12.E1',force=True)
load_skymap_sources_for_artsurvey("{}/1y_cat_2.fits".format(path),'S12.E2',force=True)
load_skymap_sources_for_artsurvey("{}/1y_cat_3.fits".format(path),'S12.E3',force=True)
load_skymap_sources_for_artsurvey("{}/1y_cat_4.fits".format(path),'S12.E4',force=True)
load_skymap_sources_for_artsurvey("{}/1h_cat_0.fits".format(path),'S1.E0',force=True)
load_skymap_sources_for_artsurvey("{}/1h_cat_1.fits".format(path),'S1.E1',force=True)
load_skymap_sources_for_artsurvey("{}/1h_cat_2.fits".format(path),'S1.E2',force=True)
load_skymap_sources_for_artsurvey("{}/1h_cat_3.fits".format(path),'S1.E3',force=True)
load_skymap_sources_for_artsurvey("{}/1h_cat_4.fits".format(path),'S1.E4',force=True)
load_skymap_sources_for_artsurvey("{}/2h_cat_0.fits".format(path),'S2.E0',force=True)
load_skymap_sources_for_artsurvey("{}/2h_cat_1.fits".format(path),'S2.E1',force=True)
load_skymap_sources_for_artsurvey("{}/2h_cat_2.fits".format(path),'S2.E2',force=True)
load_skymap_sources_for_artsurvey("{}/2h_cat_3.fits".format(path),'S2.E3',force=True)
load_skymap_sources_for_artsurvey("{}/2h_cat_4.fits".format(path),'S2.E4',force=True)
"""
"""
update_skymap_sources_for_artsurvey("{}/1y_cat_0.fits".format(path),'S12.E0')
update_skymap_sources_for_artsurvey("{}/1y_cat_1.fits".format(path),'S12.E1')
update_skymap_sources_for_artsurvey("{}/1y_cat_2.fits".format(path),'S12.E2')
update_skymap_sources_for_artsurvey("{}/1y_cat_3.fits".format(path),'S12.E3')
update_skymap_sources_for_artsurvey("{}/1y_cat_4.fits".format(path),'S12.E4')
update_skymap_sources_for_artsurvey("{}/1h_cat_0.fits".format(path),'S1.E0')
update_skymap_sources_for_artsurvey("{}/1h_cat_1.fits".format(path),'S1.E1')
update_skymap_sources_for_artsurvey("{}/1h_cat_2.fits".format(path),'S1.E2')
update_skymap_sources_for_artsurvey("{}/1h_cat_3.fits".format(path),'S1.E3')
update_skymap_sources_for_artsurvey("{}/1h_cat_4.fits".format(path),'S1.E4')
update_skymap_sources_for_artsurvey("{}/2h_cat_0.fits".format(path),'S2.E0')
update_skymap_sources_for_artsurvey("{}/2h_cat_1.fits".format(path),'S2.E1')
update_skymap_sources_for_artsurvey("{}/2h_cat_2.fits".format(path),'S2.E2')
update_skymap_sources_for_artsurvey("{}/2h_cat_3.fits".format(path),'S2.E3')
update_skymap_sources_for_artsurvey("{}/2h_cat_4.fits".format(path),'S2.E4')
"""
#load_skymap_sources_for_artsurvey("{}/15y_cat_0.fits".format(path),'S123.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/2y_cat_0.fits".format(path),'S1-4.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/3h_cat_0.fits".format(path),'S3.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/4h_cat_0.fits".format(path),'S4.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/34h_cat_0.fits".format(path),'S34.E0',force=True)
#load_skymap_sources_for_artsurvey("{}/5h_cat_0.fits".format(path),'S5.E0',force=True)
load_skymap_sources_for_artsurvey("{}/22y_cat_0.fits".format(path),'S1-5.E0',force=True)
self.stdout.write(self.style.SUCCESS('Done'))

80
srgcat/management/commands/01_load_skymap_sources.py Normal file
View File

@@ -0,0 +1,80 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
import logging
from astropy.io import fits
from django.db.models import Q
from datetime import datetime
from astropy.time import Time
from astropy.io import fits
from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource
from monthplan.models import SurveyHealpixPlate
from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources
from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir
from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed
import glob
import os
import logging
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
logging.getLogger().setLevel(logging.INFO)
load_skymap_sources_dir('/srg/work/oper/staff/kate/skymap/catalog')
update_allsky_missed()
#print('mark_new_skymap_sources_in_latest')
#mark_new_skymap_sources_in_latest()
#print('update_allsky_missed')
#update_allsky_missed()
"""
skymap = SkyMaps.objects.get(SMAPNR=230147)
dump=SrgDataDump.objects.get(name='srg_20200310_163523_000')
print(skymap)
srcs=skymap.skymapsource_set.all().filter(dump=dump).order_by('-rate')
clean_skymap_sources(srcs)
"""
"""
skymap = SkyMaps.objects.get(SMAPNR=273024)
dump=SrgDataDump.objects.get(name='srg_20200311_164642_000')
print(skymap)
srcs=skymap.skymapsource_set.all().filter(dump=dump).order_by('-rate')
for s in srcs:
print(s)
"""
#load_simbad_sources(srcs,minrad=60,maxdist=120)
#clean_skymap_sources(srcs)
return
load_skymap_sources_file('/srg/work/oper/staff/kate/skymap/catalog/srg_20200302_165649_000.fits')
""" update_all_skymap_sources_counterparts() """
self.stdout.write(self.style.SUCCESS('Done'))

47
srgcat/management/commands/01_load_skymaps.py Normal file
View File

@@ -0,0 +1,47 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from astropy.io import fits
from django.db.models import Q
from datetime import datetime
from astropy.time import Time
from srgcat.models import SkyMaps, SrgDataDump
from monthplan.models import SurveyHealpixPlate
from srglib.utils import TZ_MSK, load_srg_data_dumps
import glob
import os
import logging
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
logging.getLogger().setLevel(logging.INFO)
#dumps = SrgDataDump.objects.all()
#dumps.delete()
#return
load_srg_data_dumps('/srg/work/oper/staff/kate/skymap')
self.stdout.write(self.style.SUCCESS('Done'))

47
srgcat/management/commands/01_update_allsky_missed.py Normal file
View File

@@ -0,0 +1,47 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
import logging
from astropy.io import fits
from django.db.models import Q
from datetime import datetime
from astropy.time import Time
from astropy.io import fits
from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource, SelectAllskyMissed
from monthplan.models import SurveyHealpixPlate
from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources
from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir
from srglib.utils import update_allsky_missed
import glob
import os
import logging
class Command(BaseCommand):
help = 'Initiates data dase'
def handle(self, *args, **options):
logging.getLogger().setLevel(logging.INFO)
update_allsky_missed()
self.stdout.write(self.style.SUCCESS('Done'))

47
srgcat/management/commands/run_find_heasarc_counterparts.py Normal file
View File

@@ -0,0 +1,47 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase
from srgcat.models import ArtCat, ArtSource, SkyMaps, SkyMapSource
from astropy.table import Table
from astropy_healpix import HEALPix, neighbours
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from srglib.utils import find_heasarc_counterparts
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
"""
cats = ArtCat.objects.all()
for cat in cats:
cat.source_set = cat.artsource_set
find_heasarc_counterparts(cat)
"""
skymaps = SkyMaps.objects.all()
for cat in skymaps:
ntot = cat.skymapsource_set.count()
if (ntot):
cat.source_set = cat.skymapsource_set
find_heasarc_counterparts(cat)
self.stdout.write(self.style.SUCCESS('Done'))

55
srgcat/management/commands/run_find_simbad_counterparts.py Normal file
View File

@@ -0,0 +1,55 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase
from srgcat.models import ArtCat, ArtSource, SkyMaps, SkyMapSource, SrgDataDump
from astropy.table import Table
from astropy_healpix import HEALPix, neighbours
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from srglib.utils import find_heasarc_counterparts
from srglib.utils import load_simbad_for_skymap_sources
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
"""
cats = ArtCat.objects.all()
for cat in cats:
cat.source_set = cat.artsource_set
find_heasarc_counterparts(cat)
skymaps = SkyMaps.objects.all()
for cat in skymaps:
ntot = cat.skymapsource_set.count()
if (ntot):
cat.source_set = cat.skymapsource_set
find_heasarc_counterparts(cat)
"""
try:
dump = SrgDataDump.objects.get(name='srg_20200530_231807_000')
except:
return
srcs = dump.skymapsource_set.all()
load_simbad_for_skymap_sources(srcs,minrad=30,maxdist=120)
self.stdout.write(self.style.SUCCESS('Done'))

75
srgcat/management/commands/simple_healpix_match.py Normal file
View File

@@ -0,0 +1,75 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix, neighbours
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
radius_cut=30.0
hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
srcs = ArtSource.objects.all() # get(name__contains="J175524.3-260347")
for src in srcs:
print("***",src.name,src.healpix)
src.heasarc.clear()
c = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg")
"""
Add central pixel:
"""
try:
heasarcs = HeasarcBase.objects.filter(healpix=src.healpix)#.distinct('srcid')
except:
print("healpix not found ", src.healpix)
continue
for heasarc in heasarcs:
#print(heasarc.name)
c0 = SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg")
sep=c.separation(c0)
if(sep.arcsecond < radius_cut):
src.heasarc.add(heasarc)
src.save()
"""
Add neighbour pixels:
"""
around = neighbours(src.healpix, NSIDE_SOURCES, order=ORDER)
for index in around:
try:
heasarcs = HeasarcBase.objects.filter(healpix=index)#.distinct('srcid')
except:
print("healpix not found ", src.healpix)
continue
for heasarc in heasarcs:
#print(heasarc.name)
c0 = SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg")
sep=c.separation(c0)
if(sep.arcsecond < radius_cut):
src.heasarc.add(heasarc)
src.save()
self.stdout.write(self.style.SUCCESS('Done'))

63
srgcat/management/commands/testme_artcat.py Normal file
View File

@@ -0,0 +1,63 @@
from django.core.management.base import BaseCommand, CommandError
from datetime import date
import datetime
from django.utils import timezone
import astropy
from astropy.io import ascii
import pandas as pd
import pymysql
from sqlalchemy import create_engine
from heasarc.tdat import tDat
from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase
from srgcat.models import ArtCat, ArtSource
from astropy.table import Table
from astropy_healpix import HEALPix, neighbours
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
from astropy.config import reload_config
from astropy.config import get_config_dir
class Command(BaseCommand):
help = 'Initiates data dase'
# def add_arguments(self, parser):
# parser.add_argument('poll_id', nargs='+', type=int)
def handle(self, *args, **options):
print(get_config_dir())
return
hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
try:
src = ArtSource.objects.get(name='NRTA J175524.3-260347')
except:
print('Not Found')
return
print('SRC:',src,src.healpix)
try:
refs = HeasarcXMMSSC.objects.filter(name__contains="J175525.1-260403")
except:
print("no XMMSSC found")
print("***")
#print("NRTA J175524.3-260347",src.healpix,src.ra, src.dec)
around = neighbours(src.healpix, NSIDE_SOURCES, order=ORDER)
for index in around:
sc=hp.healpix_to_skycoord(index)
print(index,sc.ra.deg, sc.dec.deg)
for ref in refs:
print("ref ***")
print(ref.name,ref.healpix,src.ra, src.dec)
self.stdout.write(self.style.SUCCESS('Done'))