erosita/coma
1
0
Fork 0
forked from erosita/uds
Code Pull Requests Activity

endless

Browse Source
This commit is contained in:
Roman Krivonos
2025-04-21 12:44:25 +03:00
parent e5760e312f
commit 49d8e21c32
10 changed files with 2435 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
scripts/03_init_obs.py
View File

@@ -57,12 +57,16 @@ from coma.config import *
root_path=dirname(dirname(dirname(inspect.getfile(coma))))
print("Coma root path: {}".format(root_path))
cwd = os.path.dirname(os.path.realpath(__file__))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
local_run=False
run_Pool=False
keylist=keylist_survey
keylist=keylist_parts
do_init = False
do_ermask = False
@@ -81,10 +85,8 @@ do_cross_match = False
do_astro_corr = False # search optimal shift
do_astro_update = True
do_wcs_match = False # Chandra task -- DEPRECATED
do_wcs_update = False # Chandra task -- DEPRECATED
eband_selected=[0]
eband_selected=[1,2,3,4,5,6,7]
#eband_selected=[0,]
vign=False
vignetting = 'vign' if (vign==True) else 'novign'
@@ -116,11 +118,12 @@ def runme(datakey):
outfile_evtool, outfile_expmap = init_events(key=datakey, eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
local_run=local_run, cwd=cwd,
do_init=do_init,
do_obsmode=True,
do_center=True,
do_evtool=True,
do_expmap=True,
do_expmap=False,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen, width=width,
emin_kev=emin_kev[index],
@@ -373,32 +376,27 @@ def runme(datakey):
save_catprep_ds9reg(catprep,scale=60*60)
if(do_cross_match==True):
crossmatch_shu2019(catprep, dlmin=10,crval=[ra_cen, de_cen], refimage=events[ii],datakey=datakey,
catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_Coma.fits.catalog",errlim=5.0)
# full catalog, all RU sky
cross_catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_Coma.fits.catalog"
# Coma scans footprint
#cross_catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_Coma.footprint.fits.catalog"
crossmatch_shu2019(catprep, dlmin=15,crval=[ra_cen, de_cen], refimage=events[ii],datakey=datakey,devmax=15,
catalog=cross_catalog,errlim=5.0)
if(do_astro_corr==True and eband[index]==0):
""" run astro_corr for 0.3-2.3 keV only """
print("START wcs_astro_corr")
wcs_astro_corr(catprep, Nsim=20000, Rsim=10.0)
wcs_astro_corr(catprep, Nsim=5000, Rsim=10.0)
#wcs_match_ciao(catprep, method='rst',radius=12,residlim=0,residtype=0,residfac=1)
if(do_astro_update==True):
""" run astro_corr for 0.3-2.3 keV only """
attcorr=wcs_update_shift(events[ii],flog=catprep_en0.replace(".fits", ".shift.log"))
do_evtool_esass(evfile=attcorr,outfile=attcorr,rmlock=False, do_center=True, ra_cen=ra_cen, de_cen=de_cen, width=width)
do_evtool_esass(evfile=attcorr,outfile=attcorr,rmlock=False, do_center=True, ra_cen=ra_cen, de_cen=de_cen, width=width, local_run=local_run, cwd=cwd)
if(do_wcs_match==True and eband[index] == 0):
""" run wcs_match for 0.3-2.3 keV only """
wcs_match_ciao(catprep, method='trans',radius=12,residlim=5)
#wcs_match_ciao(catprep, method='rst',radius=12,residlim=0,residtype=0,residfac=1)
if(do_wcs_update==True):
""" use 0.3-2.3 keV transform matrix for all other bands """
attcorr=wcs_update_ciao(events[ii],crval=wcslist[datakey],transformfile=catprep_en0.replace(".fits", ".xfm"),clean=False)
do_evtool_esass(evfile=attcorr,outfile=attcorr,rmlock=False, do_center=True, ra_cen=ra_cen, de_cen=de_cen, width=width)
"""

75
scripts/04_mosaics.py
View File

@@ -72,11 +72,11 @@ infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
local_run = True
local_run = False
do_init = False
do_merge = False
do_merge = True
do_detmask = False
do_circles = False # experimental
do_expmap = False
@@ -96,11 +96,12 @@ do_resid = False # residuals of data and source map
do_fixcat = False # only for index=0
do_fixxmm = False # prepare forced photometry, only for index=0
do_apetool = False
do_catprep = True
do_filter_catalog = True
do_catprep = False
do_ext_bad = False
do_filter_catalog = False
do_cross_match = False
index=0
index=4
forced=False
""" If forced=True, take input catalog from energy range en0 """
@@ -111,8 +112,8 @@ comm='' # for 4XMM-DR12 forced photometry use '-xmm'
vign=True
attcorr=True
rusky=False
keylist = keylist_parts
outkey = "mosa_parts_tm0{}".format('_attcorr' if (attcorr==True) else '')
"""
@@ -122,8 +123,16 @@ mosa_partII_tm0 -- partII only (as is) (keylist_partII)
mosa_parts_tm0 -- all parts (partI and partII) (keylist_parts)
mosa_all_tm0 -- all scans (partI-partII) and survey data (keylist_all)
mosa_tms_tm0 -- all scans (scan1-7) and survey data (keylist_tms)
mosa_survey -- survey data (keylist_survey)
"""
extra_events=None
extra_expmap=None
#extra_events="/data/erosita/work/coma/products/mosa_survey_tm0_attcorr_EventList_en00.fits"
#extra_expmap="/data/erosita/work/coma/products/mosa_survey_tm0_attcorr_ExposureMap_en00.vign.fits"
vignetting = 'vign' if (vign==True) else 'novign'
events=[]
expmaps=[]
@@ -137,11 +146,12 @@ for tmkey in keylist.keys():
eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
local_run=local_run, cwd=cwd,
do_init=do_init,
do_obsmode=True,
do_center=True,
do_evtool=True,
do_expmap=False,
do_expmap=True,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen, width=width,
emin_kev=emin_kev[index],
@@ -154,6 +164,9 @@ for tmkey in keylist.keys():
outfile_evtool="{}_EventList_en{:02d}.fits".format(os.path.join(outfile_dir,outkey),
eband[index])
if (extra_events):
outfile_evtool="{} {}".format(outfile_evtool,extra_events)
if(do_merge==True):
#do_evtool_esass(events=events, outfile=outfile_evtool)
evlist="{}.evlist.txt".format(os.getpid())
@@ -163,7 +176,7 @@ if(do_merge==True):
f.close()
print(outfile_evtool)
do_check_events(events=events)
do_evtool_esass(evlist=evlist, outfile=outfile_evtool, width=width, rusky=rusky)
do_evtool_esass(evlist=evlist, outfile=outfile_evtool, width=width, rusky=rusky, local_run=local_run, cwd=cwd)
if(os.path.isfile(evlist)==True):
os.remove(evlist)
@@ -173,11 +186,10 @@ makes detmask from TM exposures
take limited mask from 'mosa_parts_tm0' for scans+survey data
"""
detmask_outkey=outkey
"""
if(do_detmask==False and outkey.startswith('mosa_all_tm0')):
if(do_detmask==False and (outkey.startswith('mosa_all_tm0') or outkey.startswith('mosa_survey_tm0'))):
print("*** Substitute DetectorMask ***")
detmask_outkey = "mosa_parts_tm0{}".format('_attcorr' if (attcorr==True) else '')
"""
detmask_slug="{}/{}_DetectorMask_en{:02d}".format(outfile_dir,detmask_outkey,eband[index])
detmask="{}{}".format(detmask_slug,outfile_post)
@@ -201,9 +213,21 @@ outfile_bkgmap="{}_BackMap_en{:02d}.{}.fits".format(os.path.join(outfile_dir,out
outfile_expmap="{}_ExposureMap_en{:02d}.{}{}".format(os.path.join(outfile_dir,outkey),
eband[index],vignetting,
outfile_post)
_emin_ev = "{}".format(emin_ev[index])
_emax_ev = "{}".format(emax_ev[index])
_ecf = "{}".format(ecf[index])
if (extra_expmap):
outfile_expmap="{} {}".format(outfile_expmap,extra_expmap)
_emin_ev="{} {}".format(emin_ev[index],emin_ev[index])
_emax_ev="{} {}".format(emax_ev[index],emax_ev[index])
_ecf="{} {}".format(ecf[index],ecf[index],)
if(do_expmap==True):
create_expmap_merged(expmaps,outfile_expmap,scale=7.0)
create_expmap_merged(expmaps,outfile_expmap,scale=7.0)
outfile_boxlist1="{}/{}_BoxList1_en{:02d}{}".format(outfile_dir,outkey, eband[index], outfile_post)
if(do_erbox1==True):
cmd=["erbox",
@@ -211,9 +235,9 @@ if(do_erbox1==True):
"boxlist=%s" %(outfile_boxlist1),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"ecf=\'{}\'".format(ecf[index]),
"emin=\'{}\'".format(_emin_ev),
"emax=\'{}\'".format(_emax_ev),
"ecf=\'{}\'".format(_ecf),
"nruns=2",
"likemin=6.0",
"boxsize=4",
@@ -406,10 +430,12 @@ if(do_ermldet==True):
remove_file(mllist)
remove_file(srcmap)
print(cmd)
runme(cmd, local_run=local_run)
runme(cmd, local_run=local_run, cwd=cwd)
print(cmd)
save_ermldet_ds9reg(mllist,scale=60*60,label='det_like')
save_ermldet_ds9reg(mllist,scale=60*60,label='id_src')
#save_ermldet_ds9reg_dl(mllist,scale=60*60,label='det_like')
#save_ermldet_ds9reg_id(mllist,scale=60*60,label='id_src')
save_ermldet_ds9reg_dl(mllist,scale=60*60, dl=10, point=True)
save_ermldet_ds9reg_id(mllist,scale=60*60, dl=10, point=True)
correct_fluxerr_ermldet_forced(mllist)
if(forced==True):
@@ -514,6 +540,9 @@ if(do_catprep==True):
remove_file(catprep)
runme(cmd, local_run=local_run)
if(do_ext_bad==True):
make_ext_bad(infile=catprep, bkg_map=outfile_backmap3, bkg_cut=1.05, exception=[388,])
if(do_catprep_circles==True):
catprep_circles(mllist,catprep)
@@ -524,8 +553,12 @@ if(do_filter_catalog==True):
#filter_catprep(catprep,expcut=5000.0,dlmin=6,dlmax=10,outkey='faint')
if(do_cross_match==True):
crossmatch_shu2019(catprep,dlmin=10,refimage=outfile_evtool,crval=[ra_cen, de_cen],
catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_Coma.fits.catalog", errlim=5.0)
# full catalog, all RU sky
cross_catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_Coma.fits.catalog"
# Coma scans footprint
#cross_catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_Coma.footprint.fits.catalog"
crossmatch_shu2019(catprep,dlmin=15,refimage=outfile_evtool,crval=[ra_cen, de_cen],
catalog=cross_catalog, errlim=5.0)
# confused sources according to XMM data
# 194.2812310 27.2174300

455
scripts/04_mosaics_joint.py Executable file
View File

@@ -0,0 +1,455 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
НАЗВАНИЕ:
04_mosaics.py
НАЗНАЧЕНИЕ:
Собирает мозайки в разных энергетических диапазонах.
ВЫЗОВ:
esass
./04_mosaics.py
УПРАВЛЕНИЕ:
Запуск отдельных команд управляется переменными, например: do_init = True
Выбранный энергетический диапазон управляется переменной index
forced=True делает принудительную фотометрию
ПАРАМЕТРЫ:
index : Выбранный энергетический диапазон
ВЫВОД:
Выходные файлы записываются в директорию outfile_dir
ИСТОРИЯ:
Роман Кривонос, ИКИ РАН, krivonos@cosmos.ru
Март 2023
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
#from pathlib import Path
import numpy as np
import glob
from os.path import dirname
import inspect
import pickle
import coma
from coma.utils import *
from coma.config import *
""" find Coma root dir """
root_path=dirname(dirname(dirname(inspect.getfile(coma))))
cwd = os.path.dirname(os.path.realpath(__file__))
"""
ftools does not like long file path names,
for this reason, we use relative path here
"""
#root_path='..'
print("Coma root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
local_run = False
# check ../products/joint.evtool
do_expmap = False
do_erbox1 = False # local mode
do_erbackmap1 = False #
do_erbox2 = False # map mode, with background map
do_erbackmap2 = False #
do_erbox3 = False # map mode, with background map
do_erbackmap3 = False #
do_ersensmap = False
do_ermldet = True
do_ermldet_ds9reg = True
do_resid = False # residuals of data and source map
do_fixcat = False # only for index=0
do_fixxmm = False # prepare forced photometry, only for index=0
do_apetool = False
do_catprep = True
do_ext_bad = True # mark extended sources as 'BAD'
do_filter_catalog = True
do_cross_match = True
index=0
forced=False
""" If forced=True, take input catalog from energy range en0 """
comm='' # for 4XMM-DR12 forced photometry use '-xmm'
vign=True
attcorr=True
outkey = "mosa_joint_tm0{}".format('_attcorr' if (attcorr==True) else '')
vignetting = 'vign' if (vign==True) else 'novign'
outfile_evtool="{}/mosa_joint_tm0_attcorr_EventList_en{:02d}.fits".format(outfile_dir,eband[index]) # created by products/joint.evtool
outfile_expmap="{}/mosa_joint_tm0_attcorr_ExposureMap_en{:02d}.{}.fits".format(outfile_dir,eband[index],vignetting)
detmask="{}/mosa_parts_tm0_attcorr_DetectorMask_en{:02d}.fits".format(outfile_dir,0)
_emin_ev = "{}".format(emin_ev[index])
_emax_ev = "{}".format(emax_ev[index])
_ecf = "{}".format(ecf[index])
if(do_expmap==True):
add_expmaps(["{}/mosa_parts_tm0_attcorr_ExposureMap_en{:02d}.{}.fits".format(outfile_dir,eband[index],vignetting),
"{}/mosa_survey_tm0_attcorr_ExposureMap_en{:02d}.{}.fits".format(outfile_dir,eband[index],vignetting)],
"{}/mosa_joint_tm0_attcorr_ExposureMap_en{:02d}.{}.fits".format(outfile_dir,eband[index],vignetting))
outfile_boxlist1="{}/{}_BoxList1_en{:02d}{}".format(outfile_dir,outkey, eband[index], outfile_post)
if(do_erbox1==True):
cmd=["erbox",
"images=\'{}\'".format(outfile_evtool),
"boxlist=%s" %(outfile_boxlist1),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"emin=\'{}\'".format(_emin_ev),
"emax=\'{}\'".format(_emax_ev),
"ecf=\'{}\'".format(_ecf),
"nruns=2",
"likemin=6.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=N",
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist1)
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist1)
""" Background map 1 """
outfile_backmap1="{}_BackMap1_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
cheese_mask="{}_CheeseMask1_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_erbackmap1==True):
do_erbackmap_esass(outfile_evtool,outfile_expmap,outfile_boxlist1,detmask,emin_ev[index],emax_ev[index],
outfile_backmap1,cheese_mask)
outfile_boxlist2="{}/{}_BoxList2_en{:02d}{}".format(outfile_dir,outkey, eband[index], outfile_post)
if(do_erbox2==True):
cmd=["erbox",
"images=\'{}\'".format(outfile_evtool),
"boxlist=%s" %(outfile_boxlist2),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"emin=\'{}\'".format(_emin_ev),
"emax=\'{}\'".format(_emax_ev),
"ecf=\'{}\'".format(_ecf),
"nruns=2",
"likemin=6.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=Y",
"bkgimages={}".format(outfile_backmap1),
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist2)
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist2)
""" Background map 2 """
outfile_backmap2="{}_BackMap2_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
cheese_mask="{}_CheeseMask2_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_erbackmap2==True):
do_erbackmap_esass(outfile_evtool,outfile_expmap,outfile_boxlist2,detmask,emin_ev[index],emax_ev[index],
outfile_backmap2,cheese_mask)
outfile_boxlist3="{}/{}_BoxList3_en{:02d}{}".format(outfile_dir,outkey, eband[index], outfile_post)
if(do_erbox3==True):
cmd=["erbox",
"images=\'{}\'".format(outfile_evtool),
"boxlist=%s" %(outfile_boxlist3),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"emin=\'{}\'".format(_emin_ev),
"emax=\'{}\'".format(_emax_ev),
"ecf=\'{}\'".format(_ecf),
"nruns=2",
"likemin=6.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=Y",
"bkgimages={}".format(outfile_backmap2),
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist3)
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist3)
""" Background map 3 """
outfile_backmap3="{}_BackMap3_en{:02d}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], vignetting, outfile_post)
cheese_mask="{}_CheeseMask3_en{:02d}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], vignetting, outfile_post)
if(do_erbackmap3==True):
boxlist3 = outfile_boxlist3 if(forced == False) else "{}/{}_BoxList3_en{:02d}{}".format(outfile_dir,outkey, eband[0], outfile_post)
do_erbackmap_esass(outfile_evtool,outfile_expmap,boxlist3,detmask,emin_ev[index],emax_ev[index],
outfile_backmap3,cheese_mask)
if(forced==True):
mllist="{}_MaxLikSourceList_en{:02d}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[index], comm, outfile_post)
srcmap="{}_SourceMap_en{:02d}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[index], comm, outfile_post)
residmap="{}_ResidMap_en{:02d}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[index], comm, outfile_post)
""" for en1,2,3,4,5,6,7 give mllist from en0 as input """
boxlist3="{}_MaxLikSourceList_en{:02d}{}{}".format(os.path.join(outfile_dir,outkey), eband[0], comm, outfile_post)
#if(index==0):
boxlist3="{}_MaxLikSourceList_en{:02d}.fixed{}{}".format(os.path.join(outfile_dir,outkey), eband[0], comm, outfile_post)
if not (os.path.exists(boxlist3)):
print("{} not found. Run do_fixcat=True, index=0, forced=False".format(boxlist3))
sys.exit()
add_specific_columns(boxlist3)
fitpos_flag="fitpos_flag=no"
fitext_flag="fitext_flag=no"
nmulsou = "nmulsou=1"
nmaxfit="nmaxfit=10"
multrad="multrad=15."
cutrad="cutrad=15."
if(index == 3 or index == 4):
""" for hard band take unvignetted background """
outfile_backmap3="{}_BackMap3_en{:02d}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], "novign", outfile_post)
else:
mllist="{}_MaxLikSourceList_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
srcmap="{}_SourceMap_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
#mllist="{}_MaxLikSourceList_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
#srcmap="{}_SourceMap_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
residmap="{}_ResidMap_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
boxlist3 = outfile_boxlist3
fitpos_flag="fitpos_flag=yes"
fitext_flag="fitext_flag=yes"
nmulsou = "nmulsou=2"
nmaxfit="nmaxfit=4"
multrad="multrad=20."
cutrad="cutrad=20."
""" allow ermpldet to split sources (no more than two) """
cmd=["ermldet",
"mllist={}".format(mllist),
"boxlist=%s" %(boxlist3),
"images=\'{}\'".format(outfile_evtool),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"bkgimages=\'{}\'".format(outfile_backmap3),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"ecf=\'{}\'".format(ecf[index]),
"hrdef=",
"likemin=0.",
"extlikemin=5.",
"compress_flag=N",
cutrad,
multrad,
"extmin=2.0",
"extmax=35.0",
#"bkgima_flag=Y", looks outdated
"expima_flag=Y",
"detmask_flag=Y",
"shapelet_flag=no", # !!!
"photon_flag=no", # !!!
"extentmodel=beta",
"thres_flag=N",
"thres_col=like",
"thres_val=30.",
nmaxfit,
nmulsou,
fitpos_flag,
fitext_flag,
"srcima_flag=yes",
"srcimages=\'{}\'".format(srcmap)
]
if(do_ersensmap==True):
methods = ['nearest',]# 'linear', 'cubic']
detlike=10
sensmap="{}_SensitivityMap_dl{}_en{:02d}{}".format(os.path.join(outfile_dir,outkey), detlike, eband[index], outfile_post)
create_sensmap(sensmap=sensmap,
areatab="{}_AreaTable_dl{}_en{:02d}{}".format(os.path.join(outfile_dir,outkey), detlike, eband[index], outfile_post),
expmap=outfile_expmap, backmap=outfile_backmap3,detlike=detlike,
detmask=detmask, emin=emin_ev[index], emax=emax_ev[index],ecf=ecf[index], local_run=local_run, cwd=cwd)
for method in methods:
print("Detlike {}, Method {}".format(detlike,method))
corrmap="{}_SensitivityMap_{}_dl{}_en{:02d}{}".format(os.path.join(outfile_dir,outkey), method, detlike, eband[index], outfile_post)
sensmap_corr(sensmap=sensmap, output=corrmap, method=method)
"""
detlike=6
sensmap="{}_SensitivityMap_dl{}_en{:02d}{}".format(os.path.join(outfile_dir,outkey), detlike, eband[index], outfile_post)
create_sensmap(sensmap=sensmap,
areatab="{}_AreaTable_dl{}_en{:02d}{}".format(os.path.join(outfile_dir,outkey), detlike, eband[index], outfile_post),
expmap=outfile_expmap, backmap=outfile_backmap3,detlike=detlike,
detmask=detmask, emin=emin_ev[index], emax=emax_ev[index],ecf=ecf[index], cwd=cwd)
for method in methods:
print("Detlike {}, Method {}".format(detlike,method))
corrmap="{}_SensitivityMap_{}_dl{}_en{:02d}{}".format(os.path.join(outfile_dir,outkey), method, detlike, eband[index], outfile_post)
sensmap_corr(sensmap=sensmap, output=corrmap, method=method)
"""
if(do_ermldet==True):
test_exe('ermldet')
if(vign==False):
print('Run ermldet with vignetted exposure!')
sys.exit()
remove_file(mllist)
remove_file(srcmap)
print(cmd)
runme(cmd, local_run=local_run, cwd=cwd)
print(cmd)
if(forced==True):
result = check_ermldet_forced(mllist)
# for a some reason, for an arbitrary energy band, ermldet break order of sources. Do this forced correction.
if(result == False):
correct_srcid_ermldet_forced(mllist)
if(do_ermldet_ds9reg==True):
save_ermldet_ds9reg_dl(mllist,scale=60*60, dl=6, point=True)
save_ermldet_ds9reg_id(mllist,scale=60*60, dl=6, point=True)
correct_fluxerr_ermldet_forced(mllist)
if(do_resid==True):
do_resid_map(data=outfile_evtool, model=srcmap, outfile=residmap)
if(do_fixcat==True):
if not index == 0:
print("ERROR: You can fix only reference catalog for en0.")
sys.exit()
if forced == True:
print("ERROR: You can fix only non-forced catalog for en0.")
sys.exit()
srcs_remove=[3200,1872,1134,2171,1219,448,357,973,4423,5195,3215,1134,119,
622,2916,2915,2315,6870,824,4027,1463,518,3726,393,4049,1876,2396,1569,
4837,4286,1870,3311,6752,3691,4273,1500,5303,6207,3327,133]
srcs_add = {'4XMM J130526.4+285519':[196.3603538, 28.9221712, 0.853],# 3200
'4XMM J130526.8+285452':[196.3617587,28.9147139, 0.938],# 3200
'4XMM J130123.8+284744':[195.3494948, 28.7956932, 0.692], # 1872
'4XMM J125555.1+283110':[193.9797853,28.5196369,1.648], # 448
'4XMM J125708.3+264926':[194.2847924,26.8239298,0.220],
#'4XMM J022026.3-050251':[35.1098619,-5.0476199,0.551],
#'4XMM J021925.4-042647':[34.8559099,-4.4465007,1.366],
#'4XMM J021910.9-045108':[34.7954311,-4.8522901,0.898],
#'4XMM J021945.2-045331':[34.9383593,-4.8919843,1.538],
#'4XMM J021733.8-051311':[34.3910215,-5.2199877,2.247],
}
fix_catalog(mllist=mllist,refimage=outfile_evtool, srcs_remove=srcs_remove, srcs_add=srcs_add)
"""
Note that fix_catalog added ID_SRC to each XMM source.
Next, we save forced XMM sources (with new ID_SRC!) for later catalog compilation
"""
with open(mllist.replace(".fits", ".xmm.pickle"), 'wb') as f:
pickle.dump(srcs_add, f)
if(do_fixxmm==True):
if not index == 0:
print("ERROR: You can fix only reference catalog for en0.")
sys.exit()
if forced == True:
print("ERROR: You can fix only non-forced catalog for en0.")
sys.exit()
fix_xmm_sources(mllist=mllist,refimage=outfile_evtool, xmm_catalog='../data/4XMM-DR12/4XMM_DR12cat_slim_v1.0_UDS.fits.catalog')
if(do_apetool==True):
psfmap="{}_PsfMap{}".format(os.path.join(outfile_dir,outkey), outfile_post)
#remove_file(psfmap)
#cmd=["apetool",
# "images=\'{}\'".format(outfile_evtool),
# "psfmaps=\'{}\'".format(psfmap),
# "psfmapflag=yes",]
#runme(cmd, local_run=local_run)
#sys.exit()
cmd=["apetool",
"mllist={}".format(mllist),
"apelistout={}".format(mllist), # give the same file
"images=\'{}\'".format(outfile_evtool),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"bkgimages=\'{}\'".format(outfile_backmap3),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"srcimages=\'{}\'".format(srcmap),
"psfmaps={}".format(psfmap),
"psfmapflag=no",
"stackflag=no",
"apexflag=yes",
"apesenseflag=no",
"eefextract=0.65",
"cutrad=15",
"eindex=1",]
runme(cmd, local_run=local_run, cwd=cwd)
if(forced==True):
catprep="{}_SourceCatalog_en{:02d}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[index], comm, outfile_post)
else:
catprep="{}_SourceCatalog_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_catprep==True):
cmd=["catprep",
"infile={}".format(mllist),
"outfile={}".format(catprep),]
remove_file(catprep)
runme(cmd, local_run=local_run)
if(do_ext_bad==True):
make_ext_bad(infile=catprep, bkg_map=outfile_backmap3, bkg_cut=1.05, exception=[388,])
if(do_filter_catalog==True):
#filter_mllist(mllist,expcut=5000.0,dlcut=10.0,dlmin=10,dlmax=10000)
""" works the same """
filter_catprep(catprep,expcut=500.0,dlmin=10,dlmax=10000000,outkey='dl10')
#filter_catprep(catprep,expcut=5000.0,dlmin=6,dlmax=10,outkey='faint')
if(do_cross_match==True):
crossmatch_shu2019(catprep,dlmin=10,refimage=outfile_evtool,crval=[ra_cen, de_cen],
catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_Coma.fits.catalog", errlim=5.0)
# confused sources according to XMM data
# 194.2812310 27.2174300

121
scripts/05_srctool.py
View File

@@ -46,11 +46,11 @@ from os.path import dirname
import inspect
import pickle
import uds
import coma
from uds.utils import *
from uds.config import *
from uds.sherpa import *
from coma.utils import *
from coma.config import *
#from coma.sherpa import *
""" find UDS root dir """
@@ -60,31 +60,26 @@ ftools does not like long file path names,
for this reason, we use relative path here
"""
root_path='..'
print("UDS root path: {}".format(root_path))
print("Coma root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
srctool_dir="{}/{}".format(outfile_dir,"srctool-products")
srctool_dir="{}/{}".format(outfile_dir,"srctool-products-joint")
create_folder(srctool_dir)
outkey="tm0"
outkey="mosa_joint_tm0_attcorr"
outfile_srctool="{}_SrcTool_".format(outkey)
do_init = False
do_merge = False
do_srctool = True
do_srctool = False
do_grppha = False
do_ecf_calc = False # for all bands
do_ecf_print = False # for all bands
do_flux_calc = False # for all bands
do_catalog = False
do_extended = False
do_ds9reg = False
do_euds_final = False
do_coma_final = False
do_euds_dr12 = False # crossmatch eUDS with DR12
do_euds_stat = False
do_euds_cds = False
@@ -108,34 +103,12 @@ vignetting = 'vign' if (vign==True) else 'novign'
events=[]
expmaps=[]
bkgmaps=[]
for tmkey in keylist_tm.keys():
print("TM{} in work... init events".format(tmkey))
for datakey in keylist_tm[tmkey]:
print("--> {}".format(datakey))
""" Подготавливаем списки событий индивидуальных наблюдений """
outfile_evtool,outfile_expmap=init_events(key=datakey,attcorr=True,
eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
do_init=do_init,
do_obsmode=False,
do_center=False,
do_evtool=False,
do_expmap=False,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen,
emin_kev=emin_kev[index],
emax_kev=emax_kev[index])
events.append(outfile_evtool)
expmaps.append(outfile_expmap)
""" Собираем общий список событий """
outfile_evtool="{}_EventList_en{}.fits".format(os.path.join(outfile_dir,outkey),
eband[index])
if(do_merge==True):
do_evtool_esass(events=events, outfile=outfile_evtool)
outfile_evtool="{}_EventList_en{:02d}.fits".format(os.path.join(outfile_dir,outkey), eband[index])
outfile_expmap="{}_ExposureMap_en{:02d}.fits".format(os.path.join(outfile_dir,outkey), eband[index])
suffix_srctool=".fits"
@@ -143,7 +116,7 @@ suffix_srctool=".fits"
If suffix contains no filename extension (does not contain a "."), then ".fits"
is also appended to the filename. """
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[0], outfile_post)
catprep="{}_SourceCatalog_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[0], outfile_post)
""" take source catalog from 0.3-2.3 keV band """
if not (os.path.isfile(catprep)==True):
@@ -155,7 +128,7 @@ if(do_srctool==True):
test_exe('srctool')
cmd=['srctool',
"todo=\'SPEC RMF ARF\'",
"insts=\'1 5 6 7\'",
"insts=\'1 2 3 4 5 6 7\'",
"eventfiles={}".format(outfile_evtool),
"prefix=\'{}\'".format(os.path.join(srctool_dir,outfile_srctool)),
"suffix=\'{}\'".format(suffix_srctool),
@@ -174,61 +147,51 @@ if(do_srctool==True):
if(do_grppha==True):
group_spectra("{}/*020_SourceSpec_*.fits".format(srctool_dir))
ecfout="{}_SampleFlux_v1.pickle".format(os.path.join(outfile_dir,outkey))
if(do_ecf_calc==True):
calc_ecf("{}/tm0_SrcTool_020_ARF_?????.fits".format(srctool_dir),
catprep=catprep, emin=emin_kev, emax=emax_kev, eband=eband, outfile=ecfout, simnum=10000)
if(do_ecf_print==True):
print_ecf(infile=ecfout, emin=emin_kev, emax=emax_kev, eband=eband, skipfrac=10.0)
fluxout="{}_SherpaFlux.pickle".format(os.path.join(outfile_dir,outkey))
if(do_flux_calc==True):
calc_flux("{}/tm0_SrcTool_020_ARF_?????.fits".format(srctool_dir),
catprep=catprep, emin=emin_kev, emax=emax_kev, eband=eband, outfile=ecfout, simnum=100)
#index=0
index=0
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
rawcat="{}_SourceCatalog_en{}.pickle".format(os.path.join(outfile_dir,outkey), eband[index])
catprep="{}_SourceCatalog_en{:02d}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
rawcat="{}_SourceCatalog_en{:02d}.dl10.pickle".format(os.path.join(outfile_dir,outkey), eband[index])
if(do_catalog==True):
forced_xmm_sources="{}_MaxLikSourceList_en{}.xmm.pickle".format(os.path.join(outfile_dir,outkey), eband[index])
"""
forced_xmm_sources="{}_MaxLikSourceList_en{:02d}.xmm.pickle".format(os.path.join(outfile_dir,outkey), eband[index])
with open(forced_xmm_sources, 'rb') as f:
print("Reading forced XMM sources from {}".format(forced_xmm_sources))
srcs_forced = pickle.load(f)
print()
print(srcs_forced)
print()
"""
make_euds_catalog(infile='../products/tm0_SourceCatalog_en0.forced.fits', rawcat=rawcat, dlmin=10.0, dlmax=100000, ext_like=1000,
emin=emin_kev[index], emax=emax_kev[index], eband=eband[index],
infile_en00cat=catprep,
infile_en01cat='../products/tm0_SourceCatalog_en1.forced.fits',
infile_en02cat='../products/tm0_SourceCatalog_en2.forced.fits',
infile_en03cat='../products/tm0_SourceCatalog_en3.forced.fits',
infile_en06cat='../products/tm0_SourceCatalog_en6.forced.fits',
infile_en00sens='../products/tm0_SensitivityMap_dl10_en0.fits',
infile_en01sens='../products/tm0_SensitivityMap_dl10_en1.fits',
infile_en02sens='../products/tm0_SensitivityMap_dl10_en2.fits',
infile_en03sens='../products/tm0_SensitivityMap_dl10_en3.fits',
infile_en06sens='../products/tm0_SensitivityMap_dl10_en6.fits',
srcs_forced=srcs_forced,
)
make_coma_catalog(infile='../products/mosa_joint_tm0_attcorr_SourceCatalog_en00.forced.fits',
rawcat=rawcat, dlmin=10.0, dlmax=10000000, ext_like=1000,
emin=emin_kev[index], emax=emax_kev[index], eband=eband[index],
infile_en00cat=catprep,
infile_en01cat='../products/mosa_joint_tm0_attcorr_SourceCatalog_en01.forced.fits',
infile_en02cat='../products/mosa_joint_tm0_attcorr_SourceCatalog_en02.forced.fits',
infile_en03cat='../products/mosa_joint_tm0_attcorr_SourceCatalog_en03.forced.fits',
infile_en04cat='../products/mosa_joint_tm0_attcorr_SourceCatalog_en04.forced.fits',
infile_en00sens='../products/mosa_joint_tm0_attcorr_SensitivityMap_nearest_dl10_en00.fits',
infile_en01sens='../products/mosa_joint_tm0_attcorr_SensitivityMap_nearest_dl10_en01.fits',
infile_en02sens='../products/mosa_joint_tm0_attcorr_SensitivityMap_nearest_dl10_en02.fits',
infile_en03sens='../products/mosa_joint_tm0_attcorr_SensitivityMap_nearest_dl10_en03.fits',
infile_en04sens='../products/mosa_joint_tm0_attcorr_SensitivityMap_nearest_dl10_en04.fits',
#srcs_forced=srcs_forced,
)
if(do_extended==True):
make_extended(infile=rawcat,outreg="{}_ExtendedCat_en{}.reg".format(os.path.join(outfile_dir,outkey), eband[index]))
make_extended(infile=rawcat,outreg="{}_ExtendedCat_en{:02d}.reg".format(os.path.join(outfile_dir,outkey), eband[index]))
if(do_ds9reg==True):
#make_final_ds9reg(infile=rawcat,outreg="{}_FinalCat_dl10.reg".format(os.path.join(outfile_dir,outkey)))
make_final_ds9reg(infile=rawcat,scale=(60*60)/10,outreg="{}_FinalCat_dl10_talk.reg".format(os.path.join(outfile_dir,outkey)))
make_final_ds9reg(infile=rawcat,scale=(60*60)/10,noname=True,outreg="{}_FinalCat_dl10_noname.reg".format(os.path.join(outfile_dir,outkey)))
if(do_euds_final==True):
""" make final eUDS catalog """
final_euds_catalog(infile=rawcat, outfile_fits='../products/eUDS.fits')
if(do_coma_final==True):
""" make final Coma catalog """
final_coma_catalog(infile=rawcat, outfile_fits='../products/eCFDS.fits')
if(do_euds_dr12==True):
crossmatch_dr12('../products/eUDS.fits', catalog=root_path+"/data/4XMM-DR12/4XMM_DR12cat_slim_v1.0_UDS.fits.catalog", devmax=15)
@@ -243,7 +206,7 @@ if(do_euds_cds==True):
if(do_cross_check==True):
""" cross check final eUDS catalog """
cross_check_euds(infile=catprep, euds='../products/eUDS.fits', outkey="../products/en{}_FinalCat_dl10".format(index))
cross_check_euds(infile=catprep, euds='../products/eUDS.fits', outkey="../products/en{:02d}_FinalCat_dl10".format(index))
if(do_xmm_catalog==True):
""" complile raw forced XMM catalog """