nezaval/nsd
1
0
Fork 0
forked from erosita/uds
Code Pull Requests Activity

update

Browse Source
This commit is contained in:
Roman Krivonos
2023-03-29 17:28:29 +03:00
parent 09deb0e4f0
commit 4e6025f357
16 changed files with 792 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
scripts/04_mosaics.py
View File

@@ -19,6 +19,7 @@
Запуск отдельных команд управляется переменными, например: do_init = True
Выбранный энергетический диапазон управляется переменной index
forced=True делает принудительную фотометрию
ПАРАМЕТРЫ:
@@ -66,30 +67,27 @@ create_folder(outfile_dir)
outkey="tm0"
do_init = False
do_merge = False
do_detmask = False
do_expmap = False
do_init = True
do_merge = True
do_detmask = True
do_expmap = True
do_erbox1 = True # local mode
do_erbackmap1 = True #
do_erbox2 = True # map mode, with background map
do_erbackmap2 = True #
do_erbox3 = True # map mode, with background map
do_erbackmap3 = True #
do_ersensmap = True
do_ermldet = False
do_catprep = False
do_filter_catalog = False
do_cross_match = False
index=1
do_erbox1 = False # local mode
do_erbackmap1 = False #
do_erbox2 = False # map mode, with background map
do_erbackmap2 = False #
do_erbox3 = False # map mode, with background map
do_erbackmap3 = False #
forced=True
""" If forced=True, take input catalog from energy range en0 """
do_ersensmap = False
do_ermldet = True
do_catprep = True
do_filter_catalog = True
#do_ermldet = False
#do_cross_match = False
index=4
vign=True
vignetting = 'vign' if (vign==True) else 'novign'
@@ -110,7 +108,7 @@ for tmkey in keylist_tm.keys():
do_obsmode=False,
do_center=False,
do_evtool=False,
do_expmap=False,
do_expmap=True,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen,
emin_kev=emin_kev[index],
@@ -138,9 +136,11 @@ if(do_detmask==True):
"""
Собираем общую карту экспозиции, обратите внимание на коэффициент 7.
Экспозиция рассчитывается на 7 телескопов.
outfile_expmap="{}_ExposureMap_en{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], vignetting)
outfile_bkgmap="{}_BackMap_en{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], vignetting)
"""
outfile_expmap="{}_ExposureMap_en{}{}".format(os.path.join(outfile_dir,outkey),
eband[index],
outfile_expmap="{}_ExposureMap_en{}.{}{}".format(os.path.join(outfile_dir,outkey),
eband[index],vignetting,
outfile_post)
if(do_expmap==True):
create_expmap_merged(expmaps,outfile_expmap,scale=7.0)
@@ -231,17 +231,38 @@ if(do_erbox3==True):
save_ds9reg(outfile_boxlist3)
""" Background map 3 """
outfile_backmap3="{}_BackMap3_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
cheese_mask="{}_CheeseMask3_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
outfile_backmap3="{}_BackMap3_en{}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], vignetting, outfile_post)
cheese_mask="{}_CheeseMask3_en{}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], vignetting, outfile_post)
if(do_erbackmap3==True):
do_erbackmap_esass(outfile_evtool,outfile_expmap,outfile_boxlist3,detmask,emin_ev[index],emax_ev[index],
outfile_backmap3,cheese_mask)
boxlist3 = outfile_boxlist3 if(forced == False) else "{}/{}_BoxList3_en{}{}".format(outfile_dir,outkey, eband[0], outfile_post)
do_erbackmap_esass(outfile_evtool,outfile_expmap,boxlist3,detmask,emin_ev[index],emax_ev[index],
outfile_backmap3,cheese_mask)
if(forced==True):
mllist="{}_MaxLikSourceList_en{}.forced{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
srcmap="{}_SourceMap_en{}.forced{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
""" give mllist from en0 as input """
boxlist3="{}_MaxLikSourceList_en{}{}".format(os.path.join(outfile_dir,outkey), eband[0], outfile_post)
fitpos_flag="fitpos_flag=no"
fitext_flag="fitext_flag=no"
nmulsou = "nmulsou=1"
""" don't allow ermpldet to split sources """
if(index == 3):
""" for hard band take unvignetted background """
outfile_backmap3="{}_BackMap3_en{}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], "novign", outfile_post)
else:
mllist="{}_MaxLikSourceList_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
srcmap="{}_SourceMap_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
boxlist3 = outfile_boxlist3
fitpos_flag="fitpos_flag=yes"
fitext_flag="fitext_flag=yes"
nmulsou = "nmulsou=2"
""" allow ermpldet to split sources (no more than two) """
mllist="{}_MaxLikSourceList_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
srcmap="{}_SourceMap_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
cmd=["ermldet",
"mllist=%s" %(mllist),
"boxlist=%s" %(outfile_boxlist3),
"boxlist=%s" %(boxlist3),
"images=\'{}\'".format(outfile_evtool),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
@@ -250,7 +271,7 @@ cmd=["ermldet",
"emax=\'{}\'".format(emax_ev[index]),
"ecf=\'{}\'".format(ecf[index]),
"hrdef=",
"likemin=5.",
"likemin=0.",
"extlikemin=6.",
"compress_flag=N",
"cutrad=10.", # was 15
@@ -267,14 +288,13 @@ cmd=["ermldet",
"thres_col=like",
"thres_val=30.",
"nmaxfit=4",
"nmulsou=2",
"fitext_flag=yes",
nmulsou,
fitpos_flag,
fitext_flag,
"srcima_flag=yes",
"srcimages=\'{}\'".format(srcmap)
]
sensmap="{}_SensitivityMap_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
area="{}_AreaTable_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_ersensmap==True):
detlike=10
create_sensmap(sensmap="{}_SensitivityMap_dl{}_en{}{}".format(os.path.join(outfile_dir,outkey), detlike,
@@ -283,15 +303,32 @@ if(do_ersensmap==True):
detlike, eband[index], outfile_post),
expmap=outfile_expmap, backmap=outfile_backmap3,detlike=detlike,
detmask=detmask, emin=emin_ev[index], emax=emax_ev[index],ecf=ecf[index])
"""
detlike=6
create_sensmap(sensmap="{}_SensitivityMap_dl{}_en{}{}".format(os.path.join(outfile_dir,outkey), detlike,
eband[index], outfile_post),
areatab="{}_AreaTable_dl{}_en{}{}".format(os.path.join(outfile_dir,outkey),
detlike, eband[index], outfile_post),
expmap=outfile_expmap, backmap=outfile_backmap3,detlike=detlike,
detmask=detmask, emin=emin_ev[index], emax=emax_ev[index],ecf=ecf[index])
"""
if(do_ermldet==True):
remove_file(mllist)
remove_file(srcmap)
os.system((" ").join(cmd))
print((" ").join(cmd))
if(vign==False):
print('Run ermldet with vignetted exposure!')
sys.exit()
remove_file(mllist)
remove_file(srcmap)
os.system((" ").join(cmd))
print((" ").join(cmd))
if(forced==True):
check_ermldet_forced(mllist)
if(forced==True):
catprep="{}_SourceCatalog_en{}.forced{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
else:
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_catprep==True):
cmd=["catprep",
"infile={}".format(mllist),
@@ -306,3 +343,6 @@ if(do_filter_catalog==True):
filter_catprep(catprep,expcut=5000.0,dlmin=10,dlmax=10000,outkey='bright')
filter_catprep(catprep,expcut=5000.0,dlmin=6,dlmax=10,outkey='faint')
if(do_cross_match==True):
crossmatch_shu2019(catprep,dlmin=10,refimage=outfile_evtool,crval=[ra_cen, de_cen],
catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_UDS.fits.catalog")

40
scripts/05_srctool.py
View File

@@ -2,32 +2,31 @@
"""
НАЗВАНИЕ:
04_mosaics.py
05_srctool.py
НАЗНАЧЕНИЕ:
Собирает мозайки в разных энергетических диапазонах.
Запускает scrtool для самого широкого канала 0.2-10 кэВ, чтобы спектры имели самое полное покрытие по энергиям. Список источников берется из 0.3-2.3 кэВ.
ВЫЗОВ:
esass
./01_mosaics.py
./05_srctool.py
УПРАВЛЕНИЕ:
Запуск отдельных команд управляется переменными, например: do_init = True
Выбранный энергетический диапазон управляется переменной index
Требуется запуск предыдущего скрипта 04_mosaics.py
ПАРАМЕТРЫ:
index : Выбранный энергетический диапазон
index=4 : Выбранный энергетический диапазон
ВЫВОД:
Выходные файлы записываются в директорию outfile_dir
Выходные файлы записываются в директорию outfile_dir/srctool_dir
ИСТОРИЯ:
@@ -49,6 +48,7 @@ import uds
from uds.utils import *
from uds.config import *
from uds.sherpa import *
""" find UDS root dir """
@@ -73,10 +73,13 @@ outfile_srctool="{}_SrcTool_".format(outkey)
do_init = False
do_merge = False
do_srctool = True
do_grppha = True
do_srctool = False
do_grppha = False
do_ecf_calc = False
do_ecf_print = False
do_catalog = True
index=4
index=1
""" работаем именно в этом диапазоне, чтобы спектры покрывали все энергии """
vign=True
@@ -122,6 +125,7 @@ is also appended to the filename. """
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[0], outfile_post)
""" take source catalog from 0.3-2.3 keV band """
if not (os.path.isfile(catprep)==True):
print("{} not found, run 04_mosaics.py?".format(catprep))
sys.exit()
@@ -146,5 +150,19 @@ if(do_srctool==True):
if(do_grppha==True):
group_spectra("{}/*_SourceSpec_*.fits".format(srctool_dir))
ecfout="{}_SampleFlux.pickle".format(os.path.join(outfile_dir,outkey))
if(do_ecf_calc==True):
calc_ecf("{}/tm0_SrcTool_020_ARF_?????.fits".format(srctool_dir),
catprep=catprep, emin=emin_kev, emax=emax_kev, eband=eband, outfile=ecfout, simnum=10000)
if(do_ecf_print==True):
print_ecf(infile=ecfout, emin=emin_kev, emax=emax_kev, eband=eband, skipfrac=10.0)
index=0
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_catalog==True):
make_catalog(infile=catprep, dlmin=10.0, dlmax=100000, ext_like=10, ecf=ecf[index],
emin=emin_kev[index], emax=emax_kev[index], eband=eband[index],
infile_en03cat='../products/tm0_SourceCatalog_en3.forced.fits',
infile_en03sens='../products/tm0_SensitivityMap_dl10_en3.fits')

189
scripts/06_plot.py Executable file
View File

@@ -0,0 +1,189 @@
#!/usr/bin/env python
"""
НАЗВАНИЕ:
05_srctool.py
НАЗНАЧЕНИЕ:
Запускает scrtool для самого широкого канала 0.2-10 кэВ, чтобы спектры имели самое полное покрытие по энергиям. Список источников берется из 0.3-2.3 кэВ.
ВЫЗОВ:
esass
./05_srctool.py
УПРАВЛЕНИЕ:
Требуется запуск предыдущего скрипта 04_mosaics.py
ПАРАМЕТРЫ:
index=4 : Выбранный энергетический диапазон
ВЫВОД:
Выходные файлы записываются в директорию outfile_dir/srctool_dir
ИСТОРИЯ:
Роман Кривонос, ИКИ РАН, krivonos@cosmos.ru
Март 2023
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
from pathlib import Path
import numpy as np
import glob
from os.path import dirname
import inspect
import uds
import matplotlib.pyplot as plt
from uds.utils import *
from uds.config import *
from uds.sherpa import *
""" find UDS root dir """
#root_path=dirname(dirname(dirname(inspect.getfile(uds))))
"""
ftools does not like long file path names,
for this reason, we use relative path here
"""
root_path='..'
print("UDS root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
srctool_dir="{}/{}".format(outfile_dir,"srctool-products")
create_folder(srctool_dir)
outkey="tm0"
outfile_srctool="{}_SrcTool_".format(outkey)
do_flux_distr = False
do_sens_curve = True
index=0
catalog = "{}_SourceCatalog_en{}.main.selected.csv".format(os.path.join(outfile_dir,outkey), eband[0])
if not (os.path.isfile(catalog)==True):
print("{} not found, run 05_srctool.py?".format(catalog))
sys.exit()
if(do_flux_distr==True):
data, logbins = get_log_distr(infile=catalog, field='ml_rate', minval=1e-3, maxval=2)
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.hist(data, bins=logbins, histtype='step', color='blue', linewidth=1, linestyle='solid')
plt.xlabel('Count rate (counts s$^{-1}$)',fontsize=14, fontweight='normal')
plt.ylabel('Number',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
plt.savefig(catalog.replace("main.selected.csv", "ml_rate.png"), bbox_inches='tight')
plt.close(fig)
data, logbins = get_log_distr(infile=catalog, field='ml_flux', minval=1e-15, maxval=2e-12)
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.hist(data, bins=logbins, histtype='step', color='blue', linewidth=1, linestyle='solid')
plt.xlabel('Energy flux (erg s$^{-1}$ cm$^{-2}$)',fontsize=14, fontweight='normal')
plt.ylabel('Number',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
plt.savefig(catalog.replace("main.selected.csv", "ml_flux.png"), bbox_inches='tight')
plt.close(fig)
if(do_sens_curve==True):
coeff = 2.4336e-13 / 3.4012e-13 # see below
areatab="{}_AreaTable_dl10_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
hdul = fits.open(areatab)
tbdata = hdul[1].data
""" convert limflux from 0.3-2.3 keV to 0.5-2 keV """
limflux_dl10 = tbdata['LIMFLUX']*coeff
area_dl10 = tbdata['SKY_AREA']
hdul.close()
areatab="{}_AreaTable_dl6_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
hdul = fits.open(areatab)
tbdata = hdul[1].data
""" convert limflux from 0.3-2.3 keV to 0.5-2 keV """
limflux_dl6 = tbdata['LIMFLUX']*coeff
area_dl6 = tbdata['SKY_AREA']
hdul.close()
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
ax.set_xlim([3e-16,1e-13])
ax.set_ylim([0.3,160])
df = pandas.read_csv("../data/surveys/eFEDS.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="brown", linestyle='solid',label="eFEDS (DL6)")
plt.plot(limflux_dl10,area_dl10, color="black", linestyle='solid',label="eUDS (DL10)")
plt.plot(limflux_dl6,area_dl6, color="black", linestyle='dashed', label="eUDS (DL6)")
df = pandas.read_csv("../data/surveys/cosmos-legacy.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="blue", linestyle='solid',label="COSMOS Legacy")
df = pandas.read_csv("../data/surveys/CDWFS.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="green", linestyle='solid', label="CDWFS")
df = pandas.read_csv("../data/surveys/XMM-RM.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="magenta", linestyle='solid',label="XMM-RM")
df = pandas.read_csv("../data/surveys/XMM-XXL-N.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="red", linestyle='solid',label="XMM-XXL-N")
ax.legend()
#plt.hist(data, bins=logbins, histtype='step', color='blue', linewidth=1, linestyle='solid')
plt.xlabel('Limiting flux (0.5-2 keV, erg s$^{-1}$ cm$^{-2}$)',fontsize=14, fontweight='normal')
plt.ylabel('Area (deg$^{2}$)',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
png="{}_AreaTable_en{}.png".format(os.path.join(outfile_dir,outkey), eband[index])
plt.savefig(png, bbox_inches='tight')
plt.close(fig)
"""
========================================================================
Model TBabs<1>*powerlaw<2> Source No.: 1 Active/On
Model Model Component Parameter Unit Value
par comp
1 1 TBabs nH 10^22 2.00000E-02 frozen
2 2 powerlaw PhoIndex 2.00000 frozen
3 2 powerlaw norm 1.14851E-04 +/- 3.83988E-06
________________________________________________________________________
Model Flux 0.00028334 photons (3.4012e-13 ergs/cm^2/s) range (0.30000 - 2.3000 keV)
Model Flux 0.0001619 photons (2.4336e-13 ergs/cm^2/s) range (0.50000 - 2.0000 keV)
"""

6
scripts/README.md
View File

@@ -33,4 +33,8 @@ source <MY PATH>/eSASS4EDR/bin/esass-init.csh
### 04_mosaics.py
Создает сборные изображения (мозайки) в разных энергетических диапазонах.
Создает сборные изображения (мозайки) в разных энергетических диапазонах.
### 05_scrtool.py
Запускает scrtool для самого широкого канала 0.2-10 кэВ, чтобы спектры имели самое полное покрытие по энергиям. Список источников берется из 0.3-2.3 кэВ.