nezaval/nsd
1
0
Fork 0
forked from erosita/uds
Code Pull Requests Activity

05_srctool added

Browse Source
This commit is contained in:
Roman Krivonos
2023-03-16 19:31:59 +03:00
parent d67146208e
commit 09deb0e4f0
9 changed files with 1017 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
uds/uds/config.py
View File

@@ -43,14 +43,15 @@ wcslist={'tm1_obs_1':[34.7279760,-5.0680267],
'tm6_scan_3':[34.5405596,-4.8088748],
'tm6_scan_4':[34.5405596,-4.8088748]}
""" Диапазоны энергий. """
emin_ev=[300, 300, 600, 2300, 200,300]
emax_ev=[2300, 600, 2300, 5000,10000,8000]
emin_kev=[0.3, 0.3, 0.6, 2.3, 0.2, 0.3]
emax_kev=[2.3, 0.6, 2.3, 5.0, 10.0, 8.0]
ecf=[1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
""" Это просто индекс диапазона для выходных файлов. """
eband=["0", "1", "2", "3", "4", "5"]
eband=[ "0", "1", "2", "3", "4", "5"]
""" Диапазоны энергий. """
emin_ev=[ 300, 300, 600, 2300, 200, 300]
emax_ev=[2300, 600, 2300, 5000, 10000,8000]
emin_kev=[0.3, 0.3, 0.6, 2.3, 0.2, 0.3]
emax_kev=[2.3, 0.6, 2.3, 5.0, 10.0, 8.0]
ecf = [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
outfile_post='.fits'

435
uds/uds/utils.py
View File

@@ -47,17 +47,35 @@ def remove_file(filename):
if(os.path.isfile(filename)==True):
os.remove(filename)
def do_evtool_esass(events=None,outfile=None,evlist=None,gti=None,region=None,emin=None,emax=None, rmlock=False):
def do_evtool_esass(evfile=None,events=None,outfile=None,evlist=None,
gti=None,region=None,emin=None,emax=None, rmlock=False,
do_center=False, ra_cen=None, de_cen=None):
eventfiles=None
if(events):
eventfiles="eventfiles=\'{}\'".format((" ").join(events))
if(evlist):
eventfiles="eventfiles=@{}".format(evlist)
if(evfile):
eventfiles="eventfiles={}".format(evfile)
if not (eventfiles):
print("ERROR: Event files not provided")
if(do_center==True and evfile):
""" re-center event file """
if not (ra_cen and de_cen):
print("Please provide center coordinates")
sys.exit()
test_exe('radec2xy')
cmd=["radec2xy",
"{}".format(evfile),
"{}".format(ra_cen),
"{}".format(de_cen)]
print((" ").join(cmd))
os.system((" ").join(cmd))
emin="emin={}".format(emin) if(emin) else ''
emax="emax={}".format(emax) if(emax) else ''
gti="gti=\'{}\'".format(gti) if(gti) else ''
@@ -83,6 +101,7 @@ def do_evtool_esass(events=None,outfile=None,evlist=None,gti=None,region=None,em
if(os.path.isfile(filename)==True):
os.remove(filename)
pass
def set_bit(value, bit):
return value | (1<<bit)
@@ -279,20 +298,21 @@ def do_expmap_esass(expmaps=None,outfile=None,emin_kev=None,emax_kev=None,refima
test_exe('expmap')
os.system((" ").join(cmd))
def do_expmap_ftools(expmaps=None,outfile=None):
""" takes first exposure as reference and merges the rest """
tmp="expmaps.txt"
def do_fimgmerge_ftools(maps=None,outfile=None):
""" takes first map as reference and merges the rest """
tmp="maps.txt"
f = open(tmp, "w")
for jj in range(1,len(expmaps)):
f.write("{},0,0\n".format(expmaps[jj]))
for jj in range(1,len(maps)):
f.write("{},0,0\n".format(maps[jj]))
f.close()
cmd=["fimgmerge",
"infile={}".format(expmaps[0]),
"infile={}".format(maps[0]),
"list=@{}".format(tmp),
"outfile={}".format(outfile),
"clobber=yes",
]
print((" ").join(cmd))
test_exe('fimgmerge')
os.system((" ").join(cmd))
def test_exe(program):
@@ -547,10 +567,10 @@ def do_adapt_ciao(infile=None,outfile=None,expmap=None,function='tophat',expcut=
if (expcut):
cntmap_data, cntmap_hdr = fits.getdata(outfile, ext=0, header=True)
expmap_data, expmap_hdr = fits.getdata(expmap, ext=0, header=True)
index = np.where(expmap_data > expcut)
index = np.where(expmap_data < expcut)
cntmap_data[index]=0.0
fits.writeto(outfile, cntmap_data, cntmap_hdr, overwrite=True)
"""
dmimgadapt artmap_ait.fits adapt.fits tophat 1 30 30 log 30 radfile=map.scale.fits sumfile=sum.fits clobber=yes
@@ -588,3 +608,400 @@ def make_rate_map(cntmap=None, expmap=None, outfile=None):
fits.writeto(outfile, rate, overwrite=True)
def wcs_match_ciao(catalog=None, method='trans',radius=15,residlim=7):
xfm=catalog.replace(".fits", ".xfm")
src_list=catalog.replace(".fits", ".shu2019.src.fits")
if not (os.path.isfile(src_list)==True):
print("Not found {}".format(src_list))
print("Did you run cross-match?")
sys.exit()
ref_list=catalog.replace(".fits", ".shu2019.ref.fits")
if not (os.path.isfile(ref_list)==True):
print("Not found {}".format(ref_list))
print("Did you run cross-match?")
sys.exit()
log=catalog.replace(".fits", ".xfm.log")
cmd=["wcs_match",
"infile={}".format(src_list),
"refsrcfile={}".format(ref_list),
"outfile={}".format(xfm),
"wcsfile={}".format(src_list),
"logfile={}".format(log),
"radius={}".format(radius),
"residlim={}".format(residlim),
"verbose=1",
"method={}".format(method),
"clobber=yes",
]
test_exe('wcs_match')
os.system((" ").join(cmd))
print((" ").join(cmd))
def wcs_update_ciao(events,ext=2,crval=None,transformfile=None,clean=True):
""" Prepares eRosita attitude file for Chandra's wcs_update command """
if not (transformfile):
print("ERROR: Please provide transformfile")
return
if not (os.path.isfile(transformfile)==True):
print("Not found {}".format(transformfile))
print("Did you run wcs_update_ciao?")
sys.exit()
fnaspsol=events.replace(".fits", ".aspsol")
fntmp=events.replace(".fits", ".tmp")
fnattcor=events.replace(".fits", ".attcorr.fits")
hdul = fits.open(events,mode='readonly')
corratt=hdul[ext]
naxis1=corratt.header['NAXIS1']
naxis2=corratt.header['NAXIS2']
corratt_extname=corratt.header['EXTNAME']
""" create new empty Q_ATT column """
q_att=np.zeros((naxis2,4))
cols = []
cols.append(fits.Column(name='q_att', format='4D', array=q_att))
orig_cols = corratt.columns
new_cols = fits.ColDefs(cols)
hdu0 = fits.PrimaryHDU()
hdu0.header['EXTNAME'] = ('ASPSOL', 'Name of this binary table extension')
hdu0.header['HDUNAME'] = ('ASPSOL', 'ASCDM block name')
if(crval):
hdu0.header['RA_NOM']=(crval[0], '[deg] Pointing RA')
hdu0.header['DEC_NOM']=(crval[1], '[deg] Pointing Dec')
hdu0.header['ROLL_NOM']=(0.0 , '[deg] Pointing Roll')
hdu0.header['RA_PNT']=(crval[0], '[deg] Nominal RA')
hdu0.header['DEC_PNT']=(crval[1], '[deg] Nominal Dec')
hdu0.header['ROLL_PNT']=(0.0 , '[deg] Nominal Roll')
hdu = fits.BinTableHDU.from_columns(orig_cols + new_cols, header=hdu0.header)
hdu.writeto(fnaspsol, overwrite=True)
"""
First, update attitude information in ASPSOL (Chandra notation),
which corresponds to CORRATT (eRosita notation)
"""
cmd=["wcs_update",
"infile={}".format(fnaspsol),
"outfile={}".format(fntmp),
"transformfile={}".format(transformfile),
"clobber=yes",
]
os.system((" ").join(cmd))
print((" ").join(cmd))
"""
Replace attitude extension
"""
shutil.copyfile(events, fnattcor)
data, hdr = getdata(fntmp, 1, header=True)
hdr['EXTNAME']=corratt_extname
update(fnattcor, data, ext, header=hdr)
"""
Second, calculate new RA/Dec for events using updated attitude, using evatt from eSASS
"""
cmd=["evatt",
"EVENTFILE={}".format(fnattcor),
"ATTFILE={}".format(fnattcor),
"GTIFILE={}".format(fnattcor),
]
os.system((" ").join(cmd))
print((" ").join(cmd))
if(clean==True):
remove_file(fnaspsol)
remove_file(fntmp)
return fnattcor
def create_detmask_merged(expmaps,outfile,minval=None):
"""
creates mask from each exposure and then stacks each masks.
Header is taken from first exposure file.
"""
tmlist={}
for expmap in expmaps:
hdul = fits.open(expmap)
emap = hdul[0].data
ehdr = hdul[0].header
instrume = ehdr['INSTRUME']
threshold = minval if(minval) else 0
mask = np.where(emap > threshold, 1, 0)
print("--- detmask {} -- {}".format(expmap,instrume))
if not (instrume in tmlist.keys()):
tmlist[instrume]=1
else:
pass
if 'merged_mask' in locals():
merged_mask = np.add(merged_mask, mask)
else:
merged_mask = mask
merged_hdr = ehdr
if 'merged_hdr' in locals():
index=1
for tm in tmlist.keys():
merged_hdr["INSTRUM{}".format(index)]=tm
index=index+1
merged_hdr['INSTRUME']='merged'
merged_hdr['NINST']=len(tmlist)
merged_hdr['OBS_MODE']=' '
if 'merged_mask' in locals() and 'merged_hdr' in locals():
fits.writeto(outfile, np.where(merged_mask > 0, 1, 0), header=merged_hdr, overwrite=True)
def create_expmap_merged(expmaps,outfile,scale=7.0):
"""
Adds exposure from TMs. Header is taken from first exposure file.
"""
tmlist={}
for expmap in expmaps:
hdul = fits.open(expmap)
emap = np.divide(hdul[0].data, scale)
ehdr = hdul[0].header
instrume = ehdr['INSTRUME']
print("--- expmap {} -- {}".format(expmap,instrume))
if not (instrume in tmlist.keys()):
tmlist[instrume]=1
else:
pass
if 'merged_map' in locals():
merged_map = np.add(merged_map, emap)
else:
merged_map = emap
merged_hdr = ehdr
if 'merged_hdr' in locals():
index=1
for tm in tmlist.keys():
merged_hdr["INSTRUM{}".format(index)]=tm
index=index+1
merged_hdr['INSTRUME']='merged'
merged_hdr['NINST']=len(tmlist)
merged_hdr['OBS_MODE']=' '
if 'merged_map' in locals() and 'merged_hdr' in locals():
fits.writeto(outfile, merged_map, header=merged_hdr, overwrite=True)
def do_erbackmap_esass(image,expimage,boxlist,detmask,emin,emax,outfile_backmap,cheese_mask):
test_exe('erbackmap')
cmd=["erbackmap",
"image=%s" %(image),
"expimage=%s" %(expimage),
"boxlist={}".format(boxlist),
"detmask=%s" %(detmask),
"emin=%s" %(emin),
"emax=%s" %(emax),
"bkgimage=%s" %(outfile_backmap),
"cheesemask=%s" %(cheese_mask),
"idband=1",
"scut=0.001",
"mlmin=6",
"maxcut=0.5",
"fitmethod=smooth smoothval=15",
"snr=40.",
]
remove_file(cheese_mask)
remove_file(outfile_backmap)
os.system((" ").join(cmd))
print((" ").join(cmd))
def filter_catprep(filename, expcut=100,dlmin=6.0,dlmax=10,scale=60*60,ext_like=0.0,outkey='main'):
if(os.path.isfile(filename)==False):
print("File not found {}".format(filename))
print("Filter catprep {}".format(filename))
fout_selected=filename.replace(".fits", ".{}.selected.reg".format(outkey))
fout_skip=filename.replace(".fits", ".{}.skip.reg".format(outkey))
fout_extended=filename.replace(".fits", ".extended.reg")
hdul = fits.open(filename)
tbdata = hdul[1].data
catsel=[]
catskip=[]
catext=[]
skip_count=0
selected_count=0
keepalive_count=0
for rec in tbdata:
if (rec['ext_like'] > ext_like):
catext.append({'ra':rec['ra'],'dec':rec['dec'],'radec_err':rec['radec_err'],
'det_like':rec['det_like_0'],'ext_like':rec['ext_like'],
'ext':rec['ext'],'ext_err':rec['ext_err'],
'src_id':rec['id_src']})
if ((rec['det_like_0'] > dlmin and rec['det_like_0'] < dlmax)):
catsel.append({'ra':rec['ra'],'dec':rec['dec'],'radec_err':rec['radec_err'],'det_like':rec['det_like_0'],
'src_id':rec['id_src']})
selected_count=selected_count + 1
else:
catskip.append({'ra':rec['ra'],'dec':rec['dec'],'radec_err':rec['radec_err'],'det_like':rec['det_like_0'],
'src_id':rec['id_src']})
skip_count = skip_count+1
print("total={} skip_count={} selected={}".format(len(tbdata),skip_count,selected_count))
with open(fout_selected, 'w') as writer:
for rec in catsel:
writer.write("fk5;circle({}, {}, {}) # color=white text={{{} {:.2f}}}\n".format(rec['ra'],rec['dec'],rec['radec_err']/scale,
rec['src_id'],rec['det_like']))
with open(fout_skip, 'w') as writer:
for rec in catskip:
writer.write("fk5;circle({}, {}, {}) # color=red text={{{} {:.2f}}}\n".format(rec['ra'],rec['dec'],rec['radec_err']/scale,
rec['src_id'],rec['det_like']))
with open(fout_extended, 'w') as writer:
for rec in catext:
writer.write("fk5;circle({}, {}, {}) # color=magenta text={{{} {:.2f} {:.2f}}}\n".format(rec['ra'],
rec['dec'],
rec['ext']/scale,
rec['src_id'],
rec['det_like'],
rec['ext_like']))
def filter_mllist(filename, expcut=100,dlcut=6.0,dlmin=6.0,dlmax=10,scale=60*60,ext_like=0.0):
if(os.path.isfile(filename)==False):
print("File not found {}".format(filename))
print("Filter mllist {}".format(filename))
fout_selected=filename.replace(".fits", ".selected.reg")
fout_skip=filename.replace(".fits", ".skip.reg")
fout_extended=filename.replace(".fits", ".extended.reg")
hdul = fits.open(filename)
tbdata = hdul[1].data
""" create list of unique source IDs """
id_src_set = set(tbdata['id_src'])
src_id_list = list(id_src_set)
catsel=[]
catskip=[]
catext=[]
skip_count=0
selected_count=0
keepalive_count=0
for src_id in src_id_list:
tab_tm0=tbdata[(tbdata['id_src'] == src_id) & (tbdata['id_inst'] == 0) & (tbdata['id_band'] == 0)]
""" averaged over all instruments and bands """
ext_tm0=tbdata[(tbdata['id_src'] == src_id) & (tbdata['id_inst'] == 0) & (tbdata['id_band'] == 0) & (tbdata['ext_like'] > ext_like)]
""" averaged over all instruments and bands """
if(len(ext_tm0)>0):
catext.append({'ra':ext_tm0['ra'][0],'dec':ext_tm0['dec'][0],'radec_err':ext_tm0['radec_err'][0],
'det_like':ext_tm0['det_like'][0],'ext_like':ext_tm0['ext_like'][0],
'ext':ext_tm0['ext'][0],'ext_err':ext_tm0['ext_err'][0],
'src_id':ext_tm0['id_src'][0]})
mask_src_id = (tbdata['id_src'] == src_id) & (tbdata['id_inst'] == 0)
""" get all source records except merged one """
tab_src=tbdata[(tbdata['id_src'] == src_id) & (tbdata['id_inst'] != 0) & (tbdata['id_band'] == 1) & (tbdata['ml_exp'] > expcut)]
""" only individual instruments, and only first energy band is selected here """
keepalive=False
mask = tab_src['det_like'] > dlcut
if (any(mask) and tab_tm0['det_like'] < dlmin):
keepalive=True
keepalive_count=keepalive_count+1
if(keepalive):
tab_src=tbdata[(tbdata['id_src'] == src_id) & (tbdata['id_inst'] != 0) & (tbdata['id_band'] == 1)]
print("KEEP ALIVE ID={} DL={:.2f} | radec: {:.4f} {:.4f} DL0={:.2f} TEXP={:.1f}".format(src_id, tab_tm0['det_like'][0], tab_tm0['ra'][0], tab_tm0['dec'][0],
tab_src['det_like'][0],tab_src['ml_exp'][0]))
if ((tab_tm0['det_like'] > dlmin and tab_tm0['det_like'] < dlmax) or keepalive):
catsel.append({'ra':tab_tm0['ra'][0],'dec':tab_tm0['dec'][0],'radec_err':tab_tm0['radec_err'][0],'det_like':tab_tm0['det_like'][0],
'src_id':tab_tm0['id_src'][0]})
selected_count=selected_count + 1
else:
catskip.append({'ra':tab_tm0['ra'][0],'dec':tab_tm0['dec'][0],'radec_err':tab_tm0['radec_err'][0],'det_like':tab_tm0['det_like'][0],
'src_id':tab_tm0['id_src'][0]})
skip_count = skip_count+1
print("total={} skip_count={} keepalive_count={} selected={}".format(len(src_id_list),skip_count,keepalive_count,selected_count))
with open(fout_selected, 'w') as writer:
for rec in catsel:
writer.write("fk5;circle({}, {}, {}) # color=white text={{{} {:.2f}}}\n".format(rec['ra'],rec['dec'],rec['radec_err']/scale,
rec['src_id'],rec['det_like']))
with open(fout_skip, 'w') as writer:
for rec in catskip:
writer.write("fk5;circle({}, {}, {}) # color=red text={{{} {:.2f}}}\n".format(rec['ra'],rec['dec'],rec['radec_err']/scale,
rec['src_id'],rec['det_like']))
with open(fout_extended, 'w') as writer:
for rec in catext:
writer.write("fk5;circle({}, {}, {}) # color=magenta text={{{} {:.2f} {:.2f}}}\n".format(rec['ra'],rec['dec'],rec['ext']/scale,
rec['src_id'],rec['det_like'],rec['ext_like']))
def create_sensmap(sensmap=None,expmap=None,backmap=None,detmask=None,areatab=None,
emin=None,emax=None,ecf=None, detlike=10):
test_exe('ersensmap')
cmd=['ersensmap',
"expimages=\'{}\'".format(expmap),
"bkgimages=\'{}\'".format(backmap),
"detmasks=\'{}\'".format(detmask),
"sensimage={}".format(sensmap),
"emin=\'{}\'".format(emin),
"emax=\'{}\'".format(emax),
"ecf=\'{}\'".format(ecf),
"area_table={}".format(areatab),
"method=APER",
"aper_type=BOX",
"aper_size=4.5",
"likemin={}".format(detlike),
"extlikemin=6.0",
"ext=6.0",
"extentmodel=beta",
"detmask_flag=yes",
"shapelet_flag=no",
"photon_flag=no",
"ext_flag=no",
"extlike_flag=no",
"compress_flag=no",
"area_flag=no",]
remove_file(sensmap)
remove_file(areatab)
os.system((" ").join(cmd))
print((" ").join(cmd))
def group_spectra(path,group_min=25):
test_exe('grppha')
flist=glob.glob(path)
for f in flist:
fout=f.replace(".fits", ".fits.grp")
print(f)
cmd=["grppha",
"infile={}".format(f),
"outfile={}".format(fout),
"comm=\'GROUP MIN {}\'".format(group_min),
"tempc=EXIT",
"clobber=yes",
"chatter=0", # with chatter <= 5 being very quite and chatter >= 20 very verbose.
]
print((" ").join(cmd))
os.system((" ").join(cmd))