integral/ridge
1
0
Fork 0
generated from erosita/uds
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Roman Krivonos
2024-04-13 14:35:10 +03:00
commit 9aac031a01
48 changed files with 65971 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
scripts/00_check.py Executable file
View File

@@ -0,0 +1,61 @@
#!/usr/bin/env python
"""Печатает информацию о наблюдениях
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
from pathlib import Path
import numpy as np
import glob
from os.path import dirname
import inspect
import uds
from uds.utils import *
from uds.config import *
outkey="mosa_tm0"
""" find UDS root dir """
root_path=dirname(dirname(dirname(inspect.getfile(uds))))
print("UDS root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
index=5
events=[]
expmaps=[]
totexp=0.0
for tmkey in keylist_tm.keys():
print("TM{} in work... init events".format(tmkey))
for datakey in keylist_tm[tmkey]:
#print("--> {}".format(datakey))
""" Запускаем полностью в холостом режиме, нам нужно получить только названия файлов """
outfile_evtool,outfile_expmap=init_events(key=datakey, eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
do_init=False,
do_obsmode=False,
do_center=False,
do_evtool=False,
do_expmap=False,
ra_cen=ra_cen, de_cen=de_cen,
emin_kev=emin_kev[index],
emax_kev=emax_kev[index])
events.append(outfile_evtool)
expmaps.append(outfile_expmap)
tstart, tstop = read_tstart_tstop(infile=outfile_evtool)
totexp=totexp+(tstop-tstart)
print("{}, {} -- {}, {} -- {}, {:.2f} ks".format(outfile_evtool,tstart,tstop,
mission2date_utc(tstart),
mission2date_utc(tstop),
(tstop-tstart)/1000))
print("\n***\n*** Total exposure: {:.1f} ks\n***".format(totexp))

58
scripts/01_init_events.py Executable file
View File

@@ -0,0 +1,58 @@
#!/usr/bin/env python
"""Создает начальные списки событий и помещает их в uds/data/processed
Оригинальные файлы со списками событий задаются в файлах uds/data/evtlists/*.txt
"""
import os
import inspect
from os.path import dirname
import uds
from uds.config import *
from uds.utils import *
""" find UDS root dir """
root_path=dirname(dirname(dirname(inspect.getfile(uds))))
print("UDS root path: {}".format(root_path))
el=root_path+'/data/evtlists/'
pr=root_path+'/data/processed/'
region="box({},{},4d,4d,0)".format(ra_cen,de_cen)
""" Selection region """
"""
# TM1
do_evtool_esass(evlist=el+'tm1.txt', outfile=pr+'tm1_obs_1.fits', gti='621296896. 621304128.', rmlock=True)
# TM5
do_evtool_esass(evlist=el+'tm5.txt', outfile=pr+'tm5_obs_1.fits', gti='620606016. 620614848.', rmlock=True)
do_evtool_esass(evlist=el+'tm5.txt', outfile=pr+'tm5_obs_2.fits', gti='620676992. 620689792.', rmlock=True)
# TM6
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_park_1.fits', gti='620174080. 620178002.620032', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_scan_1.fits', gti='620178002.620032 620192246.62720', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_park_2.fits', gti='620192448. 620194624.', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_scan_2.fits', gti='620194666.606144 620208904.673408', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_park_3.fits', gti='620209162.670976 620211316.650304', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_scan_3.fits', gti='620211328. 620225600.', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_park_4.fits', gti='620225853.609024 620227974.68832', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_scan_4.fits', gti='620227904. 620242176.', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_obs_1.fits', gti='620242432. 620258368.', rmlock=True)
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_obs_2.fits', gti='620607424. 620614656.', rmlock=True)
do_badpix_tm6(filename=pr+'tm6_obs_2.fits')
do_evtool_esass(evlist=el+'tm6.txt', outfile=pr+'tm6_obs_3.fits', gti='620676992. 620690368.', rmlock=True)
do_badpix_tm6(filename=pr+'tm6_obs_3.fits')
"""
# TM7
do_evtool_esass(evlist=el+'tm7.txt', outfile=pr+'tm7_obs_1.fits', gti='621043136. 621052416.', emin=0.2, emax=10.0, region=region, rmlock=True)
#do_evtool_esass(evlist=el+'tm7.txt', outfile=pr+'tm7_obs_2.fits', gti='621110272. 621117952.', region=region, rmlock=True)

95
scripts/02_merge_events.py Executable file
View File

@@ -0,0 +1,95 @@
#!/usr/bin/env python
"""Создает объедененный список событий и помещает его в uds/products
Этот список событий нужен, в основном для извлечения спектров с помощью srctool
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
from pathlib import Path
import numpy as np
import glob
from os.path import dirname
import inspect
import uds
from uds.utils import *
from uds.config import *
outkey="mosa_tm0"
""" find UDS root dir """
#root_path=dirname(dirname(dirname(inspect.getfile(uds))))
"""
ftools does not like long file path names, for this reason, we use relative path here
"""
root_path='..'
print("UDS root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
index=3 # select energy band
do_init = True
do_merge = True
do_rate = False
do_adapt = True # requires CIAO
vign=False
vignetting = 'vign' if (vign==True) else 'novign'
events=[]
expmaps=[]
bkgmaps=[]
for tmkey in keylist_tm.keys():
print("TM{} in work... init events".format(tmkey))
for datakey in keylist_tm[tmkey]:
#if not ("scan" in datakey):
# continue
print("--> {}".format(datakey))
""" Подготавливаем списки событий индивидуальных наблюдений """
outfile_evtool,outfile_expmap=init_events(key=datakey, eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
do_init=do_init,
do_obsmode=False,
do_center=False,
do_evtool=True,
do_expmap=False,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen,
emin_kev=emin_kev[index],
emax_kev=emax_kev[index])
events.append(outfile_evtool)
expmaps.append(outfile_expmap)
bkgmaps.append("{}_BackMap3_en{}.fits".format(os.path.join(outfile_dir,datakey), eband[0]))
""" Собираем общий список событий """
outfile_evtool="{}_EventList_en{}.fits".format(os.path.join(outfile_dir,outkey), eband[index])
outfile_expmap="{}_ExposureMap_en{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], vignetting)
outfile_bkgmap="{}_BackMap_en{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], vignetting)
if(do_merge==True):
do_evtool_esass(events=events, outfile=outfile_evtool)
do_fimgmerge_ftools(maps=expmaps, outfile=outfile_expmap)
do_fimgmerge_ftools(maps=bkgmaps, outfile=outfile_bkgmap)
outfile_rate="{}_RateMap_en{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], vignetting)
if(do_rate==True):
make_rate_map(cntmap=outfile_evtool, expmap=outfile_expmap, outfile=outfile_rate)
function='gaussian'
outfile_adapt="{}_ImageAdapt_en{}.{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], function, vignetting)
if(do_adapt==True):
do_adapt_ciao(infile=outfile_evtool, outfile=outfile_adapt, expmap=outfile_expmap, function=function, expcut=100)

424
scripts/03_init_obs.py Executable file
View File

@@ -0,0 +1,424 @@
#!/usr/bin/env python
"""
НАЗВАНИЕ:
01_init_obs.py
НАЗНАЧЕНИЕ:
Подготавливает списки событий в разных энергетических диапазонах.
Производит списки источников в наждом наблюдении и делает астрокоррекцию с помощью wcs_match/wcs_update
ВЫЗОВ:
conda activate ciao-4.15
esass
./01_init_obs.py
УПРАВЛЕНИЕ:
Запуск отдельных команд управляется переменными, например: do_init = True
Выбранные энергетические диапазоны управляется массивом eband_selected
ПАРАМЕТРЫ:
eband_selected : Выбранные энергетические диапазоны
ВЫВОД:
Выходные файлы записываются в директорию outfile_dir
ИСТОРИЯ:
Роман Кривонос, ИКИ РАН, krivonos@cosmos.ru
Март 2023
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
from pathlib import Path
import numpy as np
import glob
from multiprocessing import Pool
from os.path import dirname
import inspect
import uds
from uds.utils import *
from uds.config import *
""" find UDS root dir """
root_path=dirname(dirname(dirname(inspect.getfile(uds))))
print("UDS root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
run_Pool=False
do_init = False
do_ermask = False
do_erbox1 = False # local mode
do_erbackmap1 = False #
do_erbox2 = False # map mode, with background map
do_erbackmap2 = False #
do_erbox3 = False # map mode, with background map
do_erbackmap3 = False #
do_ermldet = False
do_catprep = False
do_cross_match = False
do_astro_corr = False # search optimal shift
do_astro_update = True
do_wcs_match = False # Chandra task -- DEPRECATED
do_wcs_update = False # Chandra task -- DEPRECATED
eband_selected=[5]
vign=True
vignetting = 'vign' if (vign==True) else 'novign'
def runme(datakey):
""" runs datakey over energy bands """
events=[]
expmaps=[]
outfile_boxlist1=[]
outfile_boxlist2=[]
outfile_boxlist3=[]
outfile_backmap1=[]
outfile_backmap2=[]
outfile_backmap3=[]
cheesemask=[]
bkgimage=[]
srcmaps=[]
print(datakey)
print('module name:', __name__)
print('parent process:', os.getppid())
print('process id:', os.getpid())
for ii in range(len(eband_selected)):
index=eband_selected[ii]
print("\t>>> Energy band en{} -- {}-{} keV".format(eband[index],emin_kev[index],emax_kev[index]))
outfile_evtool, outfile_expmap = init_events(key=datakey, eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
do_init=do_init,
do_obsmode=True,
do_center=True,
do_evtool=True,
do_expmap=True,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen,
emin_kev=emin_kev[index],
emax_kev=emax_kev[index])
expmaps.append(outfile_expmap)
events.append(outfile_evtool)
# After astrometry-corrected files (*.attcorr.fits) are obtained, one can take them as original, in order to check the full chain:
#events.append(outfile_evtool.replace(".fits", ".attcorr.fits"))
""" Detmask """
detmask="{}_DetectionMask{}".format(os.path.join(outfile_dir,datakey), outfile_post)
if(do_ermask==True):
cmd=["ermask",
"expimage=%s" %(expmaps[0]), # use the first exposure maps calculated for that skyfield, independent of the energy band
"detmask=%s" %(detmask),
"threshold1=0.01",
"threshold2=10.0",
"regionfile_flag=no"
]
remove_file(detmask)
print((" ").join(cmd))
os.system((" ").join(cmd))
for ii in range(len(eband_selected)):
index=eband_selected[ii]
print("\t>>> Energy band en{} -- {}-{} keV".format(eband[index],emin_kev[index],emax_kev[index]))
""" erbox in local mode """
outfile_boxlist1.append("{}_BoxList1_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post))
if(do_erbox1==True):
""" erbox in local mode """
cmd=["erbox",
"images=%s" %(events[ii]),
"boxlist=%s" %(outfile_boxlist1[ii]),
"expimages=%s" %(expmaps[ii]),
"detmasks=%s" %(detmask),
"emin=%s" %(emin_ev[index]),
"emax=%s" %(emax_ev[index]),
"ecf=1.0",
"nruns=2",
"likemin=6.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=N",
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist1[ii])
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist1[ii])
outfile_backmap1.append("{}_BackMap1_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post))
cheese_mask="{}_CheeseMask1_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post)
if(do_erbackmap1==True):
""" back map 1 """
cmd=["erbackmap",
"image=%s" %(events[ii]),
"expimage=%s" %(expmaps[ii]),
"boxlist=%s" %(outfile_boxlist1[ii]),
"detmask=%s" %(detmask),
"emin=%s" %(emin_ev[index]),
"emax=%s" %(emax_ev[index]),
"bkgimage=%s" %(outfile_backmap1[ii]),
"cheesemask=%s" %(cheese_mask),
"idband=1",
"scut=0.001",
"mlmin=6",
"maxcut=0.5",
"fitmethod=smooth smoothval=15",
"snr=40.",
]
remove_file(cheese_mask)
remove_file(outfile_backmap1[ii])
os.system((" ").join(cmd))
print((" ").join(cmd))
outfile_boxlist2.append("{}_BoxList2_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post))
if(do_erbox2==True):
""" erbox in background mode """
cmd=["erbox",
"images=%s" %(events[ii]),
"boxlist=%s" %(outfile_boxlist2[ii]),
"expimages=%s" %(expmaps[ii]),
"detmasks=%s" %(detmask),
"emin=%s" %(emin_ev[index]),
"emax=%s" %(emax_ev[index]),
"ecf=1.0",
"nruns=2",
"likemin=4.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=Y",
"bkgimages={}".format(outfile_backmap1[ii]),
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist2[ii])
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist2[ii])
outfile_backmap2.append("{}_BackMap2_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post))
cheese_mask="{}_CheeseMask2_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post)
if(do_erbackmap2==True):
""" back map 2 """
cmd=["erbackmap",
"image=%s" %(events[ii]),
"expimage=%s" %(expmaps[ii]),
"boxlist=%s" %(outfile_boxlist2[ii]),
"detmask=%s" %(detmask),
"emin=%s" %(emin_ev[index]),
"emax=%s" %(emax_ev[index]),
"bkgimage=%s" %(outfile_backmap2[ii]),
"cheesemask=%s" %(cheese_mask),
"idband=1",
"scut=0.001",
"mlmin=6",
"maxcut=0.5",
"fitmethod=smooth smoothval=15",
"snr=40.",
]
remove_file(cheese_mask)
remove_file(outfile_backmap2[ii])
os.system((" ").join(cmd))
print((" ").join(cmd))
outfile_boxlist3.append("{}_BoxList3_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post))
if(do_erbox3==True):
""" erbox in map mode FINAL """
cmd=["erbox",
"images=%s" %(events[ii]),
"boxlist=%s" %(outfile_boxlist3[ii]),
"expimages=%s" %(expmaps[ii]),
"detmasks=%s" %(detmask),
"emin=%s" %(emin_ev[index]),
"emax=%s" %(emax_ev[index]),
"ecf=1.0",
"nruns=2",
"likemin=4.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=Y",
"bkgimages={}".format(outfile_backmap2[ii]),
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist3[ii])
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist3[ii])
outfile_backmap3.append("{}_BackMap3_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post))
cheese_mask="{}_CheeseMask3_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post)
if(do_erbackmap3==True):
""" back map 3 FINAL """
cmd=["erbackmap",
"image=%s" %(events[ii]),
"expimage=%s" %(expmaps[ii]),
"boxlist=%s" %(outfile_boxlist3[ii]),
"detmask=%s" %(detmask),
"emin=%s" %(emin_ev[index]),
"emax=%s" %(emax_ev[index]),
"bkgimage=%s" %(outfile_backmap3[ii]),
"cheesemask=%s" %(cheese_mask),
"idband=1",
"scut=0.001",
"mlmin=6",
"maxcut=0.5",
"fitmethod=smooth smoothval=15",
"snr=40.",
]
remove_file(cheese_mask)
remove_file(outfile_backmap3[ii])
os.system((" ").join(cmd))
print((" ").join(cmd))
mllist="{}_MaxLikSourceList_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post)
srcmap="{}_SourceMap_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post)
cmd=["ermldet",
"mllist=%s" %(mllist),
"boxlist=%s" %(outfile_boxlist3[ii]),
"images=%s" %(events[ii]),
"expimages=%s" %(expmaps[ii]),
"detmasks=%s" %(detmask),
"bkgimages=%s" %(outfile_backmap3[ii]),
"emin=%s" %(emin_ev[index]),
"emax=%s" %(emax_ev[index]),
"hrdef=",
"ecf={}".format(ecf[index]),
"likemin=5.",
"extlikemin=6.",
"compress_flag=N",
"cutrad=15.",
"multrad=20.",
"extmin=2.0",
"extmax=15.0",
#"bkgima_flag=Y", looks outdated
"expima_flag=Y",
"detmask_flag=Y",
"extentmodel=beta",
"thres_flag=N",
"thres_col=like",
"thres_val=30.",
"nmaxfit=4",
"nmulsou=2",
"fitext_flag=yes",
"srcima_flag=yes",
"srcimages=%s" %(srcmap)
]
if(do_ermldet==True):
test_exe('ermldet')
remove_file(mllist)
remove_file(srcmap)
os.system((" ").join(cmd))
print((" ").join(cmd))
save_ermldet_ds9reg(mllist,scale=60*60)
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post)
catprep_en0="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,datakey), eband[0], outfile_post)
if(do_catprep==True):
cmd=["catprep",
"infile={}".format(mllist),
"outfile={}".format(catprep),]
remove_file(catprep)
os.system((" ").join(cmd))
print((" ").join(cmd))
save_catprep_ds9reg(catprep,scale=60*60)
if(do_cross_match==True):
crossmatch_shu2019(catprep,dlmin=10,refimage=events[ii],crval=wcslist[datakey],
catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_UDS.fits.catalog",errlim=5.0)
if(do_astro_corr==True and eband[index]=='0'):
""" run astro_corr for 0.3-2.3 keV only """
wcs_astro_corr(catprep)
#wcs_match_ciao(catprep, method='rst',radius=12,residlim=0,residtype=0,residfac=1)
if(do_astro_update==True):
""" run astro_corr for 0.3-2.3 keV only """
attcorr=wcs_update_shift(events[ii],flog=catprep_en0.replace(".fits", ".shift.log"))
do_evtool_esass(evfile=attcorr,outfile=attcorr,rmlock=False, do_center=True, ra_cen=ra_cen, de_cen=de_cen)
if(do_wcs_match==True and eband[index]=='0'):
""" run wcs_match for 0.3-2.3 keV only """
wcs_match_ciao(catprep, method='trans',radius=12,residlim=5)
#wcs_match_ciao(catprep, method='rst',radius=12,residlim=0,residtype=0,residfac=1)
if(do_wcs_update==True):
""" use 0.3-2.3 keV transform matrix for all other bands """
attcorr=wcs_update_ciao(events[ii],crval=wcslist[datakey],transformfile=catprep_en0.replace(".fits", ".xfm"),clean=False)
do_evtool_esass(evfile=attcorr,outfile=attcorr,rmlock=False, do_center=True, ra_cen=ra_cen, de_cen=de_cen)
"""
# individual run, testing
runme("tm7_obs_1")
runme("tm5_obs_1")
runme("tm6_scan_1")
"""
if(run_Pool==True):
# parallel run
items=[]
for tmkey in keylist_tm.keys():
for datakey in keylist_tm[tmkey]:
items.append(datakey)
with Pool() as pool:
pool.map(runme, items)
else:
# conventional run
for tmkey in keylist_tm.keys():
for datakey in keylist_tm[tmkey]:
print("--> {}".format(datakey))
runme(datakey)
#sys.exit()

458
scripts/04_mosaics.py Executable file
View File

@@ -0,0 +1,458 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
НАЗВАНИЕ:
04_mosaics.py
НАЗНАЧЕНИЕ:
Собирает мозайки в разных энергетических диапазонах.
ВЫЗОВ:
esass
./04_mosaics.py
УПРАВЛЕНИЕ:
Запуск отдельных команд управляется переменными, например: do_init = True
Выбранный энергетический диапазон управляется переменной index
forced=True делает принудительную фотометрию
ПАРАМЕТРЫ:
index : Выбранный энергетический диапазон
ВЫВОД:
Выходные файлы записываются в директорию outfile_dir
ИСТОРИЯ:
Роман Кривонос, ИКИ РАН, krivonos@cosmos.ru
Март 2023
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
#from pathlib import Path
import numpy as np
import glob
from os.path import dirname
import inspect
import pickle
import uds
from uds.utils import *
from uds.config import *
""" find UDS root dir """
#root_path=dirname(dirname(dirname(inspect.getfile(uds))))
"""
ftools does not like long file path names,
for this reason, we use relative path here
"""
root_path='..'
print("UDS root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
local_run = False
outkey="tm0"
do_init = True
do_merge = True
do_detmask = True
do_expmap = True
do_erbox1 = True # local mode
do_erbackmap1 = True #
do_erbox2 = True # map mode, with background map
do_erbackmap2 = True #
do_erbox3 = True # map mode, with background map
do_erbackmap3 = True #
do_ersensmap = False
do_ermldet = True
do_fixcat = False # only for index=0
do_fixxmm = False # prepare forced photometry, only for index=0
do_apetool = False
do_catprep = True
do_filter_catalog = False
do_cross_match = False
index=3
forced=False
""" If forced=True, take input catalog from energy range en0 """
comm='' # for 4XMM-DR12 forced photometry use '-xmm'
vign=True
vignetting = 'vign' if (vign==True) else 'novign'
attcorr=True
events=[]
expmaps=[]
bkgmaps=[]
for tmkey in keylist_tm.keys():
print("TM{} in work... init events".format(tmkey))
for datakey in keylist_tm[tmkey]:
print("--> {}".format(datakey))
""" Подготавливаем списки событий индивидуальных наблюдений """
outfile_evtool,outfile_expmap=init_events(key=datakey,attcorr=attcorr,
eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
do_init=do_init,
do_obsmode=False,
do_center=False,
do_evtool=True,
do_expmap=False,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen,
emin_kev=emin_kev[index],
emax_kev=emax_kev[index])
events.append(outfile_evtool)
expmaps.append(outfile_expmap)
""" Собираем общий список событий """
outfile_evtool="{}_EventList_en{}.fits".format(os.path.join(outfile_dir,outkey),
eband[index])
if(do_merge==True):
do_evtool_esass(events=events, outfile=outfile_evtool)
""" makes detmask from TM exposures """
detmask="{}/{}_DetectorMask_en{}{}".format(outfile_dir,
outkey,
eband[index],
outfile_post)
if(do_detmask==True):
create_detmask_merged(expmaps,detmask,minval=100)
"""
Собираем общую карту экспозиции, обратите внимание на коэффициент 7.
Экспозиция рассчитывается на 7 телескопов.
outfile_expmap="{}_ExposureMap_en{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], vignetting)
outfile_bkgmap="{}_BackMap_en{}.{}.fits".format(os.path.join(outfile_dir,outkey), eband[index], vignetting)
"""
outfile_expmap="{}_ExposureMap_en{}.{}{}".format(os.path.join(outfile_dir,outkey),
eband[index],vignetting,
outfile_post)
if(do_expmap==True):
create_expmap_merged(expmaps,outfile_expmap,scale=7.0)
outfile_boxlist1="{}/{}_BoxList1_en{}{}".format(outfile_dir,outkey, eband[index], outfile_post)
if(do_erbox1==True):
cmd=["erbox",
"images=\'{}\'".format(outfile_evtool),
"boxlist=%s" %(outfile_boxlist1),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"ecf=\'{}\'".format(ecf[index]),
"nruns=2",
"likemin=6.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=N",
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist1)
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist1)
""" Background map 1 """
outfile_backmap1="{}_BackMap1_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
cheese_mask="{}_CheeseMask1_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_erbackmap1==True):
do_erbackmap_esass(outfile_evtool,outfile_expmap,outfile_boxlist1,detmask,emin_ev[index],emax_ev[index],
outfile_backmap1,cheese_mask)
outfile_boxlist2="{}/{}_BoxList2_en{}{}".format(outfile_dir,outkey, eband[index], outfile_post)
if(do_erbox2==True):
cmd=["erbox",
"images=\'{}\'".format(outfile_evtool),
"boxlist=%s" %(outfile_boxlist2),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"ecf=\'{}\'".format(ecf[index]),
"nruns=2",
"likemin=6.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=Y",
"bkgimages={}".format(outfile_backmap1),
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist2)
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist2)
""" Background map 2 """
outfile_backmap2="{}_BackMap2_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
cheese_mask="{}_CheeseMask2_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_erbackmap2==True):
do_erbackmap_esass(outfile_evtool,outfile_expmap,outfile_boxlist2,detmask,emin_ev[index],emax_ev[index],
outfile_backmap2,cheese_mask)
outfile_boxlist3="{}/{}_BoxList3_en{}{}".format(outfile_dir,outkey, eband[index], outfile_post)
if(do_erbox3==True):
cmd=["erbox",
"images=\'{}\'".format(outfile_evtool),
"boxlist=%s" %(outfile_boxlist3),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"ecf=\'{}\'".format(ecf[index]),
"nruns=2",
"likemin=6.0",
"boxsize=4",
"compress_flag=N",
"bkgima_flag=Y",
"bkgimages={}".format(outfile_backmap2),
"expima_flag=Y",
"detmask_flag=Y"
]
remove_file(outfile_boxlist3)
print((" ").join(cmd))
os.system((" ").join(cmd))
save_ds9reg(outfile_boxlist3)
""" Background map 3 """
outfile_backmap3="{}_BackMap3_en{}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], vignetting, outfile_post)
cheese_mask="{}_CheeseMask3_en{}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], vignetting, outfile_post)
if(do_erbackmap3==True):
boxlist3 = outfile_boxlist3 if(forced == False) else "{}/{}_BoxList3_en{}{}".format(outfile_dir,outkey, eband[0], outfile_post)
do_erbackmap_esass(outfile_evtool,outfile_expmap,boxlist3,detmask,emin_ev[index],emax_ev[index],
outfile_backmap3,cheese_mask)
if(forced==True):
mllist="{}_MaxLikSourceList_en{}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[index], comm, outfile_post)
srcmap="{}_SourceMap_en{}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[index], comm, outfile_post)
""" for en1,2,3,6 give mllist from en0 as input """
#boxlist3="{}_MaxLikSourceList_en{}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[0], comm, outfile_post)
#if(index==0):
boxlist3="{}_MaxLikSourceList_en{}.fixed{}{}".format(os.path.join(outfile_dir,outkey), eband[0], comm, outfile_post)
if not (os.path.exists(boxlist3)):
print("{} not found. Run do_fixcat=True, index=0, forced=False".format(boxlist3))
sys.exit()
add_specific_columns(boxlist3)
fitpos_flag="fitpos_flag=no"
fitext_flag="fitext_flag=no"
nmulsou = "nmulsou=1"
nmaxfit="nmaxfit=10"
multrad="multrad=15."
cutrad="cutrad=15."
if(index == 3 or index == 6):
""" for hard band take unvignetted background """
outfile_backmap3="{}_BackMap3_en{}.{}{}".format(os.path.join(outfile_dir,outkey), eband[index], "novign", outfile_post)
else:
mllist="{}_MaxLikSourceList_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
srcmap="{}_SourceMap_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
boxlist3 = outfile_boxlist3
fitpos_flag="fitpos_flag=yes"
fitext_flag="fitext_flag=yes"
nmulsou = "nmulsou=2"
nmaxfit="nmaxfit=4"
multrad="multrad=20."
cutrad="cutrad=20."
""" allow ermpldet to split sources (no more than two) """
cmd=["ermldet",
"mllist={}".format(mllist),
"boxlist=%s" %(boxlist3),
"images=\'{}\'".format(outfile_evtool),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"bkgimages=\'{}\'".format(outfile_backmap3),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"ecf=\'{}\'".format(ecf[index]),
"hrdef=",
"likemin=0.",
"extlikemin=5.",
"compress_flag=N",
cutrad,
multrad,
"extmin=2.0",
"extmax=35.0",
#"bkgima_flag=Y", looks outdated
"expima_flag=Y",
"detmask_flag=Y",
"shapelet_flag=yes",
"photon_flag=yes",
"extentmodel=beta",
"thres_flag=N",
"thres_col=like",
"thres_val=30.",
nmaxfit,
nmulsou,
fitpos_flag,
fitext_flag,
"srcima_flag=yes",
"srcimages=\'{}\'".format(srcmap)
]
if(do_ersensmap==True):
detlike=10
create_sensmap(sensmap="{}_SensitivityMap_dl{}_en{}{}".format(os.path.join(outfile_dir,outkey), detlike,
eband[index], outfile_post),
areatab="{}_AreaTable_dl{}_en{}{}".format(os.path.join(outfile_dir,outkey),
detlike, eband[index], outfile_post),
expmap=outfile_expmap, backmap=outfile_backmap3,detlike=detlike,
detmask=detmask, emin=emin_ev[index], emax=emax_ev[index],ecf=ecf[index])
"""
detlike=6
create_sensmap(sensmap="{}_SensitivityMap_dl{}_en{}{}".format(os.path.join(outfile_dir,outkey), detlike,
eband[index], outfile_post),
areatab="{}_AreaTable_dl{}_en{}{}".format(os.path.join(outfile_dir,outkey),
detlike, eband[index], outfile_post),
expmap=outfile_expmap, backmap=outfile_backmap3,detlike=detlike,
detmask=detmask, emin=emin_ev[index], emax=emax_ev[index],ecf=ecf[index])
"""
if(do_ermldet==True):
test_exe('ermldet')
if(vign==False):
print('Run ermldet with vignetted exposure!')
sys.exit()
remove_file(mllist)
remove_file(srcmap)
print(cmd)
runme(cmd, local_run=local_run)
print(cmd)
save_ermldet_ds9reg(mllist,scale=60*60,label='det_like')
save_ermldet_ds9reg(mllist,scale=60*60,label='id_src')
correct_fluxerr_ermldet_forced(mllist)
if(forced==True):
result = check_ermldet_forced(mllist)
# for a some reason, for an arbitrary energy band, ermldet break order of sources. Do this forced correction.
if(result == False):
correct_srcid_ermldet_forced(mllist)
if(do_fixcat==True):
if not index == 0:
print("ERROR: You can fix only reference catalog for en0.")
sys.exit()
if forced == True:
print("ERROR: You can fix only non-forced catalog for en0.")
sys.exit()
srcs_remove=[341,446,346,96]
srcs_add = {'4XMM J021738.8-051257':[34.4117002, -5.2159135, 0.624],# 341
'4XMM J021733.8-051311':[34.3910215,-5.2199877,2.247],# 341
'4XMM J021929.4-051220':[34.8725460,-5.2056849,1.074],#446
'4XMM J021930.7-051225':[34.8782267,-5.2072112,0.624],# 446
'4XMM J021945.2-045331':[34.9383593,-4.8919843,1.538],#346
'4XMM J021929.4-043224':[34.8728586,-4.5400022,0.659555],#96
#'4XMM J021929.4-043224':[34.8728586,-4.5400022, 0.660],
#'4XMM J021831.8-050059':[34.6328841,-5.0163909,1.529],
#'4XMM J022131.1-050027':[35.3797879,-5.0075498,0.941],
#'4XMM J022129.5-045914':[35.3732136,-4.9874025,0.332],
#'4XMM J022026.3-050251':[35.1098619,-5.0476199,0.551],
#'4XMM J021925.4-042647':[34.8559099,-4.4465007,1.366],
#'4XMM J021910.9-045108':[34.7954311,-4.8522901,0.898],
#'4XMM J021945.2-045331':[34.9383593,-4.8919843,1.538],
#'4XMM J021733.8-051311':[34.3910215,-5.2199877,2.247],
}
fix_catalog(mllist=mllist,refimage=outfile_evtool, srcs_remove=srcs_remove, srcs_add=srcs_add)
"""
Note that fix_catalog added ID_SRC to each XMM source.
Next, we save forced XMM sources (with new ID_SRC!) for later catalog compilation
"""
with open(mllist.replace(".fits", ".xmm.pickle"), 'wb') as f:
pickle.dump(srcs_add, f)
if(do_fixxmm==True):
if not index == 0:
print("ERROR: You can fix only reference catalog for en0.")
sys.exit()
if forced == True:
print("ERROR: You can fix only non-forced catalog for en0.")
sys.exit()
fix_xmm_sources(mllist=mllist,refimage=outfile_evtool, xmm_catalog='../data/4XMM-DR12/4XMM_DR12cat_slim_v1.0_UDS.fits.catalog')
if(do_apetool==True):
psfmap="{}_PsfMap{}".format(os.path.join(outfile_dir,outkey), outfile_post)
#remove_file(psfmap)
#cmd=["apetool",
# "images=\'{}\'".format(outfile_evtool),
# "psfmaps=\'{}\'".format(psfmap),
# "psfmapflag=yes",]
#runme(cmd, local_run=local_run)
cmd=["apetool",
"mllist={}".format(mllist),
"apelistout={}".format(mllist), # give the same file
"images=\'{}\'".format(outfile_evtool),
"expimages=\'{}\'".format(outfile_expmap),
"detmasks=\'{}\'".format(detmask),
"bkgimages=\'{}\'".format(outfile_backmap3),
"emin=\'{}\'".format(emin_ev[index]),
"emax=\'{}\'".format(emax_ev[index]),
"srcimages=\'{}\'".format(srcmap),
"psfmaps={}".format(psfmap),
"psfmapflag=no",
"stackflag=no",
"apexflag=yes",
"apesenseflag=no",
"eefextract=0.65",
"cutrad=15",
"eindex=1",]
runme(cmd, local_run=local_run)
if(forced==True):
catprep="{}_SourceCatalog_en{}.forced{}{}".format(os.path.join(outfile_dir,outkey), eband[index], comm, outfile_post)
else:
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
if(do_catprep==True):
cmd=["catprep",
"infile={}".format(mllist),
"outfile={}".format(catprep),]
remove_file(catprep)
runme(cmd, local_run=local_run)
if(do_filter_catalog==True):
#filter_mllist(mllist,expcut=5000.0,dlcut=10.0,dlmin=10,dlmax=10000)
""" works the same """
filter_catprep(catprep,expcut=5000.0,dlmin=10,dlmax=10000,outkey='bright')
#filter_catprep(catprep,expcut=5000.0,dlmin=6,dlmax=10,outkey='faint')
if(do_cross_match==True):
crossmatch_shu2019(catprep,dlmin=10,refimage=outfile_evtool,crval=[ra_cen, de_cen],
catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_UDS.fits.catalog")

284
scripts/05_srctool.py Executable file
View File

@@ -0,0 +1,284 @@
#!/usr/bin/env python
"""
НАЗВАНИЕ:
05_srctool.py
НАЗНАЧЕНИЕ:
Запускает scrtool для самого широкого канала 0.2-10 кэВ, чтобы спектры имели самое полное покрытие по энергиям. Список источников берется из 0.3-2.3 кэВ.
ВЫЗОВ:
esass
./05_srctool.py
УПРАВЛЕНИЕ:
Требуется запуск предыдущего скрипта 04_mosaics.py
ПАРАМЕТРЫ:
index=4 : Выбранный энергетический диапазон
ВЫВОД:
Выходные файлы записываются в директорию outfile_dir/srctool_dir
ИСТОРИЯ:
Роман Кривонос, ИКИ РАН, krivonos@cosmos.ru
Март 2023
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
from pathlib import Path
import numpy as np
import glob
from os.path import dirname
import inspect
import pickle
import uds
from uds.utils import *
from uds.config import *
from uds.sherpa import *
""" find UDS root dir """
#root_path=dirname(dirname(dirname(inspect.getfile(uds))))
"""
ftools does not like long file path names,
for this reason, we use relative path here
"""
root_path='..'
print("UDS root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
srctool_dir="{}/{}".format(outfile_dir,"srctool-products")
create_folder(srctool_dir)
outkey="tm0"
outfile_srctool="{}_SrcTool_".format(outkey)
do_init = False
do_merge = False
do_srctool = False
do_grppha = False
do_ecf_calc = False # for all bands
do_ecf_print = False # for all bands
do_flux_calc = False # for all bands
do_catalog = False
do_extended = False
do_ds9reg = False
do_euds_final = False
do_euds_dr12 = False # crossmatch eUDS with DR12
do_euds_stat = False
do_euds_cds = False
do_xmm_catalog = False
do_xmm_final = False
do_xmm_xmatch = False
do_xmm_ds9reg = False
do_xmm_cds = False
do_euds_cosmatch = True
do_cross_check = False # Check whether all E3,E6 sources are detected in E0
index=0
""" чтобы спектры покрывали все энергии работаем в диапазоне 5 """
vign=True
vignetting = 'vign' if (vign==True) else 'novign'
events=[]
expmaps=[]
bkgmaps=[]
for tmkey in keylist_tm.keys():
print("TM{} in work... init events".format(tmkey))
for datakey in keylist_tm[tmkey]:
print("--> {}".format(datakey))
""" Подготавливаем списки событий индивидуальных наблюдений """
outfile_evtool,outfile_expmap=init_events(key=datakey,attcorr=True,
eband_index=eband[index],
infile_dir=infile_dir,
outfile_dir=outfile_dir,
do_init=do_init,
do_obsmode=False,
do_center=False,
do_evtool=False,
do_expmap=False,
vign=vign,
ra_cen=ra_cen, de_cen=de_cen,
emin_kev=emin_kev[index],
emax_kev=emax_kev[index])
events.append(outfile_evtool)
expmaps.append(outfile_expmap)
""" Собираем общий список событий """
outfile_evtool="{}_EventList_en{}.fits".format(os.path.join(outfile_dir,outkey),
eband[index])
if(do_merge==True):
do_evtool_esass(events=events, outfile=outfile_evtool)
suffix_srctool=".fits"
""" Output filename suffix - all output filenames appended with this string.
If suffix contains no filename extension (does not contain a "."), then ".fits"
is also appended to the filename. """
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[0], outfile_post)
""" take source catalog from 0.3-2.3 keV band """
if not (os.path.isfile(catprep)==True):
print("{} not found, run 04_mosaics.py?".format(catprep))
sys.exit()
if(do_srctool==True):
test_exe('srctool')
cmd=['srctool',
"insts=\'1 5 6 7\'",
"eventfiles={}".format(outfile_evtool),
"prefix=\'{}\'".format(os.path.join(srctool_dir,outfile_srctool)),
"suffix=\'{}\'".format(suffix_srctool),
"srccoord={}".format("../products/eUDS_for_srctool.fits"),
# the same as original file eUDS.fits, but changed ML_CTS --> ML_CTS_0, ML_BKG --> ML_BKG_0, ML_EXP --> ML_EXP_0
#"srcreg=\'fk5;circle * * 60s\'",
#"backreg=\'fk5;annulus * * 90s 120s\'",
"srcreg=AUTO",
"backreg=AUTO",
"clobber=yes",]
print((" ").join(cmd))
#os.system((" ").join(cmd))
#print((" ").join(cmd))
if(do_grppha==True):
group_spectra("{}/*020_SourceSpec_*.fits".format(srctool_dir))
ecfout="{}_SampleFlux_v1.pickle".format(os.path.join(outfile_dir,outkey))
if(do_ecf_calc==True):
calc_ecf("{}/tm0_SrcTool_020_ARF_?????.fits".format(srctool_dir),
catprep=catprep, emin=emin_kev, emax=emax_kev, eband=eband, outfile=ecfout, simnum=10000)
if(do_ecf_print==True):
print_ecf(infile=ecfout, emin=emin_kev, emax=emax_kev, eband=eband, skipfrac=10.0)
fluxout="{}_SherpaFlux.pickle".format(os.path.join(outfile_dir,outkey))
if(do_flux_calc==True):
calc_flux("{}/tm0_SrcTool_020_ARF_?????.fits".format(srctool_dir),
catprep=catprep, emin=emin_kev, emax=emax_kev, eband=eband, outfile=ecfout, simnum=100)
#index=0
catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
rawcat="{}_SourceCatalog_en{}.pickle".format(os.path.join(outfile_dir,outkey), eband[index])
if(do_catalog==True):
forced_xmm_sources="{}_MaxLikSourceList_en{}.xmm.pickle".format(os.path.join(outfile_dir,outkey), eband[index])
with open(forced_xmm_sources, 'rb') as f:
print("Reading forced XMM sources from {}".format(forced_xmm_sources))
srcs_forced = pickle.load(f)
print()
print(srcs_forced)
print()
make_euds_catalog(infile='../products/tm0_SourceCatalog_en0.forced.fits', rawcat=rawcat, dlmin=10.0, dlmax=100000, ext_like=1000,
emin=emin_kev[index], emax=emax_kev[index], eband=eband[index],
infile_en00cat=catprep,
infile_en01cat='../products/tm0_SourceCatalog_en1.forced.fits',
infile_en02cat='../products/tm0_SourceCatalog_en2.forced.fits',
infile_en03cat='../products/tm0_SourceCatalog_en3.forced.fits',
infile_en06cat='../products/tm0_SourceCatalog_en6.forced.fits',
infile_en00sens='../products/tm0_SensitivityMap_dl10_en0.fits',
infile_en01sens='../products/tm0_SensitivityMap_dl10_en1.fits',
infile_en02sens='../products/tm0_SensitivityMap_dl10_en2.fits',
infile_en03sens='../products/tm0_SensitivityMap_dl10_en3.fits',
infile_en06sens='../products/tm0_SensitivityMap_dl10_en6.fits',
srcs_forced=srcs_forced,
)
if(do_extended==True):
make_extended(infile=rawcat,outreg="{}_ExtendedCat_en{}.reg".format(os.path.join(outfile_dir,outkey), eband[index]))
if(do_ds9reg==True):
#make_final_ds9reg(infile=rawcat,outreg="{}_FinalCat_dl10.reg".format(os.path.join(outfile_dir,outkey)))
make_final_ds9reg(infile=rawcat,scale=(60*60)/10,outreg="{}_FinalCat_dl10_talk.reg".format(os.path.join(outfile_dir,outkey)))
if(do_euds_final==True):
""" make final eUDS catalog """
final_euds_catalog(infile=rawcat, outfile_fits='../products/eUDS.fits')
if(do_euds_dr12==True):
crossmatch_dr12('../products/eUDS.fits', catalog=root_path+"/data/4XMM-DR12/4XMM_DR12cat_slim_v1.0_UDS.fits.catalog", devmax=15)
if(do_euds_stat==True):
make_euds_stat(infile="../products/eUDS.fits",fluxlim=5e-14)
if(do_euds_cds==True):
make_euds_cds(infile="../products/eUDS.fits",outfile='../products/eUDS.cds')
if(do_cross_check==True):
""" cross check final eUDS catalog """
cross_check_euds(infile=catprep, euds='../products/eUDS.fits', outkey="../products/en{}_FinalCat_dl10".format(index))
if(do_xmm_catalog==True):
""" complile raw forced XMM catalog """
make_xmm_catalog(infile_en00cat='../products/tm0_SourceCatalog_en0.forced-xmm.fits',
infile_en01cat='../products/tm0_SourceCatalog_en1.forced-xmm.fits',
infile_en02cat='../products/tm0_SourceCatalog_en2.forced-xmm.fits',
infile_en03cat='../products/tm0_SourceCatalog_en3.forced-xmm.fits',
infile_en06cat='../products/tm0_SourceCatalog_en6.forced-xmm.fits',
forced_xmm_sources='../products/tm0_MaxLikSourceList_en0.fixed-xmm.pickle',
outfile='../products/tm0_4XMM-DR12.pickle')
if(do_xmm_final==True):
""" make final XMM-forced catalog """
final_xmm_catalog(infile='../products/tm0_4XMM-DR12.pickle', outfile_fits='../products/eUDS_4XMM-DR12.fits')
if(do_xmm_xmatch==True):
""" cross-match XMM-forced catalog
outfile_cvs contains 0.3-2.3 keV eUDS flux (col1=flux,col2=err) vs. 4XMM-DR12 flux (col3=flux, col4=err)
XMM flux was converted from 0.2-2.0 keV to 0.3-2.3 keV using wabs*powerlow with wabs=0.02 gamma=2.0
"""
final_xmm_xmatch(infile='../products/eUDS_4XMM-DR12.fits',
xmmslim='../data/4XMM-DR12/4XMM_DR12cat_slim_v1.0_UDS.fits.catalog',
xmmfull='../data/4XMM-DR12/4XMM_DR12cat_v1.0_UDS.fits.catalog',
xmmlim=2e-14,
outfile_flux="../products/eUDS_4XMM-DR12.flux.csv")
if(do_xmm_ds9reg==True):
""" show XMM-forced catalog """
make_xmm_ds9reg_confused(infile='../products/eUDS_4XMM-DR12.fits', outfile='../products/eUDS_4XMM-DR12.confused.reg')
""" obsolete """
if(do_xmm_cds==True):
make_xmm_cds(infile="../products/eUDS_4XMM-DR12.fits", outfile='../products/eUDS_4XMM-DR12.cds')
if(do_euds_cosmatch==True):
""" prepare eUDS catalog for CosMatch (Mescheryakov) """
make_euds_cosmatch(infile='../products/eUDS.fits', outfile='../products/eUDS-CosMatch.fits')

445
scripts/06_plot.py Executable file
View File

@@ -0,0 +1,445 @@
#!/usr/bin/env python
"""
НАЗВАНИЕ:
05_srctool.py
НАЗНАЧЕНИЕ:
Запускает scrtool для самого широкого канала 0.2-10 кэВ, чтобы спектры имели самое полное покрытие по энергиям. Список источников берется из 0.3-2.3 кэВ.
ВЫЗОВ:
esass
./05_srctool.py
УПРАВЛЕНИЕ:
Требуется запуск предыдущего скрипта 04_mosaics.py
ПАРАМЕТРЫ:
index=4 : Выбранный энергетический диапазон
ВЫВОД:
Выходные файлы записываются в директорию outfile_dir/srctool_dir
ИСТОРИЯ:
Роман Кривонос, ИКИ РАН, krivonos@cosmos.ru
Март 2023
"""
from astropy.wcs import WCS
from astropy.io import fits
import sys, os, os.path, time, subprocess
from pathlib import Path
import numpy as np
import glob
from os.path import dirname
import inspect
import uds
from scipy.stats import norm
import matplotlib.pyplot as plt
import pandas as pd
from uds.utils import *
from uds.config import *
from uds.sherpa import *
""" find UDS root dir """
#root_path=dirname(dirname(dirname(inspect.getfile(uds))))
"""
ftools does not like long file path names,
for this reason, we use relative path here
"""
root_path='..'
print("UDS root path: {}".format(root_path))
infile_dir=root_path+'/data/processed'
outfile_dir=root_path+'/products'
create_folder(outfile_dir)
srctool_dir="{}/{}".format(outfile_dir,"srctool-products")
create_folder(srctool_dir)
outkey="tm0"
outfile_srctool="{}_SrcTool_".format(outkey)
do_flux_distr = False
do_sens_curve = False
do_4xmm_ratio = False
do_euds_radec_err = False
do_euds_dr12_diff = False
do_euds_dr12_stat = True
do_print_ecf = False
index=0
catalog = "{}_SourceCatalog_en{}.main.selected.csv".format(os.path.join(outfile_dir,outkey), eband[0])
if not (os.path.isfile(catalog)==True):
print("{} not found, run 05_srctool.py?".format(catalog))
sys.exit()
if(do_flux_distr==True):
data, logbins, mean, median = get_log_distr(infile=catalog, field='ml_rate', minval=1e-3, maxval=2)
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.hist(data, bins=logbins, histtype='step', color='blue', linewidth=1, linestyle='solid')
plt.xlabel('Count rate (counts s$^{-1}$)',fontsize=14, fontweight='normal')
plt.ylabel('Number',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
plt.savefig(catalog.replace("main.selected.csv", "ml_rate.png"), bbox_inches='tight')
plt.close(fig)
data, logbins, mean, median = get_log_distr(infile=catalog, field='ml_flux', minval=1e-15, maxval=2e-12)
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.hist(data, bins=logbins, histtype='step', color='blue', linewidth=1, linestyle='solid')
plt.xlabel('Energy flux (erg s$^{-1}$ cm$^{-2}$)',fontsize=14, fontweight='normal')
plt.ylabel('Number',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
plt.savefig(catalog.replace("main.selected.csv", "ml_flux.png"), bbox_inches='tight')
plt.close(fig)
if(do_sens_curve==True):
coeff = 2.4336e-13 / 3.4012e-13 # see below
areatab="{}_AreaTable_dl10_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
hdul = fits.open(areatab)
tbdata = hdul[1].data
""" convert limflux from 0.3-2.3 keV to 0.5-2 keV """
limflux_dl10 = tbdata['LIMFLUX']*coeff
area_dl10 = tbdata['SKY_AREA']
hdul.close()
areatab="{}_AreaTable_dl6_en{}{}".format(os.path.join(outfile_dir,outkey), eband[index], outfile_post)
hdul = fits.open(areatab)
tbdata = hdul[1].data
""" convert limflux from 0.3-2.3 keV to 0.5-2 keV """
limflux_dl6 = tbdata['LIMFLUX']*coeff
area_dl6 = tbdata['SKY_AREA']
hdul.close()
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
ax.set_xlim([3e-16,1e-13])
ax.set_ylim([0.3,160])
plt.plot(limflux_dl10,area_dl10, color="black", linestyle='solid',label="eUDS (DL10)")
plt.plot(limflux_dl6,area_dl6, color="black", linestyle='dashed', label="eUDS (DL6)")
df = pandas.read_csv("../data/surveys/eFEDS.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="brown", linestyle='solid',label="eFEDS (DL6)")
df = pandas.read_csv("../data/surveys/cosmos-legacy.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="blue", linestyle='solid',label="COSMOS Legacy")
df = pandas.read_csv("../data/surveys/CDWFS.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="green", linestyle='solid', label="CDWFS")
df = pandas.read_csv("../data/surveys/XMM-RM.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="magenta", linestyle='solid',label="XMM-RM")
df = pandas.read_csv("../data/surveys/XMM-XXL-N.dat",header=None,names=['limflux','area'])
plt.plot(df['limflux'],df['area'], color="red", linestyle='solid',label="XMM-XXL-N")
ax.legend()
#plt.hist(data, bins=logbins, histtype='step', color='blue', linewidth=1, linestyle='solid')
plt.xlabel('Limiting flux (0.5-2 keV, erg s$^{-1}$ cm$^{-2}$)',fontsize=14, fontweight='normal')
plt.ylabel('Area (deg$^{2}$)',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
png="{}_AreaTable_en{}.png".format(os.path.join(outfile_dir,outkey), eband[index])
plt.savefig(png, bbox_inches='tight')
plt.close(fig)
"""
========================================================================
Model TBabs<1>*powerlaw<2> Source No.: 1 Active/On
Model Model Component Parameter Unit Value
par comp
1 1 TBabs nH 10^22 2.00000E-02 frozen
2 2 powerlaw PhoIndex 2.00000 frozen
3 2 powerlaw norm 1.14851E-04 +/- 3.83988E-06
________________________________________________________________________
Model Flux 0.00028334 photons (3.4012e-13 ergs/cm^2/s) range (0.30000 - 2.3000 keV)
Model Flux 0.0001619 photons (2.4336e-13 ergs/cm^2/s) range (0.50000 - 2.0000 keV)
"""
if(do_4xmm_ratio==True):
filename="../products/eUDS_4XMM-DR12.flux.csv"
data, logbins, mean, median = get_log_distr(infile=filename, field='ratio', minval=8e-2, maxval=60, nbin=60)
print("Median {}".format(median))
print(" Mean {}".format(mean))
print("Ntotal {}".format(len(data)))
fig, ax = plt.subplots()
#plt.figure(figsize=(5,5))
#plt.figure().set_figheight(3.6)
#plt.rcParams['figure.figsize'] = [4, 4]
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.hist(data, bins=logbins, histtype='step', color='blue', linewidth=1, linestyle='solid')
plt.axvline(x = median, color = 'black', label = 'Median value', linestyle='dashed')
plt.xlabel('Energy flux ratio',fontsize=14, fontweight='normal')
plt.ylabel('Number',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
plt.savefig(filename.replace(".csv", ".distr.png"), bbox_inches='tight')
plt.close(fig)
df = pandas.read_csv(filename)
fig, ax = plt.subplots()
#plt.rcParams['figure.figsize'] = [4, 4]
#plt.figure(figsize=(5,5))
#plt.figure().set_figheight(4)
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.plot(np.array([1e-17,1e-11]), np.array([1e-17,1e-11]), linestyle = 'solid', color='black')
plt.plot(np.array([1e-17,1e-11]), 10*np.array([1e-17,1e-11]), linestyle = 'dashed', color='black')
plt.plot(np.array([1e-17,1e-11]), 0.1*np.array([1e-17,1e-11]), linestyle = 'dotted', color='black')
#plt.plot(np.array([1e-17,1e-11]), median*np.array([1e-17,1e-11]), linestyle = 'dashed', color='blue')
plt.errorbar(df['dr12_flux'],df['euds_flux'], yerr=df['euds_flux_err'], xerr=df['dr12_flux_err'], linestyle='None', color='black', label='Errors')
plt.plot(df['dr12_flux'],df['euds_flux'], marker="o", linewidth=1, linestyle='None', markerfacecolor='Gold',markeredgecolor="black",)
#plt.axvline(x = median, color = 'black', label = 'Median value', linestyle='dashed')
plt.xlabel('4XMM-DR12 Energy flux (erg s$^{-1}$ cm$^{-2}$)',fontsize=14, fontweight='normal')
plt.ylabel('eUDS Energy flux (erg s$^{-1}$ cm$^{-2}$)',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xlim(2e-16,1e-12)
plt.ylim(1e-15,2e-12)
plt.xscale('log')
plt.yscale('log')
plt.savefig(filename.replace(".csv", ".png"), bbox_inches='tight')
plt.close(fig)
if(do_euds_radec_err==True):
filename="../products/eUDS.fits"
ecat={}
hdul = fits.open(filename)
etab = hdul[1].data
hdul.close()
x=[]
y=[]
for s in etab:
if ("XMM" in s['DR12_IAU_NAME']):
print("Skip ",s['DR12_IAU_NAME'])
continue
if (s['EXT_LIKE']>0.0):
print("Skip extended ",s['ID_SRC'])
continue
x.append(s['DET_LIKE'])
y.append(s['RADEC_ERR'])
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
ax.set_xlim([10,10000])
ax.set_ylim([0.1,40])
plt.plot(x,y,".", color="gold", markersize=12, markeredgewidth=1.5, markeredgecolor="black")
plt.xlabel('DET_LIKE',fontsize=14, fontweight='normal')
plt.ylabel('RADEC_ERR (arcsec)',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('log')
plt.yscale('log')
png="../products/en0_radec_err.png"
plt.savefig(png, bbox_inches='tight')
plt.close(fig)
stat={}
hdul = fits.open("../products/eUDS.dr12.cross.fits")
tab = hdul[1].data
hdul.close()
print(len(np.unique(tab['ID'])))
for rec in tab:
sid=rec['ID']
if sid in stat.keys():
stat[sid]=stat[sid]+1
else:
stat[sid]=1
unique=0
double=0
triple=0
for key in stat.keys():
if(stat[key]==1):
unique=unique+1
if(stat[key]==2):
double=double+1
if(stat[key]==3):
triple=triple+1
print("unique: {}, double: {}. triple: {}".format(unique, double, triple))
if(do_euds_dr12_diff==True):
# read forced first
# fieldnames = ['euds_det_like', 'euds_flux', 'euds_flux_err', 'dr12_flux', 'dr12_flux_err', 'ratio']
filename="../products/eUDS_4XMM-DR12.flux.csv"
df = pandas.read_csv(filename)
# calculate difference similar to Bruner
threshold=5e-14
# forced catalog contains CONF flag, take it to remove from histogram
hdul = fits.open("../products/eUDS_4XMM-DR12.fits")
forced = hdul[1].data
fcat={}
for rec in forced:
key=rec['DR12_SRCID']
fcat[key]={'conf':rec['CONF'],}
# read cross-match results (not forced)
hdul = fits.open('../products/eUDS.dr12.cross.fits')
tb = hdul[1].data
cross_diff=[]
cross_ratio=[]
mean=0.0
count=0
for ind,s in enumerate(tb):
key=rec['DR12_SRCID']
#if not (tb[ind]['dr12_flux']>threshold and tb[ind]['src_flux']>threshold):
if not (tb[ind]['dr12_flux']>threshold):
continue
ape_flux=(tb[ind]['ape_cts']-tb[ind]['ape_bkg'])/tb[ind]['ape_exp']/ecf[index]
cross_dr12_flux=(tb[ind]['dr12_flux'])
cross_dr12_flux_error=(tb[ind]['dr12_flux_error'])
cross_euds_flux=(tb[ind]['src_flux'])
#cross_euds_flux=ape_flux
cross_euds_flux_error=(tb[ind]['src_flux_error'])
print(cross_euds_flux,cross_euds_flux_error,tb[ind]['ape_cts'],tb[ind]['det_like'])
d=(cross_dr12_flux - cross_euds_flux) / np.sqrt(cross_dr12_flux_error**2 + cross_euds_flux_error**2)
#if(d < -15):
# continue
r=cross_euds_flux/cross_dr12_flux
cross_diff.append(d)
cross_ratio.append(r)
#if(d < -15.0):
#print("GGG",d,r,cross_euds_flux,cross_dr12_flux)
#print(tb[ind]['DR12_NAME'],tb[ind]['DR12_RA'],tb[ind]['DR12_DEC'])
mean=mean + (cross_euds_flux/cross_dr12_flux)
count=count+1
print("min={:.2f}, max={:.2f}, mean={:.2f}, median={:.2f}, std={:.2f} N={}".format(min(cross_diff),
max(cross_diff),
np.mean(cross_diff),
np.median(cross_diff),
np.std(cross_diff),
len(cross_diff)))
print("ratio mean",mean/count)
print("ratio median",np.median(cross_ratio))
nbin=50
bins=np.linspace(-30,30, nbin)
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.hist(cross_diff, bins=bins,histtype='step', color='green', linewidth=1, linestyle='solid', density=True)
#x = np.arange(-10, 10, 0.001)
#plot normal distribution with mean 0 and standard deviation 1
#plt.plot(x, norm.pdf(x, 0, 1), color='red', linewidth=2)
plt.ylabel('Relative fraction',fontsize=14, fontweight='normal')
plt.xlabel('(F$_{XMM}$-F$_{eUDS}$)/$\sqrt{\Delta F_{XMM}^{2}+\Delta F_{eUDS}^{2}}$',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('linear')
plt.yscale('linear')
ax.set_xlim([-31, 31])
ax.set_ylim([0, 0.2])
plt.savefig("../products/cross-match_diff.png", bbox_inches='tight')
plt.close(fig)
nbin=200
bins=np.linspace(-1,1.0, nbin)
fig, ax = plt.subplots()
for axis in ['top','bottom','left','right']:
ax.spines[axis].set_linewidth(1)
ax.tick_params(axis="both", width=1, labelsize=14)
plt.hist(cross_ratio, bins=bins, histtype='step', color='green', linewidth=1, linestyle='solid', density=False)
#x = np.arange(-10, 10, 0.001)
#plot normal distribution with mean 0 and standard deviation 1
#plt.plot(x, norm.pdf(x, 0, 1), color='red', linewidth=2)
plt.xlabel('eUDS and 4XMM-DR12 flux ratio',fontsize=14, fontweight='normal')
plt.ylabel('Relative fraction',fontsize=14, fontweight='normal')
plt.grid(visible=True)
plt.xscale('linear')
plt.yscale('linear')
plt.savefig("../products/cross-match_ratio.png", bbox_inches='tight')
plt.close(fig)
if(do_print_ecf==True):
filename='../data/ECF/ecf_tbabspow_g2nh0.02.pkl'
with open(filename, 'rb') as f:
ecf_table = pickle.load(f)
"""
for key in table.keys():
print("{} --> {}".format(key,table[key]))
"""
print(ecf_table[(0.3,2.3)])
print(ecf_table[(0.3,0.6)])
print(ecf_table[(0.6,2.3)])
print(ecf_table[(2.3,5.0)])
print(ecf_table[(5.0,8.0)])
print()
print(ecf_table[(0.5,1.0)]) # 4XMM-DR12 EP2 band
print(ecf_table[(1.0,2.0)]) # 4XMM-DR12 EP3 band
if(do_euds_dr12_stat==True):
dr12_stat(infile='../products/eUDS_4XMM-DR12.fits',
xmmslim='../data/4XMM-DR12/4XMM_DR12cat_slim_v1.0_UDS.fits.catalog',
xmmlim=None,
outfile_reg='../products/dr12_fluxlim.reg')

44
scripts/README.md Normal file
View File

@@ -0,0 +1,44 @@
Подготовка рабочего окружения:
```
source <MY PATH>/venv/bin/activate.csh
source <MY PATH>/eSASS4EDR/bin/esass-init.csh
```
### 01_init_events.py
Создает начальные списки событий и помещает их в ```uds/data/processed```
Оригинальные файлы со списками событий задаются в файлах ```uds/data/evtlists/*.txt```
### 02_merge_events.py
Создает объедененный список событий и помещает его в ```uds/products```. Этот список событий нужен, в основном для извлечения спектров с помощью ```srctool```.
Попутно этот скрипт унифицирует оригинальные списки событий для последующей обработки. А именно, корректируются слова OBS_MODE=POINING/SURVEY в зависимости от типа наблюдения и производится центрирование на одни и те же координаты с помощью команды ```radec2xy```.
Для запуска адаптивного сглаживания ```do_adapt = True``` требуется запустить окружение ```ciao```, так как нужна команда ```dmimgadapt```
```
conda activate ciao-4.15
source <MY PATH>/eSASS4EDR/bin/esass-init.csh
```
### 03_init_obs.py
1) Подготавливает списки событий в разных энергетических диапазонах.
2) Запускает ```erbox``` в три этапа, чтобы получить рабочий список источников для ```ermldet```.
3) Запускает ```ermldet```
4) Делает кросс-корреляцию с каталогом Gaia-unWISE ```do_cross_match=True```, которая создает три файла: ```.cross``` -- все пересечения, и ```.ref``` / ```.src``` -- входные каталоги для последующей команды ```wcs_match```
5) Делает матрицу преобразования координат и корректирует списки событий. Для запуска команд```wcs_match/wcs_update``` требуется запустить окружение ```ciao``` (см. выше)
### 04_mosaics.py
Создает сборные изображения (мозайки) в разных энергетических диапазонах.
### 05_scrtool.py
Запускает scrtool для самого широкого канала 0.2-10 кэВ, чтобы спектры имели самое полное покрытие по энергиям. Список источников берется из 0.3-2.3 кэВ.
Вычисляет ECF для всех диапазонов.
Делает принудительную фотометрию в выбранных каналах (параметр```forced=True```). Внимание! ermldet из eSASS4EDR не делает ассимитричные ошибки на потоки. Мы запускаем более последнюю версию ermldet (v1.56/2.18 esass_200412 Jul 2 12:04:46 2022). Для этого используется параметр ```local_run=True```, который высвечивает какую команду надо запустить на другой машине и ждет ввода.

55
scripts/print_radec_err.py Normal file
View File

@@ -0,0 +1,55 @@
import glob, sys
from astropy.io import fits
import statistics
import pandas as pd
import numpy as np
import csv
colnames=['Match', 'Ref', 'Dup', 'RA', 'Dec', 'Incl']
flist=glob.glob('../products/tm[1,5,6,7]*en0*src.fits')
total1=0
total2=0
for f in flist:
hdul = fits.open(f)
try:
table=hdul[1].data['RADEC_ERR']
except:
continue
hdul.close()
dref=f.replace("src.fits","ref.fits")
hdul = fits.open(dref)
try:
rtable=hdul[1].data['RA']
except:
continue
hdul.close()
err=[]
err10=[]
dfile=f.replace("shu2019.src.fits","xfm.log.dat.awk")
with open(dfile) as csvfile:
spamreader = csv.reader(csvfile, delimiter=' ')
total=0
for row in spamreader:
if(row[5] == 'N'):
continue
for ii,values in np.ndenumerate(rtable):
if(int(row[2]) == int(ii[0])):
total=total+1
#print(">>",row[5],rtable[ii],table[ii],ii[0])
if(table[ii]>0.0):
err.append(table[ii])
if(table[ii]>10.0):
err10.append(table[ii])
total1=total1+total
total2=total2+len(err)
#print("{} MIN {:.2f} MAX {:.2f} ({}/{})".format(f[12:15],min(err),max(err),len(err10),len(err)))
print("{} MIN {:.2f} MAX {:.2f} ({}/{})".format(f,min(err),max(err),len(err10),len(err)))
#sys.exit()
#print(total1,total2)

67
scripts/print_transform.py Normal file
View File

@@ -0,0 +1,67 @@
import glob, sys
from astropy.io import fits
import statistics
import pandas as pd
import numpy as np
import csv
from uds.config import *
colnames=['Match', 'Ref', 'Dup', 'RA', 'Dec', 'Incl']
flist=glob.glob('../products/tm[1,5,6,7]*en0*.xfm')
total1=0
total2=0
for f in flist:
pos = f.find("_SourceCatalog")
key=f[12:pos]
hdul = fits.open(f)
try:
tab=hdul[1].data
except:
continue
hdul.close()
dsrc=f.replace(".xfm",".shu2019.src.fits")
hdul = fits.open(dsrc)
try:
hdr=hdul[0].header
cdelt=hdr['CDELT2']*3600
except:
continue
hdul.close()
a11=tab[0]['a11']
a12=tab[0]['a12']
a22=tab[0]['a22']
a21=tab[0]['a21']
x_scale=tab[0]['x_scale']
y_scale=tab[0]['y_scale']
t1=tab[0]['t1']*cdelt
t2=tab[0]['t2']*cdelt
sx1=np.sign(a11)*np.sqrt(a11**2 + a12**2)
sy1=np.sign(a22)*np.sqrt(a21**2 + a22**2)
sx2=np.sign(a11)*np.sqrt(a11**2 + a21**2)
sy2=np.sign(a22)*np.sqrt(a12**2 + a22**2)
a1 = np.arctan2(a12/sx1,a11/sy2) * 180 / np.pi * 60
a2 = np.arctan2(a12/sx1,a22/sy2) * 180 / np.pi * 60
b1 = np.arctan2(a21,a11) * 180 / np.pi * 60
b2 = np.arctan2(a21,a22) * 180 / np.pi * 60
det=a11*a22-a12*a21
alpha=355*np.pi/180
a3 = np.arctan2(np.sin(alpha),np.cos(alpha)) * 180 / np.pi
print("{:+.8} {:+.8}".format(a11,a12))
print("{:+.8} {:+.8}".format(a21,a22))
#print("{} d {:.10f} s {:.8f} {:.8f} {:.8f} {:.8f} | {:+.8f} {:+.8f} {:+.8f} {:+.8f} t {:+.2f} {:+.2f}".format(obslist[key],np.sqrt(det)*cdelt,sx1*cdelt,sx2*cdelt,sy1*cdelt,sy2*cdelt,a1,a2,b1,b2,t1,t2))
#print("{} d {:.10f} s {:.8f} {:.8f} {:.8f} {:.8f} | {:+.8f} {:+.8f} {:+.8f} {:+.8f} t {:+.2f} {:+.2f}".format(obslist[key],np.sqrt(det)*cdelt,sx1*cdelt,sx2*cdelt,sy1*cdelt,sy2*cdelt,a1,a2,b1,b2,t1,t2))