diff --git a/.gitignore b/.gitignore index 5edc052..97f9e0f 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ venv.bak/ *.fits *.fits.gz *.evt -*.evt.gz \ No newline at end of file +*.evt.gz +*.lock \ No newline at end of file diff --git a/README.md b/README.md index 29f9d67..651ee01 100644 --- a/README.md +++ b/README.md @@ -31,5 +31,7 @@ pip install --editable uds/ После работы можете удалить проект: ``` pip uninstall uds ``` +## Работа с данными + +Непосредственная работа с данными происходит в директории scripts, где нужно последовательно запускать скрипты обработки. В данной директории находится подробное описание всех действий. -Работа начинается со списка событий, расположенных в директории \ No newline at end of file diff --git a/data/evtlists/tm1.txt b/data/evtlists/tm1.txt new file mode 100644 index 0000000..05cf3c4 --- /dev/null +++ b/data/evtlists/tm1.txt @@ -0,0 +1,3 @@ +/srg/work/krivonos/erosita/work/events/cef_43140_1_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43146_1_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43147_1_P003_00.fits.gz diff --git a/data/evtlists/tm5.txt b/data/evtlists/tm5.txt new file mode 100644 index 0000000..cf06a41 --- /dev/null +++ b/data/evtlists/tm5.txt @@ -0,0 +1,9 @@ +/srg/work/krivonos/erosita/work/events/cef_43086_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43087_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43092_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43093_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43097_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43098_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43099_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43103_5_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43104_5_P003_00.fits.gz diff --git a/data/evtlists/tm6.txt b/data/evtlists/tm6.txt new file mode 100644 index 0000000..cc3a3d8 --- /dev/null +++ b/data/evtlists/tm6.txt @@ -0,0 +1,19 @@ +/srg/work/krivonos/erosita/work/events/cef_43050_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43051_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43062_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43063_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43064_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43065_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43066_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43067_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43068_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43069_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43070_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43071_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43072_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43073_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43074_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43098_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43099_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43103_6_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43104_6_P003_00.fits.gz diff --git a/data/evtlists/tm6_img_1.txt b/data/evtlists/tm6_img_1.txt new file mode 100644 index 0000000..4b9340f --- /dev/null +++ b/data/evtlists/tm6_img_1.txt @@ -0,0 +1 @@ +/srg/work/krivonos/erosita/work/events/cef_43068_6_P003_00.fits.gz diff --git a/data/evtlists/tm7.txt b/data/evtlists/tm7.txt new file mode 100644 index 0000000..7d1b7fd --- /dev/null +++ b/data/evtlists/tm7.txt @@ -0,0 +1,6 @@ +#/srg/work/krivonos/erosita/work/events/cef_43122_7_P003_00.fits.gz +#/srg/work/krivonos/erosita/work/events/cef_43123_7_P003_00.fits.gz +#/srg/work/krivonos/erosita/work/events/cef_43127_7_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43129_7_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43133_7_P003_00.fits.gz +/srg/work/krivonos/erosita/work/events/cef_43134_7_P003_00.fits.gz diff --git a/scripts/01_init_events.py b/scripts/01_init_events.py old mode 100644 new mode 100755 index 57828a3..af3c477 --- a/scripts/01_init_events.py +++ b/scripts/01_init_events.py @@ -1,19 +1,71 @@ +#!/usr/bin/env python +"""Создает начальные списки событий и помещает их в uds/data/processed + +Оригинальные файлы со списками событий задаются в файлах uds/data/evtlists/*.txt +""" + +import os +import inspect + +from os.path import dirname + +import uds from uds.config import * from uds.utils import * -""" + +""" find UDS root dir """ +root_path=dirname(dirname(dirname(inspect.getfile(uds)))) +print("UDS root path: {}".format(root_path)) + +evtlists=root_path+'/data/evtlists' +processed=root_path+'/data/processed' + +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm1.txt'), outfile="{}/{}".format(processed,'tm1_obs_1.fits'), gti='621296896. 621304128.') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm5.txt'), outfile="{}/{}".format(processed,'tm5_obs_1.fits'), gti='620606016. 620614848.') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm5.txt'), outfile="{}/{}".format(processed,'tm5_obs_2.fits'), gti='620676992. 620689792.') + +do_evtool_esass(evlist="{}/{}".format(evtlists,'tm7.txt'), outfile="{}/{}".format(processed,'tm7_obs_1.fits'), gti='621043136. 621052416.') +do_evtool_esass(evlist="{}/{}".format(evtlists,'tm7.txt'), outfile="{}/{}".format(processed,'tm7_obs_1.fits'), gti='621110272. 621117952.') + +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_1.fits'), gti='620174080. 620178002.620032') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_1.fits'), gti='620178002.620032 620192246.62720') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_2.fits'), gti='620192448. 620194624.') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_2.fits'), gti='620194666.606144 620208904.673408') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_3.fits'), gti='620209162.670976 620211316.650304') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_3.fits'), gti='620211328. 620225600.') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_4.fits'), gti='620225853.609024 620227974.68832') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_4.fits'), gti='620227904. 620242176.') +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_obs_1.fits'), gti='620242432. 620258368.') + +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_obs_2.fits'), gti='620607424. 620614656.') +#do_badpix_tm6(filename="{}/{}".format(processed,'tm6_obs_2.fits')) +""" creates new file with added _badpix.fits """ + +#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_obs_3.fits'), gti='620676992. 620690368.') +#do_badpix_tm6(filename="{}/{}".format(processed,'tm6_obs_3.fits')) +""" creates new file with added _badpix.fits """ + + +""" original: +evtool eventfiles=@./tm1_img_orig.txt gti='621296896. 621304128.' outfile=./tm1_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes + +evtool eventfiles=@./tm5_img_orig.txt gti='620606016. 620614848.' outfile=./tm5_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes +evtool eventfiles=@./tm5_img_orig.txt gti='620676992. 620689792.' outfile=./tm5_obs_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes + +evtool eventfiles=@./tm7_img_orig.txt gti='621043136. 621052416.' outfile=./tm7_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,4d,4d,0)' +evtool eventfiles=@./tm7_img_orig.txt gti='621110272. 621117952.' outfile=./tm7_obs_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,4d,4d,0)' + evtool eventfiles=@./tm6_img_1.txt gti='620174080. 620178002.620032' outfile=./tm6_park_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes evtool eventfiles=@./tm6_img_scan1.txt gti='620178002.620032 620192246.62720' outfile=./tm6_scan_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes - evtool eventfiles=@./tm6_img_scan2.txt gti='620192448. 620194624.' outfile=./tm6_park_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes evtool eventfiles=@./tm6_img_scan2.txt gti='620194666.606144 620208904.673408' outfile=./tm6_scan_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes - evtool eventfiles=@./tm6_img_scan3.txt gti='620209162.670976 620211316.650304' outfile=./tm6_park_3.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes evtool eventfiles=@./tm6_img_scan3.txt gti='620211328. 620225600.' outfile=./tm6_scan_3.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes - evtool eventfiles=@./tm6_img_scan3.txt gti='620225853.609024 620227974.68832' outfile=./tm6_park_4.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes evtool eventfiles=@./tm6_img_scan3.txt gti='620227904. 620242176.' outfile=./tm6_scan_4.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes -""" +evtool eventfiles=@./tm6_img_2.txt gti='620242432. 620258368.' outfile=./tm6_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes +evtool eventfiles=@./tm6_img_4to5.txt gti='620607424. 620614656.' outfile=./tm6_obs_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes +evtool eventfiles=@./tm6_img_4to5.txt gti='620676992. 620690368.' outfile=./tm6_obs_3.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes -print("WORK {}".format(work_dir)) -printme() +""" diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..1c530d5 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,5 @@ +Создает начальные списки событий и помещает их в uds/data/processed + +Оригинальные файлы со списками событий задаются в файлах uds/data/evtlists/*.txt + +Работа начинается со списка событий, расположенных в директории data diff --git a/uds/uds/utils.py b/uds/uds/utils.py index c09e530..be8fc73 100644 --- a/uds/uds/utils.py +++ b/uds/uds/utils.py @@ -7,6 +7,7 @@ from astropy.wcs import WCS from astropy.io.fits import update from astropy.io.fits import getdata import glob +from fitsio import FITS from astropy.table import QTable, Table, Column from astropy import units as u @@ -19,8 +20,7 @@ from astropy.coordinates import Angle, Latitude, Longitude # Angles import statistics import shutil -def printme(): - print("hellow world FFF ASAA") + def create_folder(folder): if not (os.path.exists(folder)): @@ -30,14 +30,83 @@ def remove_file(filename): if(os.path.isfile(filename)==True): os.remove(filename) -def do_evtool_esass(events,outfile): - cmd=["evtool", - "eventfiles=\'{}\'".format((" ").join(events)), - "outfile=%s" %(outfile), +def do_evtool_esass(events=None,outfile=None,evlist=None,gti=None): + + eventfiles=None + if(events): + eventfiles="eventfiles=\'{}\'".format((" ").join(events)) + if(evlist): + eventfiles="eventfiles=@{}".format(evlist) + + if not (eventfiles): + print("ERROR: Event files not provided") + + gti="gti=\'{}\'".format(gti) if(gti) else '' + + cmd=["evtool", + eventfiles, + gti, + "outfile={}".format(outfile), "image=yes", "flag=0x2000", "pattern=15" ] - # run the command - print((" ").join(cmd)) - os.system((" ").join(cmd)) + + print((" ").join(cmd)) + os.system((" ").join(cmd)) + +def set_bit(value, bit): + return value | (1<250 """ + + nraw=384 + skip_after=250 + nskip=(nraw - skip_after) + nrows0=nraw*nskip + + fout=filename.replace(".fits", "_badpix.fits") + shutil.copyfile(filename, fout) + + hdul = fits.open(fout) + #hdul.info() + tstart=hdul[1].header['TSTART'] + tstop=hdul[1].header['TSTOP'] + nrows=hdul[1].header['NAXIS2'] + + + f = FITS(fout,'rw') + data = f['EVENTS'].read() + + #print(data.dtype) + + for row in range(nrows): + if (data['RAWX'][row] >= skip_after): + data['FLAG'][row]=set_bit(data['FLAG'][row],13) + #hdr = fits[0].read_header() + f[1].write(data) + f[1].write_comment("Changed by Roman Krivonos") + f[1].write_history("All RAWx>=250 marked as BAD 0x2000") + + data_badpix = f['BADPIX6'].read() + data4 = np.zeros(nrows0,dtype=data_badpix.dtype) + + n=0 + for i in range(skip_after,nraw): + for j in range(nraw): + data4['RAWX'][n]=i + data4['RAWY'][n]=j + data4['TIMEMIN'][n]=tstart + data4['TIMEMAX'][n]=tstop + data4['BADFLAG'][n]=3 + data4['TYPE'][n]=3 + data4['PHAMAX'][n]=12000 + data4['PHAMIN'][n]=0 + n=n+1 + + f[4].append(data4) + f.close()