First steps

.gitignore

@@ -47,4 +47,5 @@ venv.bak/
 *.fits
 *.fits.gz
 *.evt
-*.evt.gz
+*.evt.gz
+*.lock

README.md

@@ -31,5 +31,7 @@ pip install --editable uds/
 После работы можете удалить проект:
 ``` pip uninstall uds ```
 
+## Работа с данными
+
+Непосредственная работа с данными происходит в директории scripts, где нужно последовательно запускать скрипты обработки. В данной директории находится подробное описание всех действий.
 
-Работа начинается со списка событий, расположенных в директории 

data/evtlists/tm1.txt

@@ -0,0 +1,3 @@
+/srg/work/krivonos/erosita/work/events/cef_43140_1_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43146_1_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43147_1_P003_00.fits.gz

data/evtlists/tm5.txt

@@ -0,0 +1,9 @@
+/srg/work/krivonos/erosita/work/events/cef_43086_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43087_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43092_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43093_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43097_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43098_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43099_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43103_5_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43104_5_P003_00.fits.gz

data/evtlists/tm6.txt

@@ -0,0 +1,19 @@
+/srg/work/krivonos/erosita/work/events/cef_43050_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43051_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43062_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43063_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43064_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43065_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43066_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43067_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43068_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43069_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43070_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43071_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43072_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43073_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43074_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43098_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43099_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43103_6_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43104_6_P003_00.fits.gz

data/evtlists/tm6_img_1.txt

@@ -0,0 +1 @@
+/srg/work/krivonos/erosita/work/events/cef_43068_6_P003_00.fits.gz

data/evtlists/tm7.txt

@@ -0,0 +1,6 @@
+#/srg/work/krivonos/erosita/work/events/cef_43122_7_P003_00.fits.gz
+#/srg/work/krivonos/erosita/work/events/cef_43123_7_P003_00.fits.gz
+#/srg/work/krivonos/erosita/work/events/cef_43127_7_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43129_7_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43133_7_P003_00.fits.gz
+/srg/work/krivonos/erosita/work/events/cef_43134_7_P003_00.fits.gz

scripts/01_init_events.py

@@ -1,19 +1,71 @@
+#!/usr/bin/env python
+"""Создает начальные списки событий и помещает их в uds/data/processed
+
+Оригинальные файлы со списками событий задаются в файлах uds/data/evtlists/*.txt
+"""
+
+import os
+import inspect
+
+from os.path import dirname
+
+import uds
 from uds.config import *
 from uds.utils import *
 
-"""
+
+""" find UDS root dir """
+root_path=dirname(dirname(dirname(inspect.getfile(uds))))
+print("UDS root path: {}".format(root_path))
+
+evtlists=root_path+'/data/evtlists'
+processed=root_path+'/data/processed'
+
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm1.txt'), outfile="{}/{}".format(processed,'tm1_obs_1.fits'), gti='621296896. 621304128.')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm5.txt'), outfile="{}/{}".format(processed,'tm5_obs_1.fits'), gti='620606016. 620614848.')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm5.txt'), outfile="{}/{}".format(processed,'tm5_obs_2.fits'), gti='620676992. 620689792.')
+
+do_evtool_esass(evlist="{}/{}".format(evtlists,'tm7.txt'), outfile="{}/{}".format(processed,'tm7_obs_1.fits'), gti='621043136. 621052416.')
+do_evtool_esass(evlist="{}/{}".format(evtlists,'tm7.txt'), outfile="{}/{}".format(processed,'tm7_obs_1.fits'), gti='621110272. 621117952.')
+
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_1.fits'), gti='620174080. 620178002.620032')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_1.fits'), gti='620178002.620032 620192246.62720')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_2.fits'), gti='620192448. 620194624.')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_2.fits'), gti='620194666.606144 620208904.673408')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_3.fits'), gti='620209162.670976 620211316.650304')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_3.fits'), gti='620211328. 620225600.')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_park_4.fits'), gti='620225853.609024 620227974.68832')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_scan_4.fits'), gti='620227904. 620242176.')
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_obs_1.fits'),  gti='620242432. 620258368.')
+
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_obs_2.fits'),  gti='620607424. 620614656.')
+#do_badpix_tm6(filename="{}/{}".format(processed,'tm6_obs_2.fits'))
+""" creates new file with added _badpix.fits """
+
+#do_evtool_esass(evlist="{}/{}".format(evtlists,'tm6.txt'), outfile="{}/{}".format(processed,'tm6_obs_3.fits'),  gti='620676992. 620690368.')
+#do_badpix_tm6(filename="{}/{}".format(processed,'tm6_obs_3.fits'))
+""" creates new file with added _badpix.fits """
+
+
+""" original:
+evtool eventfiles=@./tm1_img_orig.txt gti='621296896. 621304128.' outfile=./tm1_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
+
+evtool eventfiles=@./tm5_img_orig.txt gti='620606016. 620614848.' outfile=./tm5_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
+evtool eventfiles=@./tm5_img_orig.txt gti='620676992. 620689792.' outfile=./tm5_obs_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
+
+evtool eventfiles=@./tm7_img_orig.txt gti='621043136. 621052416.' outfile=./tm7_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,4d,4d,0)'
+evtool eventfiles=@./tm7_img_orig.txt gti='621110272. 621117952.' outfile=./tm7_obs_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,4d,4d,0)'
+
 evtool eventfiles=@./tm6_img_1.txt gti='620174080. 620178002.620032' outfile=./tm6_park_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
 evtool eventfiles=@./tm6_img_scan1.txt gti='620178002.620032 620192246.62720' outfile=./tm6_scan_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
-
 evtool eventfiles=@./tm6_img_scan2.txt gti='620192448. 620194624.' outfile=./tm6_park_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
 evtool eventfiles=@./tm6_img_scan2.txt gti='620194666.606144 620208904.673408' outfile=./tm6_scan_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
-
 evtool eventfiles=@./tm6_img_scan3.txt gti='620209162.670976 620211316.650304' outfile=./tm6_park_3.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
 evtool eventfiles=@./tm6_img_scan3.txt  gti='620211328. 620225600.' outfile=./tm6_scan_3.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
-
 evtool eventfiles=@./tm6_img_scan3.txt gti='620225853.609024 620227974.68832' outfile=./tm6_park_4.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
 evtool eventfiles=@./tm6_img_scan3.txt gti='620227904. 620242176.' outfile=./tm6_scan_4.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
-"""
+evtool eventfiles=@./tm6_img_2.txt gti='620242432. 620258368.' outfile=./tm6_obs_1.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
+evtool eventfiles=@./tm6_img_4to5.txt gti='620607424. 620614656.' outfile=./tm6_obs_2.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
+evtool eventfiles=@./tm6_img_4to5.txt gti='620676992. 620690368.' outfile=./tm6_obs_3.fits flag=0x2000 emin=0.2 emax=10.0 region='box(34.5550100,-4.7177600,153.0',153.0',0)' image=yes
 
-print("WORK {}".format(work_dir))
+"""
-printme()

scripts/README.md

@@ -0,0 +1,5 @@
+Создает начальные списки событий и помещает их в uds/data/processed
+
+Оригинальные файлы со списками событий задаются в файлах uds/data/evtlists/*.txt
+
+Работа начинается со списка событий, расположенных в директории data

uds/uds/utils.py

@@ -7,6 +7,7 @@ from astropy.wcs import WCS
 from astropy.io.fits import update
 from astropy.io.fits import getdata
 import glob
+from fitsio import FITS
 
 from astropy.table import QTable, Table, Column
 from astropy import units as u
@@ -19,8 +20,7 @@ from astropy.coordinates import Angle, Latitude, Longitude # Angles
 import statistics
 import shutil
 
-def printme():
+
-    print("hellow world FFF ASAA")
 
 def create_folder(folder):
     if not (os.path.exists(folder)):
@@ -30,14 +30,83 @@ def remove_file(filename):
     if(os.path.isfile(filename)==True):
         os.remove(filename)            
 
-def do_evtool_esass(events,outfile):
+def do_evtool_esass(events=None,outfile=None,evlist=None,gti=None):
-   cmd=["evtool",
+
-        "eventfiles=\'{}\'".format((" ").join(events)),
+    eventfiles=None
-        "outfile=%s" %(outfile),
+    if(events):
+        eventfiles="eventfiles=\'{}\'".format((" ").join(events))
+    if(evlist):
+        eventfiles="eventfiles=@{}".format(evlist)
+
+    if not (eventfiles):
+        print("ERROR: Event files not provided")
+
+    gti="gti=\'{}\'".format(gti) if(gti) else ''
+
+    cmd=["evtool",
+         eventfiles,
+         gti,
+        "outfile={}".format(outfile),
         "image=yes",
         "flag=0x2000",
         "pattern=15"
         ]
-   # run the command
+    
-   print((" ").join(cmd))
+    print((" ").join(cmd))
-   os.system((" ").join(cmd))
+    os.system((" ").join(cmd))
+
+def set_bit(value, bit):
+    return value | (1<<bit)
+
+def clear_bit(value, bit):
+    return value & ~(1<<bit)
+
+def do_badpix_tm6(filename):
+    """ special corrrection for TM6 camera, needed for two CalPV-UDS observations, which shows high background at RAWX>250 """
+
+    nraw=384
+    skip_after=250
+    nskip=(nraw - skip_after)
+    nrows0=nraw*nskip
+    
+    fout=filename.replace(".fits", "_badpix.fits")
+    shutil.copyfile(filename, fout)
+
+    hdul = fits.open(fout)
+    #hdul.info()
+    tstart=hdul[1].header['TSTART']
+    tstop=hdul[1].header['TSTOP']
+    nrows=hdul[1].header['NAXIS2']
+
+
+    f = FITS(fout,'rw')
+    data = f['EVENTS'].read()
+
+    #print(data.dtype)
+
+    for row in range(nrows):
+        if (data['RAWX'][row] >= skip_after):
+            data['FLAG'][row]=set_bit(data['FLAG'][row],13)
+    #hdr = fits[0].read_header()
+    f[1].write(data)
+    f[1].write_comment("Changed by Roman Krivonos")
+    f[1].write_history("All RAWx>=250 marked as BAD 0x2000")
+
+    data_badpix = f['BADPIX6'].read()
+    data4 = np.zeros(nrows0,dtype=data_badpix.dtype)
+
+    n=0
+    for i in range(skip_after,nraw):
+        for j in range(nraw):
+            data4['RAWX'][n]=i
+            data4['RAWY'][n]=j
+            data4['TIMEMIN'][n]=tstart
+            data4['TIMEMAX'][n]=tstop
+            data4['BADFLAG'][n]=3
+            data4['TYPE'][n]=3
+            data4['PHAMAX'][n]=12000
+            data4['PHAMIN'][n]=0        
+            n=n+1
+        
+    f[4].append(data4)
+    f.close()