integral/ridge
1
0
Fork 0
generated from erosita/uds
Code Issues Pull Requests Packages Projects Releases Wiki Activity

+++

Browse Source
This commit is contained in:
Roman Krivonos
2024-07-07 16:07:39 +03:00
parent 2552bdb552
commit d7ddc5dc43
31 changed files with 13798 additions and 56949 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Spectra/LON-20.sim.spec
+21 -21
View File
@@ -1,21 +1,21 @@
0 25.00 27.50 80.829894 12.245058 0.0 0 25.00 27.50 97.594215 19.152866 0.0
0 27.50 30.25 76.644942 10.424311 0.0 0 27.50 30.25 88.572954 17.572310 0.0
0 30.25 33.28 76.502631 11.634459 0.0 0 30.25 33.28 98.698707 18.434761 0.0
0 33.28 36.60 68.303048 10.264405 0.0 0 33.28 36.60 76.185527 16.795524 0.0
0 36.60 40.26 47.306287 12.222633 0.0 0 36.60 40.26 74.456883 18.952182 0.0
0 40.26 44.29 50.277235 13.161144 0.0 0 40.26 44.29 68.498175 20.414274 0.0
0 44.29 48.72 38.157378 14.334599 0.0 0 44.29 48.72 60.938981 23.037130 0.0
0 48.72 53.59 38.182359 16.922607 0.0 0 48.72 53.59 78.093265 24.753731 0.0
0 53.59 58.95 32.176187 23.454953 0.0 0 53.59 58.95 62.780993 35.373011 0.0
0 58.95 64.84 9.085719 26.909627 0.0 0 58.95 64.84 54.981761 45.622070 0.0
0 64.84 71.33 4.184346 29.146344 0.0 0 64.84 71.33 31.131458 45.463755 0.0
0 71.33 78.46 -9.500519 31.479429 0.0 0 71.33 78.46 21.238334 52.216610 0.0
0 78.46 86.31 -8.236339 34.082221 0.0 0 78.46 86.31 15.370207 56.318480 0.0
0 86.31 94.94 -29.519105 36.423630 0.0 0 86.31 94.94 33.395021 60.671498 0.0
0 94.94 104.43 -29.638303 36.196842 0.0 0 94.94 104.43 33.371988 65.827159 0.0
0 104.43 114.87 -58.969197 47.170262 0.0 0 104.43 114.87 8.761509 74.581412 0.0
0 114.87 126.36 -95.503579 62.312335 0.0 0 114.87 126.36 34.274489 102.125880 0.0
0 126.36 139.00 -128.008918 86.404246 0.0 0 126.36 139.00 145.426485 137.177118 0.0
0 139.00 152.90 -254.865359 106.787744 0.0 0 139.00 152.90 -31.141499 180.263798 0.0
0 152.90 168.19 -248.310328 132.790433 0.0 0 152.90 168.19 90.530509 223.724154 0.0
0 168.19 185.01 -262.343876 146.080123 0.0 0 168.19 185.01 -16.157665 241.171948 0.0
Spectra/LON-20.spec
+21 -21
View File
@@ -1,21 +1,21 @@
0 25.00 27.50 80.868838 3.629462 0.0 0 25.00 27.50 97.893621 6.006746 0.0
0 27.50 30.25 75.581563 3.298105 0.0 0 27.50 30.25 89.649217 5.476124 0.0
0 30.25 33.28 75.798053 3.563746 0.0 0 30.25 33.28 96.877164 5.834810 0.0
0 33.28 36.60 69.671013 3.269331 0.0 0 33.28 36.60 76.973632 5.520131 0.0
0 36.60 40.26 48.079290 3.686636 0.0 0 36.60 40.26 74.815301 5.998109 0.0
0 40.26 44.29 48.567741 3.956909 0.0 0 40.26 44.29 69.366524 6.371047 0.0
0 44.29 48.72 38.679751 4.547690 0.0 0 44.29 48.72 62.652641 7.497996 0.0
0 48.72 53.59 40.698258 4.980663 0.0 0 48.72 53.59 77.111748 8.100871 0.0
0 53.59 58.95 34.223263 7.023514 0.0 0 53.59 58.95 61.348888 11.408661 0.0
0 58.95 64.84 8.573069 8.490589 0.0 0 58.95 64.84 56.159316 14.202528 0.0
0 64.84 71.33 6.375353 8.631849 0.0 0 64.84 71.33 34.170958 14.626111 0.0
0 71.33 78.46 -7.298427 9.896588 0.0 0 71.33 78.46 20.940958 15.934871 0.0
0 78.46 86.31 -5.188647 10.760612 0.0 0 78.46 86.31 12.647710 18.433726 0.0
0 86.31 94.94 -27.892689 11.373849 0.0 0 86.31 94.94 22.093487 18.552664 0.0
0 94.94 104.43 -28.106104 12.048314 0.0 0 94.94 104.43 34.076339 20.646834 0.0
0 104.43 114.87 -55.165521 14.936937 0.0 0 104.43 114.87 6.561830 24.903652 0.0
0 114.87 126.36 -98.708217 19.653155 0.0 0 114.87 126.36 36.303914 32.104944 0.0
0 126.36 139.00 -128.152693 26.863584 0.0 0 126.36 139.00 158.224658 44.397384 0.0
0 139.00 152.90 -260.097479 34.058972 0.0 0 139.00 152.90 -18.224589 59.476750 0.0
0 152.90 168.19 -247.225449 42.062017 0.0 0 152.90 168.19 85.614695 71.931770 0.0
0 168.19 185.01 -263.723359 45.821742 0.0 0 168.19 185.01 -2.233538 80.500729 0.0
Spectra/fits/LON-20.B01.fits
+710 -2784
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B02.fits
+584 -2798
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B03.fits
+628 -2652
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B04.fits
+666 -2746
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B05.fits
+571 -2482
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B06.fits
+538 -2573
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B07.fits
+710 -2929
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B08.fits
+665 -2441
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B09.fits
+524 -2450
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B10.fits
+576 -2526
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B11.fits
+616 -2681
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B12.fits
+591 -2560
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B13.fits
+720 -2816
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B14.fits
+580 -2483
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B15.fits
+791 -2620
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B16.fits
+766 -2764
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B17.fits
+770 -2628
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B18.fits
+686 -2868
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B19.fits
+706 -3028
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B20.fits
+554 -2741
View File
File diff suppressed because one or more lines are too long
Spectra/fits/LON-20.B21.fits
+658 -3241
View File
File diff suppressed because one or more lines are too long
ridge/ridge/utils.py
+4 -2
View File
@@ -136,8 +136,10 @@ def get_syserror(fn):
#fn="detcnts.{}.{}.resid.fits".format(enkey,key) #fn="detcnts.{}.{}.resid.fits".format(enkey,key)
d = fits.getdata(fn) #d = fits.getdata(fn)
df=pd.DataFrame(np.array(d).byteswap().newbyteorder()) #df=pd.DataFrame(np.array(d).byteswap().newbyteorder())
with fits.open(fn) as data:
df = pd.DataFrame(data[1].data)
#print(fn) #print(fn)
#print(df.columns) #print(df.columns)
scripts/01_bgdmodel.py
+5 -2
View File
@@ -30,9 +30,12 @@ enkey = sys.argv[1]
fn="detcnts.{}.fits".format(enkey) fn="detcnts.{}.fits".format(enkey)
print("Reading {}".format(datadir+fn)) print("Reading {}".format(datadir+fn))
d = fits.getdata(datadir+fn) #d = fits.getdata(datadir+fn)
df=pd.DataFrame(np.array(d).byteswap().newbyteorder()) #df=pd.DataFrame(np.array(d).byteswap().newbyteorder())
with fits.open(datadir+fn) as data:
df = pd.DataFrame(data[1].data)
print(df.columns) print(df.columns)
#interp_a = np.interp(50, [0,100], [0, 1]) #interp_a = np.interp(50, [0,100], [0, 1])
scripts/01_bgdmodel.sh
+26 -26
View File
@@ -1,30 +1,30 @@
./01_bgdmodel.py E01 #./01_bgdmodel.py E01
./01_bgdmodel.py E02 #./01_bgdmodel.py E02
./01_bgdmodel.py E03 #./01_bgdmodel.py E03
./01_bgdmodel.py E04 #./01_bgdmodel.py E04
./01_bgdmodel.py E05 #./01_bgdmodel.py E05
./01_bgdmodel.py E06 #./01_bgdmodel.py E06
./01_bgdmodel.py E07 #./01_bgdmodel.py E07
./01_bgdmodel.py E08 #./01_bgdmodel.py E08
./01_bgdmodel.py E09 #./01_bgdmodel.py E09
./01_bgdmodel.py E10 #./01_bgdmodel.py E10
./01_bgdmodel.py E11 #./01_bgdmodel.py E11
./01_bgdmodel.py E12 #./01_bgdmodel.py E12
./01_bgdmodel.py E13 #./01_bgdmodel.py E13
./01_bgdmodel.py E14 #./01_bgdmodel.py E14
./01_bgdmodel.py E15 #./01_bgdmodel.py E15
./01_bgdmodel.py A01 #./01_bgdmodel.py A01
./01_bgdmodel.py A02 #./01_bgdmodel.py A02
./01_bgdmodel.py A03 #./01_bgdmodel.py A03
./01_bgdmodel.py A04 #./01_bgdmodel.py A04
./01_bgdmodel.py A05 #./01_bgdmodel.py A05
./01_bgdmodel.py A06 #./01_bgdmodel.py A06
./01_bgdmodel.py A07 #./01_bgdmodel.py A07
./01_bgdmodel.py A08 #./01_bgdmodel.py A08
./01_bgdmodel.py A09 #./01_bgdmodel.py A09
./01_bgdmodel.py A10 #./01_bgdmodel.py A10
./01_bgdmodel.py A11 #./01_bgdmodel.py A11
./01_bgdmodel.py B01 ./01_bgdmodel.py B01
./01_bgdmodel.py B02 ./01_bgdmodel.py B02
scripts/01_crabmodel.py
+6 -3
View File
@@ -63,10 +63,13 @@ fn="detcnts.{}.fits".format(enkey)
with open(proddir+fn.replace(".fits",".ignored_scw.pkl"), 'rb') as fp: with open(proddir+fn.replace(".fits",".ignored_scw.pkl"), 'rb') as fp:
ignored_scw = pickle.load(fp) ignored_scw = pickle.load(fp)
d = fits.getdata(datadir+fn) #d = fits.getdata(datadir+fn)
df=pd.DataFrame(np.array(d).byteswap().newbyteorder()) #df=pd.DataFrame(np.array(d).byteswap().newbyteorder())
#print(df.columns) #print(df.columns)
with fits.open(datadir+fn) as data:
df = pd.DataFrame(data[1].data)
crab_crd = SkyCoord(crab_ra, crab_dec, frame=FK5(), unit="deg") crab_crd = SkyCoord(crab_ra, crab_dec, frame=FK5(), unit="deg")
plotme=False plotme=False
@@ -94,7 +97,7 @@ totx=[]
toty=[] toty=[]
for i,rec in df.iterrows(): for i,rec in df.iterrows():
obsid = rec['OBSID'].decode("utf-8") obsid = rec['OBSID']#.decode("utf-8")
if(obsid in ignore_scws): if(obsid in ignore_scws):
print("Remove {}".format(obsid)) print("Remove {}".format(obsid))
df.drop(index=i, inplace=True) df.drop(index=i, inplace=True)
scripts/01_crabmodel.sh
+26 -26
View File
@@ -1,30 +1,30 @@
./01_crabmodel.py E01 #./01_crabmodel.py E01
./01_crabmodel.py E02 #./01_crabmodel.py E02
./01_crabmodel.py E03 #./01_crabmodel.py E03
./01_crabmodel.py E04 #./01_crabmodel.py E04
./01_crabmodel.py E05 #./01_crabmodel.py E05
./01_crabmodel.py E06 #./01_crabmodel.py E06
./01_crabmodel.py E07 #./01_crabmodel.py E07
./01_crabmodel.py E08 #./01_crabmodel.py E08
./01_crabmodel.py E09 #./01_crabmodel.py E09
./01_crabmodel.py E10 #./01_crabmodel.py E10
./01_crabmodel.py E11 #./01_crabmodel.py E11
./01_crabmodel.py E12 #./01_crabmodel.py E12
./01_crabmodel.py E13 #./01_crabmodel.py E13
./01_crabmodel.py E14 #./01_crabmodel.py E14
./01_crabmodel.py E15 #./01_crabmodel.py E15
./01_crabmodel.py A01 #./01_crabmodel.py A01
./01_crabmodel.py A02 #./01_crabmodel.py A02
./01_crabmodel.py A03 #./01_crabmodel.py A03
./01_crabmodel.py A04 #./01_crabmodel.py A04
./01_crabmodel.py A05 #./01_crabmodel.py A05
./01_crabmodel.py A06 #./01_crabmodel.py A06
./01_crabmodel.py A07 #./01_crabmodel.py A07
./01_crabmodel.py A08 #./01_crabmodel.py A08
./01_crabmodel.py A09 #./01_crabmodel.py A09
./01_crabmodel.py A10 #./01_crabmodel.py A10
./01_crabmodel.py A11 #./01_crabmodel.py A11
./01_crabmodel.py B01 ./01_crabmodel.py B01
./01_crabmodel.py B02 ./01_crabmodel.py B02
scripts/02_grxe_resid.py
+31 -9
View File
@@ -26,10 +26,10 @@ from numpy import arange
from ridge.utils import * from ridge.utils import *
from ridge.config import * from ridge.config import *
plotme=False plotme=True
enkey = sys.argv[1] enkey = sys.argv[1]
outkey = sys.argv[2] #outkey = sys.argv[2]
#outkey = "ALL" outkey = "ALL"
fn="detcnts.{}.fits".format(enkey) fn="detcnts.{}.fits".format(enkey)
@@ -83,8 +83,11 @@ base0=[]
c0=[] c0=[]
texp0=[] texp0=[]
d = fits.getdata(datadir+fn) #d = fits.getdata(datadir+fn)
df = pd.DataFrame(np.array(d).byteswap().newbyteorder()) #df = pd.DataFrame(np.array(d).byteswap().newbyteorder())
with fits.open(datadir+fn) as data:
df = pd.DataFrame(data[1].data)
# BKG # BKG
if(outkey == 'BKG'): if(outkey == 'BKG'):
@@ -100,7 +103,7 @@ if(outkey=='ALL'):
for i, row in df.iterrows(): for i, row in df.iterrows():
orbit=row['REV'] orbit=row['REV']
obsid=row['OBSID'].decode("UTF-8") obsid=row['OBSID']#.decode("UTF-8")
if not (orbit > revmin and orbit < revmax): if not (orbit > revmin and orbit < revmax):
print("Skip orbit",orbit,row['OBSID']) print("Skip orbit",orbit,row['OBSID'])
@@ -170,7 +173,6 @@ for r in range(rev_min,rev_max):
ind = np.nonzero(orbits == r) ind = np.nonzero(orbits == r)
if not (len(resid_arr[ind])>10): if not (len(resid_arr[ind])>10):
continue continue
#print(r,np.mean(resid_arr[ind]), len(resid_arr[ind]))
distr_val.append(np.mean(resid_arr[ind])) distr_val.append(np.mean(resid_arr[ind]))
distr_rev.append(r) distr_rev.append(r)
@@ -183,13 +185,14 @@ filtered_data = sigma_clip(dval, sigma=sigma, maxiters=10, return_bounds=True)
filtered_arr=filtered_data[0] filtered_arr=filtered_data[0]
filtered_min=filtered_data[1] filtered_min=filtered_data[1]
filtered_max=filtered_data[2] filtered_max=filtered_data[2]
print("Sigma clipping: min {:.2e} max {:.2e}".format(filtered_min,filtered_max))
print("length orig: {} taken: {} filtered: {}".format(len(dval),len(dval[filtered_arr.mask==False]),len(dval[filtered_arr.mask==True]))) print("length orig: {} taken: {} filtered: {}".format(len(dval),len(dval[filtered_arr.mask==False]),len(dval[filtered_arr.mask==True])))
sg_mean, sg_med, sg_std = sigma_clipped_stats(distr_val, sigma=sigma, maxiters=10) sg_mean, sg_med, sg_std = sigma_clipped_stats(distr_val, sigma=sigma, maxiters=10)
sg_sem = sem(dval[filtered_arr.mask==False]) sg_sem = sem(dval[filtered_arr.mask==False])
print("Sigma clipping: mean {:.2f} med {:.2f} std {:.2f} ".format(sg_mean, sg_med, sg_std)) print("Sigma clipping: mean {:.2e} med {:.2e} std {:.2e} ".format(sg_mean, sg_med, sg_std))
print(len(drev[filtered_arr.mask==True])) print(len(drev[filtered_arr.mask==True]))
@@ -208,7 +211,7 @@ if(plotme):
with open(proddir+fn.replace(".fits",".ignored_rev.resid.pkl"), 'wb') as fp: with open(proddir+fn.replace(".fits",".ignored_rev.resid.pkl"), 'wb') as fp:
pickle.dump(drev[filtered_arr.mask==True], fp, protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(drev[filtered_arr.mask==True], fp, protocol=pickle.HIGHEST_PROTOCOL)
print("Removed REVs:",ignored_rev) print("Removed REVs:",drev[filtered_arr.mask==True])
indices = sorted( indices = sorted(
@@ -256,3 +259,22 @@ with fits.open(proddir+fout, mode='update') as hdus:
coldefs = fits.ColDefs([
fits.Column(name='REV', format='J', unit='', array=drev),
fits.Column(name='RESID', format='D', unit='cts/s', array=dval),
fits.Column(name='MASK', format='L', unit='', array=filtered_arr.mask),
])
fout = fn.replace(".fits",".{}.resid_by_rev.fits".format(outkey))
hdu = fits.BinTableHDU.from_columns(coldefs, name='GRXE')
hdu.header['MISSION'] = ('INTEGRAL', '')
hdu.header['TELESCOP'] = (outkey, '')
hdu.header['INSTITUT'] = ('IKI', 'Affiliation')
hdu.header['AUTHOR'] = ('Roman Krivonos', 'Responsible person')
hdu.header['EMAIL'] = ('krivonos@cosmos.ru', 'E-mail')
#hdu.add_checksum()
print(hdu.columns)
hdu.writeto(proddir+fout, overwrite=True)
with fits.open(proddir+fout, mode='update') as hdus:
hdus[1].add_checksum()
scripts/02_grxe_resid.sh
+21 -21
View File
@@ -48,24 +48,24 @@
#./02_grxe_resid.py B20 ALL #./02_grxe_resid.py B20 ALL
#./02_grxe_resid.py B21 ALL #./02_grxe_resid.py B21 ALL
./02_grxe_resid.py B01 BKG ./02_grxe_resid.py B01
./02_grxe_resid.py B02 BKG ./02_grxe_resid.py B02
./02_grxe_resid.py B03 BKG ./02_grxe_resid.py B03
./02_grxe_resid.py B04 BKG ./02_grxe_resid.py B04
./02_grxe_resid.py B05 BKG ./02_grxe_resid.py B05
./02_grxe_resid.py B06 BKG ./02_grxe_resid.py B06
./02_grxe_resid.py B07 BKG ./02_grxe_resid.py B07
./02_grxe_resid.py B08 BKG ./02_grxe_resid.py B08
./02_grxe_resid.py B09 BKG ./02_grxe_resid.py B09
./02_grxe_resid.py B10 BKG ./02_grxe_resid.py B10
./02_grxe_resid.py B11 BKG ./02_grxe_resid.py B11
./02_grxe_resid.py B12 BKG ./02_grxe_resid.py B12
./02_grxe_resid.py B13 BKG ./02_grxe_resid.py B13
./02_grxe_resid.py B14 BKG ./02_grxe_resid.py B14
./02_grxe_resid.py B15 BKG ./02_grxe_resid.py B15
./02_grxe_resid.py B16 BKG ./02_grxe_resid.py B16
./02_grxe_resid.py B17 BKG ./02_grxe_resid.py B17
./02_grxe_resid.py B18 BKG ./02_grxe_resid.py B18
./02_grxe_resid.py B19 BKG ./02_grxe_resid.py B19
./02_grxe_resid.py B20 BKG ./02_grxe_resid.py B20
./02_grxe_resid.py B21 BKG ./02_grxe_resid.py B21
scripts/03_grxe_spec.py
+27 -7
View File
@@ -139,7 +139,15 @@ with open(proddir+'detcnts.B21.ignored_rev.resid.pkl', 'rb') as fp:
print(ignored_rev) print(ignored_rev)
ign=ignored_rev.tolist() ign=ignored_rev.tolist()
"""
if(1091 in ign):
print("Removed")
else:
print("Taken")
sys.exit() sys.exit()
"""
nsim=1000 nsim=1000
for skey in skeys: for skey in skeys:
@@ -147,23 +155,35 @@ for skey in skeys:
print("{} not found in {}".format(skey,list(skyreg.keys()))) print("{} not found in {}".format(skey,list(skyreg.keys())))
sys.exit() sys.exit()
for enkey in ebands0.keys(): for enkey in ebands0.keys():
bkg_fn="detcnts.{}.BKG.resid.fits".format(enkey,inkey) #bkg_fn="detcnts.{}.BKG.resid.fits".format(enkey,inkey)
syserr, bkg_sem = get_syserror(proddir+bkg_fn) #syserr, bkg_sem = get_syserror(proddir+bkg_fn)
fn="detcnts.{}.{}.resid.fits".format(enkey,inkey) fn="detcnts.{}.{}.resid.fits".format(enkey,inkey)
d = fits.getdata(proddir+fn) #d1 = fits.getdata(proddir+fn)
df=pd.DataFrame(np.array(d).byteswap().newbyteorder()) #d2=np.array(d1)
#print(df.columns) #df=pd.DataFrame(d2.view(d2.dtype.newbyteorder()))
#df=pd.DataFrame(np.array(d).byteswap().newbyteorder())
#with fits.open(proddir+fn) as data:
# df = pd.DataFrame(data[1].data)
dat = Table.read(proddir+fn)
df = dat.to_pandas()
print(df.columns)
#sys.exit()
#df = df.query("REV == @ign") #df = df.query("REV == @ign")
df = df.query("LON > {} & LON < {} & LAT > {} & LAT < {} & REV != @ign".format( query = "LON > {} & LON < {} & LAT > {} & LAT < {} & REV != @ign".format(
skyreg[skey]['lon'] - skyreg[skey]['wlon']/2, skyreg[skey]['lon'] - skyreg[skey]['wlon']/2,
skyreg[skey]['lon'] + skyreg[skey]['wlon']/2, skyreg[skey]['lon'] + skyreg[skey]['wlon']/2,
skyreg[skey]['lat'] - skyreg[skey]['wlat']/2, skyreg[skey]['lat'] - skyreg[skey]['wlat']/2,
skyreg[skey]['lat'] + skyreg[skey]['wlat']/2) skyreg[skey]['lat'] + skyreg[skey]['wlat']/2)
)
print(query)
#sys.exit()
df = df.query(query)
t = Table.from_pandas(df) t = Table.from_pandas(df)