generated from erosita/uds
maps
This commit is contained in:
parent
1401747f7d
commit
bbdf6a3420
BIN
data/modelrxte_ait_3to20keV_flux_1deg.fits
Normal file
BIN
data/modelrxte_ait_3to20keV_flux_1deg.fits
Normal file
Binary file not shown.
BIN
data/modelrxte_ait_3to20keV_flux_2deg.fits
Normal file
BIN
data/modelrxte_ait_3to20keV_flux_2deg.fits
Normal file
Binary file not shown.
@ -5,6 +5,7 @@
|
|||||||
"""
|
"""
|
||||||
datadir="../data/"
|
datadir="../data/"
|
||||||
proddir="../products/"
|
proddir="../products/"
|
||||||
|
mapsdir="../products/maps/"
|
||||||
|
|
||||||
""" Определение областей "Галактика" и "Внегалактика" """
|
""" Определение областей "Галактика" и "Внегалактика" """
|
||||||
bmax=30.0
|
bmax=30.0
|
||||||
@ -39,3 +40,6 @@ SEM означает standatd error on mean (~RMS/sqrt(n))
|
|||||||
"""
|
"""
|
||||||
sem_cut=98
|
sem_cut=98
|
||||||
|
|
||||||
|
""" Координаты Краба """
|
||||||
|
crab_ra = 83.6287
|
||||||
|
crab_dec = 22.014
|
||||||
|
|||||||
@ -63,7 +63,7 @@ def plot_best_fit(X, y, model):
|
|||||||
y_pred = model.predict(X)
|
y_pred = model.predict(X)
|
||||||
resid = y - y_pred
|
resid = y - y_pred
|
||||||
err = np.sqrt(np.sum((resid)**2))/len(resid)
|
err = np.sqrt(np.sum((resid)**2))/len(resid)
|
||||||
print("ax+b: a={:.2e}, b={:.2e}, std={:.2f}".format(a,b,err))
|
print("ax+b: a={:.2e}, b={:.2e}, std={:.2e}".format(a,b,err))
|
||||||
"""
|
"""
|
||||||
plt.scatter(X, y)
|
plt.scatter(X, y)
|
||||||
xaxis = arange(X.min(), X.max(), 0.01)
|
xaxis = arange(X.min(), X.max(), 0.01)
|
||||||
|
|||||||
@ -7,7 +7,7 @@ import numpy as np
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
from astropy.io import fits
|
from astropy.io import fits
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import math, sys
|
import math, sys, os
|
||||||
import pickle
|
import pickle
|
||||||
|
|
||||||
from sklearn.linear_model import LinearRegression
|
from sklearn.linear_model import LinearRegression
|
||||||
@ -42,6 +42,9 @@ nrev=0
|
|||||||
bgdmodel={}
|
bgdmodel={}
|
||||||
ignored_scw=[]
|
ignored_scw=[]
|
||||||
|
|
||||||
|
if not os.path.exists(proddir):
|
||||||
|
os.makedirs(proddir)
|
||||||
|
|
||||||
for rev in range(revmin,revmax):
|
for rev in range(revmin,revmax):
|
||||||
|
|
||||||
# if not (rev==341):
|
# if not (rev==341):
|
||||||
|
|||||||
307
scripts/01_crabmodel.py
Executable file
307
scripts/01_crabmodel.py
Executable file
@ -0,0 +1,307 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
__author__ = "Roman Krivonos"
|
||||||
|
__copyright__ = "Space Research Institute (IKI)"
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
from astropy.io import fits
|
||||||
|
from astropy import wcs
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
import math, sys, os
|
||||||
|
import pickle
|
||||||
|
from numpy.polynomial import Polynomial
|
||||||
|
|
||||||
|
from sklearn.linear_model import LinearRegression
|
||||||
|
from sklearn.linear_model import HuberRegressor
|
||||||
|
from sklearn.linear_model import RANSACRegressor
|
||||||
|
from sklearn.linear_model import TheilSenRegressor
|
||||||
|
from sklearn.model_selection import cross_val_score
|
||||||
|
from sklearn.model_selection import RepeatedKFold
|
||||||
|
|
||||||
|
|
||||||
|
from astropy.coordinates import SkyCoord # High-level coordinates
|
||||||
|
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
|
||||||
|
from astropy.coordinates import Angle, Latitude, Longitude # Angles
|
||||||
|
import astropy.units as u
|
||||||
|
|
||||||
|
#from statsmodels.robust.scale import huber
|
||||||
|
from astropy.stats import sigma_clip
|
||||||
|
from numpy import absolute
|
||||||
|
from numpy import arange
|
||||||
|
|
||||||
|
from ridge.utils import *
|
||||||
|
from ridge.config import *
|
||||||
|
|
||||||
|
if not os.path.exists(proddir):
|
||||||
|
os.makedirs(proddir)
|
||||||
|
|
||||||
|
enkey = sys.argv[1]
|
||||||
|
|
||||||
|
sigma=3
|
||||||
|
|
||||||
|
# some static revs/scws to be removed
|
||||||
|
ignore_orbits=[352,834,912,1019,1021,1028,2275,2405,2493]
|
||||||
|
ignore_scws=['066600420020','066600420030','132800350010','090200390010','269500190010']
|
||||||
|
|
||||||
|
fn="detcnts.{}.fits".format(enkey)
|
||||||
|
|
||||||
|
with open(proddir+fn.replace(".fits",".ignored_scw.pkl"), 'rb') as fp:
|
||||||
|
ignored_scw = pickle.load(fp)
|
||||||
|
|
||||||
|
d = fits.getdata(datadir+fn)
|
||||||
|
df=pd.DataFrame(np.array(d).byteswap().newbyteorder())
|
||||||
|
#print(df.columns)
|
||||||
|
|
||||||
|
crab_crd = SkyCoord(crab_ra, crab_dec, frame=FK5(), unit="deg")
|
||||||
|
|
||||||
|
plotme=False
|
||||||
|
|
||||||
|
npix = 50
|
||||||
|
sw = 30.0 # deg
|
||||||
|
pix = sw/npix # pixel size in degrees
|
||||||
|
|
||||||
|
|
||||||
|
mean_map = [[0.0 for i in range(npix)] for j in range(npix)]
|
||||||
|
count_map = [[0 for i in range(npix)] for j in range(npix)]
|
||||||
|
|
||||||
|
|
||||||
|
crabmodel={}
|
||||||
|
rota_arr=[]
|
||||||
|
a0=[]
|
||||||
|
b0=[]
|
||||||
|
a_full0=[]
|
||||||
|
b_full0=[]
|
||||||
|
b_est0=[]
|
||||||
|
err0=[]
|
||||||
|
rev0=[]
|
||||||
|
|
||||||
|
totx=[]
|
||||||
|
toty=[]
|
||||||
|
|
||||||
|
for i,rec in df.iterrows():
|
||||||
|
obsid = rec['OBSID'].decode("utf-8")
|
||||||
|
if(obsid in ignore_scws):
|
||||||
|
print("Remove {}".format(obsid))
|
||||||
|
df.drop(index=i, inplace=True)
|
||||||
|
|
||||||
|
if (obsid in ignored_scw):
|
||||||
|
print("Skip ScW",obsid)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# accumulate full data set
|
||||||
|
for rev in range(revmin,revmax):
|
||||||
|
if(rev in ignore_orbits):
|
||||||
|
continue
|
||||||
|
if(enkey == 'E02'):
|
||||||
|
if(rev > 800):
|
||||||
|
continue
|
||||||
|
if(enkey == 'E03'):
|
||||||
|
if(rev > 800):
|
||||||
|
continue
|
||||||
|
#if(rev > 1000):
|
||||||
|
# continue
|
||||||
|
df0 = df.query('SRC > 0.0 & REV == {} & PHASE > {} & PHASE < {} & CRAB_SEP < {}'.format(rev,phmin,phmax,crab_sep_max))
|
||||||
|
nobs=len(df0)
|
||||||
|
if not (nobs> crab_nmax):
|
||||||
|
continue
|
||||||
|
for n in df0['CRAB_SEP'].values:
|
||||||
|
totx.append(n)
|
||||||
|
for n in df0['SRC'].values:
|
||||||
|
toty.append(n)
|
||||||
|
#if(np.min(df0['SRC'])<1e-3):
|
||||||
|
# print(rev)
|
||||||
|
# sys.exit()
|
||||||
|
|
||||||
|
|
||||||
|
x = np.array(totx)
|
||||||
|
y = np.array(toty)
|
||||||
|
x = x.reshape((-1, 1))
|
||||||
|
model = LinearRegression()
|
||||||
|
#model = TheilSenRegressor()
|
||||||
|
results = evaluate_model(x, y, model)
|
||||||
|
a_full,b_full,err_full = plot_best_fit(x, y, model)
|
||||||
|
if(plotme):
|
||||||
|
plot_ab(x, y, a_full, b_full, err_full, title="REGRESSION")
|
||||||
|
|
||||||
|
# go over orbits
|
||||||
|
poly_x=[]
|
||||||
|
poly_y=[]
|
||||||
|
|
||||||
|
ntotal=0
|
||||||
|
nrev=0
|
||||||
|
for rev in range(revmin,revmax):
|
||||||
|
if(rev in ignore_orbits):
|
||||||
|
continue
|
||||||
|
df0 = df.query('SRC > 0.0 & REV == {} & PHASE > {} & PHASE < {} & CRAB_SEP < {}'.format(rev,phmin,phmax,crab_sep_max))
|
||||||
|
nobs=len(df0)
|
||||||
|
if not (nobs> crab_nmax):
|
||||||
|
continue
|
||||||
|
cen_ra = np.array(df0['RA'].values)
|
||||||
|
cen_dec = np.array(df0['DEC'].values)
|
||||||
|
print("*** Orbit ",rev)
|
||||||
|
x = np.array(df0['CRAB_SEP'].values)
|
||||||
|
y = np.array(df0['SRC'].values)
|
||||||
|
rota_deg = np.array(df0['ROTA'].values)
|
||||||
|
rota = np.array(df0['ROTA'].values) * np.pi / 180. # in radians
|
||||||
|
detx = np.cos(rota)*x
|
||||||
|
dety = np.sin(rota)*x
|
||||||
|
|
||||||
|
for i in range(rota_deg.shape[0]):
|
||||||
|
rota_arr.append(rota_deg[i])
|
||||||
|
|
||||||
|
w = wcs.WCS(naxis=2)
|
||||||
|
w.wcs.crpix = [npix/2, npix/2]
|
||||||
|
w.wcs.cdelt = np.array([-pix, pix])
|
||||||
|
w.wcs.crota = [rota_deg[i], rota_deg[i]]
|
||||||
|
w.wcs.crval = [cen_ra[i], cen_dec[i]]
|
||||||
|
w.wcs.ctype = ["RA---TAN", "DEC--TAN"]
|
||||||
|
#w.wcs.set_pv([(2, 1, 45.0)])
|
||||||
|
|
||||||
|
#header = w.to_header()
|
||||||
|
#hdu = fits.PrimaryHDU(header=header)
|
||||||
|
|
||||||
|
|
||||||
|
pixcrd = w.wcs_world2pix([(crab_ra,crab_dec)], 1)
|
||||||
|
ix=int(np.round(pixcrd[0][0]))
|
||||||
|
iy=int(np.round(pixcrd[0][1]))
|
||||||
|
#print(ix,iy)
|
||||||
|
mean_map[ix][iy] += y[i]
|
||||||
|
count_map[ix][iy] += 1
|
||||||
|
|
||||||
|
x = x.reshape((-1, 1))
|
||||||
|
model = LinearRegression()
|
||||||
|
#model = TheilSenRegressor()
|
||||||
|
results = evaluate_model(x, y, model)
|
||||||
|
a,b,err = plot_best_fit(x, y, model)
|
||||||
|
|
||||||
|
b_est = np.mean(y - a_full*x)
|
||||||
|
|
||||||
|
if(enkey in ['E10','E11','E12',] or a > 0.0):
|
||||||
|
filtered_data = sigma_clip(y, sigma=sigma, maxiters=10, return_bounds=True)
|
||||||
|
filtered_y = filtered_data[0]
|
||||||
|
filtered_min = filtered_data[1]
|
||||||
|
filtered_max = filtered_data[2]
|
||||||
|
b = np.mean(filtered_y)
|
||||||
|
a = 0.0
|
||||||
|
err = np.sqrt(np.sum((b-filtered_y)**2))/len(filtered_y)
|
||||||
|
|
||||||
|
b_est = b
|
||||||
|
a_full = a
|
||||||
|
|
||||||
|
a_full0.append(a_full)
|
||||||
|
b_full0.append(b_full)
|
||||||
|
b_est0.append(b_est)
|
||||||
|
|
||||||
|
#if(b>2.0e-4):
|
||||||
|
# print(rev)
|
||||||
|
# sys.exit()
|
||||||
|
|
||||||
|
a0.append(a)
|
||||||
|
b0.append(b)
|
||||||
|
err0.append(err)
|
||||||
|
rev0.append(rev)
|
||||||
|
|
||||||
|
|
||||||
|
poly_x.append(rev)
|
||||||
|
poly_y.append(b_est)
|
||||||
|
|
||||||
|
if(enkey in ['E04','E05','E06','E07','E08','E09','E10','E11','E12']):
|
||||||
|
crabmodel[rev]={'a':a_full, 'b':b_est, 'err':err}
|
||||||
|
if(plotme):
|
||||||
|
plot_ab(x, y, a_full, b_est, err, title="REGRESSION rev {}".format(rev))
|
||||||
|
|
||||||
|
if(enkey in ['E02', 'E03']):
|
||||||
|
crabmodel[rev]={'a':a, 'b':b, 'err':err}
|
||||||
|
if(plotme):
|
||||||
|
plot_ab(x, y, a, b, err, title="REGRESSION rev {}".format(rev))
|
||||||
|
|
||||||
|
print("ax+b: a={:.2e}, b={:.2e}, std={:.2e}\n".format(a,b,err))
|
||||||
|
ntotal=ntotal+nobs
|
||||||
|
nrev=nrev+1
|
||||||
|
|
||||||
|
print("Orbits: {}, obs: {}".format(nrev,ntotal))
|
||||||
|
|
||||||
|
for i in range(npix):
|
||||||
|
for j in range(npix):
|
||||||
|
if(count_map[i][j]>0):
|
||||||
|
mean_map[i][j]=mean_map[i][j]/count_map[i][j]
|
||||||
|
|
||||||
|
w = wcs.WCS(naxis=2)
|
||||||
|
w.wcs.crpix = [npix/2, npix/2]
|
||||||
|
w.wcs.cdelt = np.array([-pix, pix])
|
||||||
|
w.wcs.crval = [0.0, 0.0]
|
||||||
|
w.wcs.ctype = ["RA---TAN", "DEC--TAN"]
|
||||||
|
|
||||||
|
header = w.to_header()
|
||||||
|
hdu = fits.PrimaryHDU(header=header)
|
||||||
|
|
||||||
|
hdu.data=mean_map
|
||||||
|
hdu.writeto(proddir+fn.replace(".fits",".crab_mean_map.fits"), overwrite=True)
|
||||||
|
hdu.data=count_map
|
||||||
|
hdu.writeto(proddir+fn.replace(".fits",".crab_count_map.fits"), overwrite=True)
|
||||||
|
|
||||||
|
npoly=4
|
||||||
|
if(enkey in ['E11','E12',]):
|
||||||
|
npoly=0
|
||||||
|
|
||||||
|
z = np.polyfit(poly_x, poly_y, npoly)
|
||||||
|
|
||||||
|
p = np.poly1d(z)
|
||||||
|
poly_z=[]
|
||||||
|
for t in poly_x:
|
||||||
|
poly_z.append(p(t))
|
||||||
|
plt.scatter(poly_x, poly_y)
|
||||||
|
plt.plot(poly_x, poly_z, color='r')
|
||||||
|
plt.title("Crab detector count rate evolution")
|
||||||
|
plt.ylabel("Crab count rate cts/s/pix")
|
||||||
|
plt.xlabel("INTEGRAL orbit")
|
||||||
|
#plt.show()
|
||||||
|
plt.savefig(proddir+fn.replace(".fits",".crab_rate.png"))
|
||||||
|
|
||||||
|
indices = sorted(
|
||||||
|
range(len(a0)),
|
||||||
|
key=lambda index: a0[index]
|
||||||
|
)
|
||||||
|
|
||||||
|
coldefs = fits.ColDefs([
|
||||||
|
#fits.Column(name='OBSID', format='11A', array=[obs_id[index] for index in indices]),
|
||||||
|
#fits.Column(name='RA', format='D', unit='deg', array=[ra[index] for index in indices]),
|
||||||
|
#fits.Column(name='DEC', format='D', unit='deg', array=[dec[index] for index in indices]),
|
||||||
|
#fits.Column(name='LON', format='D', unit='deg', array=[lon0[index] for index in indices]),
|
||||||
|
#fits.Column(name='LAT', format='D', unit='deg', array=[lat0[index] for index in indices]),
|
||||||
|
fits.Column(name='REV', format='J', unit='', array=[rev0[index] for index in indices]),
|
||||||
|
#fits.Column(name='MJD', format='D', unit='', array=[mjd0[index] for index in indices]),
|
||||||
|
#fits.Column(name='PHASE', format='D', unit='', array=[phase0[index] for index in indices]),
|
||||||
|
#fits.Column(name='CLEAN', format='D', unit='cts/s', array=[clean0[index] for index in indices]),
|
||||||
|
#fits.Column(name='MODEL', format='D', unit='cts/s', array=[model0[index] for index in indices]),
|
||||||
|
#fits.Column(name='RESID', format='D', unit='cts/s', array=[resid0[index] for index in indices]),
|
||||||
|
fits.Column(name='A', format='D', unit='', array=[a0[index] for index in indices]),
|
||||||
|
fits.Column(name='B', format='D', unit='', array=[b0[index] for index in indices]),
|
||||||
|
fits.Column(name='ERR', format='D', unit='', array=[err0[index] for index in indices]),
|
||||||
|
fits.Column(name='A_FULL', format='D', unit='', array=[a_full0[index] for index in indices]),
|
||||||
|
fits.Column(name='B_FULL', format='D', unit='', array=[b_full0[index] for index in indices]),
|
||||||
|
fits.Column(name='B_EST', format='D', unit='', array=[b_est0[index] for index in indices]),
|
||||||
|
fits.Column(name='B_POLY', format='D', unit='', array=[poly_z[index] for index in indices]),
|
||||||
|
])
|
||||||
|
|
||||||
|
fout = fn.replace(".fits",".crabmodel.fits")
|
||||||
|
hdu = fits.BinTableHDU.from_columns(coldefs, name='GRXE')
|
||||||
|
hdu.header['MISSION'] = ('INTEGRAL', '')
|
||||||
|
hdu.header['TELESCOP'] = ('IBIS', '')
|
||||||
|
hdu.header['INSTITUT'] = ('IKI', 'Affiliation')
|
||||||
|
hdu.header['AUTHOR'] = ('Roman Krivonos', 'Responsible person')
|
||||||
|
hdu.header['EMAIL'] = ('krivonos@cosmos.ru', 'E-mail')
|
||||||
|
#hdu.add_checksum()
|
||||||
|
print(hdu.columns)
|
||||||
|
hdu.writeto(proddir+fout, overwrite=True)
|
||||||
|
|
||||||
|
|
||||||
|
with open(proddir+fn.replace(".fits",".crabmodel.pkl"), 'wb') as fp:
|
||||||
|
pickle.dump([crabmodel, z], fp, protocol=pickle.HIGHEST_PROTOCOL)
|
||||||
|
|
||||||
|
|
||||||
|
with open(proddir+fn.replace(".fits",".rota.dat"), 'w') as fp:
|
||||||
|
for i in range(len(rota_arr)):
|
||||||
|
fp.write("{} {}\n".format(i,rota_arr[i]))
|
||||||
|
|
||||||
11
scripts/01_crabmodel.sh
Executable file
11
scripts/01_crabmodel.sh
Executable file
@ -0,0 +1,11 @@
|
|||||||
|
./01_crabmodel.py E02
|
||||||
|
./01_crabmodel.py E03
|
||||||
|
./01_crabmodel.py E04
|
||||||
|
./01_crabmodel.py E05
|
||||||
|
./01_crabmodel.py E06
|
||||||
|
./01_crabmodel.py E07
|
||||||
|
./01_crabmodel.py E08
|
||||||
|
./01_crabmodel.py E09
|
||||||
|
./01_crabmodel.py E10
|
||||||
|
./01_crabmodel.py E11
|
||||||
|
./01_crabmodel.py E12
|
||||||
164
scripts/02_grxe_resid.py
Executable file
164
scripts/02_grxe_resid.py
Executable file
@ -0,0 +1,164 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
__author__ = "Roman Krivonos"
|
||||||
|
__copyright__ = "Space Research Institute (IKI)"
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
from astropy.io import fits
|
||||||
|
from astropy.table import Table, Column
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
import math, sys
|
||||||
|
import pickle
|
||||||
|
|
||||||
|
from sklearn.linear_model import LinearRegression
|
||||||
|
from sklearn.linear_model import HuberRegressor
|
||||||
|
from sklearn.linear_model import RANSACRegressor
|
||||||
|
from sklearn.linear_model import TheilSenRegressor
|
||||||
|
from sklearn.model_selection import cross_val_score
|
||||||
|
from sklearn.model_selection import RepeatedKFold
|
||||||
|
|
||||||
|
#from statsmodels.robust.scale import huber
|
||||||
|
from astropy.stats import sigma_clip
|
||||||
|
from numpy import absolute
|
||||||
|
from numpy import arange
|
||||||
|
|
||||||
|
from ridge.utils import *
|
||||||
|
from ridge.config import *
|
||||||
|
|
||||||
|
enkey = sys.argv[1]
|
||||||
|
outkey = sys.argv[2]
|
||||||
|
|
||||||
|
fn="detcnts.{}.fits".format(enkey)
|
||||||
|
|
||||||
|
with open(proddir+fn.replace(".fits",".pkl"), 'rb') as fp:
|
||||||
|
bgdmodel = pickle.load(fp)
|
||||||
|
|
||||||
|
with open(proddir+fn.replace(".fits",".ignored_scw.pkl"), 'rb') as fp:
|
||||||
|
ignored_scw = pickle.load(fp)
|
||||||
|
|
||||||
|
with open(proddir+fn.replace(".fits",".crabmodel.pkl"), 'rb') as fp:
|
||||||
|
crabmodel, z = pickle.load(fp)
|
||||||
|
p = np.poly1d(z)
|
||||||
|
#print(crabmodel)
|
||||||
|
|
||||||
|
crab_rev_max = np.max(list(crabmodel.keys()))
|
||||||
|
print("Crab is defined untill orbit {}".format(crab_rev_max))
|
||||||
|
|
||||||
|
with fits.open(datadir+fn) as hdul:
|
||||||
|
data=hdul[1].data
|
||||||
|
|
||||||
|
#print(data.columns)
|
||||||
|
|
||||||
|
rev = data.field('rev')
|
||||||
|
mjd = data.field('mjd')
|
||||||
|
clean = data.field('clean')
|
||||||
|
phase = data.field('phase')
|
||||||
|
|
||||||
|
|
||||||
|
rev0=[]
|
||||||
|
phase0=[]
|
||||||
|
clean0=[]
|
||||||
|
model0=[]
|
||||||
|
resid0=[] # residuals in cts/s
|
||||||
|
grxe0=[] # mCrab
|
||||||
|
crab0=[] # Crab count rate
|
||||||
|
mjd0=[]
|
||||||
|
a0=[]
|
||||||
|
b0=[]
|
||||||
|
err0=[]
|
||||||
|
lon0=[]
|
||||||
|
lat0=[]
|
||||||
|
|
||||||
|
d = fits.getdata(datadir+fn)
|
||||||
|
df = pd.DataFrame(np.array(d).byteswap().newbyteorder())
|
||||||
|
|
||||||
|
# BKG
|
||||||
|
if(outkey == 'BKG'):
|
||||||
|
df = df.query('CLEAN > 0.0 & ( abs(LAT) > {} | abs(LON) > {}) & PHASE > {} & PHASE < {}'.format(bmax,lmax,phmin,phmax))
|
||||||
|
|
||||||
|
# GAL
|
||||||
|
if(outkey=='GAL'):
|
||||||
|
df = df.query('CLEAN > 0.0 & abs(LAT) < {} & abs(LON) < {} & PHASE > {} & PHASE < {}'.format(bmax,lmax,phmin,phmax))
|
||||||
|
|
||||||
|
# ALL
|
||||||
|
if(outkey=='ALL'):
|
||||||
|
df = df.query('CLEAN > 0.0 & PHASE > {} & PHASE < {}'.format(phmin,phmax))
|
||||||
|
|
||||||
|
for i, row in df.iterrows():
|
||||||
|
orbit=row['REV']
|
||||||
|
obsid=row['OBSID'].decode("UTF-8")
|
||||||
|
|
||||||
|
if not (orbit > revmin and orbit < revmax):
|
||||||
|
print("Skip orbit",orbit,row['OBSID'])
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not (orbit < crab_rev_max):
|
||||||
|
print("Skip orbit",orbit,row['OBSID'])
|
||||||
|
continue
|
||||||
|
|
||||||
|
if (obsid in ignored_scw):
|
||||||
|
print("Skip ScW",obsid)
|
||||||
|
continue
|
||||||
|
|
||||||
|
a = bgdmodel[orbit]['a']
|
||||||
|
b = bgdmodel[orbit]['b']
|
||||||
|
err = bgdmodel[orbit]['err']
|
||||||
|
m = a*row['PHASE']+b
|
||||||
|
|
||||||
|
clean0.append(clean[i])
|
||||||
|
mjd0.append(mjd[i])
|
||||||
|
model0.append(m)
|
||||||
|
resid0.append(clean[i]-m)
|
||||||
|
grxe0.append(1000*(clean[i]-m)/p(orbit))
|
||||||
|
crab0.append(p(orbit))
|
||||||
|
|
||||||
|
a0.append(a)
|
||||||
|
b0.append(b)
|
||||||
|
err0.append(err)
|
||||||
|
phase0.append(row['PHASE'])
|
||||||
|
rev0.append(orbit)
|
||||||
|
lon0.append(row['LON'])
|
||||||
|
lat0.append(row['LAT'])
|
||||||
|
|
||||||
|
|
||||||
|
indices = sorted(
|
||||||
|
range(len(mjd0)),
|
||||||
|
key=lambda index: mjd0[index]
|
||||||
|
)
|
||||||
|
|
||||||
|
coldefs = fits.ColDefs([
|
||||||
|
#fits.Column(name='OBSID', format='11A', array=[obs_id[index] for index in indices]),
|
||||||
|
#fits.Column(name='RA', format='D', unit='deg', array=[ra[index] for index in indices]),
|
||||||
|
#fits.Column(name='DEC', format='D', unit='deg', array=[dec[index] for index in indices]),
|
||||||
|
fits.Column(name='LON', format='D', unit='deg', array=[lon0[index] for index in indices]),
|
||||||
|
fits.Column(name='LAT', format='D', unit='deg', array=[lat0[index] for index in indices]),
|
||||||
|
fits.Column(name='REV', format='J', unit='', array=[rev0[index] for index in indices]),
|
||||||
|
fits.Column(name='MJD', format='D', unit='', array=[mjd0[index] for index in indices]),
|
||||||
|
fits.Column(name='PHASE', format='D', unit='', array=[phase0[index] for index in indices]),
|
||||||
|
fits.Column(name='CLEAN', format='D', unit='cts/s', array=[clean0[index] for index in indices]),
|
||||||
|
fits.Column(name='MODEL', format='D', unit='cts/s', array=[model0[index] for index in indices]),
|
||||||
|
fits.Column(name='RESID', format='D', unit='cts/s', array=[resid0[index] for index in indices]),
|
||||||
|
fits.Column(name='GRXE', format='D', unit='mCrab', array=[grxe0[index] for index in indices]),
|
||||||
|
fits.Column(name='CRAB', format='D', unit='cts/s', array=[crab0[index] for index in indices]),
|
||||||
|
fits.Column(name='A', format='D', unit='', array=[a0[index] for index in indices]),
|
||||||
|
fits.Column(name='B', format='D', unit='', array=[b0[index] for index in indices]),
|
||||||
|
fits.Column(name='ERR', format='D', unit='', array=[err0[index] for index in indices]),
|
||||||
|
])
|
||||||
|
|
||||||
|
fout = fn.replace(".fits",".{}.resid.fits".format(outkey))
|
||||||
|
hdu = fits.BinTableHDU.from_columns(coldefs, name='GRXE')
|
||||||
|
hdu.header['MISSION'] = ('INTEGRAL', '')
|
||||||
|
hdu.header['TELESCOP'] = (outkey, '')
|
||||||
|
hdu.header['INSTITUT'] = ('IKI', 'Affiliation')
|
||||||
|
hdu.header['AUTHOR'] = ('Roman Krivonos', 'Responsible person')
|
||||||
|
hdu.header['EMAIL'] = ('krivonos@cosmos.ru', 'E-mail')
|
||||||
|
#hdu.add_checksum()
|
||||||
|
print(hdu.columns)
|
||||||
|
hdu.writeto(proddir+fout, overwrite=True)
|
||||||
|
|
||||||
|
with fits.open(proddir+fout, mode='update') as hdus:
|
||||||
|
hdus[1].add_checksum()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
35
scripts/02_grxe_resid.sh
Executable file
35
scripts/02_grxe_resid.sh
Executable file
@ -0,0 +1,35 @@
|
|||||||
|
./02_grxe_resid.py E02 GAL
|
||||||
|
./02_grxe_resid.py E03 GAL
|
||||||
|
./02_grxe_resid.py E04 GAL
|
||||||
|
./02_grxe_resid.py E05 GAL
|
||||||
|
./02_grxe_resid.py E06 GAL
|
||||||
|
./02_grxe_resid.py E07 GAL
|
||||||
|
./02_grxe_resid.py E08 GAL
|
||||||
|
./02_grxe_resid.py E09 GAL
|
||||||
|
./02_grxe_resid.py E10 GAL
|
||||||
|
./02_grxe_resid.py E11 GAL
|
||||||
|
./02_grxe_resid.py E12 GAL
|
||||||
|
|
||||||
|
./02_grxe_resid.py E02 BKG
|
||||||
|
./02_grxe_resid.py E03 BKG
|
||||||
|
./02_grxe_resid.py E04 BKG
|
||||||
|
./02_grxe_resid.py E05 BKG
|
||||||
|
./02_grxe_resid.py E06 BKG
|
||||||
|
./02_grxe_resid.py E07 BKG
|
||||||
|
./02_grxe_resid.py E08 BKG
|
||||||
|
./02_grxe_resid.py E09 BKG
|
||||||
|
./02_grxe_resid.py E10 BKG
|
||||||
|
./02_grxe_resid.py E11 BKG
|
||||||
|
./02_grxe_resid.py E12 BKG
|
||||||
|
|
||||||
|
./02_grxe_resid.py E02 ALL
|
||||||
|
./02_grxe_resid.py E03 ALL
|
||||||
|
./02_grxe_resid.py E04 ALL
|
||||||
|
./02_grxe_resid.py E05 ALL
|
||||||
|
./02_grxe_resid.py E06 ALL
|
||||||
|
./02_grxe_resid.py E07 ALL
|
||||||
|
./02_grxe_resid.py E08 ALL
|
||||||
|
./02_grxe_resid.py E09 ALL
|
||||||
|
./02_grxe_resid.py E10 ALL
|
||||||
|
./02_grxe_resid.py E11 ALL
|
||||||
|
./02_grxe_resid.py E12 ALL
|
||||||
155
scripts/03_grxe_map.py
Executable file
155
scripts/03_grxe_map.py
Executable file
@ -0,0 +1,155 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
__author__ = "Roman Krivonos"
|
||||||
|
__copyright__ = "Space Research Institute (IKI)"
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import numpy.ma as ma
|
||||||
|
import pandas as pd
|
||||||
|
from astropy.wcs import WCS
|
||||||
|
from astropy import wcs
|
||||||
|
from astropy.io import fits
|
||||||
|
from astropy.table import Table, Column
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
import math, sys, os
|
||||||
|
import pickle
|
||||||
|
|
||||||
|
from astropy.coordinates import SkyCoord # High-level coordinates
|
||||||
|
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
|
||||||
|
from astropy.coordinates import Angle, Latitude, Longitude # Angles
|
||||||
|
import astropy.units as u
|
||||||
|
|
||||||
|
from sklearn.linear_model import LinearRegression
|
||||||
|
from sklearn.linear_model import HuberRegressor
|
||||||
|
from sklearn.linear_model import RANSACRegressor
|
||||||
|
from sklearn.linear_model import TheilSenRegressor
|
||||||
|
from sklearn.model_selection import cross_val_score
|
||||||
|
from sklearn.model_selection import RepeatedKFold
|
||||||
|
|
||||||
|
from astropy.stats import sigma_clip
|
||||||
|
from astropy.stats import sigma_clipped_stats
|
||||||
|
|
||||||
|
from scipy.stats import norm
|
||||||
|
from scipy.stats import describe
|
||||||
|
from scipy.stats import sem
|
||||||
|
|
||||||
|
|
||||||
|
from numpy import absolute
|
||||||
|
from numpy import arange
|
||||||
|
|
||||||
|
from ridge.utils import *
|
||||||
|
from ridge.config import *
|
||||||
|
|
||||||
|
enkey = sys.argv[1]
|
||||||
|
key = sys.argv[2]
|
||||||
|
#key="ALL"
|
||||||
|
fn='detcnts.{}.{}.resid.fits'.format(enkey,key)
|
||||||
|
|
||||||
|
d = fits.getdata(proddir+fn)
|
||||||
|
df=pd.DataFrame(np.array(d).byteswap().newbyteorder())
|
||||||
|
#print(df.columns)
|
||||||
|
|
||||||
|
#df = df.query('abs(LAT) < {} & abs(LON) < {}'.format(5,5))
|
||||||
|
|
||||||
|
n_bins = 80
|
||||||
|
minmax=[-300,800]
|
||||||
|
sigma=3
|
||||||
|
maxiters=10
|
||||||
|
|
||||||
|
modelrxte="modelrxte_ait_3to20keV_flux_2deg.fits"
|
||||||
|
hdulist = fits.open(datadir+modelrxte)
|
||||||
|
w = wcs.WCS(hdulist[0].header)
|
||||||
|
smap =hdulist[0].data
|
||||||
|
|
||||||
|
sx=int(hdulist[0].header['NAXIS1'])
|
||||||
|
sy=int(hdulist[0].header['NAXIS2'])
|
||||||
|
|
||||||
|
# fill AITOF map indexes
|
||||||
|
ds9x=[]
|
||||||
|
ds9y=[]
|
||||||
|
for i,row in df.iterrows():
|
||||||
|
lon=row['LON']
|
||||||
|
lat=row['LAT']
|
||||||
|
world = SkyCoord(lon,lat, frame=Galactic, unit="deg")
|
||||||
|
ra=world.fk5.ra.deg
|
||||||
|
dec=world.fk5.dec.deg
|
||||||
|
pixcrd = w.wcs_world2pix([(lon,lat)], 1)
|
||||||
|
y=int(pixcrd[0][0])
|
||||||
|
x=int(pixcrd[0][1])
|
||||||
|
ds9x.append(x)
|
||||||
|
ds9y.append(y)
|
||||||
|
#print(x,y,smap[y-1,x-1])
|
||||||
|
df['DS9Y']=ds9x
|
||||||
|
df['DS9X']=ds9y
|
||||||
|
|
||||||
|
mean_map = np.array([[0.0 for i in range(sx)] for j in range(sy)])
|
||||||
|
sign_map = np.array([[0.0 for i in range(sx)] for j in range(sy)])
|
||||||
|
sem_map = np.array([[0.0 for i in range(sx)] for j in range(sy)])
|
||||||
|
cnt_map = np.array([[0 for i in range(sx)] for j in range(sy)])
|
||||||
|
|
||||||
|
for i in range(sx):
|
||||||
|
for j in range(sy):
|
||||||
|
world = w.wcs_pix2world([(i+1,j+1)], 1)
|
||||||
|
lon = world[0][0]
|
||||||
|
lat = world[0][1]
|
||||||
|
if(np.isnan(lon) or np.isnan(lat)):
|
||||||
|
continue
|
||||||
|
ds9i=i+1
|
||||||
|
ds9j=j+1
|
||||||
|
df0 = df.query('DS9X == {} & DS9Y == {}'.format(ds9i,ds9j))
|
||||||
|
if (len(df0) <= nscw_min):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# check coordinates
|
||||||
|
#print("***",i+1,j+1,lon,lat,smap[j][i])
|
||||||
|
#for i0,row0 in df0.iterrows():
|
||||||
|
# print(row0['LON'],row0['LAT'],row0['GRXE'])
|
||||||
|
|
||||||
|
grxe = np.array(df0['GRXE'])
|
||||||
|
|
||||||
|
sg_mean, sg_med, sg_std = sigma_clipped_stats(grxe, sigma=sigma, maxiters=maxiters)
|
||||||
|
filtered_data = sigma_clip(grxe, sigma=sigma, maxiters=maxiters, return_bounds=True)
|
||||||
|
filtered_grxe = filtered_data[0]
|
||||||
|
filtered_min = filtered_data[1]
|
||||||
|
filtered_max = filtered_data[2]
|
||||||
|
|
||||||
|
# final error on flux measurement ~RMS/sqrt(n)
|
||||||
|
sg_sem = sem(grxe[filtered_grxe.mask==False])
|
||||||
|
|
||||||
|
#print("Sigma clipping: mean {:.2f} med {:.2f} std {:.2f} sem {:.2f}".format(sg_mean, sg_med, sg_std, sg_sem))
|
||||||
|
|
||||||
|
#plt.hist(grxe, bins=n_bins, range=minmax)
|
||||||
|
#plt.hist(grxe[filtered_grxe.mask], bins=n_bins, range=minmax)
|
||||||
|
#plt.show()
|
||||||
|
|
||||||
|
mean_map[j][i] = sg_mean
|
||||||
|
sem_map[j][i] = sg_sem
|
||||||
|
sign_map[j][i] = sg_mean/sg_sem
|
||||||
|
cnt_map[j][i] = df0.shape[0]
|
||||||
|
|
||||||
|
|
||||||
|
# Calculate the percentiles across the x and y dimension
|
||||||
|
perc = np.percentile(sem_map, sem_cut, axis=(0, 1), keepdims=False)
|
||||||
|
|
||||||
|
print("{}: {}% cut of SEM map: {:.2f} mCrab".format(enkey,sem_cut,perc))
|
||||||
|
|
||||||
|
idx=np.where(sem_map > perc)
|
||||||
|
mean_map[idx]=0.0
|
||||||
|
sem_map[idx]=0.0
|
||||||
|
cnt_map[idx]=0
|
||||||
|
sign_map[idx]=0.0
|
||||||
|
|
||||||
|
if not os.path.exists(mapsdir):
|
||||||
|
os.makedirs(mapsdir)
|
||||||
|
|
||||||
|
hdulist[0].data=mean_map
|
||||||
|
hdulist.writeto(mapsdir+fn.replace(".fits",".mean.fits"),overwrite=True)
|
||||||
|
|
||||||
|
hdulist[0].data=sem_map
|
||||||
|
hdulist.writeto(mapsdir+fn.replace(".fits",".error.fits"),overwrite=True)
|
||||||
|
|
||||||
|
hdulist[0].data=cnt_map
|
||||||
|
hdulist.writeto(mapsdir+fn.replace(".fits",".cnt.fits"),overwrite=True)
|
||||||
|
|
||||||
|
hdulist[0].data=sign_map
|
||||||
|
hdulist.writeto(mapsdir+fn.replace(".fits",".sign.fits"),overwrite=True)
|
||||||
11
scripts/03_grxe_map.sh
Normal file
11
scripts/03_grxe_map.sh
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
./03_grxe_map.py E02 ALL
|
||||||
|
./03_grxe_map.py E03 ALL
|
||||||
|
./03_grxe_map.py E04 ALL
|
||||||
|
./03_grxe_map.py E05 ALL
|
||||||
|
./03_grxe_map.py E06 ALL
|
||||||
|
./03_grxe_map.py E07 ALL
|
||||||
|
./03_grxe_map.py E08 ALL
|
||||||
|
./03_grxe_map.py E09 ALL
|
||||||
|
./03_grxe_map.py E10 ALL
|
||||||
|
./03_grxe_map.py E11 ALL
|
||||||
|
./03_grxe_map.py E12 ALL
|
||||||
@ -3,41 +3,20 @@
|
|||||||
source <MY PATH>/venv/bin/activate.csh
|
source <MY PATH>/venv/bin/activate.csh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Номер скрипта означает последовательность выполнения. Результаты работы скриптов помещаются в ```ridge/products/```.
|
||||||
|
|
||||||
### 01_init_events.py
|
### 01_bgdmodel.py / 01_bgdmodel.sh
|
||||||
|
|
||||||
Создает начальные списки событий и помещает их в ```uds/data/processed```
|
Создает модель фона
|
||||||
Оригинальные файлы со списками событий задаются в файлах ```uds/data/evtlists/*.txt```
|
|
||||||
|
|
||||||
### 02_merge_events.py
|
### 01_crabmodel.py / 01_crabmodel.sh
|
||||||
|
|
||||||
Создает объедененный список событий и помещает его в ```uds/products```. Этот список событий нужен, в основном для извлечения спектров с помощью ```srctool```.
|
Создает модель скорости счета Краба
|
||||||
|
|
||||||
Попутно этот скрипт унифицирует оригинальные списки событий для последующей обработки. А именно, корректируются слова OBS_MODE=POINING/SURVEY в зависимости от типа наблюдения и производится центрирование на одни и те же координаты с помощью команды ```radec2xy```.
|
### 02_grxe_resid.py / 01_grxe_resid.sh
|
||||||
|
|
||||||
Для запуска адаптивного сглаживания ```do_adapt = True``` требуется запустить окружение ```ciao```, так как нужна команда ```dmimgadapt```
|
Вычисляет остатки после вычитания модели из данных в единицах мКраб
|
||||||
|
|
||||||
```
|
### 02_grxe_map.py / 01_grxe_map.sh
|
||||||
conda activate ciao-4.15
|
|
||||||
source <MY PATH>/eSASS4EDR/bin/esass-init.csh
|
|
||||||
```
|
|
||||||
|
|
||||||
### 03_init_obs.py
|
Создает карту остатков в единицах мКраб.
|
||||||
|
|
||||||
1) Подготавливает списки событий в разных энергетических диапазонах.
|
|
||||||
2) Запускает ```erbox``` в три этапа, чтобы получить рабочий список источников для ```ermldet```.
|
|
||||||
3) Запускает ```ermldet```
|
|
||||||
4) Делает кросс-корреляцию с каталогом Gaia-unWISE ```do_cross_match=True```, которая создает три файла: ```.cross``` -- все пересечения, и ```.ref``` / ```.src``` -- входные каталоги для последующей команды ```wcs_match```
|
|
||||||
5) Делает матрицу преобразования координат и корректирует списки событий. Для запуска команд```wcs_match/wcs_update``` требуется запустить окружение ```ciao``` (см. выше)
|
|
||||||
|
|
||||||
### 04_mosaics.py
|
|
||||||
|
|
||||||
Создает сборные изображения (мозайки) в разных энергетических диапазонах.
|
|
||||||
|
|
||||||
### 05_scrtool.py
|
|
||||||
|
|
||||||
Запускает scrtool для самого широкого канала 0.2-10 кэВ, чтобы спектры имели самое полное покрытие по энергиям. Список источников берется из 0.3-2.3 кэВ.
|
|
||||||
|
|
||||||
Вычисляет ECF для всех диапазонов.
|
|
||||||
|
|
||||||
Делает принудительную фотометрию в выбранных каналах (параметр```forced=True```). Внимание! ermldet из eSASS4EDR не делает ассимитричные ошибки на потоки. Мы запускаем более последнюю версию ermldet (v1.56/2.18 esass_200412 Jul 2 12:04:46 2022). Для этого используется параметр ```local_run=True```, который высвечивает какую команду надо запустить на другой машине и ждет ввода.
|
|
||||||
Loading…
x
Reference in New Issue
Block a user