integral/ridge
1
0
Fork 0
generated from erosita/uds
Code Issues Pull Requests Packages Projects Releases Wiki Activity

referee

Browse Source
This commit is contained in:
Roman Krivonos
2024-11-06 18:50:06 +03:00
parent 83d2f62e80
commit 8121da0f23
14 changed files with 1107 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
scripts/00_scox1_lightcurve.py Executable file
View File

@@ -0,0 +1,209 @@
#!/usr/bin/env python
__author__ = "Roman Krivonos"
__copyright__ = "Space Research Institute (IKI)"
import numpy as np
import pandas as pd
from astropy.io import fits
from astropy import wcs
import matplotlib.pyplot as plt
import math, sys, os
import pickle
from numpy.polynomial import Polynomial
from astropy.table import Table, Column
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import HuberRegressor
from sklearn.linear_model import RANSACRegressor
from sklearn.linear_model import TheilSenRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedKFold
from astropy.coordinates import SkyCoord # High-level coordinates
from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
from astropy.coordinates import Angle, Latitude, Longitude # Angles
import astropy.units as u
#from statsmodels.robust.scale import huber
from astropy.stats import sigma_clip
from numpy import absolute
from numpy import arange
from ridge.utils import *
from ridge.config import *
if not os.path.exists(proddir):
os.makedirs(proddir)
#enkey = sys.argv[1]
enkey='A01'
inkey="ALL"
fn="detcnts.{}.{}.resid.fits".format(enkey,inkey)
dat = Table.read(proddir+fn, unit_parse_strict='silent')
df = dat.to_pandas()
with open(proddir+"detcnts.{}.ignored_scw.pkl".format(enkey), 'rb') as fp:
ignored_scw = pickle.load(fp)
sco_crd = SkyCoord(sco_ra, sco_dec, frame=FK5(), unit="deg")
plotme=False
npix = 50
sw = 30.0 # deg
pix = sw/npix # pixel size in degrees
crabmodel={}
rota_arr=[]
a0=[]
b0=[]
a_full0=[]
b_full0=[]
b_est0=[]
err0=[]
rev0=[]
totx=[]
toty=[]
for i,rec in df.iterrows():
obsid = rec['OBSID']#.decode("utf-8")
if (obsid in ignored_scw):
print("Skip ScW",obsid)
continue
# accumulate full data set
for rev in range(revmin,revmax):
df0 = df.query('SRC > 0.0 & REV == {} & PHASE > {} & PHASE < {} & SCO_SEP < {}'.format(rev,phmin,phmax,crab_sep_max))
nobs=len(df0)
if not (nobs> crab_nmax):
continue
print(rev,nobs)
for n in df0['SCO_SEP'].values:
totx.append(n)
for n in df0['SRC'].values:
toty.append(n)
x = np.array(totx)
y = np.array(toty)
x = x.reshape((-1, 1))
model = LinearRegression()
#model = TheilSenRegressor()
results = evaluate_model(x, y, model)
a_full,b_full,err_full = plot_best_fit(x, y, model)
if(plotme):
plot_ab(x, y, a_full, b_full, err_full, title="REGRESSION")
# go over orbits
poly_x=[]
poly_y=[]
ntotal=0
nrev=0
for rev in range(revmin,revmax):
df0 = df.query('SRC > 0.0 & REV == {} & PHASE > {} & PHASE < {} & SCO_SEP < {}'.format(rev,phmin,phmax,crab_sep_max))
nobs=len(df0)
if not (nobs> crab_nmax):
continue
#cen_ra = np.array(df0['RA'].values)
#cen_dec = np.array(df0['DEC'].values)
print("*** Orbit ",rev)
x = np.array(df0['SCO_SEP'].values)
y = np.array(df0['SRC'].values)
#rota_deg = np.array(df0['ROTA'].values)
#rota = np.array(df0['ROTA'].values) * np.pi / 180. # in radians
#detx = np.cos(rota)*x
#dety = np.sin(rota)*x
x = x.reshape((-1, 1))
model = LinearRegression()
#model = TheilSenRegressor()
results = evaluate_model(x, y, model)
a,b,err = plot_best_fit(x, y, model)
b_est = np.mean(y - a_full*x)
a_full0.append(a_full)
b_full0.append(b_full)
b_est0.append(b_est)
a0.append(a)
b0.append(b)
err0.append(err)
rev0.append(rev)
poly_x.append(rev)
poly_y.append(b_est)
crabmodel[rev]={'a':a_full, 'b':b_est, 'err':err}
if(plotme):
plot_ab(x, y, a_full, b_est, err, title="REGRESSION rev {}".format(rev))
print("ax+b: a={:.2e}, b={:.2e}, std={:.2e}\n".format(a,b,err))
ntotal=ntotal+nobs
nrev=nrev+1
print("Orbits: {}, obs: {}".format(nrev,ntotal))
npoly=4
z = np.polyfit(poly_x, poly_y, npoly)
p = np.poly1d(z)
poly_z=[]
for t in poly_x:
poly_z.append(p(t))
plt.scatter(poly_x, poly_y)
plt.plot(poly_x, poly_z, color='r')
plt.title("Crab detector count rate evolution")
plt.ylabel("Crab count rate cts/s/pix")
plt.xlabel("INTEGRAL orbit")
#plt.show()
plt.savefig(proddir+fn.replace(".fits",".sco_rate.png"))
indices = sorted(
range(len(rev0)),
key=lambda index: rev0[index]
)
coldefs = fits.ColDefs([
fits.Column(name='REV', format='J', unit='', array=[rev0[index] for index in indices]),
fits.Column(name='A', format='D', unit='', array=[a0[index] for index in indices]),
fits.Column(name='B', format='D', unit='', array=[b0[index] for index in indices]),
fits.Column(name='ERR', format='D', unit='', array=[err0[index] for index in indices]),
fits.Column(name='A_FULL', format='D', unit='', array=[a_full0[index] for index in indices]),
fits.Column(name='B_FULL', format='D', unit='', array=[b_full0[index] for index in indices]),
fits.Column(name='B_EST', format='D', unit='', array=[b_est0[index] for index in indices]),
fits.Column(name='B_POLY', format='D', unit='', array=[poly_z[index] for index in indices]),
])
fout = fn.replace(".fits",".scox1.fits")
hdu = fits.BinTableHDU.from_columns(coldefs, name='GRXE')
hdu.header['MISSION'] = ('INTEGRAL', '')
hdu.header['TELESCOP'] = ('IBIS', '')
hdu.header['INSTITUT'] = ('IKI', 'Affiliation')
hdu.header['AUTHOR'] = ('Roman Krivonos', 'Responsible person')
hdu.header['EMAIL'] = ('krivonos@cosmos.ru', 'E-mail')
#hdu.add_checksum()
print(hdu.columns)
hdu.writeto(proddir+fout, overwrite=True)

16
scripts/02_grxe_resid.py
View File

@@ -36,6 +36,8 @@ from numpy import arange
from ridge.utils import *
from ridge.config import *
sco_crd = SkyCoord(sco_ra, sco_dec, frame=FK5(), unit="deg")
plotme=False
enkey = sys.argv[1]
outkey = sys.argv[2]
@@ -71,6 +73,8 @@ clean = data.field('clean')
phase = data.field('phase')
texp = data.field('exposure')
src = data.field('src') # for Sco X-1 testing
obsid0=[]
rev0=[]
phase0=[]
@@ -90,8 +94,10 @@ lat0=[]
base0=[]
c0=[]
texp0=[]
src0=[]
sco_sep0=[]
hdulist = fits.open(datadir+modelrxte)
hdulist = fits.open(modelsdir+modelrxte)
w = wcs.WCS(hdulist[0].header)
smap = hdulist[0].data
sx=int(hdulist[0].header['NAXIS1'])
@@ -191,6 +197,12 @@ for i, row in df.iterrows():
rev0.append(orbit)
lon0.append(row['LON'])
lat0.append(row['LAT'])
src0.append(1000*(float(row['SRC'])/p(orbit)))
ra=float(row['RA'])
dec=float(row['DEC'])
sc = SkyCoord(ra, dec, frame=FK5(), unit="deg")
sco_sep0.append(sco_crd.separation(sc).deg)
lon=row['LON']
lat=row['LAT']
@@ -276,6 +288,7 @@ coldefs = fits.ColDefs([
fits.Column(name='TEXP', format='D', unit='', array=[texp0[index] for index in indices]),
fits.Column(name='PHASE', format='D', unit='', array=[phase0[index] for index in indices]),
fits.Column(name='CLEAN', format='D', unit='cts/s', array=[clean0[index] for index in indices]),
fits.Column(name='SRC', format='D', unit='cts/s', array=[src0[index] for index in indices]),
fits.Column(name='MODEL', format='D', unit='cts/s', array=[model0[index] for index in indices]),
fits.Column(name='MODEL_ERR', format='D', unit='', array=[model_err0[index] for index in indices]),
fits.Column(name='RESID', format='D', unit='cts/s', array=[resid0[index] for index in indices]),
@@ -288,6 +301,7 @@ coldefs = fits.ColDefs([
fits.Column(name='C', format='D', unit='', array=[c0[index] for index in indices]),
fits.Column(name='DS9X', format='D', unit='', array=[ds9x[index] for index in indices]),
fits.Column(name='DS9Y', format='D', unit='', array=[ds9y[index] for index in indices]),
fits.Column(name='SCO_SEP', format='D', unit='', array=[sco_sep0[index] for index in indices]),
])
fout = fn.replace(".fits",".{}.resid.fits".format(outkey))

8
scripts/02_grxe_resid.sh
View File

@@ -1,14 +1,14 @@
for band in A01 A02 A03
do
./02_grxe_resid.py $band ALL
./02_grxe_resid.py $band BKG
./02_grxe_resid.py $band GAL
#./02_grxe_resid.py $band BKG
#./02_grxe_resid.py $band GAL
done
for band in B01 B02 B03 B04 B05 B06 B07 B08 B09 B10 B11 B12 B13 B14 B15 B16 B17 B18 B19 B20 B21
do
./02_grxe_resid.py $band ALL
./02_grxe_resid.py $band BKG
./02_grxe_resid.py $band GAL
#./02_grxe_resid.py $band BKG
#./02_grxe_resid.py $band GAL
done

125
scripts/03_scox1_spec.py Executable file
View File

@@ -0,0 +1,125 @@
#!/usr/bin/env python
__author__ = "Roman Krivonos"
__copyright__ = "Space Research Institute (IKI)"
import numpy as np
import pandas as pd
from astropy.io import fits
from astropy.table import Table, Column
from astropy import units as u
import matplotlib.pyplot as plt
import math, sys, os
import pickle
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import HuberRegressor
from sklearn.linear_model import RANSACRegressor
from sklearn.linear_model import TheilSenRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedKFold
#from statsmodels.robust.scale import huber
from astropy.stats import sigma_clip
from astropy.stats import sigma_clipped_stats
from scipy.stats import norm
from scipy.stats import describe
from scipy.stats import sem
import subprocess
from numpy import absolute
from numpy import arange
from ridge.utils import *
from ridge.config import *
inkey="ALL"
skey="SCOX1"
plotme=False
ebands0={'B01':[0.0,0.0],
'B02':[0.0,0.0],
'B03':[0.0,0.0],
'B04':[0.0,0.0],
'B05':[0.0,0.0],
'B06':[0.0,0.0],
'B07':[0.0,0.0],
'B08':[0.0,0.0],
'B09':[0.0,0.0],
'B10':[0.0,0.0],
'B11':[0.0,0.0],
'B12':[0.0,0.0],
'B13':[0.0,0.0],
'B14':[0.0,0.0],
'B15':[0.0,0.0],
'B16':[0.0,0.0],
'B17':[0.0,0.0],
'B18':[0.0,0.0],
'B19':[0.0,0.0],
'B20':[0.0,0.0],
'B21':[0.0,0.0],
}
"""
ebands0={'A01':[0.0,0.0],
'A02':[0.0,0.0],
'A03':[0.0,0.0],
}
"""
mcrab=u.def_unit('mCrab')
ctss=u.def_unit('cts/s')
u.add_enabled_units([mcrab,ctss])
if not os.path.exists(specdir):
os.makedirs(specdir)
fitsdir = "{}fits/".format(specdir)
if not os.path.exists(fitsdir):
os.makedirs(fitsdir)
for enkey in ebands0.keys():
fn="detcnts.{}.{}.resid.fits".format(enkey,inkey)
dat = Table.read(proddir+fn, unit_parse_strict='silent')
df = dat.to_pandas()
#crab_sep_max=2.0
query = 'PHASE > {} & PHASE < {} & SCO_SEP < {}'.format(phmin,phmax,crab_sep_max)
df = df.query(query)
print("{}: {} N={}".format(enkey, query, df.shape[0]))
t = Table.from_pandas(df)
t.write("{}fits/SCOX1.{}.fits".format(specdir,enkey),overwrite=True)
if not (df.shape[0]>0):
print("continue")
continue
sg_mean,sg_sem,skew_val,skew_err = get_spec_src(df, sigma=3, grxe_err_cut=grxe_err_cut, enkey=enkey, plotme=True, gaussfit=True)
ebands0[enkey]=[sg_mean,sg_sem]
fspec="{}{}.spec".format(specdir,skey)
with open(fspec, 'w') as fp:
for enkey in ebands0.keys():
flux=ebands0[enkey][0]
err=ebands0[enkey][1]
print("[DATA] {}: {} {:.6f} {:.6f}".format(skey,enkey,flux,err))
fp.write("0 {} {:.6f} {:.6f} 0.0\n".format(bands[enkey],flux,err))
subprocess.run(["perl", "do_pha_igr_v3_mCrab.pl", fspec])
try:
for remfile in ["cols","cols1","cols2","header",]:
os.remove(remfile)
except OSError:
pass

2
scripts/_00_scan_detcnts.py
View File

@@ -3,7 +3,7 @@
__author__ = "Roman Krivonos"
__copyright__ = "Space Research Institute (IKI)"
""" Работает только в архиве """
""" TO BE RUN IN ARCHIVE ONLY """
import re
import sys