generated from erosita/uds
256 lines
7.1 KiB
Python
Executable File
256 lines
7.1 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
__author__ = "Roman Krivonos"
|
|
__copyright__ = "Space Research Institute (IKI)"
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from astropy.io import fits
|
|
from astropy.table import Table, Column
|
|
import matplotlib.pyplot as plt
|
|
import math, sys, os
|
|
import pickle
|
|
|
|
from sklearn.linear_model import LinearRegression
|
|
from sklearn.linear_model import HuberRegressor
|
|
from sklearn.linear_model import RANSACRegressor
|
|
from sklearn.linear_model import TheilSenRegressor
|
|
from sklearn.model_selection import cross_val_score
|
|
from sklearn.model_selection import RepeatedKFold
|
|
|
|
#from statsmodels.robust.scale import huber
|
|
from astropy.stats import sigma_clip
|
|
from astropy.stats import sigma_clipped_stats
|
|
|
|
from scipy.stats import norm
|
|
from scipy.stats import describe
|
|
from scipy.stats import sem
|
|
import subprocess
|
|
|
|
from numpy import absolute
|
|
from numpy import arange
|
|
|
|
from ridge.utils import *
|
|
from ridge.config import *
|
|
|
|
inkey="ALL"
|
|
|
|
n_bins = 80
|
|
sigma=3
|
|
plotme=False
|
|
|
|
"""
|
|
ebands0={#'E02':[0.0,0.0],
|
|
'E03':[0.0,0.0],
|
|
'E04':[0.0,0.0],
|
|
'E05':[0.0,0.0],
|
|
'E06':[0.0,0.0],
|
|
'E07':[0.0,0.0],
|
|
'E08':[0.0,0.0],
|
|
'E09':[0.0,0.0],
|
|
'E10':[0.0,0.0],
|
|
'E11':[0.0,0.0],
|
|
'E12':[0.0,0.0],}
|
|
"""
|
|
|
|
ebands0={'B01':[0.0,0.0],
|
|
'B02':[0.0,0.0],
|
|
'B03':[0.0,0.0],
|
|
'B04':[0.0,0.0],
|
|
'B05':[0.0,0.0],
|
|
'B06':[0.0,0.0],
|
|
'B07':[0.0,0.0],
|
|
'B08':[0.0,0.0],
|
|
'B09':[0.0,0.0],
|
|
'B10':[0.0,0.0],
|
|
'B11':[0.0,0.0],
|
|
'B12':[0.0,0.0],
|
|
'B13':[0.0,0.0],
|
|
'B14':[0.0,0.0],
|
|
'B15':[0.0,0.0],
|
|
'B16':[0.0,0.0],
|
|
'B17':[0.0,0.0],
|
|
'B18':[0.0,0.0],
|
|
'B19':[0.0,0.0],
|
|
'B20':[0.0,0.0],
|
|
'B21':[0.0,0.0],
|
|
}
|
|
|
|
"""
|
|
ebands_sim={'B01':[[],[]],
|
|
'B02':[[],[]],
|
|
'B03':[[],[]],
|
|
'B04':[[],[]],
|
|
'B05':[[],[]],
|
|
'B06':[[],[]],
|
|
'B07':[[],[]],
|
|
'B08':[[],[]],
|
|
'B09':[[],[]],
|
|
'B10':[[],[]],
|
|
'B11':[[],[]],
|
|
'B12':[[],[]],
|
|
'B13':[[],[]],
|
|
'B14':[[],[]],
|
|
'B15':[[],[]],
|
|
'B16':[[],[]],
|
|
'B17':[[],[]],
|
|
'B18':[[],[]],
|
|
'B19':[[],[]],
|
|
'B20':[[],[]],
|
|
'B21':[[],[]],
|
|
}
|
|
"""
|
|
ebands_sim={'B01':[],
|
|
'B02':[],
|
|
'B03':[],
|
|
'B04':[],
|
|
'B05':[],
|
|
'B06':[],
|
|
'B07':[],
|
|
'B08':[],
|
|
'B09':[],
|
|
'B10':[],
|
|
'B11':[],
|
|
'B12':[],
|
|
'B13':[],
|
|
'B14':[],
|
|
'B15':[],
|
|
'B16':[],
|
|
'B17':[],
|
|
'B18':[],
|
|
'B19':[],
|
|
'B20':[],
|
|
'B21':[],
|
|
}
|
|
|
|
|
|
#skey='Geminga'
|
|
if len(sys.argv) > 1:
|
|
skeys = [sys.argv[1]]
|
|
else:
|
|
skeys = list(skyreg.keys())
|
|
|
|
|
|
if not os.path.exists(specdir):
|
|
os.makedirs(specdir)
|
|
|
|
with open(proddir+'detcnts.B21.ignored_rev.resid.pkl', 'rb') as fp:
|
|
ignored_rev = pickle.load(fp)
|
|
print(ignored_rev)
|
|
|
|
ign=ignored_rev.tolist()
|
|
sys.exit()
|
|
nsim=1000
|
|
|
|
for skey in skeys:
|
|
if not skey in skyreg.keys():
|
|
print("{} not found in {}".format(skey,list(skyreg.keys())))
|
|
sys.exit()
|
|
for enkey in ebands0.keys():
|
|
bkg_fn="detcnts.{}.BKG.resid.fits".format(enkey,inkey)
|
|
syserr, bkg_sem = get_syserror(proddir+bkg_fn)
|
|
|
|
fn="detcnts.{}.{}.resid.fits".format(enkey,inkey)
|
|
d = fits.getdata(proddir+fn)
|
|
df=pd.DataFrame(np.array(d).byteswap().newbyteorder())
|
|
#print(df.columns)
|
|
|
|
#df = df.query("REV == @ign")
|
|
|
|
|
|
df = df.query("LON > {} & LON < {} & LAT > {} & LAT < {} & REV != @ign".format(
|
|
skyreg[skey]['lon'] - skyreg[skey]['wlon']/2,
|
|
skyreg[skey]['lon'] + skyreg[skey]['wlon']/2,
|
|
skyreg[skey]['lat'] - skyreg[skey]['wlat']/2,
|
|
skyreg[skey]['lat'] + skyreg[skey]['wlat']/2)
|
|
)
|
|
|
|
|
|
t = Table.from_pandas(df)
|
|
t.write("{}fits/{}.{}.fits".format(specdir,skey,enkey),overwrite=True)
|
|
|
|
texp = np.array(df['TEXP'])
|
|
print("{} Number of ScWs: {}, {:.1f} Ms".format(skey,df.shape[0],np.sum(texp)/1e6))
|
|
if not (df.shape[0]>0):
|
|
continue
|
|
|
|
#plt.scatter(df['LON'],df['LAT'])
|
|
#plt.show()
|
|
print("*** Data frame size {} ***".format(df.size))
|
|
sg_mean,sg_sem = get_spec(df, grxe_err_cut=grxe_err_cut, skey=skey, enkey=enkey)
|
|
ebands0[enkey]=[sg_mean,sg_sem]
|
|
|
|
nsel = int(df.shape[0]/10)
|
|
for n in range(nsim):
|
|
df0=df.sample(nsel)
|
|
sg_mean,sg_sem = get_spec(df0, grxe_err_cut=grxe_err_cut, skey=skey, enkey=enkey)
|
|
ebands_sim[enkey].append(sg_mean)
|
|
#ebands_sim[enkey][1].append(sg_sem)
|
|
|
|
|
|
|
|
if(plotme):
|
|
k=1.2
|
|
plt.hist(grxe, bins=n_bins, range=[filtered_min*k, filtered_max*k])
|
|
plt.hist(grxe[filtered_grxe.mask], bins=n_bins, range=[filtered_min*k, filtered_max*k])
|
|
plt.axvline(sg_mean, color="black")
|
|
plt.axvline(sg_mean+sg_sem, color="black", linestyle="dashed")
|
|
plt.axvline(sg_mean-sg_sem, color="black", linestyle="dashed")
|
|
plt.axvline(sg_mean+sg_std, color="blue", linestyle="dashed")
|
|
plt.axvline(sg_mean-sg_std, color="blue", linestyle="dashed")
|
|
plt.xlabel("mCrab")
|
|
plt.show()
|
|
|
|
###
|
|
fspec="{}{}.spec".format(specdir,skey)
|
|
with open(fspec, 'w') as fp:
|
|
for enkey in ebands0.keys():
|
|
fp.write("0 {} {:.6f} {:.6f} 0.0\n".format(bands[enkey],ebands0[enkey][0],ebands0[enkey][1]))
|
|
subprocess.run(["perl", "do_pha_igr_v3_mCrab.pl", fspec])
|
|
|
|
###
|
|
|
|
|
|
|
|
fspec="{}{}.sim.spec".format(specdir,skey)
|
|
with open(fspec, 'w') as fp:
|
|
for enkey in ebands_sim.keys():
|
|
data=ebands_sim[enkey]
|
|
#print(type(data))
|
|
|
|
(mu, sg) = norm.fit(data)
|
|
#n, bins, patches = plt.hist(data, 60, density=True, facecolor='green', alpha=0.75)
|
|
|
|
if(plotme):
|
|
|
|
# add a 'best fit' line
|
|
y = norm.pdf( bins, mu, sg)
|
|
l = plt.plot(bins, y, 'r--', linewidth=2)
|
|
|
|
#plot
|
|
plt.xlabel('Flux')
|
|
plt.ylabel('Probability')
|
|
#plt.title(r'$\mathrm{Histogram\ of\ IQ:}\ \mu=%.3f,\ \sigma=%.3f$' %(mu, sg))
|
|
plt.title("{} {:.2f} {:.2f}".format(enkey, mu, sg))
|
|
plt.grid(True)
|
|
plt.show()
|
|
|
|
print(mu,sg)
|
|
|
|
filtered_data = sigma_clip(data, sigma=sigma, maxiters=10, return_bounds=True)
|
|
filtered_arr=filtered_data[0]
|
|
filtered_min=filtered_data[1]
|
|
filtered_max=filtered_data[2]
|
|
sg_mean, sg_med, sg_std = sigma_clipped_stats(data, sigma=sigma, maxiters=10)
|
|
sg_sem = sem(data)
|
|
|
|
fp.write("0 {} {:.6f} {:.6f} 0.0\n".format(bands[enkey],sg_mean,sg_std))
|
|
subprocess.run(["perl", "do_pha_igr_v3_mCrab.pl", fspec])
|
|
|
|
|
|
try:
|
|
for remfile in ["cols","cols1","cols2","header",]:
|
|
os.remove(remfile)
|
|
except OSError:
|
|
pass
|