This commit is contained in:
Roman Krivonos
2024-07-06 17:35:25 +03:00
parent 265e153465
commit 2552bdb552
156 changed files with 158161 additions and 224 deletions

View File

@@ -26,9 +26,10 @@ from numpy import arange
from ridge.utils import *
from ridge.config import *
plotme=False
enkey = sys.argv[1]
#outkey = sys.argv[2]
outkey = "ALL"
outkey = sys.argv[2]
#outkey = "ALL"
fn="detcnts.{}.fits".format(enkey)
@@ -60,6 +61,7 @@ rev = data.field('rev')
mjd = data.field('mjd')
clean = data.field('clean')
phase = data.field('phase')
texp = data.field('exposure')
obsid0=[]
rev0=[]
@@ -79,6 +81,7 @@ lon0=[]
lat0=[]
base0=[]
c0=[]
texp0=[]
d = fits.getdata(datadir+fn)
df = pd.DataFrame(np.array(d).byteswap().newbyteorder())
@@ -139,6 +142,7 @@ for i, row in df.iterrows():
base0.append(abs(orbit - int(np.min([r1,r2]))))
clean0.append(clean[i])
mjd0.append(mjd[i])
texp0.append(texp[i])
model0.append(m)
resid0.append(clean[i]-m)
grxe0.append(1000*(clean[i]-m)/p(orbit))
@@ -156,6 +160,57 @@ for i, row in df.iterrows():
lat0.append(row['LAT'])
rev_min=np.min(rev0)
rev_max=np.max(rev0)
orbits=np.array(rev0)
resid_arr=np.array(resid0)
distr_val=[]
distr_rev=[]
for r in range(rev_min,rev_max):
ind = np.nonzero(orbits == r)
if not (len(resid_arr[ind])>10):
continue
#print(r,np.mean(resid_arr[ind]), len(resid_arr[ind]))
distr_val.append(np.mean(resid_arr[ind]))
distr_rev.append(r)
dval=np.array(distr_val)
drev=np.array(distr_rev)
sigma=2
n_bins=20
filtered_data = sigma_clip(dval, sigma=sigma, maxiters=10, return_bounds=True)
filtered_arr=filtered_data[0]
filtered_min=filtered_data[1]
filtered_max=filtered_data[2]
print("length orig: {} taken: {} filtered: {}".format(len(dval),len(dval[filtered_arr.mask==False]),len(dval[filtered_arr.mask==True])))
sg_mean, sg_med, sg_std = sigma_clipped_stats(distr_val, sigma=sigma, maxiters=10)
sg_sem = sem(dval[filtered_arr.mask==False])
print("Sigma clipping: mean {:.2f} med {:.2f} std {:.2f} ".format(sg_mean, sg_med, sg_std))
print(len(drev[filtered_arr.mask==True]))
if(plotme):
k=1.2
plt.hist(dval, bins=n_bins, range=[filtered_min*k, filtered_max*k])
plt.hist(dval[filtered_arr.mask], bins=n_bins, range=[filtered_min*k, filtered_max*k])
plt.axvline(sg_mean, color="black")
#plt.axvline(sg_mean+sg_sem, color="black", linestyle="dashed")
#plt.axvline(sg_mean-sg_sem, color="black", linestyle="dashed")
plt.axvline(sg_mean+sg_std, color="blue", linestyle="dashed")
plt.axvline(sg_mean-sg_std, color="blue", linestyle="dashed")
plt.xlabel("Resid")
plt.show()
with open(proddir+fn.replace(".fits",".ignored_rev.resid.pkl"), 'wb') as fp:
pickle.dump(drev[filtered_arr.mask==True], fp, protocol=pickle.HIGHEST_PROTOCOL)
print("Removed REVs:",ignored_rev)
indices = sorted(
range(len(mjd0)),
key=lambda index: mjd0[index]
@@ -170,6 +225,7 @@ coldefs = fits.ColDefs([
fits.Column(name='REV', format='J', unit='', array=[rev0[index] for index in indices]),
fits.Column(name='BASE', format='J', unit='', array=[base0[index] for index in indices]), # nearest BKG
fits.Column(name='MJD', format='D', unit='', array=[mjd0[index] for index in indices]),
fits.Column(name='TEXP', format='D', unit='', array=[texp0[index] for index in indices]),
fits.Column(name='PHASE', format='D', unit='', array=[phase0[index] for index in indices]),
fits.Column(name='CLEAN', format='D', unit='cts/s', array=[clean0[index] for index in indices]),
fits.Column(name='MODEL', format='D', unit='cts/s', array=[model0[index] for index in indices]),