#!/usr/bin/env python __author__ = "Roman Krivonos" __copyright__ = "Space Research Institute (IKI)" import numpy as np import pandas as pd from astropy.io import fits from astropy.table import Table, Column import matplotlib.pyplot as plt import math, sys, os import pickle from sklearn.linear_model import LinearRegression from sklearn.linear_model import HuberRegressor from sklearn.linear_model import RANSACRegressor from sklearn.linear_model import TheilSenRegressor from sklearn.model_selection import cross_val_score from sklearn.model_selection import RepeatedKFold #from statsmodels.robust.scale import huber from astropy.stats import sigma_clip from astropy.stats import sigma_clipped_stats from scipy.stats import norm from scipy.stats import describe from scipy.stats import sem import subprocess from numpy import absolute from numpy import arange from ridge.utils import * from ridge.config import * inkey="ALL" n_bins = 80 sigma=3 plotme=False """ ebands0={#'E02':[0.0,0.0], 'E03':[0.0,0.0], 'E04':[0.0,0.0], 'E05':[0.0,0.0], 'E06':[0.0,0.0], 'E07':[0.0,0.0], 'E08':[0.0,0.0], 'E09':[0.0,0.0], 'E10':[0.0,0.0], 'E11':[0.0,0.0], 'E12':[0.0,0.0],} """ ebands0={'B01':[0.0,0.0], 'B02':[0.0,0.0], 'B03':[0.0,0.0], 'B04':[0.0,0.0], 'B05':[0.0,0.0], 'B06':[0.0,0.0], 'B07':[0.0,0.0], 'B08':[0.0,0.0], 'B09':[0.0,0.0], 'B10':[0.0,0.0], 'B11':[0.0,0.0], 'B12':[0.0,0.0], 'B13':[0.0,0.0], 'B14':[0.0,0.0], 'B15':[0.0,0.0], 'B16':[0.0,0.0], 'B17':[0.0,0.0], 'B18':[0.0,0.0], 'B19':[0.0,0.0], 'B20':[0.0,0.0], 'B21':[0.0,0.0], } """ ebands_sim={'B01':[[],[]], 'B02':[[],[]], 'B03':[[],[]], 'B04':[[],[]], 'B05':[[],[]], 'B06':[[],[]], 'B07':[[],[]], 'B08':[[],[]], 'B09':[[],[]], 'B10':[[],[]], 'B11':[[],[]], 'B12':[[],[]], 'B13':[[],[]], 'B14':[[],[]], 'B15':[[],[]], 'B16':[[],[]], 'B17':[[],[]], 'B18':[[],[]], 'B19':[[],[]], 'B20':[[],[]], 'B21':[[],[]], } """ ebands_sim={'B01':[], 'B02':[], 'B03':[], 'B04':[], 'B05':[], 'B06':[], 'B07':[], 'B08':[], 'B09':[], 'B10':[], 'B11':[], 'B12':[], 'B13':[], 'B14':[], 'B15':[], 'B16':[], 'B17':[], 'B18':[], 'B19':[], 'B20':[], 'B21':[], } #skey='Geminga' if len(sys.argv) > 1: skeys = [sys.argv[1]] else: skeys = list(skyreg.keys()) if not os.path.exists(specdir): os.makedirs(specdir) with open(proddir+'detcnts.B21.ignored_rev.resid.pkl', 'rb') as fp: ignored_rev = pickle.load(fp) print(ignored_rev) ign=ignored_rev.tolist() """ if(1091 in ign): print("Removed") else: print("Taken") sys.exit() """ nsim=1000 for skey in skeys: if not skey in skyreg.keys(): print("{} not found in {}".format(skey,list(skyreg.keys()))) sys.exit() for enkey in ebands0.keys(): #bkg_fn="detcnts.{}.BKG.resid.fits".format(enkey,inkey) #syserr, bkg_sem = get_syserror(proddir+bkg_fn) fn="detcnts.{}.{}.resid.fits".format(enkey,inkey) #d1 = fits.getdata(proddir+fn) #d2=np.array(d1) #df=pd.DataFrame(d2.view(d2.dtype.newbyteorder())) #df=pd.DataFrame(np.array(d).byteswap().newbyteorder()) #with fits.open(proddir+fn) as data: # df = pd.DataFrame(data[1].data) dat = Table.read(proddir+fn) df = dat.to_pandas() print(df.columns) #sys.exit() #df = df.query("REV == @ign") query = "LON > {} & LON < {} & LAT > {} & LAT < {} & REV != @ign".format( skyreg[skey]['lon'] - skyreg[skey]['wlon']/2, skyreg[skey]['lon'] + skyreg[skey]['wlon']/2, skyreg[skey]['lat'] - skyreg[skey]['wlat']/2, skyreg[skey]['lat'] + skyreg[skey]['wlat']/2) print(query) #sys.exit() df = df.query(query) t = Table.from_pandas(df) t.write("{}fits/{}.{}.fits".format(specdir,skey,enkey),overwrite=True) texp = np.array(df['TEXP']) print("{} Number of ScWs: {}, {:.1f} Ms".format(skey,df.shape[0],np.sum(texp)/1e6)) if not (df.shape[0]>0): continue #plt.scatter(df['LON'],df['LAT']) #plt.show() print("*** Data frame size {} ***".format(df.size)) sg_mean,sg_sem = get_spec(df, grxe_err_cut=grxe_err_cut, skey=skey, enkey=enkey) ebands0[enkey]=[sg_mean,sg_sem] nsel = int(df.shape[0]/10) for n in range(nsim): df0=df.sample(nsel) sg_mean,sg_sem = get_spec(df0, grxe_err_cut=grxe_err_cut, skey=skey, enkey=enkey) ebands_sim[enkey].append(sg_mean) #ebands_sim[enkey][1].append(sg_sem) if(plotme): k=1.2 plt.hist(grxe, bins=n_bins, range=[filtered_min*k, filtered_max*k]) plt.hist(grxe[filtered_grxe.mask], bins=n_bins, range=[filtered_min*k, filtered_max*k]) plt.axvline(sg_mean, color="black") plt.axvline(sg_mean+sg_sem, color="black", linestyle="dashed") plt.axvline(sg_mean-sg_sem, color="black", linestyle="dashed") plt.axvline(sg_mean+sg_std, color="blue", linestyle="dashed") plt.axvline(sg_mean-sg_std, color="blue", linestyle="dashed") plt.xlabel("mCrab") plt.show() ### fspec="{}{}.spec".format(specdir,skey) with open(fspec, 'w') as fp: for enkey in ebands0.keys(): fp.write("0 {} {:.6f} {:.6f} 0.0\n".format(bands[enkey],ebands0[enkey][0],ebands0[enkey][1])) subprocess.run(["perl", "do_pha_igr_v3_mCrab.pl", fspec]) ### fspec="{}{}.sim.spec".format(specdir,skey) with open(fspec, 'w') as fp: for enkey in ebands_sim.keys(): data=ebands_sim[enkey] #print(type(data)) (mu, sg) = norm.fit(data) #n, bins, patches = plt.hist(data, 60, density=True, facecolor='green', alpha=0.75) if(plotme): # add a 'best fit' line y = norm.pdf( bins, mu, sg) l = plt.plot(bins, y, 'r--', linewidth=2) #plot plt.xlabel('Flux') plt.ylabel('Probability') #plt.title(r'$\mathrm{Histogram\ of\ IQ:}\ \mu=%.3f,\ \sigma=%.3f$' %(mu, sg)) plt.title("{} {:.2f} {:.2f}".format(enkey, mu, sg)) plt.grid(True) plt.show() print(mu,sg) filtered_data = sigma_clip(data, sigma=sigma, maxiters=10, return_bounds=True) filtered_arr=filtered_data[0] filtered_min=filtered_data[1] filtered_max=filtered_data[2] sg_mean, sg_med, sg_std = sigma_clipped_stats(data, sigma=sigma, maxiters=10) sg_sem = sem(data) fp.write("0 {} {:.6f} {:.6f} 0.0\n".format(bands[enkey],sg_mean,sg_std)) subprocess.run(["perl", "do_pha_igr_v3_mCrab.pl", fspec]) try: for remfile in ["cols","cols1","cols2","header",]: os.remove(remfile) except OSError: pass