#!/usr/bin/env python

__author__      = "Roman Krivonos"
__copyright__   = "Space Research Institute (IKI)"

import numpy as np
import pandas as pd
from astropy.io import fits
from astropy.table import Table, Column
from astropy import units as u

import matplotlib.pyplot as plt
import math, sys, os
import pickle

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import HuberRegressor
from sklearn.linear_model import RANSACRegressor
from sklearn.linear_model import TheilSenRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedKFold

#from statsmodels.robust.scale import huber
from astropy.stats import sigma_clip
from astropy.stats import sigma_clipped_stats

from scipy.stats import norm
from scipy.stats import describe
from scipy.stats import sem
import subprocess

from numpy import absolute
from numpy import arange

from ridge.utils import *
from ridge.config import *

inkey="BKG"
skey="GRXE-BKG"

sigma=3
plotme=False


with open(proddir+'detcnts.B21.ignored_rev.resid.pkl', 'rb') as fp:
    ignored_rev = pickle.load(fp)
    print("{} orbits ignored".format(len(ignored_rev)))

ign=ignored_rev.tolist()

enkey="E01"
fn="detcnts.{}.{}.resid.fits".format(enkey,inkey)
dat = Table.read(proddir+fn, unit_parse_strict='silent')
df = dat.to_pandas()
print("N={}".format(df.shape[0]))
query = "REV != @ign"

df = df.query(query)
print("{} N={}".format(query, df.shape[0]))

t = Table.from_pandas(df)
t.write("{}/{}.{}.resid_filtered_rev.fits".format(proddir,inkey,enkey),overwrite=True)
 
sg_mean,sg_sem = get_spec(df, sigma=sigma,  grxe_err_cut=grxe_err_cut, skey=skey, enkey=enkey, plotme=True,
                          fout="{}/{}.{}.resid_filtered_spec.fits".format(proddir,inkey,enkey))