HEASARC

heasarc/management/commands/00_clean_gaia.py

@@ -0,0 +1,34 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcBase, HeasarcGAIADR2, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        tables = HeasarcGAIADR2.objects.all()
+        tables.delete()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_2sxps.py

@@ -0,0 +1,149 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, Heasarc2SXPS, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy.time import Time, TimezoneInfo
+
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    data = astropy.table.Table.read(filename, format='ascii.csv',encoding='latin1')
+
+    """
+    keywords = data.meta['keywords']
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+    """
+
+    tables = Heasarc2SXPS.objects.all()
+    tables.delete()
+
+    utc_moscow = TimezoneInfo(utc_offset=3*u.hour)
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+
+        tm = Time(item['FirstObsDate'], format='iso', scale='utc')
+        FirstObsDate_dt  = tm.to_datetime(timezone=utc_moscow)
+
+        tm = Time(item['LastObsDate'], format='iso', scale='utc')
+        LastObsDate_dt  = tm.to_datetime(timezone=utc_moscow)
+
+        tm = Time(item['FirstDetDate'], format='iso', scale='utc')
+        FirstDetDate_dt  = tm.to_datetime(timezone=utc_moscow)
+
+        tm = Time(item['LastDetDate'], format='iso', scale='utc')
+        LastDetDate_dt  = tm.to_datetime(timezone=utc_moscow)
+ 
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['RA']) or ma.is_masked(item['Decl'])):
+            ra = float(item['RA'])
+            dec = float(item['Decl'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+        else:
+            continue
+        
+        obj = Heasarc2SXPS.objects.create(healpix=healpix, 
+                                          ra=ra,
+                                          dec=dec,
+                                          lii=item['l'],
+                                          bii=item['b'],
+                                          error_radius=item['Err90'],
+                                          name=item['IAUName'],
+                                          Exposure = item['Exposure'],
+                                          LastObsDate = LastObsDate_dt,
+                                          FirstObsDate = FirstObsDate_dt,
+                                          FirstDetDate = FirstDetDate_dt,
+                                          LastDetDate = LastDetDate_dt,
+                                          BestDetectionID = item['BestDetectionID'],
+                                          DetFlag = item['DetFlag'],
+                                          Rate_band0 = item['Rate_band0'],
+                                          Rate_band0_pos = item['Rate_band0_pos'],
+                                          Rate_band0_neg = item['Rate_band0_neg'],
+                                          Rate_band1 = item['Rate_band1'],
+                                          Rate_band1_pos = item['Rate_band1_pos'],
+                                          Rate_band1_neg = item['Rate_band1_neg'],
+                                          Rate_band2 = item['Rate_band2'],
+                                          Rate_band2_pos = item['Rate_band2_pos'],
+                                          Rate_band2_neg = item['Rate_band2_neg'],
+                                          Rate_band3 = item['Rate_band3'],
+                                          Rate_band3_pos = item['Rate_band3_pos'],
+                                          Rate_band3_neg = item['Rate_band3_neg'],
+                                          PeakRate_band0_pos = item['PeakRate_band0_pos'],
+                                          PeakRate_band0_neg = item['PeakRate_band0_neg'],
+                                          PeakRate_band1 = item['PeakRate_band1'],
+                                          PeakRate_band1_pos = item['PeakRate_band1_pos'],
+                                          PeakRate_band1_neg = item['PeakRate_band1_neg'],
+                                          PeakRate_band2 = item['PeakRate_band2'],
+                                          PeakRate_band2_pos = item['PeakRate_band2_pos'],
+                                          PeakRate_band2_neg = item['PeakRate_band2_neg'],
+                                          PeakRate_band3 = item['PeakRate_band3'],
+                                          PeakRate_band3_pos = item['PeakRate_band3_pos'],
+                                          PeakRate_band3_neg = item['PeakRate_band3_neg'],
+                                          PowFlux =item['PowFlux'],
+                                          PowFlux_pos= item['PowFlux_pos'],
+                                          PowFlux_neg= item['PowFlux_neg'],
+                                          APECFlux= item['APECFlux'],
+                                          APECFlux_pos= item['APECFlux_pos'],
+                                          APECFlux_neg= item['APECFlux_neg'],
+                                          PowPeakFlux=item['PowPeakFlux'],
+                                          PowPeakFlux_pos=item['PowPeakFlux_pos'],
+                                          PowPeakFlux_neg=item['PowPeakFlux_neg'],
+                                          FittedPowNH=item['FittedPowNH'],
+                                          FittedPowRedChi=item['FittedPowReducedChi2'],
+                                          FittedAPECNH=item['FittedAPECNH'],
+                                          FittedAPECRedChi =item['FittedAPECReducedChi2'])
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('/data/2SXPS/2SXPS_Sources.csv.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_3maxi.py

@@ -0,0 +1,85 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, Heasarc3MAXI, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+from astropy.io import fits
+import math
+
+def load_heasarc_table(filename):
+    """
+    ['Seq', '3MAXI', 'RAdeg', 'DEdeg', 'ePos', 's4-10keV', 'F4-10keV', 'e_F4-10keV', 's3-4keV', 'F3-4keV', 'l_F3-4keV', 'e_F3-4keV', 's10-20keV', 'F10-20keV', 'l_F10-20keV', 'e_F10-20keV', 'F3-10keV', 'e_F3-10keV', 'HR1', 'e_HR1', 'HR2', 'e_HR2', 'HR3', 'e_HR3', 'TSVar', 'XVA', 'e_XVA']
+    """
+
+    hdul = fits.open(filename)
+    data = hdul[1].data
+    cols = hdul[1].columns
+    print(cols.names)
+
+    #tables = HeasarcSwiftBAT105m.objects.all()
+    #tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+        if not (ma.is_masked(item['RAdeg']) or ma.is_masked(item['DEdeg'])):
+            ra = float(item['RAdeg'])
+            dec = float(item['DEdeg'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            lon = crd.galactic.l.value
+            lat = crd.galactic.b.value
+            healpix = hp.skycoord_to_healpix(crd)
+            healpix_plate = hp_plate.skycoord_to_healpix(crd)
+        else:
+            continue
+
+        sign=item['s4-10keV']
+
+        obj = Heasarc3MAXI.objects.create(healpix=healpix, 
+                                          healpix_plate=healpix_plate, 
+                                          ra=ra,
+                                          dec=dec,
+                                          lii = lon,
+                                          bii = lat,
+                                          name="3MAXI {}".format(item['3MAXI']),
+                                          error_radius = float(item['ePos'])*60*60,
+                                          sign = sign,
+                                          flux = float(item['F4-10keV']),)
+
+                
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('/data/VizieR/3MAXI/table1.fits')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_4xmm.py

@@ -0,0 +1,149 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, Heasarc4XMMDR12, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+
+
+    data = astropy.table.Table.read(filename, format='ascii.csv',encoding='latin1')
+    print(data.info)
+    
+    """
+    keywords = data.meta['keywords']
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    ,0.525479,0,,-0.523192,,,0,56086.7429513889,56087.5881944444,,,1,f,http://xmm-catalog.irap.omp.eu/source/206931901010113/
+
+    """
+    
+
+    tables = Heasarc4XMMDR12.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+        
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+        url=''
+        if not (ma.is_masked(item['webpage_url'])):
+            url=item['webpage_url']
+
+        n_detections=0
+        if not (ma.is_masked(item['n_detections'])):
+            m_detections=item['n_detections']
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+        if not (ma.is_masked(item['sc_ra']) or ma.is_masked(item['sc_dec'])):
+            ra = float(item['sc_ra'])
+            dec = float(item['sc_dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            lon = crd.galactic.l.value
+            lat = crd.galactic.b.value
+            
+            healpix = hp.skycoord_to_healpix(crd)
+        else:
+            continue
+        
+        obj = Heasarc4XMMDR12.objects.create(healpix=healpix, 
+                                            ra=ra,
+                                            dec=dec,
+                                            lii = lon,
+                                            bii = lat,
+                                            name=item['iauname'],
+                                            error_radius = item['sc_poserr'],
+                                            SRCID = item['srcid'],
+                                            radec_error = item['sc_poserr'],
+                                            DET_ML = item['sc_det_ml'],
+                                            EP_1_FLUX = item['sc_ep_1_flux'],
+                                            EP_1_FLUX_ERR = item['sc_ep_1_flux_err'],
+                                            EP_2_FLUX= item['sc_ep_2_flux'],
+                                            EP_2_FLUX_ERR =item['sc_ep_2_flux_err'],
+                                            EP_3_FLUX = item['sc_ep_3_flux'],
+                                            EP_3_FLUX_ERR = item['sc_ep_3_flux_err'],
+                                            EP_4_FLUX = item['sc_ep_4_flux'],
+                                            EP_4_FLUX_ERR = item['sc_ep_4_flux_err'],
+                                            EP_5_FLUX = item['sc_ep_5_flux'],
+                                            EP_5_FLUX_ERR = item['sc_ep_5_flux_err'],
+                                            EP_8_FLUX = item['sc_ep_8_flux'],
+                                            EP_8_FLUX_ERR = item['sc_ep_8_flux_err'],
+                                            EP_9_FLUX = item['sc_ep_9_flux'],
+                                            EP_9_FLUX_ERR = item['sc_ep_9_flux_err'],
+                                            HR1	 = item['sc_hr1'],
+                                            HR1_ERR = item['sc_hr1_err'],
+                                            HR2 = item['sc_hr2'],
+                                            HR2_ERR = item['sc_hr2_err'],
+                                            HR3	= item['sc_hr3'],
+                                            HR3_ERR = item['sc_hr3_err'],
+                                            HR4 = item['sc_hr4'],
+                                            HR4_ERR = item['sc_hr4_err'],
+                                            EXTENT = item['sc_extent'],
+                                            EXT_ERR = item['sc_ext_err'],
+                                            #EXT_ML = item['sc_ext_ml'],
+                                            #CHI2PROB = item['sc_chi2prob'],
+                                            FVAR = item['sc_fvar'],
+                                            FVARERR = item['sc_fvarerr'],
+                                            #VAR_FLAG = item['sc_var_flag'],
+                                            SUM_FLAG = item['sc_sum_flag'],
+                                            EP_8_FMIN = item['sc_ep_8_fmin'],
+                                            EP_8_FMIN_ERR = item['sc_ep_8_fmin_err'],
+                                            EP_8_FMAX = item['sc_ep_8_fmax'],
+                                            EP_8_FMAX_ERR = item['sc_ep_8_fmax_err'],
+                                            MJD_FIRST = item['mjd_first'],
+                                            MJD_LAST = item['mjd_last'],
+                                            N_DETECTIONS = n_detections,
+                                            CONFUSED = item['confused'],
+                                            WEBPAGE_URL = url,)
+                
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+
+        tables = Heasarc4XMMDR12.objects.all()
+        
+        print("Selected {} in total".format(tables.count()))
+
+        #load_heasarc_table('/data/4XMM/4XMM_DR9cat_slim_v1.0.csv.gz')
+        #load_heasarc_table('/data/4XMM/4XMM_DR10cat_slim_v1.0.csv.gz')
+        load_heasarc_table('/data/4XMM/4XMM_DR12cat_slim_v1.0.csv.gz')
+        
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_4xmm_print.py

@@ -0,0 +1,158 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, Heasarc4XMMDR9, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+
+
+    data = astropy.table.Table.read(filename, format='ascii.csv',encoding='latin1')
+    print(data.info)
+    
+    """
+    keywords = data.meta['keywords']
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    ,0.525479,0,,-0.523192,,,0,56086.7429513889,56087.5881944444,,,1,f,http://xmm-catalog.irap.omp.eu/source/206931901010113/
+
+    """
+
+    tables = Heasarc4XMMDR9.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+        
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+        url=''
+        if not (ma.is_masked(item['webpage_url'])):
+            url=item['webpage_url']
+
+        n_detections=0
+        if not (ma.is_masked(item['n_detections'])):
+            url=item['n_detections']
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+        if not (ma.is_masked(item['sc_ra']) or ma.is_masked(item['sc_dec'])):
+            ra = float(item['sc_ra'])
+            dec = float(item['sc_dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            lon = crd.galactic.l.value
+            lat = crd.galactic.b.value
+            
+            healpix = hp.skycoord_to_healpix(crd)
+        else:
+            continue
+
+        obj = Heasarc4XMMDR9.objects.create(healpix=healpix, 
+                                            ra=ra,
+                                            dec=dec,
+                                            lii = lon,
+                                            bii = lat,
+                                            name=item['iauname'],
+                                            error_radius = item['sc_poserr'],
+                                            SRCID = item['srcid'],
+                                            POSERR = item['sc_poserr'],
+                                            DET_ML = item['sc_det_ml'],
+                                            EP_1_FLUX = item['sc_ep_1_flux'],
+                                            EP_1_FLUX_ERR = item['sc_ep_1_flux_err'],
+                                            EP_2_FLUX= item['sc_ep_2_flux'],
+                                            EP_2_FLUX_ERR =item['sc_ep_2_flux_err'],
+                                            EP_3_FLUX = item['sc_ep_3_flux'],
+                                            EP_3_FLUX_ERR = item['sc_ep_3_flux_err'],
+                                            EP_4_FLUX = item['sc_ep_4_flux'],
+                                            EP_4_FLUX_ERR = item['sc_ep_4_flux_err'],
+                                            EP_5_FLUX = item['sc_ep_5_flux'],
+                                            EP_5_FLUX_ERR = item['sc_ep_5_flux_err'],
+                                            EP_8_FLUX = item['sc_ep_8_flux'],
+                                            EP_8_FLUX_ERR = item['sc_ep_8_flux_err'],
+                                            EP_9_FLUX = item['sc_ep_9_flux'],
+                                            EP_9_FLUX_ERR = item['sc_ep_9_flux_err'],
+                                            HR1	 = item['sc_hr1'],
+                                            HR1_ERR = item['sc_hr1_err'],
+                                            HR2 = item['sc_hr2'],
+                                            HR2_ERR = item['sc_hr2_err'],
+                                            HR3	= item['sc_hr3'],
+                                            HR3_ERR = item['sc_hr3_err'],
+                                            HR4 = item['sc_hr4'],
+                                            HR4_ERR = item['sc_hr4_err'],
+                                            EXTENT = item['sc_extent'],
+                                            EXT_ERR = item['sc_ext_err'],
+                                            #EXT_ML = item['sc_ext_ml'],
+                                            #CHI2PROB = item['sc_chi2prob'],
+                                            FVAR = item['sc_fvar'],
+                                            FVARERR = item['sc_fvarerr'],
+                                            #VAR_FLAG = item['sc_var_flag'],
+                                            SUM_FLAG = item['sc_sum_flag'],
+                                            EP_8_FMIN = item['sc_ep_8_fmin'],
+                                            EP_8_FMIN_ERR = item['sc_ep_8_fmin_err'],
+                                            EP_8_FMAX = item['sc_ep_8_fmax'],
+                                            EP_8_FMAX_ERR = item['sc_ep_8_fmax_err'],
+                                            MJD_FIRST = item['mjd_first'],
+                                            MJD_LAST = item['mjd_last'],
+                                            N_DETECTIONS = n_detections,
+                                            CONFUSED = item['confused'],
+                                            WEBPAGE_URL = url,)
+                
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        ra_map=34.577746
+        dec_map=-4.7238375
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        crd = SkyCoord(ra_map, dec_map, frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        hlist = hp.cone_search_skycoord(crd,u.Quantity(2.0,unit="deg"))
+        xmm = Heasarc4XMMDR9.objects.all().filter(healpix__in=hlist)
+        for src in xmm:
+            print("fk5;point({}, {}) # point=cross".format(src.ra,src.dec))
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_allwiseagn.py

@@ -0,0 +1,138 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcALLWISEAGN, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    # convert data frame to pandas                                                                                                           
+    # df = data.to_pandas()
+    # sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    # result = engine.execute(sql)
+    # Insert whole DataFrame into MySQL                                                                            
+    # df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    
+    print("*** Delete all ALLWISEAGN objects ***")
+    tables = HeasarcALLWISEAGN.objects.all()
+    tables.delete()
+
+    print("*** Create new ALLWISEAGN objects ***")
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+        redshift_flag=''
+        if not (ma.is_masked(item['redshift_flag'])):
+            redshift_flag=item['redshift_flag']
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+        
+        obj = HeasarcALLWISEAGN.objects.create(healpix=healpix, 
+                                               ra=ra,
+                                               dec=dec,
+                                               lii=item['lii'],
+                                               bii=item['bii'],
+                                               name=item['name'],
+                                               w1w2_color =  item['w1w2_color'],
+                                               w2w3_color = item['w2w3_color'],
+                                               w1_mag = item['w1_mag'],
+                                               gmag = item['gmag'],
+                                               redshift = item['redshift'],
+                                               redshift_flag = redshift_flag,
+                                               lqac2_name = item['lqac2_name'],
+                                               dr12q_name = item['dr12q_name'],
+                                               milliquas_name = item['milliquas_name'])
+                                             
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_allwiseagn.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_chanmaster.py

@@ -0,0 +1,146 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcCHANMASTER, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    # convert data frame to pandas                                                                                                           
+    # df = data.to_pandas()
+    # sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    # result = engine.execute(sql)
+    # Insert whole DataFrame into MySQL                                                                            
+    # df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    
+    tables = HeasarcCHANMASTER.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+
+        public_date = 0
+        if not (ma.is_masked(item['public_date'])):
+            public_date = int(item['public_date'])
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+
+        
+        obj = HeasarcCHANMASTER.objects.create(healpix=healpix, 
+                                               ra=ra,
+                                               dec=dec,
+                                               lii=item['lii'],
+                                               bii=item['bii'],
+                                               name=item['name'],
+                                               obsid=item['obsid'],
+                                               time = float(item['time']),
+                                               status=item['status'],
+                                               detector=item['detector'],
+                                               grating=item['grating'],
+                                               exposure = float(item['exposure']),
+                                               obstype=item['type'],
+                                               pi=item['pi'],
+                                               cycle = int(item['cycle']),
+                                               proposal=item['proposal'],
+                                               public_date = public_date,
+                                               sequence_number=item['sequence_number'],
+                                               data_mode=item['data_mode'],
+                                               category=item['category'],
+                                               class_id  = int(item['class']),)
+
+
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_chanmaster.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_csc.py

@@ -0,0 +1,139 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcCSC, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    # convert data frame to pandas                                                                                                           
+    # df = data.to_pandas()
+    # sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    # result = engine.execute(sql)
+    # Insert whole DataFrame into MySQL                                                                            
+    # df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    
+    tables = HeasarcCSC.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+        
+        conf_flag=''
+        if(item['conf_flag'] == 'NULL'):
+            conf_flag = 'N'
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+        
+        obj = HeasarcCSC.objects.create(healpix=healpix, 
+                                        ra=ra,
+                                        dec=dec,
+                                        lii=item['lii'],
+                                        bii=item['bii'],
+                                        name=item['name'],
+                                        significance=item['significance'],
+                                        extent_flag=item['extent_flag'],
+                                        conf_flag=conf_flag,
+                                        error_ellipse_r0=item['error_ellipse_r0'],
+                                        error_ellipse_r1=item['error_ellipse_r1'],
+                                        error_ellipse_angle=item['error_ellipse_angle'],
+                                        b_flux_ap=item['b_flux_ap'],
+                                        b_flux_ap_hi=item['b_flux_ap_hi'],
+                                        b_flux_ap_lo=item['b_flux_ap_lo'],
+                                        m_flux_ap=item['m_flux_ap'],
+                                        m_flux_ap_hi=item['m_flux_ap_hi'],
+                                        m_flux_ap_lo=item['m_flux_ap_lo'],
+                                        s_flux_ap = item['s_flux_ap'], 
+                                        s_flux_ap_hi = item['s_flux_ap_hi'],
+                                        m_photflux_ap=item['m_photflux_ap'],
+                                        m_photflux_ap_hi=item['m_photflux_ap_hi'],
+                                        m_photflux_ap_lo=item['m_photflux_ap_lo'],                                        
+                                        s_photflux_ap=item['s_photflux_ap'],
+                                        s_photflux_ap_hi=item['s_photflux_ap_hi'],
+                                        s_photflux_ap_lo=item['s_photflux_ap_lo'],)
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_csc.tdat.gz')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_fermi.py

@@ -0,0 +1,110 @@
+from django.core.management.base import BaseCommand, CommandError
+import math, sys
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, Heasarc4FGL, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+            
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    
+    #sys.exit()
+    tables = Heasarc4FGL.objects.all()
+    tables.delete()
+
+    
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+
+        if not (ma.is_masked(item['semi_major_axis_68']) or ma.is_masked(item['semi_minor_axis_68'])):
+            sx=float(item['semi_major_axis_68'])*3600
+            sy=float(item['semi_minor_axis_68'])*3600
+        else:
+            continue
+
+        error_radius = max(sx,sy)
+
+        name=item['name']
+
+        koeff=1.42 # Paper I
+
+        R68=math.sqrt(-2*math.log(1-0.68))
+        R95=math.sqrt(-2*math.log(1-0.95))
+        R98=math.sqrt(-2*math.log(1-0.98))
+
+        obj = Heasarc4FGL.objects.create(healpix=healpix, 
+                                           healpix_plate=healpix_plate, 
+                                           ra=item['ra'],
+                                           dec=item['dec'],
+                                           lii=item['lii'],
+                                           bii=item['bii'],
+                                           error_radius=error_radius*R98,
+                                           name=item['name'],
+                                           flux=item['energy_flux'],sign=item['detection_significance'],)
+                                               
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_fermilpsc.tdat.gz')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_gaia.py

@@ -0,0 +1,216 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcBase, HeasarcGAIADR2
+from heasarc.models import HeasarcObjectClass, GaiaSourceFile
+from heasarc.models import NSIDE_SOURCES, ORDER
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+from django.db import DatabaseError, transaction
+
+import os
+import glob
+import logging
+
+def load_heasarc_table_atomic(filepath):
+
+    filename_w_ext = os.path.basename(filepath)
+    filename, file_extension = os.path.splitext(filename_w_ext)
+
+    #print(filename_w_ext)
+
+    try:
+        gaia = GaiaSourceFile.objects.get(filename=filename)
+        print("Gaia: %s is already loaded, skip" % filename)
+        return 0
+    except GaiaSourceFile.DoesNotExist:
+        print("Gaia: %s is not loaded" % filename)
+        pass
+
+    gaia = GaiaSourceFile(filename=filename)
+    gaia.save()
+
+    return
+
+    data = astropy.table.Table.read(filepath, format='ascii.csv',encoding='latin1')
+    print(data.info)
+    
+    tables = HeasarcGAIADR2.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+
+    try:
+        with transaction.atomic():
+            for item in data:
+                crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+                healpix = hp.skycoord_to_healpix(crd)
+                obj = HeasarcGAIADR2.objects.create(healpix=healpix, 
+                                                    error_radius = max(item['ra_error'], item['dec_error']),
+                                                    ra=item['ra'],
+                                                    dec=item['dec'],
+                                                    lii = item['l'],
+                                                    bii = item['b'],
+                                                    name=item['designation'],
+                                                    solution_id = item['solution_id'],
+                                                    source_id = item['source_id'],
+                                                    ref_epoch = item['ref_epoch'],
+                                                    ra_error = item['ra_error'],
+                                                    dec_error = item['dec_error'],
+                                                    parallax = item['parallax'],
+                                                    parallax_error = item['parallax_error'],
+                                                    pmra = item['pmra'],
+                                                    pmra_error = item['pmra_error'],
+                                                    pmdec = item['pmdec'],
+                                                    pmdec_error = item['pmdec_error'],
+                                                    phot_g_mean_mag = item['phot_g_mean_mag'],
+                                                    phot_bp_mean_mag = item['phot_bp_mean_mag'],
+                                                    phot_rp_mean_mag = item['phot_rp_mean_mag'])
+                obj.save()
+
+    except DatabaseError:
+        print('--> Database error "%s"' % filename)
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+
+def load_heasarc_table(filename):
+    data = astropy.table.Table.read(filename, format='ascii.csv',encoding='latin1')
+    print(data.info)
+    
+    tables = HeasarcGAIADR2.objects.all()
+    tables.delete()
+
+    
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+        
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            
+            healpix = hp.skycoord_to_healpix(crd)
+        else:
+            continue
+
+        obj = HeasarcGAIADR2.objects.create(healpix=healpix, 
+                                            ra=ra,
+                                            dec=dec,
+                                            lii = item['l'],
+                                            bii = item['b'],
+                                            name=item['designation'],
+                                            solution_id = item['solution_id'],
+                                            source_id = item['source_id'],
+                                            ref_epoch = item['ref_epoch'],
+                                            ra_error = item['ra_error'],
+                                            dec_error = item['dec_error'],
+                                            parallax = item['parallax'],
+                                            parallax_error = item['parallax_error'],
+                                            pmra = item['pmra'],
+                                            pmra_error = item['pmra_error'],
+                                            pmdec = item['pmdec'],
+                                            pmdec_error = item['pmdec_error'],
+                                            phot_g_mean_mag = item['phot_g_mean_mag'],
+                                            phot_bp_mean_mag = item['phot_bp_mean_mag'],
+                                            phot_rp_mean_mag = item['phot_rp_mean_mag'])
+
+
+
+
+                
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+def load_heasarc_table_bulk(filename):
+    data = astropy.table.Table.read(filename, format='ascii.csv',encoding='latin1')
+    print(data.info)
+    
+    tables = HeasarcGAIADR2.objects.all()
+    tables.delete()
+    
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+
+
+    batch_size = 200
+    objs = (HeasarcGAIADR2(ra=data[i]['ra'],
+                           dec=data[i]['dec'],
+                           name=data[i]['designation'],
+                           solution_id = data[i]['solution_id'],
+                           source_id = data[i]['source_id'],
+                           ref_epoch = data[i]['ref_epoch'],
+                           ra_error = data[i]['ra_error'],
+                           dec_error = data[i]['dec_error'],
+                           parallax = data[i]['parallax'],
+                           parallax_error = data[i]['parallax_error'],
+                           pmra = data[i]['pmra'],
+                           pmra_error = data[i]['pmra_error'],
+                           pmdec = data[i]['pmdec'],
+                           pmdec_error = data[i]['pmdec_error'],
+                           phot_g_mean_mag = data[i]['phot_g_mean_mag'],
+                           phot_bp_mean_mag = data[i]['phot_bp_mean_mag'],
+                           phot_rp_mean_mag = data[i]['phot_rp_mean_mag'],)
+            for i in range(len(data)))
+
+    HeasarcBase.objects.bulk_create(objs,batch_size)
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        #load_heasarc_table_atomic('/data/Gaia/gaia_source/csv/GaiaSource_970579492690449408_970786789287579392.csv.gz')
+
+        files = []
+        for file in glob.glob("/data/Gaia/gaia_source/csv/*.csv.gz"):
+            files.append(file)
+
+        for file in files:
+            print("Loading ",file)
+            load_heasarc_table_atomic(file)
+
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_healpix_plate.py

@@ -0,0 +1,36 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcBase, HeasarcTable, TableColumn, HeasarcIntegral9, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        
+        hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+        srcs = HeasarcBase.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix_plate = hp_plate.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_integral2020.py

@@ -0,0 +1,106 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcIntegral2020, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+import math
+
+def load_heasarc_table(filename):
+
+
+    data = astropy.table.Table.read(filename,format='ascii.csv')
+    print(data.info)
+    
+    tables = HeasarcIntegral2020.objects.all()
+    tables.delete()
+
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+    
+
+    for item in data:
+
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+
+        
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+
+        ra = float(item['RA'])
+        dec = float(item['Dec'])
+        crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+        lon = crd.galactic.l.value
+        lat = crd.galactic.b.value
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+ 
+
+        sign=item['SIGN']
+        #error_radius = math.sqrt((30.5/sign)**2 + 0.1*0.1)
+        """ See Eq. 1 in 2018ApJS..235....4O """
+
+        error_radius = 2.1*60
+
+        """
+        if(sign <= 10):
+            error_radius = 2.1*60
+        elif(sign > 10 and sign <= 20):
+            error_radius = 1.5*60
+        else:
+            error_radius = 0.8*60
+        """
+        #print(ra,dec,sign,error_radius)
+        #continue
+
+        P=0.98
+        koeff=math.sqrt(-2*math.log(1-P))
+        """ makes 90% interval for 2D case """
+
+        obj = HeasarcIntegral2020.objects.create(healpix=healpix, 
+                                                 healpix_plate=healpix_plate, 
+                                                 ra=ra,
+                                                 dec=dec,
+                                                 lii = lon,
+                                                 bii = lat,
+                                                 name=item['NAME'],
+                                                 error_radius = error_radius*koeff,
+                                                 sign = item['SIGN'],
+                                                 flux = float(item['FLUX']),
+                                                 flux_error = float(item['ERR']),
+                                                 ref_id = int(item['ID']))
+
+                
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('/export/django/srg/data/integral2020/e0017_0060.joint_20210818-112734.csv')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_intibisgal.py

@@ -0,0 +1,110 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcIntegral9, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    tables = HeasarcIntegral9.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        class_id=9999
+        if item['class'] != '':
+           class_id = int(item['class'])
+        try:
+            object_class = HeasarcObjectClass.objects.get(class_id=class_id)
+        except:
+            print("HeasarcObjectClass not found for class_id=", class_id)
+            object_class=None
+
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+        print(item['name'], healpix_plate)
+        obj = HeasarcIntegral9.objects.create(healpix=healpix, 
+                                              healpix_plate=healpix_plate, 
+                                              ra=item['ra'],
+                                              dec=item['dec'],
+                                              lii=item['lii'],
+                                              bii=item['bii'],
+                                              error_radius=240,
+                                              name=item['name'],
+                                              source_type=item['source_type'],
+                                              flux=item['flux'],
+                                              flux_error=item['flux'],
+                                              class_id=class_id,
+                                              object_class=object_class)
+                                               
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_intibisgal.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXrayMaster.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_intrefcat.py

@@ -0,0 +1,107 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcIntRefCat, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    tables = HeasarcIntRefCat.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        class_id=9999
+        if item['class'] != '':
+           class_id = int(item['class'])
+        try:
+            object_class = HeasarcObjectClass.objects.get(class_id=class_id)
+        except:
+            print("HeasarcObjectClass not found for class_id=", class_id)
+            object_class=None
+
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+       
+        obj = HeasarcIntRefCat.objects.create(healpix=healpix, 
+                                              healpix_plate=healpix_plate, 
+                                              ra=item['ra'],
+                                              dec=item['dec'],
+                                              lii=item['lii'],
+                                              bii=item['bii'],
+                                              error_radius=item['error_radius'],
+                                              name=item['name'],
+                                              class_id=class_id,
+                                              object_class=object_class)
+                                               
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_intrefcat.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXrayMaster.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_match_gaia3.py

@@ -0,0 +1,125 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.core.paginator import Paginator
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+from multiprocessing import Pool, cpu_count
+
+from datetime import date
+import datetime
+import time
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.models import HeasarcBase
+from artsurvey.models import ArtSurveySource, ArtSurvey
+from srglib.utils import load_vizier_nvss
+from srglib.utils import load_vizier_first
+from srglib.utils import load_vizier_sumss
+
+from astrobasis.models import GAIADR3
+from astrobasis.models import GLIMPSE
+
+from srglib.utils import find_counterparts
+from srglib.utils import load_vizier_allwise
+from srglib.utils import load_simbad_for_skymap_sources
+
+from heasarc.models import NSIDE_SOURCES, ORDER
+
+NPARTS=10
+
+def do_match():
+
+    minrad=5
+    maxdist=30
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    
+    """
+    g=GLIMPSE.objects.all()
+    g2=g.filter(healpix__exact=0)
+    print("Total {} healpix=NULL {}".format(g.count(),g2.count()))
+    print(g2[0].ra,g2[0].dec,g2[0].healpix)
+    return
+    """
+
+    srcs = HeasarcBase.objects.all().filter(error_radius__lte=minrad)#.order_by("-id")
+
+
+    pages = Paginator(srcs, 100000)
+    pages_count = pages.count
+    pages_num = pages.num_pages
+    pages_range = pages.page_range
+
+    gaia3_all = GAIADR3.objects.all()
+
+    for i in pages_range:
+        page = pages.page(i)
+        start_time0 = time.time()
+        for src in page.object_list:
+            crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg")      
+            heal = hp.cone_search_skycoord(crd, radius=maxdist*u.arcsecond)
+
+            src.gaia3.clear()
+            try:
+                gaia3_all_cone = gaia3_all.filter(healpix__in=heal)
+            except:
+                continue
+            gaia_list=[]
+            gaia_crd_cone = SkyCoord(gaia3_all_cone.values_list('ra', flat=True), 
+                                     gaia3_all_cone.values_list('dec', flat=True), 
+                                     frame="fk5", unit="deg")
+            sep_cone = crd.separation(gaia_crd_cone).arcsecond
+            isel = (sep_cone <= minrad)
+        
+            index=0
+            for gaia in gaia3_all_cone:
+                #gaia_crd = SkyCoord(gaia.ra, gaia.dec, frame="fk5", unit="deg")
+                #sep=crd.separation(gaia_crd).arcsecond
+                if(isel[index]):
+                    gaia_list.append(gaia)
+                index=index+1
+            
+            if(gaia_list):
+                src.gaia3.add(*gaia_list)
+        
+        elapsed = time.time() - start_time0
+        estimated = (elapsed*(pages_num-i))/60/60
+        print("Page {}/{} for {:.2f} sec, est. {:.2f} hours".format(i,pages_num,elapsed,estimated))
+        
+
+    
+    
+
+    #load_simbad_for_skymap_sources(srcs,minrad=minrad,maxdist=maxdist)
+    #load_vizier_allwise(srcs, minrad=minrad, maxdist=maxdist)
+    #load_vizier_nvss(srcs, minrad=minrad, maxdist=maxdist)
+    #load_vizier_first(srcs, minrad=minrad, maxdist=maxdist)
+    #load_vizier_sumss(srcs, minrad=minrad, maxdist=maxdist)        
+    #find_counterparts(srcs, GAIADR3.objects.all(), "gaia3", maxdist=maxdist, minrad=minrad)
+    #for src in srcs:
+    #    src.heasarc.clear()
+    
+    #find_counterparts(srcs, GAIADR3.objects.all(),'gaia3', maxdist=maxdist, minrad=minrad)
+    #find_counterparts(srcs, GLIMPSE.objects.all(),'glimpse', maxdist=maxdist, minrad=minrad)
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+
+        start_time = time.time()
+        
+        do_match()
+        hours = (time.time() - start_time)/60/60
+        print("--- {:.2f} hours ---".format(hours))
+
+        self.stdout.write(self.style.SUCCESS('Done'))
+        

heasarc/management/commands/00_heasarc_maxi7yr.py

@@ -0,0 +1,111 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy, math
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcMAXI7YR, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    
+
+    tables = HeasarcMAXI7YR.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+        """
+        this is correct only for 1 dimention
+
+        erf(1.65/sqrt(2)) corresponds to 90%
+
+        Чтобы получить 90% из 1 сигма, надо использовать коэффициент 1.42 (sqrt(4.61/2.3))
+
+        """
+        P=0.98
+        koeff=math.sqrt(-2*math.log(1-P))
+        sign = float(item['detsig_4_10'])
+        error_radius_1sigma=4800/sign
+        if not (ma.is_masked(item['error_radius'])):
+                error_radius_1sigma=float(item['error_radius'])
+
+        #if  (ma.is_masked(item['error_radius'])):
+        print("{} {} {} --> {}".format(item['name'], item['error_radius'], item['catalog'], error_radius_1sigma*koeff))
+
+        """
+        We save 90% confidence interval for error_radius
+        """
+        obj = HeasarcMAXI7YR.objects.create(healpix=healpix, 
+                                           healpix_plate=healpix_plate, 
+                                           ra=item['ra'],
+                                           dec=item['dec'],
+                                           lii=item['lii'],
+                                           bii=item['bii'],
+                                           error_radius=error_radius_1sigma*koeff,
+                                           name=item['name'],
+                                           flux=item['flux_4_10'],sign=item['detsig_4_10'],)
+                                               
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_maxigsc7yr.tdat.gz')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_maxigschgl.py

@@ -0,0 +1,143 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcMAXIGSCHGL, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES,  ORDER 
+from monthplan.models import  NSIDE_PLATES
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table_author = (keywords['table_author'][:36]) if len(keywords['table_author']) > 36 else keywords['table_author']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=table_author
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    print("*** Delete all MAXI objects ***")
+    tables = HeasarcMAXIGSCHGL.objects.all()
+    tables.delete()
+
+    print("*** Create new MAXI objects ***")
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+    for item in data:
+
+        class_id = int(item['class'])
+        try:
+            object_class = HeasarcObjectClass.objects.get(class_id=class_id)
+        except:
+            object_class = None
+            pass
+
+        redshift=0.0
+        if not (ma.is_masked(item['redshift'])):
+            redshift=float(item['redshift'])
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+            healpix_plate = hp_plate.skycoord_to_healpix(crd)    
+        obj = HeasarcMAXIGSCHGL.objects.create(healpix=healpix,
+                                               healpix_plate=healpix_plate,
+                                               ra=ra,
+                                               dec=dec,
+                                               lii=item['lii'],
+                                               bii=item['bii'],
+                                               name=item['name'],
+                                               error_radius=float(item['error_radius']),
+                                               redshift = redshift,
+                                               object_class = object_class,
+                                               class_id = int(item['class']),
+                                               hardness_ratio = item['hardness_ratio'],
+                                               hardness_ratio_error = item['hardness_ratio_error'],
+                                               hb_flux = item['hb_flux'],
+                                               hb_flux_error = item['hb_flux_error'],
+                                               hb_significance = item['hb_significance'],
+                                               ctrpart_name = item['ctrpart_name'])
+
+                                             
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_maxigschgl.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_median_error.py

@@ -0,0 +1,51 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcXMMSL2, HeasarcObjectClass
+from heasarc.models import Heasarc4FGL
+from heasarc.models import HeasarcIntegral2020, HeasarcSwiftBAT105m, HeasarcMAXI7YR, HeasarcRASS2RXS, Heasarc4FGL
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+import pandas as pd
+import numpy as np
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        srcs = Heasarc4FGL.objects.all()
+
+        error=[]
+        for s in srcs:
+            error.append(s.error_radius)
+            if(s.error_radius> 4000):
+                print(s,s.error_radius)
+
+        s = pd.DataFrame(error)
+        print("total {} median {:.1f} mean {:.1f} min {:.1f} max {:.1f}".format(srcs.count(),np.median(error),np.mean(error),np.min(error),np.max(error)))
+
+        #srcs = HeasarcIntegral2020.objects.all().filter(name__contains="SLX")
+        #for s in srcs:
+        #    print(s.name,s.ra,s.dec,s.error_radius)
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_object_class.py

@@ -0,0 +1,88 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass
+
+def load_heasarc_table(filename, engine):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+        # convert data frame to pandas                                                                                                           
+    df = data.to_pandas()
+    sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    result = engine.execute(sql)
+
+    # Insert whole DataFrame into MySQL                                                                            
+    df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    
+    for item in data:
+        obj = HeasarcObjectClass(table=table, 
+                                 class_id=int(item['class_id']), 
+                                 class_name=item['class_name'])
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_class.tdat.gz', engine)
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_rass2rxs.py

@@ -0,0 +1,159 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcRASS2RXS, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import NSIDE_PLATES
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    # convert data frame to pandas                                                                                                           
+    # df = data.to_pandas()
+    # sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    # result = engine.execute(sql)
+    # Insert whole DataFrame into MySQL                                                                            
+    # df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    
+    tables = HeasarcRASS2RXS.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+        
+        source_quality_flag=0
+        if not (ma.is_masked(item['source_quality_flag'])):
+            source_quality_flag=int(item['source_quality_flag'])
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+        
+        obj = HeasarcRASS2RXS.objects.create(healpix=healpix, 
+                                             ra=ra,
+                                             dec=dec,
+                                             lii=item['lii'],
+                                             bii=item['bii'],
+                                             name=item['name'],
+                                             detection_likelihood = item['detection_likelihood'],
+                                             count_rate = item['count_rate'],
+                                             count_rate_error =  item['count_rate_error'],
+                                             exposure =  item['exposure'],
+                                             source_extent =  item['source_extent'],
+                                             source_extent_prob =  item['source_extent_prob'],
+                                             source_quality_flag =  source_quality_flag,
+                                             hardness_ratio_1 =  item['hardness_ratio_1'],
+                                             hardness_ratio_2 =  item['hardness_ratio_2'],
+                                             plaw_flux = item['plaw_flux'],
+                                             plaw_chi2_reduced = item['plaw_chi2_reduced'],
+                                             plaw_nh = item['plaw_nh'],
+                                             mekal_flux = item['mekal_flux'],
+                                             mekal_chi2_reduced = item['mekal_chi2_reduced'],
+                                             mekal_nh = item['mekal_nh'],
+                                             bb_flux = item['bb_flux'],
+                                             bb_chi2_reduced = item['bb_chi2_reduced'],
+                                             bb_nh = item['bb_nh'],
+                                             x_pixel_error = item['x_pixel_error'],
+                                             y_pixel_error = item['y_pixel_error'],
+                                             time = item['time'])
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        
+        tables = HeasarcRASS2RXS.objects.all()
+        for src in tables:
+            print(src.error_radius)
+            return
+
+
+        #load_heasarc_table('heasarc_rass2rxs.tdat.gz')
+        
+        # 14547279 | 28571450 | 180.63796 | 18.45888 | 247.99462296 | 75.7992302 |  11.33246061 | 2RXS J120233.1+182731 |                   44 | 1.6219809792e-13 |          6975
+
+        hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+        crd = SkyCoord(248.52271, -44.04758, frame=FK5(), unit="deg")
+        h = hp_plate.skycoord_to_healpix(crd)
+        print(h)
+        return
+
+        # update healpix only
+        srcs = HeasarcRASS2RXS.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix_plate = hp_plate.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_simple_class.py

@@ -0,0 +1,34 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from heasarc.models import  HeasarcSimpleClass
+
+def load_heasarc_table(filename):
+    data = astropy.table.Table.read(filename, format='ascii.fast_no_header', delimiter=',')
+    
+    cls = HeasarcSimpleClass.objects.all()
+    cls.delete()
+
+    for item in data:
+        obj = HeasarcSimpleClass(class_id=int(item[0]), 
+                                 class_name=item[1])
+        obj.save()
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('/export/django/srg/data/heasarc_object_class_cut.csv')
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_swiftbat105m.py

@@ -0,0 +1,113 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcSwiftBAT105m, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+import math
+
+def load_heasarc_table(filename):
+
+
+    data = astropy.table.Table.read(filename, delimiter='|',format='ascii.fast_no_header', data_start=2)
+    print(data.info)
+    
+    tables = HeasarcSwiftBAT105m.objects.all()
+    tables.delete()
+
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        
+        #crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        #healpix = hp.skycoord_to_healpix(crd)
+
+        
+        redshift=0.0
+        if not (ma.is_masked(item['col18'])):
+            redshift=float(item['col18'])
+
+        lum=0.0
+        if not (ma.is_masked(item['col19'])):
+            lum=float(item['col19'])
+        
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+        if not (ma.is_masked(item['col3']) or ma.is_masked(item['col4'])):
+            ra = float(item['col3'])
+            dec = float(item['col4'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            lon = crd.galactic.l.value
+            lat = crd.galactic.b.value
+            healpix = hp.skycoord_to_healpix(crd)
+            healpix_plate = hp_plate.skycoord_to_healpix(crd)
+        else:
+            continue
+
+        sign=item['col5']
+        error_radius = math.sqrt((30.5/sign)**2 + 0.1*0.1)*60
+        """ See Eq. 1 in 2018ApJS..235....4O """
+        """
+        erf(1.65/sqrt(2)) corresponds to 90%
+        """
+        #koeff=1.42
+        P=0.98
+        koeff=math.sqrt(-2*math.log(1-P))
+
+        #print(item['col1'],item['col2'],ra,dec,sign,error_radius)
+        #continue
+
+        obj = HeasarcSwiftBAT105m.objects.create(healpix=healpix, 
+                                                 healpix_plate=healpix_plate, 
+                                                 ra=ra,
+                                                 dec=dec,
+                                                 lii = lon,
+                                                 bii = lat,
+                                                 name=item['col2'],
+                                                 error_radius = error_radius*koeff,
+                                                 snr = sign,
+                                                 counterpart_name = item['col6'],
+                                                 flux = float(item['col10']),
+                                                 flux_lo = float(item['col11']),
+                                                 flux_hi = float(item['col12']),
+                                                 redshift = redshift,
+                                                 lum = lum,
+                                                 class_id = int(item['col21']),
+                                                 ref_id = int(item['col1']),
+                                                 otype = item['col22'])
+
+                
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('/data/Swift/BAT_105m_catalog_07jul2019.txt')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_xmm_sl2_clean.py

@@ -0,0 +1,121 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcXMMSL2, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    
+    
+    tables = HeasarcXMMSL2.objects.all()
+    tables.delete()
+    
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        class_id=9999
+        if item['class'] != '':
+           class_id = int(item['class'])
+        try:
+            object_class = HeasarcObjectClass.objects.get(class_id=class_id)
+        except:
+            print("HeasarcObjectClass not found for class_id=", class_id)
+            object_class=None
+
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+       
+        """ Skip if source with the same name exists """
+        try:
+            src=HeasarcXMMSL2.objects.get(name__exact=item['xmmslew_name'])
+            #print("*** Skip {}".format(item['xmmslew_name']))
+
+            continue
+        except:
+            pass
+        #print("Create {}".format(item['xmmslew_name']))
+
+        obj = HeasarcXMMSL2.objects.create(healpix=healpix, 
+                                           healpix_plate=healpix_plate, 
+                                           ra=item['ra'],
+                                           dec=item['dec'],
+                                           lii=item['lii'],
+                                           bii=item['bii'],
+                                           error_radius=float(item['radec_error'])*1.42,
+                                           name="{}".format(item['xmmslew_name']),
+                                           class_id=class_id,object_class=object_class,
+                                           flux_b8=item['flux_b8'],
+                                           flux_b7=item['flux_b7'],
+                                           obsid=item['obsid'],
+                                           ver_inext = item['ver_inext'],
+                                           ver_halo = item['ver_halo'],
+                                           ver_hibgnd = item['ver_hibgnd'],
+                                           ver_nredg = item['ver_nredg'],
+                                           ver_psusp = item['ver_psusp'],
+                                           ver_false = item['ver_false'],
+                                           flag_comment = item['flag_comment'],
+                                           sourcenum=item['sourcenum'])
+                                               
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_xmmslewcln.tdat.gz')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_xmmsl2.py

@@ -0,0 +1,110 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime, math
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcXMMSL2, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import  NSIDE_PLATES
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    tables = HeasarcXMMSL2.objects.all()
+    tables.delete()
+    
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+
+    for item in data:
+        class_id=9999
+        if item['class'] != '':
+           class_id = int(item['class'])
+        try:
+            object_class = HeasarcObjectClass.objects.get(class_id=class_id)
+        except:
+            print("HeasarcObjectClass not found for class_id=", class_id)
+            object_class=None
+
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+       
+        """ Skip if source with the same name exists """
+        try:
+            src=HeasarcXMMSL2.objects.get(name__exact=item['xmmslew_name'])
+            continue
+        except:
+            pass
+        P=0.98
+        koeff=math.sqrt(-2*math.log(1-P))
+        obj = HeasarcXMMSL2.objects.create(healpix=healpix, 
+                                           healpix_plate=healpix_plate, 
+                                           ra=item['ra'],
+                                           dec=item['dec'],
+                                           lii=item['lii'],
+                                           bii=item['bii'],
+                                           error_radius=float(item['radec_error'])*koeff,
+                                           name="{}".format(item['xmmslew_name']),
+                                           class_id=class_id,object_class=object_class,
+                                           flux_b8=item['flux_b8'],
+                                           flux_b7=item['flux_b7'],
+                                           obsid=item['obsid'],
+                                           sourcenum=item['sourcenum'])
+                                               
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_xmmslewful.tdat.gz')
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_xmmssc.py

@@ -0,0 +1,150 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcXMMSSC, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    # convert data frame to pandas                                                                                                           
+    # df = data.to_pandas()
+    # sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    # result = engine.execute(sql)
+    # Insert whole DataFrame into MySQL                                                                            
+    # df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    
+    tables = HeasarcXMMSSC.objects.all()
+    tables.delete()
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+ 
+        obj = HeasarcXMMSSC.objects.create(healpix=healpix, 
+                                           ra=item['ra'],
+                                           dec=item['dec'],
+                                           lii=item['lii'],
+                                           bii=item['bii'],
+                                           error_radius=item['error_radius'],
+                                           name=item['name'],
+                                           detid=item['detid'],
+                                           srcid=item['srcid'],
+                                           time = item['time'],
+                                           end_time = item['end_time'],
+                                           ep_8_flux = item['ep_8_flux'],
+                                           ep_8_flux_error = item['ep_8_flux_error'],
+                                           pn_8_flux = item['pn_8_flux'],
+                                           pn_8_flux_error = item['pn_8_flux_error'],
+                                           m1_8_flux = item['m1_8_flux'],
+                                           m1_8_flux_error = item['m1_8_flux_error'],
+                                           m2_8_flux = item['m2_8_flux'],
+                                           m2_8_flux_error = item['m2_8_flux_error'],
+                                           ep_1_flux = item['ep_1_flux'],
+                                           ep_1_flux_error = item['ep_1_flux_error'],
+                                           ep_2_flux = item['ep_2_flux'],
+                                           ep_2_flux_error = item['ep_2_flux_error'],
+                                           ep_3_flux = item['ep_3_flux'],
+                                           ep_3_flux_error = item['ep_3_flux_error'],
+                                           sum_flag = item['sum_flag'],
+                                           sc_extent = item['sc_extent'],
+                                           sc_ext_ml = item['sc_ext_ml'])
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        # create sqlalchemy engine 
+#        engine = create_engine("mysql+pymysql://{user}:{pw}@localhost/{db}"
+#                               .format(user="heauser",
+#                                       pw="srg2019@L2_heasarc",
+#                                       db="heasarc_db"))
+
+#        engine = create_engine("postgresql://{user}:{pw}@localhost/{db}"
+#                               .format(user="heauser",
+#                                       pw="srg2019@L2_heasarc",
+#                                       db="heasarc_db"))
+
+
+
+        load_heasarc_table('heasarc_xmmssc.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_xray_master.py

@@ -0,0 +1,144 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcXrayMaster, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    # convert data frame to pandas                                                                                                           
+    # df = data.to_pandas()
+    # sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    # result = engine.execute(sql)
+    # Insert whole DataFrame into MySQL                                                                            
+    # df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    
+    tables = HeasarcXrayMaster.objects.all()
+    tables.delete()
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+        class_id=9999
+        if item['class'] != '':
+           class_id = int(item['class'])
+        try:
+            object_class = HeasarcObjectClass.objects.get(class_id=class_id)
+        except:
+            print("HeasarcObjectClass not found for class_id=", class_id)
+            object_class=None
+
+        crd = SkyCoord(item['ra'], item['dec'], frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+ 
+        obj = HeasarcXrayMaster.objects.create(healpix=healpix, 
+                                               ra=item['ra'],
+                                               dec=item['dec'],
+                                               lii=item['lii'],
+                                               bii=item['bii'],
+                                               error_radius=item['error_radius'],
+                                               name=item['name'],
+                                               database_table=item['database_table'], 
+                                               count_rate=item['count_rate'],
+                                               count_rate_error=item['count_rate_error'],
+                                               flux=item['flux'],
+                                               exposure=item['exposure'],
+                                               class_id=class_id,
+                                               object_class=object_class,
+                                               observatory=item['observatory'])
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        # create sqlalchemy engine 
+#        engine = create_engine("mysql+pymysql://{user}:{pw}@localhost/{db}"
+#                               .format(user="heauser",
+#                                       pw="srg2019@L2_heasarc",
+#                                       db="heasarc_db"))
+
+#        engine = create_engine("postgresql://{user}:{pw}@localhost/{db}"
+#                               .format(user="heauser",
+#                                       pw="srg2019@L2_heasarc",
+#                                       db="heasarc_db"))
+
+
+
+        load_heasarc_table('heasarc_xray.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXrayMaster.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarc_xteasscat.py

@@ -0,0 +1,131 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcXTEASSCAT, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES,  ORDER 
+from monthplan.models import  NSIDE_PLATES
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table_author = (keywords['table_author'][:36]) if len(keywords['table_author']) > 36 else keywords['table_author']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=table_author
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+    print("*** Delete all XTEASSCAT objects ***")
+    tables = HeasarcXTEASSCAT.objects.all()
+    tables.delete()
+
+    print("*** Create new XTEASSCAT objects ***")
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    hp_plate = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5())
+    for item in data:
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+            healpix_plate = hp_plate.skycoord_to_healpix(crd)    
+        obj = HeasarcXTEASSCAT.objects.create(healpix=healpix,
+                                              healpix_plate=healpix_plate,
+                                              ra=ra,
+                                              dec=dec,
+                                              lii=item['lii'],
+                                              bii=item['bii'],
+                                              name=item['name'],
+                                              error_radius=float(item['error_radius'])*60,
+                                              redshift = item['redshift'],
+                                              rate1 = item['count_rate_1'],
+                                              rate1_error = item['count_rate_1_error'],
+                                              rate2 = item['count_rate_2'],
+                                              rate2_error = item['count_rate_2_error'],
+                                              alt_names= item['alt_names'],
+                                              broad_type = item['broad_type'],
+                                              detailed_type = item['detailed_type'],
+                                              class_id = int(item['class']))
+                                             
+        obj.save()
+
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_xteasscat.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/00_heasarcbase_match.py

@@ -0,0 +1,33 @@
+from django.core.management.base import BaseCommand, CommandError
+
+import time
+
+from astrobasis.models import VLASS
+from astrobasis.models import GLIMPSE
+from astrobasis.models import GAIADR3
+from astrobasis.models import TwoMASS
+from astrobasis.models import AllWise
+
+
+from heasarc.utils import heasarcbase_match
+from heasarc.models import Heasarc4XMMDR12
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+
+        start_time = time.time()
+        
+        heasarcbase_match(VLASS.objects.all(),"vlass", survey=Heasarc4XMMDR12)
+        heasarcbase_match(GLIMPSE.objects.all(),"glimpse", survey=Heasarc4XMMDR12)
+        heasarcbase_match(GAIADR3.objects.all(),"gaia3", survey=Heasarc4XMMDR12)
+        heasarcbase_match(TwoMASS.objects.all(),"twomass", survey=Heasarc4XMMDR12)
+        heasarcbase_match(AllWise.objects.all(),"allwise", survey=Heasarc4XMMDR12)
+
+        hours = (time.time() - start_time)/60/60
+        print("--- {:.2f} hours ---".format(hours))
+
+        self.stdout.write(self.style.SUCCESS('Done'))
+        

heasarc/management/commands/01_heasarc_rass2rxs_update.py

@@ -0,0 +1,64 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcRASS2RXS, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from monthplan.models import NSIDE_PLATES
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+import math
+
+def update_heasarc_table():
+    
+    srcs = HeasarcRASS2RXS.objects.all()
+    """
+    for src in srcs:
+        dx=src.x_pixel_error
+        dy=src.y_pixel_error
+        error_radius=math.sqrt(dx*dx+dy*dy)*45*1.42
+        print("{} {}".format(error_radius,src.error_radius))
+    """
+    
+    objs = []
+    pixsize=45 # arcseconds
+    for src in srcs:
+        dx=src.x_pixel_error
+        dy=src.y_pixel_error
+        #src.error_radius = math.sqrt(dx*dx+dy*dy)*pixsize*1.42
+
+        P=0.98
+        src.error_radius = max(dx,dy)*pixsize*math.sqrt(-2*math.log(1-P))
+
+        objs.append(src)
+    HeasarcRASS2RXS.objects.bulk_update(objs, ['error_radius'], batch_size=1000)
+    """
+    The X image coordinate of the source from the local sliding cell detection on an X-ray image of 512 by 512 pixels 
+    with 45 arcsecond square pixels. 
+    
+    The 1-sigma error in the X image coordinate of the source. 
+
+    R(P)_2D = Sigma_1D * sqrt (-2 * ln(1-P))
+
+    """
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        
+        update_heasarc_table()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/heasarc_load_tdat.py

@@ -0,0 +1,116 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn
+
+def load_heasarc_table(filename, engine):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+    return
+
+    keywords = data.meta['keywords']
+    for key, value in keywords.items():
+            print(key, '->', value)
+
+            try:
+                table=HeasarcTable.objects.get(name__exact=keywords['table_name'])
+                table.delete()
+            except:
+                pass
+
+            table_name=keywords['table_name']
+            table = HeasarcTable(name=keywords['table_name'])
+            table.description=keywords['table_description'].replace('\"', '')
+            table.document_url=keywords['table_document_url']
+            if 'default_search_radius' in keywords:
+                table.search_radius=int(keywords['default_search_radius'])
+            if 'frequency_regime' in keywords:
+                table.frequency_regime=keywords['frequency_regime']
+            table.observatory_name=keywords['observatory_name']
+            table.security=keywords['table_security']
+            if 'table_author' in keywords:
+                table.author=keywords['table_author']
+            if 'catalog_bibcode' in keywords:
+                table.bibcode=keywords['catalog_bibcode']
+            if 'declination' in keywords:
+                table.declination=keywords['declination'].replace('@', '')
+            if 'right_ascension' in keywords:
+                table.right_ascension=keywords['right_ascension'].replace('@', '')
+            table.observatory_name = keywords['observatory_name']
+            if 'parameter_defaults' in keywords:
+                table.parameter_defaults = keywords['parameter_defaults']
+            table.save()
+
+    cols = data.meta['cols']
+    for key, value in cols.items():
+        column = TableColumn(table=table)
+        column.name=key
+        column.tdat_type=value['type']
+        column.description=value['description']
+        column.save()
+        print(key, '->', value)
+
+        # convert data frame to pandas                                                                                                           
+    df = data.to_pandas()
+    sql = 'DROP TABLE IF EXISTS '+table_name+';'
+    result = engine.execute(sql)
+
+    # Insert whole DataFrame into MySQL                                                                            
+    df.to_sql(table_name, con = engine, if_exists = 'append', chunksize = 200000)
+    print('--> Successfully loaded "%s"' % table_name)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        # create sqlalchemy engine 
+#        engine = create_engine("mysql+pymysql://{user}:{pw}@localhost/{db}"
+#                               .format(user="heauser",
+#                                       pw="srg2019@L2_heasarc",
+#                                       db="heasarc_db"))
+
+        engine = create_engine("postgresql://{user}:{pw}@localhost/{db}"
+                               .format(user="heauser",
+                                       pw="srg2019@L2_heasarc",
+                                       db="heasarc_db"))
+
+
+
+        #load_heasarc_table('heasarc_rass2rxs.tdat.gz', engine)
+        #load_heasarc_table('/heasarc/heasarc_messier.tdat', engine)
+        #load_heasarc_table('heasarc_iraspscz.tdat.gz', engine)
+        #load_heasarc_table('/heasarc/heasarc_ngc2000.tdat', engine)
+        #load_heasarc_table('heasarc_ascamaster.tdat.gz', engine)
+        #load_heasarc_table('/heasarc/heasarc_batsepulsr.tdat', engine)
+        #load_heasarc_table('/heasarc/heasarc_egrcat.tdat', engine)
+        #load_heasarc_table('/heasarc/heasarc_egret3.tdat', engine)
+        load_heasarc_table('heasarc_chanmaster.tdat.gz', engine) # failed
+        #load_heasarc_table('/heasarc/heasarc_fermilpsc.tdat.gz', engine)
+        #load_heasarc_table('/heasarc/heasarc_twomassrsc.tdat.gz', engine)
+        #load_heasarc_table('/heasarc/heasarc_abellzcat.tdat.gz', engine)
+        #load_heasarc_table('/heasarc/heasarc_qorgcat.tdat.gz', engine)
+        #load_heasarc_table('heasarc_allwiseagn.tdat.gz', engine)
+        #load_heasarc_table('heasarc_glxsdssqs2.tdat.gz', engine)
+        #load_heasarc_table('heasarc_intibisag2.tdat.gz', engine)
+        #load_heasarc_table('heasarc_rosnepagn.tdat.gz', engine)
+        #load_heasarc_table('heasarc_tevcat.tdat.gz', engine)
+        #load_heasarc_table('heasarc_intibisgal.tdat.gz', engine)
+        #load_heasarc_table('heasarc_xray.tdat.gz', engine) # Don't runm here, use 01_heasarc_xray_master
+        #load_heasarc_table('heasarc_xmmssc.tdat.gz', engine)
+        #load_heasarc_table('heasarc_class.tdat.gz', engine) # Don't run here, use 00_heasarc_object_class
+        #self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/heasarc_match_gaia.py

@@ -0,0 +1,44 @@
+from django.core.management.base import BaseCommand, CommandError
+import time
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcBase
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from astrobasis.models import GAIADR3
+from heasarc.models import Heasarc4XMMDR10 
+from srglib.utils import find_counterparts
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+
+        srcs = Heasarc4XMMDR10.objects.all()#.filter(EP_1_FLUX__gt=1e-13)
+
+        maxdist=60.0
+        minrad=5.0
+
+        print("Taken from HEASARC: {}".format(srcs.count()))
+        start_time = time.time()
+        find_counterparts(srcs, GAIADR3.objects.all(), "gaia3", maxdist=maxdist, minrad=minrad, verbose=True)
+        print("--- %s seconds ---" % (time.time() - start_time))
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/heasarc_reference_paper.py

@@ -0,0 +1,41 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from nasa.models import Category
+from nasa.models import Ads
+
+from heasarc.models import HeasarcCSC
+from heasarc.models import Heasarc4XMMDR12
+from heasarc.models import HeasarcXMMSSC
+from heasarc.models import HeasarcRASS2RXS
+from heasarc.models import HeasarcXTEASSCAT
+from heasarc.models import Heasarc2SXPS
+from heasarc.models import Heasarc4XMMDR9
+from heasarc.models import Heasarc4XMMDR10
+
+def reference_paper(bibcode,survey):
+    category_open = Category.objects.get(slug="open")
+    #print(category_open)
+        
+    ads = Ads(bibcode=bibcode)
+    ads.save()
+    ads.category.add(category_open)
+
+    srcs = survey.objects.all()
+    print ("Selected {} total from {}".format(srcs.count(), survey))
+    srcs.update(ads=ads)
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        #reference_paper("2010ApJS..189...37E", HeasarcCSC)
+        #reference_paper("2020A&A...641A.136W",Heasarc4XMMDR12)
+        reference_paper("2020A&A...641A.136W",HeasarcXMMSSC)
+        reference_paper("2016A&A...588A.103B",HeasarcRASS2RXS)
+        reference_paper("2004A&A...418..927R",HeasarcXTEASSCAT)
+        reference_paper("2020ApJS..247...54E",Heasarc2SXPS)
+        reference_paper("2020A&A...641A.137T",Heasarc4XMMDR9)
+        reference_paper("2020A&A...641A.136W",Heasarc4XMMDR10)
+        self.stdout.write(self.style.SUCCESS('Done'))
+        

heasarc/management/commands/print_rass2rxs.py

@@ -0,0 +1,60 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcRASS2RXS, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def load_heasarc_table(filename):
+    tdatfile=INPUT_DATA_DIR+'/dump/'+filename
+    data = astropy.table.Table.read(tdatfile, format='ascii.tdat',encoding='latin1')
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        if not (ma.is_masked(item['ra']) or ma.is_masked(item['dec'])):
+            ra = float(item['ra'])
+            dec = float(item['dec'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            healpix = hp.skycoord_to_healpix(crd)
+        print("fk5;circle({}, {},0.01)".format(ra,dec))
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        load_heasarc_table('heasarc_rass2rxs.tdat.gz')
+        return
+
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+        # update healpix only
+        srcs = HeasarcXMMSSC.objects.all()
+        for src in srcs:
+            crd = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg")
+            src.healpix = hp.skycoord_to_healpix(crd)
+            src.save()
+
+        self.stdout.write(self.style.SUCCESS('Done'))

heasarc/management/commands/testme.py

@@ -0,0 +1,33 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcXrayMaster, HeasarcObjectClass
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+        
+        class_count=9999
+        try:
+            object_class = HeasarcObjectClass.objects.get(class_id=class_count)
+        except:
+            print("HeasarcObjectClass not found for class_id=", class_count)
+
+        print(object_class.class_name)
+
+        self.stdout.write(self.style.SUCCESS('Done'))