more apps

astrobasis/management/commands/00_astrobasis_2mass_healpix.py

@@ -0,0 +1,11 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from astrobasis.utils import astrobasis_assign_healpix
+from astrobasis.models import TwoMASS
+
+class Command(BaseCommand):
+    help = 'Initiates healpix for 2MASS'
+
+    def handle(self, *args, **options):
+        astrobasis_assign_healpix(TwoMASS)
+        self.stdout.write(self.style.SUCCESS('Done'))

astrobasis/management/commands/00_astrobasis_2mass_init.py

@@ -0,0 +1,88 @@
+from django.core.management.base import BaseCommand, CommandError
+import time
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from astrobasis.models import TwoMASS
+
+from srglib.utils import notnan
+from srglib.utils import boolean_notnan
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+
+def bulk_load_heasarc_table(filename,reset=False):
+    batch_size = 500
+
+    print("Delete all 2MASS")
+    if(reset):
+        srcs = TwoMASS.objects.all()
+        srcs.delete()
+
+    print("Loading {}".format(filename))
+    data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1')
+    print(data.info)
+    
+    print("Bulk load to DB")
+
+    """
+    cols = data.columns
+    for key, value in cols.items():
+        print(key, '->', value)
+    """
+
+ 
+    objs = (TwoMASS(name=data[i]['2MASS'],
+                    ra=float(data[i]['RAJ2000']),
+                    dec=float(data[i]['DEJ2000']),
+                    jmag = notnan(data[i]['Jmag']),
+                    e_jmag = notnan(data[i]['e_Jmag']),
+                    hmag = notnan(data[i]['Hmag']),
+                    e_hmag = notnan(data[i]['e_Hmag']),
+                    kmag = notnan(data[i]['Kmag']),
+                    e_kmag = notnan(data[i]['e_Kmag']),
+                    qkfg = data[i]['Qflg'],
+                    rkfg = data[i]['Rflg'],
+                    bkfg = data[i]['Bflg'],
+                    ckfg = data[i]['Cflg'],
+                    prox = notnan(data[i]['prox']),
+                    xflg = boolean_notnan(data[i]['Xflg']),
+                    aflg = boolean_notnan(data[i]['Aflg']),)
+            for i in range(len(data)))
+    TwoMASS.objects.bulk_create(objs,batch_size)
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+
+        start_time = time.time()
+        bulk_load_heasarc_table('/data/2MASS/l_0_180.tsv',reset=True)
+        print("--- %s seconds ---" % (time.time() - start_time))
+
+        start_time = time.time()
+        bulk_load_heasarc_table('/data/2MASS/l_180_360.tsv')
+        print("--- %s seconds ---" % (time.time() - start_time))
+
+        self.stdout.write(self.style.SUCCESS('Done'))

astrobasis/management/commands/00_astrobasis_glimpse_healpix.py

@@ -0,0 +1,11 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from astrobasis.utils import astrobasis_assign_healpix
+from astrobasis.models import GLIMPSE
+
+class Command(BaseCommand):
+    help = 'Initiates healpix for GLIMPSE'
+
+    def handle(self, *args, **options):
+        astrobasis_assign_healpix(GLIMPSE)
+        self.stdout.write(self.style.SUCCESS('Done'))

astrobasis/management/commands/00_astrobasis_glimpse_init.py

@@ -0,0 +1,159 @@
+from django.core.management.base import BaseCommand, CommandError
+import time
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from astrobasis.models import GLIMPSE
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def bulk_load_heasarc_table(filename,reset=False):
+    batch_size = 300
+
+    print("Delete all GLIMPSE")
+    if(reset):
+        tables = GLIMPSE.objects.all()
+        tables.delete()
+
+    print("Loading {}".format(filename))
+    data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1')
+    print(data.info)
+    print("Bulk load to DB")
+
+    """
+    cols = data.columns
+    for key, value in cols.items():
+        print(key, '->', value)
+    """
+
+    start_time = time.time()
+ 
+    objs = (GLIMPSE(name=data[i]['GLIMPSE'],
+                    ra=float(data[i]['RAJ2000']),
+                    dec=float(data[i]['DEJ2000']),
+                    twomass = data[i]['2MASS'],
+                    jmag = data[i]['Jmag'],
+                    hmag = data[i]['Hmag'],
+                    kmag = data[i]['Kmag'],
+                    b1mag = data[i]['3.6mag'],
+                    e_b1mag = data[i]['e_3.6mag'],
+                    b2mag = data[i]['4.5mag'],
+                    e_b2mag = data[i]['e_4.5mag'],
+                    b3mag = data[i]['5.8mag'],
+                    e_b3mag = data[i]['e_5.8mag'],
+                    b4mag = data[i]['8.0mag'],
+                    e_b4mag = data[i]['e_8.0mag'],)
+            for i in range(len(data)))
+    GLIMPSE.objects.bulk_create(objs,batch_size)
+    print("--- %s seconds ---" % (time.time() - start_time))
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+def bulk_healpix_update_tooslow():
+    glimpse_all = GLIMPSE.objects.all()
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    objs = []
+    start_time = time.time()
+    for obj in glimpse_all:
+        crd = SkyCoord(obj.ra, obj.dec, frame=FK5(), unit="deg")
+        lon = crd.galactic.l.value
+        lat = crd.galactic.b.value
+        healpix = hp.skycoord_to_healpix(crd)
+        obj.healpix = healpix
+        obj.glon = lon
+        obj.glat = lat
+        objs.append(obj)
+    GLIMPSE.objects.bulk_update(objs, ['healpix','glon','glat'], batch_size=1000)
+    print("--- %s seconds ---" % (time.time() - start_time))
+        
+def load_heasarc_table(filename,reset=False):
+
+    if(reset):
+        tables = GLIMPSE.objects.all()
+        tables.delete()
+
+    return
+
+    data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1')
+    print(data.info)
+    
+    """
+    cols = data.columns
+    for key, value in cols.items():
+        print(key, '->', value)
+    """
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+        
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+        if not (ma.is_masked(item['RAJ2000']) or ma.is_masked(item['DEJ2000'])):
+            ra = float(item['DEJ2000'])
+            dec = float(item['DEJ2000'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            lon = crd.galactic.l.value
+            lat = crd.galactic.b.value
+            healpix = hp.skycoord_to_healpix(crd)
+        else:
+            continue
+        
+
+        obj = GLIMPSE.objects.create(healpix=healpix, 
+                                            ra=ra,
+                                            dec=dec,
+                                            glon = lon,
+                                            glat = lat,
+                                            name=item['GLIMPSE'])
+        twomass = item['2MASS']
+        jmag = item['Jmag']
+        hmag = item['Hmag']
+        kmag = item['Kmag']
+        b1mag = item['3.6mag']
+        e_b1mag = item['e_3.6mag']
+        b2mag = item['4.5mag']
+        e_b2mag = item['e_4.5mag']
+        b3mag = item['5.8mag']
+        e_b3mag = item['e_5.8mag']
+        b4mag = item['8.0mag']
+        e_b4mag = item['e_8.0mag']
+        obj.save()
+#C;GLIMPSE;2MASS;RAJ2000;DEJ2000;Jmag;Hmag;Kmag;3.6mag;e_3.6mag;4.5mag;e_4.5mag;5.8mag;e_5.8mag;8.0mag;e_8.0mag
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+
+        start_time = time.time()
+        bulk_load_heasarc_table('/data/GLIMPSE/glimpse_spitzer.tsv',reset=True)
+        print("--- %s seconds ---" % (time.time() - start_time))
+        # Model.objects.filter(amount__isnull=True).update(amount=F('pre_tax') * 1.2)
+
+
+        self.stdout.write(self.style.SUCCESS('Done'))

astrobasis/management/commands/00_astrobasis_vlass_healpix.py

@@ -0,0 +1,76 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.core.paginator import Paginator
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import VLASS
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+from heasarc.models import NSIDE_SOURCES, ORDER
+
+import os
+import glob
+import logging
+import time
+
+class Command(BaseCommand):
+    help = 'Initiates healpix for Gaia'
+
+    def handle(self, *args, **options):
+        start_time = time.time()
+        hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5)
+
+        
+        #srcs = VLASS.objects.all().filter(healpix__gt=0)[:50]
+        #for src in srcs:
+        #    crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg")
+        #    heal = hp.skycoord_to_healpix(crd)
+        #    print(src.ra,src.dec,src.healpix,heal)
+        #return
+        
+
+        srcs = VLASS.objects.all()#.filter(healpix__exact=0)
+        paginator = Paginator(srcs, 50000)
+        for page_idx in range(1, paginator.num_pages):
+            start_time0 = time.time()
+            ra=paginator.page(page_idx).object_list.values_list('ra', flat=True)
+            dec=paginator.page(page_idx).object_list.values_list('dec', flat=True)
+            crd = SkyCoord(ra, dec, frame="fk5", unit="deg")
+            lon = crd.galactic.l.value
+            lat = crd.galactic.b.value
+            heal = hp.skycoord_to_healpix(crd)
+            index=0
+            updates=[]
+            for src in paginator.page(page_idx).object_list:
+                # here you can do what you want with the row
+                #crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg")
+                #lon = crd.galactic.l.value
+                #lat = crd.galactic.b.value
+                #heal = hp.skycoord_to_healpix(crd)
+                src.healpix=heal[index]
+                src.glon=lon[index]
+                src.glat=lat[index]
+                #src.save()
+                updates.append(src)
+                #print(src.ra,src.dec,heal)
+                index=index+1
+            VLASS.objects.bulk_update(updates, ["healpix","glon","glat"], batch_size=200)
+            elapsed = time.time() - start_time0
+            estimated = (elapsed*(paginator.num_pages-page_idx))/60/60
+            print("Page {}/{} for {:.2f} sec, est. {:.2f} hours".format(page_idx,paginator.num_pages,elapsed,estimated))
+
+        print("--- {:.2f} hours ---".format((time.time() - start_time)/60/60))

astrobasis/management/commands/00_astrobasis_vlass_init.py

@@ -0,0 +1,161 @@
+from django.core.management.base import BaseCommand, CommandError
+import time
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+from heasarc.tdat import tDat
+from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass
+from heasarc.models import NSIDE_SOURCES, ORDER 
+from astrobasis.models import VLASS
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+def bulk_load_heasarc_table(filename,reset=False):
+    batch_size = 500
+
+    print("Delete all VLASS")
+
+    if(reset):
+        tables = VLASS.objects.all()
+        tables.delete()
+
+    print("Loading {}".format(filename))
+    data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1')
+    #print(data.info)
+    
+
+    
+    for col in data.columns:
+        print("--> ",col)
+    
+    
+    start_time = time.time()
+    print("Bulk load to DB")
+
+    objs = (VLASS(name=data[i]['CompName'],
+                  glon=float(data[i]['_Glon']),
+                  glat=float(data[i]['_Glat']),
+                  ra_m=float(data[i]['RAMdeg']),
+                  dec_m=float(data[i]['DEMdeg']),
+                  ra=float(data[i]['RAJ2000']),
+                  e_ra=float(data[i]['e_RAJ2000']),
+                  dec=float(data[i]['DEJ2000']),
+                  e_dec=float(data[i]['e_DEJ2000']),
+                  ftot = data[i]['Ftot'],
+                  e_ftot = data[i]['e_Ftot'],
+                  fpeak = data[i]['Fpeak'],
+                  e_fpeak = data[i]['e_Fpeak'],
+                  dupflag = int(data[i]['DupFlag']),
+                  qualflag = int(data[i]['QualFlag']))
+            for i in range(len(data)))
+    VLASS.objects.bulk_create(objs,batch_size)
+    print("--- %s seconds ---" % (time.time() - start_time))
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+def bulk_healpix_update_tooslow():
+    glimpse_all = GLIMPSE.objects.all()
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    objs = []
+    start_time = time.time()
+    for obj in glimpse_all:
+        crd = SkyCoord(obj.ra, obj.dec, frame=FK5(), unit="deg")
+        lon = crd.galactic.l.value
+        lat = crd.galactic.b.value
+        healpix = hp.skycoord_to_healpix(crd)
+        obj.healpix = healpix
+        obj.glon = lon
+        obj.glat = lat
+        objs.append(obj)
+    GLIMPSE.objects.bulk_update(objs, ['healpix','glon','glat'], batch_size=1000)
+    print("--- %s seconds ---" % (time.time() - start_time))
+        
+def load_heasarc_table(filename,reset=False):
+
+    if(reset):
+        tables = GLIMPSE.objects.all()
+        tables.delete()
+
+    return
+
+    data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1')
+    print(data.info)
+    
+    """
+    cols = data.columns
+    for key, value in cols.items():
+        print(key, '->', value)
+    """
+
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5())
+    for item in data:
+        
+        healpix = 0
+        ra = 0.0
+        dec = 0.0
+        lon = 0.0
+        lat = 0.0
+        if not (ma.is_masked(item['RAJ2000']) or ma.is_masked(item['DEJ2000'])):
+            ra = float(item['DEJ2000'])
+            dec = float(item['DEJ2000'])
+            crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+            lon = crd.galactic.l.value
+            lat = crd.galactic.b.value
+            healpix = hp.skycoord_to_healpix(crd)
+        else:
+            continue
+        
+
+        obj = GLIMPSE.objects.create(healpix=healpix, 
+                                            ra=ra,
+                                            dec=dec,
+                                            glon = lon,
+                                            glat = lat,
+                                            name=item['GLIMPSE'])
+        twomass = item['2MASS']
+        jmag = item['Jmag']
+        hmag = item['Hmag']
+        kmag = item['Kmag']
+        b1mag = item['3.6mag']
+        e_b1mag = item['e_3.6mag']
+        b2mag = item['4.5mag']
+        e_b2mag = item['e_4.5mag']
+        b3mag = item['5.8mag']
+        e_b3mag = item['e_5.8mag']
+        b4mag = item['8.0mag']
+        e_b4mag = item['e_8.0mag']
+        obj.save()
+#C;GLIMPSE;2MASS;RAJ2000;DEJ2000;Jmag;Hmag;Kmag;3.6mag;e_3.6mag;4.5mag;e_4.5mag;5.8mag;e_5.8mag;8.0mag;e_8.0mag
+
+    print('--> Successfully loaded "%s"' % filename)
+    pass
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+#    def add_arguments(self, parser):
+#        parser.add_argument('poll_id', nargs='+', type=int)
+
+    def handle(self, *args, **options):
+
+        start_time = time.time()
+        bulk_load_heasarc_table('/data/VizieR/VLASS_radio.tsv',reset=True)
+        #bulk_healpix_update_tooslow()
+        print("--- %s seconds ---" % (time.time() - start_time))
+        # Model.objects.filter(amount__isnull=True).update(amount=F('pre_tax') * 1.2)
+
+
+        self.stdout.write(self.style.SUCCESS('Done'))

astrobasis/management/commands/00_init_gaia.py

@@ -0,0 +1,54 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import GaiaSourceFileDR3, GAIADR3
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+
+import os
+import glob
+import logging
+
+def init_gaia(filepath):
+
+    filename_w_ext = os.path.basename(filepath)
+    filename, file_extension = os.path.splitext(filename_w_ext)
+
+    gaia = GaiaSourceFileDR3(filename=filename)
+    gaia.save()
+
+    # print('--> Successfully initiated "%s"' % filename)
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+
+        gaia_all = GAIADR3.objects.all()
+        gaia_all.delete()
+
+        files = []
+        for file in glob.glob("/data/tmp/george/Gaia/gedr3/gaia_source/fits/fits/*.fits"):
+            files.append(file)
+
+        for file in files:
+            print(file)
+            init_gaia(file)
+
+        self.stdout.write(self.style.SUCCESS('Done'))

astrobasis/management/commands/00_load_ztf_alerce.py

@@ -0,0 +1,179 @@
+from django.core.management.base import BaseCommand, CommandError
+
+import os
+import io
+import gzip
+import tarfile
+import warnings
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from avro.datafile import DataFileReader, DataFileWriter
+from avro.io import DatumReader, DatumWriter
+import fastavro
+
+from astropy.time import Time
+from astropy.time import Time, TimezoneInfo
+from astropy.io import fits
+import astropy.units as u
+import aplpy
+
+from astropy.time import Time
+from django.utils.timezone import localdate, localtime, now
+
+from collections import defaultdict
+
+import logging
+import requests 
+import json
+import pprint
+
+from astrobasis.models import ALeRCE
+from srglib.ztf import find_ztf_in_survey, load_ztf_alerce
+
+from astropy.coordinates import SkyCoord # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude # Angles
+import astropy.units as u
+import re
+import json
+import pandas as pd
+from astropy_healpix import HEALPix
+from erotrans.models import EroTransSource, EroTrans
+from erosurvey.models import NSIDE_PLATES, ORDER_PLATES
+from heasarc.models import NSIDE_SOURCES, ORDER
+from monthplan.models import Survey
+
+def load_ztf_alerce_to_django_local(item):
+    utc_moscow = TimezoneInfo(utc_offset=3*u.hour)
+    hp = HEALPix(nside=NSIDE_SOURCES,
+                 order=ORDER, frame=FK5())
+
+    hp_plates = HEALPix(nside=NSIDE_PLATES,
+                 order=ORDER_PLATES, frame=FK5())
+
+    try:
+        ztf = ALeRCE.objects.get(oid=item['oid'])
+        print("ALeRCE ZTF alert %s is already loaded, skip" % item['oid'])
+        return
+    except ALeRCE.DoesNotExist:
+        print("ALeRCE ZTF alert %s is not loaded" % item['oid'])
+        pass
+    
+    ra = float(item['meanra'])
+    dec = float(item['meandec'])
+    crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+    healpix = hp.skycoord_to_healpix(crd)
+    healpix_plate = hp_plates.skycoord_to_healpix(crd)
+
+    first_tm = Time(item['firstmjd'], format='mjd', scale='utc')
+    last_tm  = Time(item['lastmjd'], format='mjd', scale='utc')
+    first_dt = first_tm.to_datetime(timezone=utc_moscow)
+    last_dt  = last_tm.to_datetime(timezone=utc_moscow)
+
+    ztf = ALeRCE(
+        healpix  = healpix,
+        healpix_plate  = healpix_plate,
+        classearly = item['classearly'],
+        classrf = item['classrf'],
+        oid = item['oid'],
+        firstdate = first_dt,
+        lastdate = last_dt,
+        firstmjd = item['firstmjd'],
+        lastmjd = item['lastmjd'],
+        mean_magap_g = item['mean_magap_g'],
+        mean_magap_r = item['mean_magap_r'],
+        mean_magpsf_g = item['mean_magpsf_g'],
+        mean_magpsf_r = item['mean_magpsf_r'],
+        dec = dec,
+        ra = ra,
+        nobs = item['nobs'],
+        pclassearly = item['pclassearly'],
+        pclassrf = item['pclassrf'],
+        sigma_magap_g = item['sigma_magap_g'],
+        sigma_magap_r = item['sigma_magap_r'],
+        sigma_magpsf_g = item['sigma_magpsf_g'],
+        sigma_magpsf_r = item['sigma_magpsf_r'],
+        sigmadec = item['sigmadec'],
+        sigmara = item['sigmara'])
+        
+    ztf.save()
+    find_ztf_in_survey(ztf)
+
+def load_ztf_alerce_local(days, extra=None):
+    classrf = "sn ia"
+    pclassrf = 0.1
+    pclassearly = 0.1
+    sortBy = "firstmjd"
+    nobsmin = 2
+    nobsmax = 40
+    url = "https://ztf.alerce.online/query"
+    
+    current_utc = now()
+    current_mjd = Time(current_utc).mjd
+    firstmjd_min = current_mjd - days
+
+    post={
+        "sortBy": sortBy,
+        "records_per_pages": 100,
+        "query_parameters":{
+            "filters": {
+            "nobs": { 
+                "min": nobsmin,
+                "max": nobsmax
+            },
+            #"classrf": classrf,
+            #"pclassrf": pclassrf,
+            'pclassearly':pclassearly,
+            },
+            "dates":{
+                "firstmjd":{
+                    "min": firstmjd_min
+                }
+            }
+        }
+    }
+    
+    if(extra):
+        post['query_parameters']['filters'].update(extra)
+
+    #pprint.pprint(post)
+    
+    r = requests.post(url = url, json = post) 
+    data = r.json() 
+    for item in data['result']:
+        """ loads first page """
+        packet=data['result'][item]
+        #pprint.pprint(packet['oid'])
+        load_ztf_alerce_to_django_local(packet)
+
+    total=int(data['total'])
+    num_pages=int(data['num_pages'])
+    page=int(data['page'])
+    
+    print('total ',total,'num_pages ',num_pages,'page ', page)
+    #return
+    pages = list(range(2,num_pages+1))
+    for page in pages:
+        """ loads all other pages, staring from 2 """
+        post.update( {'page':page,} )
+        pprint.pprint(post)
+        r = requests.post(url = url, json = post) 
+        data = r.json() 
+        for item in data['result']:
+            packet=data['result'][item]
+            #pprint.pprint(packet['oid'])
+            load_ztf_alerce_to_django_local(packet)
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        #all = ALeRCE.objects.all()
+        #all.delete()
+
+        load_ztf_alerce_local(2)
+
+        print('Done')

astrobasis/management/commands/00_load_ztf_archive.py

@@ -0,0 +1,256 @@
+from django.core.management.base import BaseCommand, CommandError
+
+import os
+import io
+import gzip
+import tarfile
+import warnings
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from avro.datafile import DataFileReader, DataFileWriter
+from avro.io import DatumReader, DatumWriter
+import fastavro
+
+from astropy.time import Time
+from astropy.io import fits
+import astropy.units as u
+import aplpy
+
+from collections import defaultdict
+
+import logging
+import requests 
+import json
+import pprint
+
+from astrobasis.models import ZTFAlert
+from srglib.utils import find_ztf_in_survey
+
+from astropy.coordinates import SkyCoord # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude # Angles
+import astropy.units as u
+import re
+import json
+import pandas as pd
+from astropy_healpix import HEALPix
+from erotrans.models import EroTransSource, EroTrans
+from erosurvey.models import NSIDE_PLATES, ORDER_PLATES
+from heasarc.models import NSIDE_SOURCES, ORDER
+from monthplan.models import Survey
+
+
+"""
+https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html
+
+Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample:
+
+rb >= 0.65 and
+nbad = 0 and
+fwhm <= 5 and
+elong <= 1.2 and
+abs(magdiff) <= 0.1
+"""
+
+def plot_lightcurve(dflc, ax=None, days_ago=True):
+    
+    filter_color = {1:'green', 2:'red', 3:'pink'}
+    if days_ago:
+        now = Time.now().jd
+        t = dflc.jd - now
+        xlabel = 'Days Ago'
+    else:
+        t = dflc.jd
+        xlabel = 'Time (JD)'
+    
+    if ax is None:
+        plt.figure()
+    for fid, color in filter_color.items():
+        # plot detections in this filter:
+        w = (dflc.fid == fid) & ~dflc.magpsf.isnull()
+        if np.sum(w):
+            plt.errorbar(t[w],dflc.loc[w,'magpsf'], dflc.loc[w,'sigmapsf'],fmt='.',color=color)
+        wnodet = (dflc.fid == fid) & dflc.magpsf.isnull()
+        if np.sum(wnodet):
+            plt.scatter(t[wnodet],dflc.loc[wnodet,'diffmaglim'], marker='v',color=color,alpha=0.25)
+    
+    plt.gca().invert_yaxis()
+    plt.xlabel(xlabel)
+    plt.ylabel('Magnitude')
+    plt.title(dflc.objectId)
+
+def plot_cutout(stamp, fig=None, subplot=None, **kwargs):
+    with gzip.open(io.BytesIO(stamp), 'rb') as f:
+        with fits.open(io.BytesIO(f.read())) as hdul:
+            if fig is None:
+                fig = plt.figure(figsize=(4,4))
+            if subplot is None:
+                subplot = (1,1,1)
+            ffig = aplpy.FITSFigure(hdul[0],figure=fig, subplot=subplot, **kwargs)
+            ffig.show_grayscale(stretch='arcsinh')
+    return ffig
+
+def show_stamps(packet):
+    #fig, axes = plt.subplots(1,3, figsize=(12,4))
+    fig = plt.figure(figsize=(12,4))
+    for i, cutout in enumerate(['Science','Template','Difference']):
+        stamp = packet['cutout{}'.format(cutout)]['stampData']
+        ffig = plot_cutout(stamp, fig=fig, subplot = (1,3,i+1))
+        ffig.set_title(cutout)
+
+def show_all(packet):
+    fig = plt.figure(figsize=(16,4))
+    dflc = make_dataframe(packet)
+    plot_lightcurve(dflc,ax = plt.subplot(1,4,1))
+    for i, cutout in enumerate(['Science','Template','Difference']):
+        stamp = packet['cutout{}'.format(cutout)]['stampData']
+        ffig = plot_cutout(stamp, fig=fig, subplot = (1,4,i+2))
+        ffig.set_title(cutout)
+
+def make_dataframe(packet):
+    dfc = pd.DataFrame(packet['candidate'], index=[0])
+    df_prv = pd.DataFrame(packet['prv_candidates'])
+    dflc = pd.concat([dfc,df_prv], ignore_index=True, sort=False)
+    # we'll attach some metadata--not this may not be preserved after all operations
+    # https://stackoverflow.com/questions/14688306/adding-meta-information-metadata-to-pandas-dataframe
+    dflc.objectId = packet['objectId']
+    dflc.candid = packet['candid']
+    return dflc
+
+def is_alert_pure(packet):
+    pure = True
+    pure &= packet['candidate']['rb'] >= 0.65
+    pure &= packet['candidate']['nbad'] == 0
+    pure &= packet['candidate']['fwhm'] <= 5
+    pure &= packet['candidate']['elong'] <= 1.2
+    pure &= np.abs(packet['candidate']['magdiff']) <= 0.1
+    return pure
+
+def generate_dictionaries(root_dir):
+    for fname in find_files(root_dir):
+        for packet in open_avro(fname):
+            yield packet
+
+def open_avro(fname):
+    with open(fname,'rb') as f:
+        freader = fastavro.reader(f)
+        # in principle there can be multiple packets per file
+        for packet in freader:
+            yield packet
+
+def find_files(root_dir):
+    for dir_name, subdir_list, file_list in os.walk(root_dir, followlinks=True):
+        for fname in file_list:
+            if fname.endswith('.avro'):
+                yield dir_name+'/'+fname
+
+def is_transient(dflc):
+    
+    candidate = dflc.loc[0]
+    
+    is_positive_sub = candidate['isdiffpos'] == 't'
+    
+    if (candidate['distpsnr1'] is None) or (candidate['distpsnr1'] > 1.5):
+        no_pointsource_counterpart = True
+    else:
+        if candidate['sgscore1'] < 0.5:
+            no_pointsource_counterpart = True
+        else:
+            no_pointsource_counterpart = False
+            
+    where_detected = (dflc['isdiffpos'] == 't') # nondetections will be None
+    if np.sum(where_detected) >= 2:
+        detection_times = dflc.loc[where_detected,'jd'].values
+        dt = np.diff(detection_times)
+        not_moving = np.max(dt) >= (30*u.minute).to(u.day).value
+    else:
+        not_moving = False
+    
+    no_ssobject = (candidate['ssdistnr'] is None) or (candidate['ssdistnr'] < 0) or (candidate['ssdistnr'] > 5)
+    
+    return is_positive_sub and no_pointsource_counterpart and not_moving and no_ssobject
+
+def load_ztf_packet(item):
+    hp = HEALPix(nside=NSIDE_SOURCES,
+                 order=ORDER, frame=FK5())
+
+    hp_plate = HEALPix(nside=NSIDE_PLATES,
+                       order=ORDER_PLATES,
+                       frame=FK5())
+
+    name = item['objectId']
+    
+    
+    try:
+        ztf = ZTFAlert.objects.get(objectId=name)
+        print("ZTF alert ID %s is already loaded, skip" % name)
+        return
+    except ZTFAlert.DoesNotExist:
+        print("ZTF alert ID %s is not loaded" % name)
+        pass
+        #date_string = item['candidate']['wall_time']
+        #dt = datetime.strptime(date_string, format_string)
+        ra = item['candidate']['ra']
+        dec = item['candidate']['dec']
+        crd = SkyCoord(ra, dec, frame=FK5(), unit="deg")
+        healpix = hp.skycoord_to_healpix(crd)
+        healpix_plate = hp_plate.skycoord_to_healpix(crd)
+        ztf = ZTFAlert(
+            objectId = item['objectId'],
+            fid = item['candidate']['fid'],
+            lco_id = item['candid'],
+            healpix  = healpix, healpix_plate  = healpix_plate, 
+            ra = ra, dec = dec,
+            programid = item['candidate']['programid'],
+            magpsf = item['candidate']['magpsf'],
+            sigmapsf = item['candidate']['sigmapsf'],
+            magap = item['candidate']['magap'],
+            magdiff = item['candidate']['magdiff'],
+            nbad = item['candidate']['nbad'],
+            fwhm = item['candidate']['fwhm'],
+            elong = item['candidate']['elong'],
+            sigmagap = item['candidate']['sigmagap'],
+            #wall_time = datetime.strptime(date_string, format_string),
+            diffmaglim = item['candidate']['diffmaglim'],
+            #deltamagref = item['candidate']['deltamagref'],
+            #deltamaglatest = item['candidate']['deltamaglatest'],
+            rb = item['candidate']['rb'])
+        ztf.save()
+        
+        find_ztf_in_survey(ztf)
+
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+
+        tar_archive = '/data/ZTF/ztf_public_20200126.tar.gz'
+
+        output_dir = tar_archive.split('/')[-1].split('.')[-3]
+
+                
+        #archive = tarfile.open(tar_archive,'r:gz')
+        #archive.extractall(path=output_dir)
+        #archive.close()
+
+        print (output_dir)
+        print('{} has {} avro files'.format(output_dir, len(list(find_files(output_dir)))))
+
+        transient_alerts = []
+        for packet in filter(is_alert_pure,generate_dictionaries(output_dir)):
+            dflc = make_dataframe(packet)
+            if is_transient(dflc):
+                transient_alerts.append(packet)
+                load_ztf_packet(packet)
+                
+
+        """
+        for packet in transient_alerts:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                show_all(packet)
+        """

astrobasis/management/commands/00_load_ztf_mars.py

@@ -0,0 +1,65 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from astropy.coordinates import SkyCoord # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude # Angles
+import astropy.units as u
+import re
+import json
+import pandas as pd
+from astropy_healpix import HEALPix
+
+import os
+import io
+import gzip
+import tarfile
+import warnings
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from avro.datafile import DataFileReader, DataFileWriter
+from avro.io import DatumReader, DatumWriter
+import fastavro
+
+from astropy.time import Time
+from astropy.io import fits
+import astropy.units as u
+import aplpy
+
+from astrobasis.models import ztf_filter
+
+from collections import defaultdict
+
+import logging
+import requests 
+import json
+import pprint
+
+from astrobasis.models import ZTFAlert
+from heasarc.models import NSIDE_SOURCES, ORDER, HeasarcBase
+from srglib.utils import load_ztf_alerts
+
+"""
+https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html
+
+Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample:
+
+rb >= 0.65 and
+nbad = 0 and
+fwhm <= 5 and
+elong <= 1.2 and
+abs(magdiff) <= 0.1
+"""
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        
+
+        all = ZTFAlert.objects.all()
+        all.delete()
+
+        #load_ztf_alerts()

astrobasis/management/commands/00_status_gaia.py

@@ -0,0 +1,52 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import GaiaSourceFileDR3, GAIADR3
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+
+import os
+import glob
+import logging
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+
+        total_empty=GaiaSourceFileDR3.objects.filter(status__exact='empty').count()
+        print("Empty %d" % total_empty)
+
+        total_ready=GaiaSourceFileDR3.objects.filter(status__exact='ready').count()
+        print("Ready %d" % total_ready)
+        
+        total_loaded=GaiaSourceFileDR3.objects.filter(status__exact='loaded').count()
+        print("Loaded %d" % total_loaded)
+
+        total_healpix=GaiaSourceFileDR3.objects.filter(status__exact='healpix').count()
+        print("Healpix %d" % total_healpix)
+
+        total=GaiaSourceFileDR3.objects.count()
+        print("Total %d = %d" % (total, total_empty+total_loaded+total_healpix))
+
+        #total=GAIADR3.objects.count()
+        #print("Total Gaia %d" % total)
+
+        return
+
+

astrobasis/management/commands/00_test_gaia.py

@@ -0,0 +1,39 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import GaiaSourceFile, GAIADR3
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+import time
+
+
+import os
+import glob
+import logging
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+
+        start_time = time.time()
+        srcs=GAIADR3.objects.filter(healpix=38837101).count()
+        #for src in srcs:            
+        #    print(src.name)
+        print("Total Gaia found %d" % srcs)
+        print("--- %s seconds ---" % (time.time() - start_time))
+

astrobasis/management/commands/00_test_ztf.py

@@ -0,0 +1,221 @@
+from django.core.management.base import BaseCommand, CommandError
+
+import os
+import io
+import gzip
+import tarfile
+import warnings
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from avro.datafile import DataFileReader, DataFileWriter
+from avro.io import DatumReader, DatumWriter
+import fastavro
+
+from astropy.time import Time
+from astropy.io import fits
+import astropy.units as u
+import aplpy
+
+from collections import defaultdict
+
+import requests 
+import json
+import pprint
+
+"""
+https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html
+
+Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample:
+
+rb >= 0.65 and
+nbad = 0 and
+fwhm <= 5 and
+elong <= 1.2 and
+abs(magdiff) <= 0.1
+"""
+
+def plot_lightcurve(dflc, ax=None, days_ago=True):
+    
+    filter_color = {1:'green', 2:'red', 3:'pink'}
+    if days_ago:
+        now = Time.now().jd
+        t = dflc.jd - now
+        xlabel = 'Days Ago'
+    else:
+        t = dflc.jd
+        xlabel = 'Time (JD)'
+    
+    if ax is None:
+        plt.figure()
+    for fid, color in filter_color.items():
+        # plot detections in this filter:
+        w = (dflc.fid == fid) & ~dflc.magpsf.isnull()
+        if np.sum(w):
+            plt.errorbar(t[w],dflc.loc[w,'magpsf'], dflc.loc[w,'sigmapsf'],fmt='.',color=color)
+        wnodet = (dflc.fid == fid) & dflc.magpsf.isnull()
+        if np.sum(wnodet):
+            plt.scatter(t[wnodet],dflc.loc[wnodet,'diffmaglim'], marker='v',color=color,alpha=0.25)
+    
+    plt.gca().invert_yaxis()
+    plt.xlabel(xlabel)
+    plt.ylabel('Magnitude')
+    plt.title(dflc.objectId)
+
+def plot_cutout(stamp, fig=None, subplot=None, **kwargs):
+    with gzip.open(io.BytesIO(stamp), 'rb') as f:
+        with fits.open(io.BytesIO(f.read())) as hdul:
+            if fig is None:
+                fig = plt.figure(figsize=(4,4))
+            if subplot is None:
+                subplot = (1,1,1)
+            ffig = aplpy.FITSFigure(hdul[0],figure=fig, subplot=subplot, **kwargs)
+            ffig.show_grayscale(stretch='arcsinh')
+    return ffig
+
+def show_stamps(packet):
+    #fig, axes = plt.subplots(1,3, figsize=(12,4))
+    fig = plt.figure(figsize=(12,4))
+    for i, cutout in enumerate(['Science','Template','Difference']):
+        stamp = packet['cutout{}'.format(cutout)]['stampData']
+        ffig = plot_cutout(stamp, fig=fig, subplot = (1,3,i+1))
+        ffig.set_title(cutout)
+
+def show_all(packet):
+    fig = plt.figure(figsize=(16,4))
+    dflc = make_dataframe(packet)
+    plot_lightcurve(dflc,ax = plt.subplot(1,4,1))
+    for i, cutout in enumerate(['Science','Template','Difference']):
+        stamp = packet['cutout{}'.format(cutout)]['stampData']
+        ffig = plot_cutout(stamp, fig=fig, subplot = (1,4,i+2))
+        ffig.set_title(cutout)
+
+def make_dataframe(packet):
+    dfc = pd.DataFrame(packet['candidate'], index=[0])
+    df_prv = pd.DataFrame(packet['prv_candidates'])
+    dflc = pd.concat([dfc,df_prv], ignore_index=True, sort=False)
+    # we'll attach some metadata--not this may not be preserved after all operations
+    # https://stackoverflow.com/questions/14688306/adding-meta-information-metadata-to-pandas-dataframe
+    dflc.objectId = packet['objectId']
+    dflc.candid = packet['candid']
+    return dflc
+
+def is_alert_pure(packet):
+    pure = True
+    pure &= packet['candidate']['rb'] >= 0.65
+    pure &= packet['candidate']['nbad'] == 0
+    pure &= packet['candidate']['fwhm'] <= 5
+    pure &= packet['candidate']['elong'] <= 1.2
+    pure &= np.abs(packet['candidate']['magdiff']) <= 0.1
+    return pure
+
+def generate_dictionaries(root_dir):
+    for fname in find_files(root_dir):
+        for packet in open_avro(fname):
+            yield packet
+
+def open_avro(fname):
+    with open(fname,'rb') as f:
+        freader = fastavro.reader(f)
+        # in principle there can be multiple packets per file
+        for packet in freader:
+            yield packet
+
+def find_files(root_dir):
+    for dir_name, subdir_list, file_list in os.walk(root_dir, followlinks=True):
+        for fname in file_list:
+            if fname.endswith('.avro'):
+                yield dir_name+'/'+fname
+
+def is_transient(dflc):
+    
+    candidate = dflc.loc[0]
+    
+    is_positive_sub = candidate['isdiffpos'] == 't'
+    
+    if (candidate['distpsnr1'] is None) or (candidate['distpsnr1'] > 1.5):
+        no_pointsource_counterpart = True
+    else:
+        if candidate['sgscore1'] < 0.5:
+            no_pointsource_counterpart = True
+        else:
+            no_pointsource_counterpart = False
+            
+    where_detected = (dflc['isdiffpos'] == 't') # nondetections will be None
+    if np.sum(where_detected) >= 2:
+        detection_times = dflc.loc[where_detected,'jd'].values
+        dt = np.diff(detection_times)
+        not_moving = np.max(dt) >= (30*u.minute).to(u.day).value
+    else:
+        not_moving = False
+    
+    no_ssobject = (candidate['ssdistnr'] is None) or (candidate['ssdistnr'] < 0) or (candidate['ssdistnr'] > 5)
+    
+    return is_positive_sub and no_pointsource_counterpart and not_moving and no_ssobject
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+
+        query_86400='https://mars.lco.global/?format=json&sort_value=jd&sort_order=desc&objectId=&candid=&time__gt=&time__lt=&time__since=86400&jd__gt=&jd__lt=&filter=&cone=&objectcone=&objectidps=&ra__gt=&ra__lt=&dec__gt=&dec__lt=&l__gt=&l__lt=&b__gt=&b__lt=&magpsf__gte=&magpsf__lte=&sigmapsf__lte=&magap__gte=&magap__lte=&distnr__gte=&distnr__lte=&deltamaglatest__gte=&deltamaglatest__lte=&deltamagref__gte=&deltamagref__lte=&elong__lte=&nbad__lte=&rb__gte=&drb__gte=&classtar__gte=&classtar__lte=&fwhm__lte='
+
+        URL = "https://mars.lco.global/"
+  
+        # defining a params dict for the parameters to be sent to the API 
+        #data =  {"queries": [{"time_since": 100,},]}
+        
+        PARAMS={'format':'json', 'time__since':86400}
+
+        # sending get request and saving the response as response object 
+        r = requests.get(url = URL, params = PARAMS) 
+
+
+        # extracting data in json format 
+        data = r.json() 
+        for d in data:
+            print(d)
+            
+
+        #pprint.pprint(data['results'])
+
+        for item in data['results']:
+            print(item['lco_id'],item['objectId'],item['candidate']['wall_time'])
+
+        return
+
+        tar_archive = '/export/django/srg/data/ZTF/ztf_public_20200125.tar.gz'
+        output_dir = tar_archive.split('/')[-1].split('.')[-3]
+        print (output_dir)
+
+        print('{} has {} avro files'.format(output_dir, len(list(find_files(output_dir)))))
+        
+        
+        #archive = tarfile.open(tar_archive,'r:gz')
+        #archive.extractall(path=output_dir)
+        #archive.close()
+
+        programs = defaultdict(int)
+        #for packet in generate_dictionaries(output_dir):
+        #    programs[packet['candidate']['programid']] += 1
+        #print(programs)        
+        
+        for packet in filter(is_alert_pure,generate_dictionaries(output_dir)):
+            programs[packet['candidate']['programid']] += 1
+        print(programs)
+
+        transient_alerts = []
+        for packet in filter(is_alert_pure,generate_dictionaries(output_dir)):
+            dflc = make_dataframe(packet)
+            if is_transient(dflc):
+                print(packet['objectId'],packet['candidate']['ra'],packet['candidate']['dec'])
+                transient_alerts.append(packet)
+
+        """
+        for packet in transient_alerts:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                show_all(packet)
+        """

astrobasis/management/commands/01_load_gaia.py

@@ -0,0 +1,71 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import GaiaSourceFile, GAIADR2
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+import os
+import glob
+import logging
+
+def load_gaia_bulk():
+
+    path='/data/Gaia/gaia_source/csv/'
+    batch_size = 500
+    gaia_empty=GaiaSourceFile.objects.filter(status__exact='empty')
+    for gaia in gaia_empty:
+        srcs = gaia.gaiadr2_set.all()
+        if(srcs):
+            srcs.delete()
+            print("Clear ",gaia.filename," back ref: ",len(srcs))
+        
+        filepath=path+gaia.filename+'.gz'
+        data = astropy.table.Table.read(filepath, format='ascii.csv',encoding='latin1')
+        gaia.nrows = len(data)
+        gaia.save()
+        print("Bulk create for %s" % gaia.filename)
+        objs = (GAIADR2(filename=gaia,
+                        ra=data[i]['ra'],
+                        dec=data[i]['dec'],
+                        name=data[i]['designation'],
+                        solution_id = data[i]['solution_id'],
+                        source_id = data[i]['source_id'],
+                        ref_epoch = data[i]['ref_epoch'],
+                        ra_error = data[i]['ra_error'],
+                        dec_error = data[i]['dec_error'],
+                        parallax = data[i]['parallax'],
+                        parallax_error = data[i]['parallax_error'],
+                        pmra = data[i]['pmra'],
+                        pmra_error = data[i]['pmra_error'],
+                        pmdec = data[i]['pmdec'],
+                        pmdec_error = data[i]['pmdec_error'],
+                        phot_g_mean_mag = data[i]['phot_g_mean_mag'],
+                        phot_bp_mean_mag = data[i]['phot_bp_mean_mag'],
+                        phot_rp_mean_mag = data[i]['phot_rp_mean_mag'],)
+                for i in range(len(data)))
+
+        GAIADR2.objects.bulk_create(objs,batch_size)
+
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        load_gaia_bulk()
+        

astrobasis/management/commands/01_load_gaia_dr3.py

@@ -0,0 +1,79 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+import time
+
+from astrobasis.models import GaiaSourceFileDR3, GAIADR3
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+import os
+import glob
+import logging
+
+def load_gaia_bulk():
+    start_time = time.time()
+    path='/data/tmp/george/Gaia/gedr3/gaia_source/fits/fits/'
+    batch_size = 500
+    gaia_empty=GaiaSourceFileDR3.objects.filter(status__exact='empty')
+
+    for gaia in gaia_empty:
+        srcs = gaia.gaiadr3_set.all()
+        if(srcs):
+            srcs.delete()
+            print("Clear ",gaia.filename," back ref: ",len(srcs))
+        
+        filepath=path+gaia.filename+'.fits'
+        data = astropy.table.Table.read(filepath, format='fits')
+        
+        
+        gaia.nrows = len(data)
+        gaia.save()
+        print("Bulk create for %s" % gaia.filename)
+        objs = (GAIADR3(filename=gaia,
+                        ra=data[i]['ra'],
+                        dec=data[i]['dec'],
+                        name=data[i]['designation'],
+                        #solution_id = data[i]['solution_id'],
+                        source_id = data[i]['source_id'],
+                        #ref_epoch = data[i]['ref_epoch'],
+                        ra_error = data[i]['ra_error'],
+                        dec_error = data[i]['dec_error'],
+                        parallax = data[i]['parallax'],
+                        parallax_error = data[i]['parallax_error'],
+                        pmra = data[i]['pmra'],
+                        pmra_error = data[i]['pmra_error'],
+                        pmdec = data[i]['pmdec'],
+                        pmdec_error = data[i]['pmdec_error'],
+                        phot_g_mean_mag = data[i]['phot_g_mean_mag'],
+                        phot_bp_mean_mag = data[i]['phot_bp_mean_mag'],
+                        phot_rp_mean_mag = data[i]['phot_rp_mean_mag'],)
+                for i in range(len(data)))
+
+        GAIADR3.objects.bulk_create(objs,batch_size)
+        gaia.status="ready"
+        gaia.save()
+        
+
+    print("--- %s seconds ---" % (time.time() - start_time))
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        load_gaia_bulk()
+        

astrobasis/management/commands/02_delete_gaia.py

@@ -0,0 +1,52 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import GaiaSourceFile, GAIADR2
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+
+import os
+import glob
+import logging
+
+def delete_gaia():
+
+    gaia_loaded = GaiaSourceFile.objects.all() #filter(status__exact='loaded')
+    total_loaded = gaia_loaded.count()
+    total = 0
+    for gaia in gaia_loaded:
+        total_linked = gaia.gaiadr2_set.all().count()
+        print("%s %s linked: %d nrows: %d" % (gaia.filename, gaia.status,total_linked,gaia.nrows))
+
+        linked = gaia.gaiadr2_set.all()
+        linked.delete()
+        gaia.status='empty'
+        gaia.save()
+        total+=1
+
+    print("Total %d removed from %d" % (total, total_loaded,))
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        delete_gaia()
+        
+
+

astrobasis/management/commands/02_mark_gaia.py

@@ -0,0 +1,65 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import GaiaSourceFile, GAIADR2
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+
+import os
+import glob
+import logging
+
+def mark_gaia():
+
+    gaia_empty = GaiaSourceFile.objects.filter(status__exact='empty')
+    total_empty = gaia_empty.count()
+    total = 0
+    for gaia in gaia_empty:
+        total_linked = gaia.gaiadr2_set.all().count()
+        #print("%s %s linked: %d nrows: %d" % (gaia.filename, gaia.status,total_linked,gaia.nrows))
+
+        if (gaia.nrows == 0):
+            """ empty and not still started to load """
+            continue
+
+        if (total_linked == 0):
+            """ looks like currently loading """
+            continue
+
+        if (total_linked == gaia.nrows):
+            print(gaia.filename+" -- loaded")
+            gaia.status='loaded'
+            gaia.save()
+            total+=1
+        else:
+            """ delete leftovers if any """
+            #linked = gaia.gaiadr2_set.all()
+            #linked.delete()
+            print(gaia.filename+" "+str(total_linked)+" != "+str(gaia.nrows))
+
+    print("Total %d marked as loaded from %d empty selected" % (total, total_empty,))
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        mark_gaia()
+        
+
+

astrobasis/management/commands/03_healpix_gaia.py

@@ -0,0 +1,108 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+from heasarc.models import INPUT_DATA_DIR
+import datetime
+from django.utils import timezone
+import astropy
+from astropy.io import ascii
+import pandas as pd
+import pymysql
+from sqlalchemy import create_engine
+import numpy.ma as ma
+
+from astrobasis.models import GaiaSourceFileDR3, GAIADR3
+
+from astropy_healpix import HEALPix
+from astropy.coordinates import SkyCoord  # High-level coordinates
+from astropy.coordinates import ICRS, Galactic, FK4, FK5  # Low-level frames
+from astropy.coordinates import Angle, Latitude, Longitude  # Angles
+import astropy.units as u
+
+from heasarc.models import NSIDE_SOURCES, ORDER
+
+import os
+import glob
+import logging
+import time
+
+from multiprocessing import Pool, cpu_count
+
+
+USE_NCORES=1
+
+def myfunc(n):
+    print(n)
+    
+    batch_size=500
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5)
+    
+    plates=GaiaSourceFileDR3.objects.filter(status__exact="ready")
+    
+    total_ready = len(plates)
+    print('Total <ready> Gaia files: '+str(total_ready))
+    
+    step=int(total_ready/USE_NCORES)+1
+    idx1 = step*n
+    idx2 = step*(n+1)
+    if idx2 > total_ready:
+        idx2=total_ready
+    for i in range(idx1,idx2):
+        print(str(n)+": "+plates[i].filename)
+        phots=plates[i].gaiadr3_set.exclude(healpix__gt=0.0)
+        ra=phots.values_list('ra', flat=True)
+        dec=phots.values_list('dec', flat=True)
+        crd = SkyCoord(ra, dec, frame="fk5", unit="deg")
+        heal = hp.skycoord_to_healpix(crd)
+        for j, phot in enumerate(phots):
+            phot.healpix=heal[j]
+        GAIADR3.objects.bulk_update(phots, ['healpix'], batch_size=batch_size)
+
+        plates[i].status='healpix'
+        plates[i].save()
+
+    pass
+
+def run_healpix():
+    
+    start_time = time.time()
+
+    batch_size=500
+    hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5)
+    
+    plates=GaiaSourceFileDR3.objects.filter(status__exact="ready")
+    
+    total_ready = len(plates)
+    
+    print('Total <ready> Gaia files: '+str(total_ready))
+    
+    for plate in plates:
+        print(plate.filename)
+        phots=plate.gaiadr3_set.exclude(healpix__gt=0.0)
+        ra=phots.values_list('ra', flat=True)
+        dec=phots.values_list('dec', flat=True)
+        crd = SkyCoord(ra, dec, frame="fk5", unit="deg")
+        heal = hp.skycoord_to_healpix(crd)
+        for j, phot in enumerate(phots):
+            phot.healpix=heal[j]
+        GAIADR3.objects.bulk_update(phots, ['healpix'], batch_size=batch_size)
+
+        plate.status='healpix'
+        plate.save()
+
+    print("--- %s seconds ---" % (time.time() - start_time))
+    pass
+
+class Command(BaseCommand):
+    help = 'Initiates healpix for Gaia'
+
+    def handle(self, *args, **options):
+
+        run_healpix()
+        
+        """
+        does not work for multitheading for some reason
+
+        with Pool(10) as p:
+            p.map(myfunc, range(USE_NCORES))
+        """

astrobasis/management/commands/show_celery.py

@@ -0,0 +1,15 @@
+from django.core.management.base import BaseCommand, CommandError
+from celery.task.control import inspect
+
+class Command(BaseCommand):
+    help = 'Initiates data dase'
+
+    def handle(self, *args, **options):
+        i = inspect()
+        
+        print('Show the items that have an ETA or are scheduled for later processing')
+        print(i.scheduled())
+        print('Show tasks that are currently active.')
+        print(i.active())
+        print('Show tasks that have been claimed by workers')
+        print(i.reserved())