From 69a2160eb771866c87e22d045f0418dc565ef7bd Mon Sep 17 00:00:00 2001 From: romakrivonos Date: Fri, 26 Apr 2024 12:43:00 +0300 Subject: [PATCH] more apps --- artsurvey/__init__.py | 0 artsurvey/admin.py | 9 + artsurvey/apps.py | 5 + artsurvey/forms.py | 105 + .../commands/00_artsurvey_addsource.py | 55 + .../management/commands/00_artsurvey_all.py | 117 + .../commands/00_artsurvey_catalog_nfalse.py | 178 + .../commands/00_artsurvey_catalog_sigma.py | 93 + .../management/commands/00_artsurvey_cds.py | 21 + .../commands/00_artsurvey_crossmatch.py | 80 + .../commands/00_artsurvey_find_orig.py | 70 + .../commands/00_artsurvey_find_skymaps.py | 33 + .../commands/00_artsurvey_heasarc.py | 133 + .../management/commands/00_artsurvey_init.py | 105 + .../00_artsurvey_mark_turkpossible.py | 33 + .../commands/00_artsurvey_mark_xrays.py | 176 + .../commands/00_artsurvey_metasource_check.py | 60 + .../00_artsurvey_metasource_manual_setup.py | 98 + .../management/commands/00_artsurvey_radio.py | 41 + .../commands/00_artsurvey_replica.py | 125 + .../commands/00_artsurvey_subquery.py | 77 + .../commands/00_artsurvey_sync_notes.py | 57 + .../commands/00_artsurvey_transfer.old | 175 + .../commands/00_artsurvey_transfer.py | 188 + .../commands/00_artsurvey_update.py | 76 + .../commands/00_artsurvey_update_marshall.py | 67 + .../00_artsurvey_update_metasource.py | 137 + .../commands/00_init_energy_bands.py | 49 + .../commands/01_artsurvey_heasarc.py | 36 + .../01_artsurvey_load_erositamatch.py | 199 + .../commands/02_artsurvey_load_wise.py | 196 + .../commands/02_artsurvey_simbad.py | 37 + .../commands/03_arsurvey_allwise.py | 37 + .../management/commands/03_artsurvey_gaia.py | 42 + .../management/commands/04_artsurvey_nvss.py | 37 + .../management/commands/05_artsurvey_first.py | 37 + .../management/commands/06_artsurvey_sumss.py | 37 + artsurvey/management/commands/` | 1 + artsurvey/migrations/0001_initial.py | 45 + .../migrations/0002_artsurveysource_cname.py | 18 + .../migrations/0003_auto_20200619_1818.py | 25 + .../migrations/0004_artsurveysource_ads.py | 19 + .../migrations/0005_artsurveysource_refid.py | 18 + ...006_remove_artsurveysource_object_class.py | 17 + .../0007_artsurveysource_object_class.py | 20 + .../migrations/0008_artsurveysource_owner.py | 21 + .../migrations/0009_artsurveysource_notes.py | 18 + artsurvey/migrations/0010_artsurveyparams.py | 24 + .../0011_artsurveyparams_class_startswith.py | 18 + .../migrations/0012_auto_20200624_1800.py | 18 + artsurvey/migrations/0013_erositamatch.py | 28 + .../migrations/0014_auto_20200625_0914.py | 23 + .../0015_artsurveysource_name_orig.py | 18 + artsurvey/migrations/0016_artsurvey.py | 20 + .../migrations/0017_artsurveysource_survey.py | 19 + .../migrations/0018_auto_20200625_0957.py | 19 + .../migrations/0019_auto_20200625_0959.py | 18 + .../migrations/0020_auto_20200625_1146.py | 23 + .../migrations/0021_auto_20200625_1148.py | 41 + .../migrations/0022_auto_20200625_1159.py | 348 ++ .../migrations/0023_auto_20200625_1251.py | 18 + .../migrations/0024_auto_20200625_1252.py | 23 + .../migrations/0025_auto_20200625_1406.py | 33 + .../migrations/0026_opticalcounterpart.py | 21 + .../migrations/0027_auto_20200625_1908.py | 49 + .../migrations/0028_artsurveysource_gaia.py | 19 + artsurvey/migrations/0029_category.py | 27 + .../0030_artsurveysource_category.py | 19 + .../migrations/0031_auto_20200626_1427.py | 22 + .../0032_artsurveysource_redshift.py | 18 + .../migrations/0033_auto_20200630_1521.py | 18 + .../migrations/0034_artsurveysource_parent.py | 19 + .../migrations/0035_artsurveyparams_survey.py | 19 + .../0036_artsurveysource_gaia_primary.py | 20 + .../migrations/0037_artsurveyparams_sky.py | 18 + .../migrations/0038_auto_20200711_1750.py | 23 + artsurvey/migrations/0039_wisecounterpart.py | 80 + .../0040_artsurveysource_wise_data.py | 18 + .../migrations/0041_delete_wisecounterpart.py | 16 + artsurvey/migrations/0042_wisecounterpart.py | 81 + .../migrations/0043_auto_20200722_1737.py | 23 + .../migrations/0044_auto_20200722_1747.py | 57 + .../0045_wisecounterpart_primarydetection.py | 18 + .../migrations/0046_auto_20200723_0856.py | 18 + .../migrations/0047_auto_20200723_0907.py | 38 + .../0048_artsurveysource_allwise.py | 19 + .../0049_artsurveysource_allwise_primary.py | 20 + .../migrations/0050_artsurveysource_nvss.py | 19 + .../migrations/0051_artsurveysource_first.py | 19 + .../migrations/0052_artsurveysource_sumss.py | 19 + .../0053_artsurveysource_nvss_primary.py | 20 + .../migrations/0054_auto_20200806_2017.py | 25 + artsurvey/migrations/0055_othername.py | 22 + artsurvey/migrations/0056_othername_owner.py | 21 + .../0057_artsurveysource_class_tentative.py | 18 + .../migrations/0058_artsurveyparams_band.py | 18 + artsurvey/migrations/0059_energyband.py | 22 + .../migrations/0060_artsurveysource_band.py | 19 + artsurvey/migrations/0061_artsurvey_active.py | 18 + .../0062_artsurveyparams_category.py | 19 + .../migrations/0063_auto_20210311_1458.py | 188 + .../migrations/0064_auto_20210312_1440.py | 22 + artsurvey/migrations/0065_artsurvey_inwork.py | 18 + .../migrations/0066_artsurveysource_gaia3.py | 19 + .../0067_artsurveysource_gaia3_primary.py | 20 + .../migrations/0068_artsurveymatchparams.py | 27 + .../migrations/0069_auto_20210312_1841.py | 33 + .../migrations/0070_auto_20210312_1848.py | 18 + .../migrations/0071_auto_20210312_1924.py | 18 + .../migrations/0072_auto_20210312_1924.py | 18 + .../migrations/0073_auto_20210312_1927.py | 23 + .../migrations/0074_auto_20210312_1945.py | 21 + .../migrations/0075_auto_20210312_1946.py | 23 + artsurvey/migrations/0076_metasource.py | 31 + .../0077_artsurveysource_metasource.py | 19 + .../0078_artsurveysource_metasource_sep.py | 18 + artsurvey/migrations/0079_artsurvey_label.py | 18 + .../migrations/0080_auto_20210317_1721.py | 18 + .../migrations/0081_auto_20210317_1726.py | 23 + .../migrations/0082_auto_20210317_1726.py | 23 + .../migrations/0083_auto_20210319_1225.py | 23 + .../migrations/0084_auto_20210319_1225.py | 18 + .../migrations/0085_auto_20210319_1231.py | 18 + .../migrations/0086_auto_20210415_1544.py | 23 + .../migrations/0087_auto_20210415_1607.py | 23 + .../migrations/0088_artsurveysource_exti.py | 18 + .../0089_remove_artsurveysource_ext.py | 17 + .../migrations/0090_auto_20210416_1322.py | 18 + .../migrations/0091_auto_20210416_1322.py | 23 + .../0092_artsurveyparams_history.py | 18 + .../migrations/0093_artsurveysource_smapnr.py | 18 + .../migrations/0094_auto_20210430_1033.py | 28 + .../migrations/0095_auto_20210430_1036.py | 28 + .../0096_artsurveyparams_exclude_survey.py | 19 + .../0097_artsurveyparams_exclude_band.py | 18 + .../0098_artsurveysource_turkish.py | 18 + .../0099_artsurveysource_follow_up.py | 18 + .../migrations/0100_artsurveysource_ext_id.py | 18 + .../migrations/0101_auto_20210526_1619.py | 23 + .../migrations/0102_auto_20210526_1621.py | 18 + .../0103_artsurveysource_log_nfalse.py | 18 + .../0104_artsurveyparams_gaia_primary.py | 18 + .../0105_artsurveyparams_allwise_primary.py | 18 + .../0106_artsurveysource_catalog_version.py | 18 + artsurvey/migrations/0107_catalog.py | 21 + .../0108_artsurveysource_catalog.py | 18 + ..._remove_artsurveysource_catalog_version.py | 17 + .../migrations/0110_metasource_catalog.py | 19 + ...0111_artsurveyparams_exclude_log_nfalse.py | 18 + .../migrations/0112_auto_20210604_1059.py | 28 + .../migrations/0113_auto_20210604_1143.py | 22 + .../migrations/0114_auto_20210607_1736.py | 19 + .../migrations/0115_auto_20210607_1739.py | 19 + .../migrations/0116_auto_20210607_1740.py | 19 + .../migrations/0117_auto_20210607_1741.py | 19 + .../migrations/0118_auto_20210607_1743.py | 19 + .../migrations/0119_auto_20210607_1746.py | 19 + .../migrations/0120_auto_20210607_1748.py | 18 + .../migrations/0121_auto_20210607_1750.py | 18 + .../migrations/0122_auto_20210615_0915.py | 33 + .../migrations/0123_auto_20210701_1724.py | 33 + .../migrations/0124_auto_20210706_1008.py | 33 + ...0125_remove_artsurveyparams_ecl_lat_min.py | 17 + .../migrations/0126_auto_20210706_1010.py | 23 + .../migrations/0127_artsurveysource_xrays.py | 18 + .../migrations/0128_auto_20210826_1325.py | 23 + artsurvey/migrations/0129_catalog_test.py | 18 + .../migrations/0130_remove_catalog_test.py | 17 + .../migrations/0131_auto_20211228_1925.py | 18 + artsurvey/migrations/0132_artsurvey_parent.py | 19 + .../migrations/0133_auto_20211229_1259.py | 73 + .../migrations/0134_auto_20220112_1202.py | 21 + artsurvey/migrations/0135_artsurvey_name.py | 18 + .../0136_artsurveysource_turkpossible.py | 18 + .../0137_artsurveyparams_turkpossible.py | 18 + ...0138_alter_artsurveyparams_turkpossible.py | 18 + .../migrations/0139_auto_20220216_1735.py | 22 + ...140_alter_artsurveyparams_turk_possible.py | 18 + .../0141_alter_artsurveysource_name_orig.py | 18 + .../migrations/0142_auto_20220303_1837.py | 273 ++ .../migrations/0143_auto_20220404_1254.py | 25 + .../migrations/0144_auto_20220404_1642.py | 21 + .../migrations/0145_auto_20220404_1643.py | 25 + .../0146_artsurveysource_heasarc_linked.py | 19 + artsurvey/migrations/0147_upload.py | 29 + .../migrations/0148_upload_artsources.py | 18 + .../0149_alter_upload_artsources.py | 18 + .../migrations/0150_alter_upload_filefield.py | 20 + artsurvey/migrations/0151_upload_notes.py | 18 + artsurvey/migrations/0152_artselection.py | 24 + .../0153_alter_artselection_owner.py | 21 + .../0154_remove_artselection_owner.py | 17 + .../migrations/0155_artselection_owner.py | 21 + .../migrations/0156_auto_20220415_1934.py | 23 + .../migrations/0157_auto_20220415_1938.py | 33 + .../migrations/0158_upload_metasource.py | 19 + artsurvey/migrations/0159_artsurvey_hidden.py | 18 + .../0160_alter_artsurveysource_options.py | 17 + artsurvey/migrations/0161_metacomment.py | 29 + .../migrations/0162_alter_metacomment_body.py | 18 + .../migrations/0163_alter_metacomment_body.py | 18 + ...64_alter_artsurveyparams_log_nfalse_min.py | 18 + .../0165_artsurveyparams_exclude_category.py | 19 + ..._alter_artsurveyparams_exclude_category.py | 19 + artsurvey/migrations/0167_artsurvey_notes.py | 18 + ...0168_artsurveysource_metasource_primary.py | 19 + ...move_artsurveysource_metasource_primary.py | 17 + ...0170_artsurveysource_metasource_primary.py | 19 + .../0171_artsurveysource_lastvisited.py | 20 + ...0172_remove_artsurveysource_lastvisited.py | 17 + artsurvey/migrations/0173_upload_survey.py | 19 + artsurvey/migrations/0174_upload_primary.py | 18 + .../0175_artsurveysource_bibtex_paper.py | 18 + .../migrations/0176_auto_20220624_1826.py | 28 + .../migrations/0177_auto_20220707_1819.py | 23 + .../migrations/0178_auto_20220707_1825.py | 23 + .../0179_artsurveyparams_exposure_min.py | 18 + .../migrations/0180_auto_20221107_1529.py | 23 + .../migrations/0181_artsurvey_modified.py | 18 + .../0182_artsurveysource_ads_hidden.py | 19 + artsurvey/migrations/0183_artbasket.py | 27 + .../0184_artsurveysource_modified.py | 18 + .../0185_artsurveyparams_cname_startswith.py | 18 + ...artswith_artsurveyparams_cname_contains.py | 18 + .../migrations/0187_auto_20230516_1536.py | 63 + .../0188_artsurveysource_log_ml_nfalse.py | 18 + .../migrations/0189_auto_20230516_1704.py | 23 + .../migrations/0190_auto_20231023_1559.py | 38 + .../migrations/0191_artsurveysource_bj2021.py | 19 + .../0192_artsurveysource_gaiadr3.py | 19 + .../migrations/0193_auto_20231025_1735.py | 25 + .../migrations/0194_auto_20231026_1224.py | 33 + ...5_artsurveyparams_exclude_log_ml_nfalse.py | 18 + ...lter_artsurveyparams_exclude_log_nfalse.py | 18 + ...lter_artsurveyparams_exclude_log_nfalse.py | 18 + .../migrations/0198_auto_20231026_1613.py | 24 + .../migrations/0199_auto_20231028_1500.py | 23 + .../0200_artsurveysource_turkish_date.py | 18 + .../migrations/0201_auto_20231130_1125.py | 23 + .../migrations/0202_auto_20231130_1127.py | 23 + .../0203_artsurveysource_marshall.py | 18 + .../0204_artsurveyparams_marshall.py | 18 + .../0205_alter_artsurveyparams_marshall.py | 18 + .../0206_alter_artsurveyparams_marshall.py | 18 + .../0207_alter_artsurveyparams_marshall.py | 18 + artsurvey/migrations/__init__.py | 0 artsurvey/models.py | 1285 ++++++ artsurvey/templatetags/__init__.py | 0 artsurvey/templatetags/artsurvey_templates.py | 57 + artsurvey/tests.py | 3 + artsurvey/urls.py | 157 + artsurvey/utils.py | 683 ++++ artsurvey/views.py | 3480 +++++++++++++++++ astrobasis/__init__.py | 0 astrobasis/admin.py | 3 + astrobasis/apps.py | 5 + .../commands/00_astrobasis_2mass_healpix.py | 11 + .../commands/00_astrobasis_2mass_init.py | 88 + .../commands/00_astrobasis_glimpse_healpix.py | 11 + .../commands/00_astrobasis_glimpse_init.py | 159 + .../commands/00_astrobasis_vlass_healpix.py | 76 + .../commands/00_astrobasis_vlass_init.py | 161 + .../management/commands/00_init_gaia.py | 54 + .../management/commands/00_load_ztf_alerce.py | 179 + .../commands/00_load_ztf_archive.py | 256 ++ .../management/commands/00_load_ztf_mars.py | 65 + .../management/commands/00_status_gaia.py | 52 + .../management/commands/00_test_gaia.py | 39 + astrobasis/management/commands/00_test_ztf.py | 221 ++ .../management/commands/01_load_gaia.py | 71 + .../management/commands/01_load_gaia_dr3.py | 79 + .../management/commands/02_delete_gaia.py | 52 + .../management/commands/02_mark_gaia.py | 65 + .../management/commands/03_healpix_gaia.py | 108 + astrobasis/management/commands/show_celery.py | 15 + astrobasis/migrations/0001_initial.py | 49 + astrobasis/migrations/0002_ztfalert.py | 35 + .../0003_remove_ztfalert_wall_time.py | 17 + .../migrations/0004_ztfalert_wall_time.py | 18 + .../migrations/0005_auto_20200128_2138.py | 23 + astrobasis/migrations/0006_delete_ztfalert.py | 16 + astrobasis/migrations/0007_ztfalert.py | 36 + .../migrations/0008_ztfalert_healpix_plate.py | 18 + astrobasis/migrations/0009_ztfalert_survey.py | 19 + .../migrations/0010_auto_20200129_1339.py | 33 + .../0011_remove_ztfalert_wall_time.py | 17 + .../migrations/0012_ztfalert_wall_time.py | 19 + astrobasis/migrations/0013_alerce.py | 44 + .../migrations/0014_auto_20200129_1948.py | 18 + .../migrations/0015_auto_20200129_1949.py | 33 + .../migrations/0016_auto_20200129_1949.py | 23 + .../migrations/0017_auto_20200129_1950.py | 33 + .../migrations/0018_auto_20200130_1300.py | 18 + .../migrations/0019_auto_20200130_1455.py | 23 + .../migrations/0020_auto_20200130_1726.py | 24 + .../migrations/0021_gaiadr2_error_radius.py | 18 + .../migrations/0022_alerce_error_radius.py | 18 + .../migrations/0023_auto_20200131_1555.py | 21 + .../migrations/0024_auto_20200131_1556.py | 23 + astrobasis/migrations/0025_sdssdr12spec.py | 29 + astrobasis/migrations/0026_flesch_z.py | 27 + astrobasis/migrations/0027_delete_flesch_z.py | 16 + astrobasis/migrations/0028_flesch.py | 27 + astrobasis/migrations/0029_simbad.py | 28 + .../migrations/0030_auto_20200223_0814.py | 18 + astrobasis/migrations/0031_simbad_healpix.py | 18 + .../migrations/0032_auto_20200223_1650.py | 18 + .../migrations/0033_auto_20200225_1604.py | 18 + .../migrations/0034_auto_20200225_1608.py | 18 + .../migrations/0035_auto_20200225_1625.py | 18 + astrobasis/migrations/0036_wise.py | 78 + astrobasis/migrations/0037_delete_wise.py | 16 + astrobasis/migrations/0038_allwise.py | 29 + .../0039_remove_allwise_image_url.py | 17 + astrobasis/migrations/0040_nvss.py | 26 + .../migrations/0041_auto_20200804_1457.py | 23 + astrobasis/migrations/0042_first.py | 29 + astrobasis/migrations/0043_sumss.py | 32 + .../migrations/0044_auto_20200804_1703.py | 23 + .../migrations/0045_auto_20200804_1707.py | 23 + astrobasis/migrations/0046_gaiadr3.py | 40 + .../0047_remove_gaiadr3_filename.py | 17 + astrobasis/migrations/0048_delete_gaiadr3.py | 16 + astrobasis/migrations/0049_gaiadr3.py | 40 + astrobasis/migrations/0050_delete_gaiadr3.py | 16 + .../0051_gaiadr3_gaiasourcefiledr3.py | 49 + .../0052_remove_gaiadr3_solution_id.py | 17 + .../0053_remove_gaiadr3_ref_epoch.py | 17 + .../migrations/0054_auto_20210812_1858.py | 23 + .../migrations/0055_auto_20211229_1259.py | 78 + astrobasis/migrations/0056_twomass.py | 39 + astrobasis/migrations/0057_glimpse.py | 38 + .../migrations/0058_alter_glimpse_healpix.py | 18 + .../migrations/0059_auto_20220315_1125.py | 23 + .../migrations/0060_alter_glimpse_options.py | 17 + astrobasis/migrations/0061_vlass.py | 28 + .../migrations/0062_auto_20220325_1731.py | 38 + .../migrations/0063_auto_20220325_1742.py | 28 + .../migrations/0064_auto_20220325_1752.py | 23 + .../migrations/0065_auto_20220330_1313.py | 25 + .../migrations/0066_alter_twomass_name.py | 18 + astrobasis/migrations/0067_vlassfromvizier.py | 39 + .../migrations/0068_auto_20220404_1639.py | 21 + .../migrations/0069_vlassfromvizier_recno.py | 18 + .../migrations/0070_alter_twomass_options.py | 17 + .../migrations/0071_astrobasisusercat.py | 30 + .../0072_delete_astrobasisusercat.py | 16 + .../migrations/0073_astrobasisusercat.py | 30 + astrobasis/migrations/0074_magnitude.py | 24 + .../0075_astrobasisusercat_magnitudes.py | 18 + astrobasis/migrations/0076_magnitude_error.py | 18 + .../migrations/0077_alter_magnitude_band.py | 18 + .../migrations/0078_alter_simbad_obj_class.py | 18 + astrobasis/migrations/0079_bjfromvizier.py | 33 + .../migrations/0080_auto_20231023_1633.py | 23 + .../0081_alter_bjfromvizier_error_radius.py | 18 + .../0082_alter_bjfromvizier_error_radius.py | 18 + .../0083_remove_bjfromvizier_recno.py | 17 + .../0084_alter_bjfromvizier_flag.py | 18 + .../0085_remove_bjfromvizier_flag.py | 17 + .../migrations/0086_gaiadr3fromvizier.py | 37 + .../migrations/0087_auto_20231025_1924.py | 43 + astrobasis/migrations/__init__.py | 0 astrobasis/models.py | 882 +++++ astrobasis/templatetags/__init__.py | 1 + astrobasis/templatetags/simbadescape.py | 7 + astrobasis/tests.py | 3 + astrobasis/urls.py | 24 + astrobasis/utils.py | 48 + astrobasis/views.py | 267 ++ genericsource/__init__.py | 0 genericsource/admin.py | 3 + genericsource/apps.py | 5 + genericsource/migrations/0001_initial.py | 105 + .../migrations/0002_auto_20200601_2248.py | 22 + .../migrations/0003_auto_20200626_1000.py | 29 + .../migrations/0004_auto_20200626_1523.py | 18 + .../0005_genericsource_src_class.py | 18 + genericsource/migrations/0006_srcauxdata.py | 28 + .../migrations/0007_srcauxdata_specimg.py | 18 + .../migrations/0008_genericcatalogsgroup.py | 21 + .../migrations/0009_srcauxdata_xcm.py | 18 + .../migrations/0010_auto_20201023_1729.py | 17 + .../migrations/0011_auto_20201103_1325.py | 28 + .../migrations/0012_auto_20201104_1915.py | 23 + .../migrations/0013_auto_20201106_1817.py | 18 + .../migrations/0014_auto_20201112_1303.py | 23 + .../migrations/0015_gsbibliogrpahy.py | 27 + .../migrations/0016_auto_20211229_1259.py | 58 + genericsource/migrations/__init__.py | 0 genericsource/models.py | 127 + genericsource/tests.py | 3 + genericsource/urls.py | 8 + genericsource/views.py | 34 + monthplan/__init__.py | 0 monthplan/admin.py | 15 + monthplan/apps.py | 5 + monthplan/forms.py | 4 + .../management/commands/00_load_surveypath.py | 29 + monthplan/management/commands/00_watchdog.py | 76 + .../management/commands/01_init_monthplan.py | 19 + .../management/commands/02_load_monthplan.py | 123 + .../management/commands/03_load_flightplan.py | 127 + .../management/commands/04_load_datadump.py | 126 + .../commands/05_load_survey_track.py | 239 ++ .../commands/05_load_survey_track.py.bak | 181 + .../management/commands/06_load_scan_track.py | 280 ++ .../commands/init_survey_healpix_plates.py | 36 + .../init_survey_healpix_sub_plates.py | 46 + .../commands/mark_survey_healpix_plates.py | 54 + .../management/commands/real_time_track.py | 91 + .../management/commands/show_user_emails.py | 34 + monthplan/migrations/0001_initial.py | 206 + .../migrations/0002_auto_20191209_2233.py | 30 + .../migrations/0003_auto_20200210_1503.py | 17 + monthplan/migrations/0004_survey_loaded.py | 18 + .../migrations/0005_auto_20200306_1022.py | 33 + .../migrations/0006_surveyhealpixsubplate.py | 24 + .../migrations/0007_auto_20211229_1259.py | 78 + monthplan/migrations/__init__.py | 0 monthplan/models.py | 524 +++ monthplan/templatetags/__init__.py | 0 monthplan/templatetags/monthplan_templates.py | 29 + monthplan/tests.py | 3 + monthplan/urls.py | 44 + monthplan/utils.py | 275 ++ monthplan/views.py | 553 +++ plan/__init__.py | 0 plan/admin.py | 11 + plan/apps.py | 5 + plan/migrations/0001_initial.py | 86 + plan/migrations/0002_auto_20211229_1259.py | 38 + plan/migrations/__init__.py | 0 plan/models.py | 262 ++ plan/tests.py | 3 + plan/urls.py | 17 + plan/views.py | 196 + srgcat/__init__.py | 0 srgcat/admin.py | 8 + srgcat/apps.py | 5 + srgcat/forms.py | 32 + srgcat/management/commands/00_init_skymaps.py | 183 + srgcat/management/commands/00_load_art_gc.py | 90 + srgcat/management/commands/00_load_gyro.py | 144 + .../01_load_galcenter_for_artsurvey.py | 90 + .../commands/01_load_skymap_erosita.py | 50 + .../commands/01_load_skymap_for_artsurvey.py | 100 + .../commands/01_load_skymap_sources.py | 80 + srgcat/management/commands/01_load_skymaps.py | 47 + .../commands/01_update_allsky_missed.py | 47 + .../commands/run_find_heasarc_counterparts.py | 47 + .../commands/run_find_simbad_counterparts.py | 55 + .../commands/simple_healpix_match.py | 75 + srgcat/management/commands/testme_artcat.py | 63 + srgcat/migrations/0001_initial.py | 43 + srgcat/migrations/0002_skymaps.py | 44 + .../0003_remove_skymaps_healpix_plate.py | 17 + .../0004_skymaps_survey_healpix_plate.py | 19 + srgcat/migrations/0005_skymaps_neighbours.py | 18 + srgcat/migrations/0006_srgdatadump.py | 29 + srgcat/migrations/0007_srgdatadump_version.py | 18 + srgcat/migrations/0008_skymapsource.py | 44 + srgcat/migrations/0009_skymapsource_simbad.py | 19 + srgcat/migrations/0010_skymapsource_gaia.py | 19 + srgcat/migrations/0011_skymapsource_dump.py | 19 + .../0012_srgdatadump_catalog_loaded.py | 18 + srgcat/migrations/0013_skymapsource_new.py | 18 + srgcat/migrations/0014_skymapsource_bad.py | 18 + .../0015_skymapsource_healpix_plate.py | 18 + srgcat/migrations/0016_skymapsource_good.py | 18 + srgcat/migrations/0017_selectallskymissed.py | 22 + srgcat/migrations/0018_selection.py | 26 + srgcat/migrations/0019_delete_selection.py | 16 + srgcat/migrations/0020_selection.py | 27 + srgcat/migrations/0021_auto_20200325_2157.py | 18 + .../migrations/0022_skymapsource_catalog.py | 19 + srgcat/migrations/0023_erositamatch.py | 28 + .../0024_remove_erositamatch_source.py | 17 + srgcat/migrations/0025_delete_erositamatch.py | 16 + srgcat/migrations/0026_erositamatch.py | 28 + .../migrations/0027_usersessionparameters.py | 24 + srgcat/migrations/0028_auto_20200503_1438.py | 18 + srgcat/migrations/0029_auto_20200505_0947.py | 23 + srgcat/migrations/0030_auto_20200505_1338.py | 18 + srgcat/migrations/0031_trackstats.py | 27 + srgcat/migrations/0032_auto_20200515_1706.py | 18 + srgcat/migrations/0033_trackstats_group.py | 20 + .../0034_skymapsource_simbad_failed.py | 18 + srgcat/migrations/0035_auto_20200601_2317.py | 18 + .../0036_skymapsource_simbad_notfound.py | 18 + .../0037_skymapsource_artsurvey_source.py | 20 + srgcat/migrations/0038_auto_20210311_1313.py | 18 + srgcat/migrations/0039_skymapsource_ext.py | 18 + .../migrations/0040_skymapsource_cnts_tot.py | 18 + srgcat/migrations/0041_auto_20210311_1338.py | 48 + srgcat/migrations/0042_auto_20210311_1340.py | 18 + srgcat/migrations/0043_auto_20210311_1342.py | 153 + srgcat/migrations/0044_auto_20210315_1542.py | 19 + .../0045_srgdatadump_gyro_loaded.py | 18 + srgcat/migrations/0046_gyro.py | 21 + srgcat/migrations/0047_auto_20210426_1746.py | 53 + srgcat/migrations/0048_auto_20210426_1805.py | 29 + srgcat/migrations/0049_gyro_healpix.py | 18 + srgcat/migrations/0050_auto_20210426_1836.py | 33 + srgcat/migrations/0051_skymapsource_ext_id.py | 18 + srgcat/migrations/0052_auto_20210615_0831.py | 33 + srgcat/migrations/0053_auto_20210615_0902.py | 18 + srgcat/migrations/0054_auto_20211229_1259.py | 68 + .../0055_remove_skymapsource_gaia.py | 17 + .../0056_alter_skymapsource_name_orig.py | 18 + srgcat/migrations/0057_auto_20220303_1826.py | 263 ++ .../0058_alter_skymapsource_ext_id.py | 18 + srgcat/migrations/0059_auto_20230516_1536.py | 63 + srgcat/migrations/__init__.py | 0 srgcat/models.py | 367 ++ srgcat/templatetags/__init__.py | 0 srgcat/templatetags/srgcat_templates.py | 10 + srgcat/tests.py | 3 + srgcat/urls.py | 66 + srgcat/utils.py | 45 + srgcat/views.py | 1150 ++++++ srglib/__init__.py | 0 srglib/admin.py | 3 + srglib/apps.py | 5 + srglib/check_transient.py | 92 + srglib/correlate_utils.py | 536 +++ srglib/genericsourceutils.py | 894 +++++ srglib/load_transinets.py | 278 ++ srglib/migrations/__init__.py | 0 srglib/models.py | 3 + srglib/tasks.py | 104 + srglib/tests.py | 3 + srglib/update_erotrans_tables.py | 44 + srglib/utils.py | 2458 ++++++++++++ srglib/views.py | 3 + srglib/ztf.py | 291 ++ 536 files changed, 33118 insertions(+) create mode 100644 artsurvey/__init__.py create mode 100644 artsurvey/admin.py create mode 100644 artsurvey/apps.py create mode 100644 artsurvey/forms.py create mode 100644 artsurvey/management/commands/00_artsurvey_addsource.py create mode 100644 artsurvey/management/commands/00_artsurvey_all.py create mode 100644 artsurvey/management/commands/00_artsurvey_catalog_nfalse.py create mode 100644 artsurvey/management/commands/00_artsurvey_catalog_sigma.py create mode 100644 artsurvey/management/commands/00_artsurvey_cds.py create mode 100644 artsurvey/management/commands/00_artsurvey_crossmatch.py create mode 100644 artsurvey/management/commands/00_artsurvey_find_orig.py create mode 100644 artsurvey/management/commands/00_artsurvey_find_skymaps.py create mode 100644 artsurvey/management/commands/00_artsurvey_heasarc.py create mode 100644 artsurvey/management/commands/00_artsurvey_init.py create mode 100644 artsurvey/management/commands/00_artsurvey_mark_turkpossible.py create mode 100644 artsurvey/management/commands/00_artsurvey_mark_xrays.py create mode 100644 artsurvey/management/commands/00_artsurvey_metasource_check.py create mode 100644 artsurvey/management/commands/00_artsurvey_metasource_manual_setup.py create mode 100644 artsurvey/management/commands/00_artsurvey_radio.py create mode 100644 artsurvey/management/commands/00_artsurvey_replica.py create mode 100644 artsurvey/management/commands/00_artsurvey_subquery.py create mode 100644 artsurvey/management/commands/00_artsurvey_sync_notes.py create mode 100644 artsurvey/management/commands/00_artsurvey_transfer.old create mode 100644 artsurvey/management/commands/00_artsurvey_transfer.py create mode 100644 artsurvey/management/commands/00_artsurvey_update.py create mode 100644 artsurvey/management/commands/00_artsurvey_update_marshall.py create mode 100644 artsurvey/management/commands/00_artsurvey_update_metasource.py create mode 100644 artsurvey/management/commands/00_init_energy_bands.py create mode 100644 artsurvey/management/commands/01_artsurvey_heasarc.py create mode 100644 artsurvey/management/commands/01_artsurvey_load_erositamatch.py create mode 100644 artsurvey/management/commands/02_artsurvey_load_wise.py create mode 100644 artsurvey/management/commands/02_artsurvey_simbad.py create mode 100644 artsurvey/management/commands/03_arsurvey_allwise.py create mode 100644 artsurvey/management/commands/03_artsurvey_gaia.py create mode 100644 artsurvey/management/commands/04_artsurvey_nvss.py create mode 100644 artsurvey/management/commands/05_artsurvey_first.py create mode 100644 artsurvey/management/commands/06_artsurvey_sumss.py create mode 100644 artsurvey/management/commands/` create mode 100644 artsurvey/migrations/0001_initial.py create mode 100644 artsurvey/migrations/0002_artsurveysource_cname.py create mode 100644 artsurvey/migrations/0003_auto_20200619_1818.py create mode 100644 artsurvey/migrations/0004_artsurveysource_ads.py create mode 100644 artsurvey/migrations/0005_artsurveysource_refid.py create mode 100644 artsurvey/migrations/0006_remove_artsurveysource_object_class.py create mode 100644 artsurvey/migrations/0007_artsurveysource_object_class.py create mode 100644 artsurvey/migrations/0008_artsurveysource_owner.py create mode 100644 artsurvey/migrations/0009_artsurveysource_notes.py create mode 100644 artsurvey/migrations/0010_artsurveyparams.py create mode 100644 artsurvey/migrations/0011_artsurveyparams_class_startswith.py create mode 100644 artsurvey/migrations/0012_auto_20200624_1800.py create mode 100644 artsurvey/migrations/0013_erositamatch.py create mode 100644 artsurvey/migrations/0014_auto_20200625_0914.py create mode 100644 artsurvey/migrations/0015_artsurveysource_name_orig.py create mode 100644 artsurvey/migrations/0016_artsurvey.py create mode 100644 artsurvey/migrations/0017_artsurveysource_survey.py create mode 100644 artsurvey/migrations/0018_auto_20200625_0957.py create mode 100644 artsurvey/migrations/0019_auto_20200625_0959.py create mode 100644 artsurvey/migrations/0020_auto_20200625_1146.py create mode 100644 artsurvey/migrations/0021_auto_20200625_1148.py create mode 100644 artsurvey/migrations/0022_auto_20200625_1159.py create mode 100644 artsurvey/migrations/0023_auto_20200625_1251.py create mode 100644 artsurvey/migrations/0024_auto_20200625_1252.py create mode 100644 artsurvey/migrations/0025_auto_20200625_1406.py create mode 100644 artsurvey/migrations/0026_opticalcounterpart.py create mode 100644 artsurvey/migrations/0027_auto_20200625_1908.py create mode 100644 artsurvey/migrations/0028_artsurveysource_gaia.py create mode 100644 artsurvey/migrations/0029_category.py create mode 100644 artsurvey/migrations/0030_artsurveysource_category.py create mode 100644 artsurvey/migrations/0031_auto_20200626_1427.py create mode 100644 artsurvey/migrations/0032_artsurveysource_redshift.py create mode 100644 artsurvey/migrations/0033_auto_20200630_1521.py create mode 100644 artsurvey/migrations/0034_artsurveysource_parent.py create mode 100644 artsurvey/migrations/0035_artsurveyparams_survey.py create mode 100644 artsurvey/migrations/0036_artsurveysource_gaia_primary.py create mode 100644 artsurvey/migrations/0037_artsurveyparams_sky.py create mode 100644 artsurvey/migrations/0038_auto_20200711_1750.py create mode 100644 artsurvey/migrations/0039_wisecounterpart.py create mode 100644 artsurvey/migrations/0040_artsurveysource_wise_data.py create mode 100644 artsurvey/migrations/0041_delete_wisecounterpart.py create mode 100644 artsurvey/migrations/0042_wisecounterpart.py create mode 100644 artsurvey/migrations/0043_auto_20200722_1737.py create mode 100644 artsurvey/migrations/0044_auto_20200722_1747.py create mode 100644 artsurvey/migrations/0045_wisecounterpart_primarydetection.py create mode 100644 artsurvey/migrations/0046_auto_20200723_0856.py create mode 100644 artsurvey/migrations/0047_auto_20200723_0907.py create mode 100644 artsurvey/migrations/0048_artsurveysource_allwise.py create mode 100644 artsurvey/migrations/0049_artsurveysource_allwise_primary.py create mode 100644 artsurvey/migrations/0050_artsurveysource_nvss.py create mode 100644 artsurvey/migrations/0051_artsurveysource_first.py create mode 100644 artsurvey/migrations/0052_artsurveysource_sumss.py create mode 100644 artsurvey/migrations/0053_artsurveysource_nvss_primary.py create mode 100644 artsurvey/migrations/0054_auto_20200806_2017.py create mode 100644 artsurvey/migrations/0055_othername.py create mode 100644 artsurvey/migrations/0056_othername_owner.py create mode 100644 artsurvey/migrations/0057_artsurveysource_class_tentative.py create mode 100644 artsurvey/migrations/0058_artsurveyparams_band.py create mode 100644 artsurvey/migrations/0059_energyband.py create mode 100644 artsurvey/migrations/0060_artsurveysource_band.py create mode 100644 artsurvey/migrations/0061_artsurvey_active.py create mode 100644 artsurvey/migrations/0062_artsurveyparams_category.py create mode 100644 artsurvey/migrations/0063_auto_20210311_1458.py create mode 100644 artsurvey/migrations/0064_auto_20210312_1440.py create mode 100644 artsurvey/migrations/0065_artsurvey_inwork.py create mode 100644 artsurvey/migrations/0066_artsurveysource_gaia3.py create mode 100644 artsurvey/migrations/0067_artsurveysource_gaia3_primary.py create mode 100644 artsurvey/migrations/0068_artsurveymatchparams.py create mode 100644 artsurvey/migrations/0069_auto_20210312_1841.py create mode 100644 artsurvey/migrations/0070_auto_20210312_1848.py create mode 100644 artsurvey/migrations/0071_auto_20210312_1924.py create mode 100644 artsurvey/migrations/0072_auto_20210312_1924.py create mode 100644 artsurvey/migrations/0073_auto_20210312_1927.py create mode 100644 artsurvey/migrations/0074_auto_20210312_1945.py create mode 100644 artsurvey/migrations/0075_auto_20210312_1946.py create mode 100644 artsurvey/migrations/0076_metasource.py create mode 100644 artsurvey/migrations/0077_artsurveysource_metasource.py create mode 100644 artsurvey/migrations/0078_artsurveysource_metasource_sep.py create mode 100644 artsurvey/migrations/0079_artsurvey_label.py create mode 100644 artsurvey/migrations/0080_auto_20210317_1721.py create mode 100644 artsurvey/migrations/0081_auto_20210317_1726.py create mode 100644 artsurvey/migrations/0082_auto_20210317_1726.py create mode 100644 artsurvey/migrations/0083_auto_20210319_1225.py create mode 100644 artsurvey/migrations/0084_auto_20210319_1225.py create mode 100644 artsurvey/migrations/0085_auto_20210319_1231.py create mode 100644 artsurvey/migrations/0086_auto_20210415_1544.py create mode 100644 artsurvey/migrations/0087_auto_20210415_1607.py create mode 100644 artsurvey/migrations/0088_artsurveysource_exti.py create mode 100644 artsurvey/migrations/0089_remove_artsurveysource_ext.py create mode 100644 artsurvey/migrations/0090_auto_20210416_1322.py create mode 100644 artsurvey/migrations/0091_auto_20210416_1322.py create mode 100644 artsurvey/migrations/0092_artsurveyparams_history.py create mode 100644 artsurvey/migrations/0093_artsurveysource_smapnr.py create mode 100644 artsurvey/migrations/0094_auto_20210430_1033.py create mode 100644 artsurvey/migrations/0095_auto_20210430_1036.py create mode 100644 artsurvey/migrations/0096_artsurveyparams_exclude_survey.py create mode 100644 artsurvey/migrations/0097_artsurveyparams_exclude_band.py create mode 100644 artsurvey/migrations/0098_artsurveysource_turkish.py create mode 100644 artsurvey/migrations/0099_artsurveysource_follow_up.py create mode 100644 artsurvey/migrations/0100_artsurveysource_ext_id.py create mode 100644 artsurvey/migrations/0101_auto_20210526_1619.py create mode 100644 artsurvey/migrations/0102_auto_20210526_1621.py create mode 100644 artsurvey/migrations/0103_artsurveysource_log_nfalse.py create mode 100644 artsurvey/migrations/0104_artsurveyparams_gaia_primary.py create mode 100644 artsurvey/migrations/0105_artsurveyparams_allwise_primary.py create mode 100644 artsurvey/migrations/0106_artsurveysource_catalog_version.py create mode 100644 artsurvey/migrations/0107_catalog.py create mode 100644 artsurvey/migrations/0108_artsurveysource_catalog.py create mode 100644 artsurvey/migrations/0109_remove_artsurveysource_catalog_version.py create mode 100644 artsurvey/migrations/0110_metasource_catalog.py create mode 100644 artsurvey/migrations/0111_artsurveyparams_exclude_log_nfalse.py create mode 100644 artsurvey/migrations/0112_auto_20210604_1059.py create mode 100644 artsurvey/migrations/0113_auto_20210604_1143.py create mode 100644 artsurvey/migrations/0114_auto_20210607_1736.py create mode 100644 artsurvey/migrations/0115_auto_20210607_1739.py create mode 100644 artsurvey/migrations/0116_auto_20210607_1740.py create mode 100644 artsurvey/migrations/0117_auto_20210607_1741.py create mode 100644 artsurvey/migrations/0118_auto_20210607_1743.py create mode 100644 artsurvey/migrations/0119_auto_20210607_1746.py create mode 100644 artsurvey/migrations/0120_auto_20210607_1748.py create mode 100644 artsurvey/migrations/0121_auto_20210607_1750.py create mode 100644 artsurvey/migrations/0122_auto_20210615_0915.py create mode 100644 artsurvey/migrations/0123_auto_20210701_1724.py create mode 100644 artsurvey/migrations/0124_auto_20210706_1008.py create mode 100644 artsurvey/migrations/0125_remove_artsurveyparams_ecl_lat_min.py create mode 100644 artsurvey/migrations/0126_auto_20210706_1010.py create mode 100644 artsurvey/migrations/0127_artsurveysource_xrays.py create mode 100644 artsurvey/migrations/0128_auto_20210826_1325.py create mode 100644 artsurvey/migrations/0129_catalog_test.py create mode 100644 artsurvey/migrations/0130_remove_catalog_test.py create mode 100644 artsurvey/migrations/0131_auto_20211228_1925.py create mode 100644 artsurvey/migrations/0132_artsurvey_parent.py create mode 100644 artsurvey/migrations/0133_auto_20211229_1259.py create mode 100644 artsurvey/migrations/0134_auto_20220112_1202.py create mode 100644 artsurvey/migrations/0135_artsurvey_name.py create mode 100644 artsurvey/migrations/0136_artsurveysource_turkpossible.py create mode 100644 artsurvey/migrations/0137_artsurveyparams_turkpossible.py create mode 100644 artsurvey/migrations/0138_alter_artsurveyparams_turkpossible.py create mode 100644 artsurvey/migrations/0139_auto_20220216_1735.py create mode 100644 artsurvey/migrations/0140_alter_artsurveyparams_turk_possible.py create mode 100644 artsurvey/migrations/0141_alter_artsurveysource_name_orig.py create mode 100644 artsurvey/migrations/0142_auto_20220303_1837.py create mode 100644 artsurvey/migrations/0143_auto_20220404_1254.py create mode 100644 artsurvey/migrations/0144_auto_20220404_1642.py create mode 100644 artsurvey/migrations/0145_auto_20220404_1643.py create mode 100644 artsurvey/migrations/0146_artsurveysource_heasarc_linked.py create mode 100644 artsurvey/migrations/0147_upload.py create mode 100644 artsurvey/migrations/0148_upload_artsources.py create mode 100644 artsurvey/migrations/0149_alter_upload_artsources.py create mode 100644 artsurvey/migrations/0150_alter_upload_filefield.py create mode 100644 artsurvey/migrations/0151_upload_notes.py create mode 100644 artsurvey/migrations/0152_artselection.py create mode 100644 artsurvey/migrations/0153_alter_artselection_owner.py create mode 100644 artsurvey/migrations/0154_remove_artselection_owner.py create mode 100644 artsurvey/migrations/0155_artselection_owner.py create mode 100644 artsurvey/migrations/0156_auto_20220415_1934.py create mode 100644 artsurvey/migrations/0157_auto_20220415_1938.py create mode 100644 artsurvey/migrations/0158_upload_metasource.py create mode 100644 artsurvey/migrations/0159_artsurvey_hidden.py create mode 100644 artsurvey/migrations/0160_alter_artsurveysource_options.py create mode 100644 artsurvey/migrations/0161_metacomment.py create mode 100644 artsurvey/migrations/0162_alter_metacomment_body.py create mode 100644 artsurvey/migrations/0163_alter_metacomment_body.py create mode 100644 artsurvey/migrations/0164_alter_artsurveyparams_log_nfalse_min.py create mode 100644 artsurvey/migrations/0165_artsurveyparams_exclude_category.py create mode 100644 artsurvey/migrations/0166_alter_artsurveyparams_exclude_category.py create mode 100644 artsurvey/migrations/0167_artsurvey_notes.py create mode 100644 artsurvey/migrations/0168_artsurveysource_metasource_primary.py create mode 100644 artsurvey/migrations/0169_remove_artsurveysource_metasource_primary.py create mode 100644 artsurvey/migrations/0170_artsurveysource_metasource_primary.py create mode 100644 artsurvey/migrations/0171_artsurveysource_lastvisited.py create mode 100644 artsurvey/migrations/0172_remove_artsurveysource_lastvisited.py create mode 100644 artsurvey/migrations/0173_upload_survey.py create mode 100644 artsurvey/migrations/0174_upload_primary.py create mode 100644 artsurvey/migrations/0175_artsurveysource_bibtex_paper.py create mode 100644 artsurvey/migrations/0176_auto_20220624_1826.py create mode 100644 artsurvey/migrations/0177_auto_20220707_1819.py create mode 100644 artsurvey/migrations/0178_auto_20220707_1825.py create mode 100644 artsurvey/migrations/0179_artsurveyparams_exposure_min.py create mode 100644 artsurvey/migrations/0180_auto_20221107_1529.py create mode 100644 artsurvey/migrations/0181_artsurvey_modified.py create mode 100644 artsurvey/migrations/0182_artsurveysource_ads_hidden.py create mode 100644 artsurvey/migrations/0183_artbasket.py create mode 100644 artsurvey/migrations/0184_artsurveysource_modified.py create mode 100644 artsurvey/migrations/0185_artsurveyparams_cname_startswith.py create mode 100644 artsurvey/migrations/0186_rename_cname_startswith_artsurveyparams_cname_contains.py create mode 100644 artsurvey/migrations/0187_auto_20230516_1536.py create mode 100644 artsurvey/migrations/0188_artsurveysource_log_ml_nfalse.py create mode 100644 artsurvey/migrations/0189_auto_20230516_1704.py create mode 100644 artsurvey/migrations/0190_auto_20231023_1559.py create mode 100644 artsurvey/migrations/0191_artsurveysource_bj2021.py create mode 100644 artsurvey/migrations/0192_artsurveysource_gaiadr3.py create mode 100644 artsurvey/migrations/0193_auto_20231025_1735.py create mode 100644 artsurvey/migrations/0194_auto_20231026_1224.py create mode 100644 artsurvey/migrations/0195_artsurveyparams_exclude_log_ml_nfalse.py create mode 100644 artsurvey/migrations/0196_alter_artsurveyparams_exclude_log_nfalse.py create mode 100644 artsurvey/migrations/0197_alter_artsurveyparams_exclude_log_nfalse.py create mode 100644 artsurvey/migrations/0198_auto_20231026_1613.py create mode 100644 artsurvey/migrations/0199_auto_20231028_1500.py create mode 100644 artsurvey/migrations/0200_artsurveysource_turkish_date.py create mode 100644 artsurvey/migrations/0201_auto_20231130_1125.py create mode 100644 artsurvey/migrations/0202_auto_20231130_1127.py create mode 100644 artsurvey/migrations/0203_artsurveysource_marshall.py create mode 100644 artsurvey/migrations/0204_artsurveyparams_marshall.py create mode 100644 artsurvey/migrations/0205_alter_artsurveyparams_marshall.py create mode 100644 artsurvey/migrations/0206_alter_artsurveyparams_marshall.py create mode 100644 artsurvey/migrations/0207_alter_artsurveyparams_marshall.py create mode 100644 artsurvey/migrations/__init__.py create mode 100644 artsurvey/models.py create mode 100644 artsurvey/templatetags/__init__.py create mode 100644 artsurvey/templatetags/artsurvey_templates.py create mode 100644 artsurvey/tests.py create mode 100644 artsurvey/urls.py create mode 100644 artsurvey/utils.py create mode 100644 artsurvey/views.py create mode 100644 astrobasis/__init__.py create mode 100644 astrobasis/admin.py create mode 100644 astrobasis/apps.py create mode 100644 astrobasis/management/commands/00_astrobasis_2mass_healpix.py create mode 100644 astrobasis/management/commands/00_astrobasis_2mass_init.py create mode 100644 astrobasis/management/commands/00_astrobasis_glimpse_healpix.py create mode 100644 astrobasis/management/commands/00_astrobasis_glimpse_init.py create mode 100644 astrobasis/management/commands/00_astrobasis_vlass_healpix.py create mode 100644 astrobasis/management/commands/00_astrobasis_vlass_init.py create mode 100644 astrobasis/management/commands/00_init_gaia.py create mode 100644 astrobasis/management/commands/00_load_ztf_alerce.py create mode 100644 astrobasis/management/commands/00_load_ztf_archive.py create mode 100644 astrobasis/management/commands/00_load_ztf_mars.py create mode 100644 astrobasis/management/commands/00_status_gaia.py create mode 100644 astrobasis/management/commands/00_test_gaia.py create mode 100644 astrobasis/management/commands/00_test_ztf.py create mode 100644 astrobasis/management/commands/01_load_gaia.py create mode 100644 astrobasis/management/commands/01_load_gaia_dr3.py create mode 100644 astrobasis/management/commands/02_delete_gaia.py create mode 100644 astrobasis/management/commands/02_mark_gaia.py create mode 100644 astrobasis/management/commands/03_healpix_gaia.py create mode 100644 astrobasis/management/commands/show_celery.py create mode 100644 astrobasis/migrations/0001_initial.py create mode 100644 astrobasis/migrations/0002_ztfalert.py create mode 100644 astrobasis/migrations/0003_remove_ztfalert_wall_time.py create mode 100644 astrobasis/migrations/0004_ztfalert_wall_time.py create mode 100644 astrobasis/migrations/0005_auto_20200128_2138.py create mode 100644 astrobasis/migrations/0006_delete_ztfalert.py create mode 100644 astrobasis/migrations/0007_ztfalert.py create mode 100644 astrobasis/migrations/0008_ztfalert_healpix_plate.py create mode 100644 astrobasis/migrations/0009_ztfalert_survey.py create mode 100644 astrobasis/migrations/0010_auto_20200129_1339.py create mode 100644 astrobasis/migrations/0011_remove_ztfalert_wall_time.py create mode 100644 astrobasis/migrations/0012_ztfalert_wall_time.py create mode 100644 astrobasis/migrations/0013_alerce.py create mode 100644 astrobasis/migrations/0014_auto_20200129_1948.py create mode 100644 astrobasis/migrations/0015_auto_20200129_1949.py create mode 100644 astrobasis/migrations/0016_auto_20200129_1949.py create mode 100644 astrobasis/migrations/0017_auto_20200129_1950.py create mode 100644 astrobasis/migrations/0018_auto_20200130_1300.py create mode 100644 astrobasis/migrations/0019_auto_20200130_1455.py create mode 100644 astrobasis/migrations/0020_auto_20200130_1726.py create mode 100644 astrobasis/migrations/0021_gaiadr2_error_radius.py create mode 100644 astrobasis/migrations/0022_alerce_error_radius.py create mode 100644 astrobasis/migrations/0023_auto_20200131_1555.py create mode 100644 astrobasis/migrations/0024_auto_20200131_1556.py create mode 100644 astrobasis/migrations/0025_sdssdr12spec.py create mode 100644 astrobasis/migrations/0026_flesch_z.py create mode 100644 astrobasis/migrations/0027_delete_flesch_z.py create mode 100644 astrobasis/migrations/0028_flesch.py create mode 100644 astrobasis/migrations/0029_simbad.py create mode 100644 astrobasis/migrations/0030_auto_20200223_0814.py create mode 100644 astrobasis/migrations/0031_simbad_healpix.py create mode 100644 astrobasis/migrations/0032_auto_20200223_1650.py create mode 100644 astrobasis/migrations/0033_auto_20200225_1604.py create mode 100644 astrobasis/migrations/0034_auto_20200225_1608.py create mode 100644 astrobasis/migrations/0035_auto_20200225_1625.py create mode 100644 astrobasis/migrations/0036_wise.py create mode 100644 astrobasis/migrations/0037_delete_wise.py create mode 100644 astrobasis/migrations/0038_allwise.py create mode 100644 astrobasis/migrations/0039_remove_allwise_image_url.py create mode 100644 astrobasis/migrations/0040_nvss.py create mode 100644 astrobasis/migrations/0041_auto_20200804_1457.py create mode 100644 astrobasis/migrations/0042_first.py create mode 100644 astrobasis/migrations/0043_sumss.py create mode 100644 astrobasis/migrations/0044_auto_20200804_1703.py create mode 100644 astrobasis/migrations/0045_auto_20200804_1707.py create mode 100644 astrobasis/migrations/0046_gaiadr3.py create mode 100644 astrobasis/migrations/0047_remove_gaiadr3_filename.py create mode 100644 astrobasis/migrations/0048_delete_gaiadr3.py create mode 100644 astrobasis/migrations/0049_gaiadr3.py create mode 100644 astrobasis/migrations/0050_delete_gaiadr3.py create mode 100644 astrobasis/migrations/0051_gaiadr3_gaiasourcefiledr3.py create mode 100644 astrobasis/migrations/0052_remove_gaiadr3_solution_id.py create mode 100644 astrobasis/migrations/0053_remove_gaiadr3_ref_epoch.py create mode 100644 astrobasis/migrations/0054_auto_20210812_1858.py create mode 100644 astrobasis/migrations/0055_auto_20211229_1259.py create mode 100644 astrobasis/migrations/0056_twomass.py create mode 100644 astrobasis/migrations/0057_glimpse.py create mode 100644 astrobasis/migrations/0058_alter_glimpse_healpix.py create mode 100644 astrobasis/migrations/0059_auto_20220315_1125.py create mode 100644 astrobasis/migrations/0060_alter_glimpse_options.py create mode 100644 astrobasis/migrations/0061_vlass.py create mode 100644 astrobasis/migrations/0062_auto_20220325_1731.py create mode 100644 astrobasis/migrations/0063_auto_20220325_1742.py create mode 100644 astrobasis/migrations/0064_auto_20220325_1752.py create mode 100644 astrobasis/migrations/0065_auto_20220330_1313.py create mode 100644 astrobasis/migrations/0066_alter_twomass_name.py create mode 100644 astrobasis/migrations/0067_vlassfromvizier.py create mode 100644 astrobasis/migrations/0068_auto_20220404_1639.py create mode 100644 astrobasis/migrations/0069_vlassfromvizier_recno.py create mode 100644 astrobasis/migrations/0070_alter_twomass_options.py create mode 100644 astrobasis/migrations/0071_astrobasisusercat.py create mode 100644 astrobasis/migrations/0072_delete_astrobasisusercat.py create mode 100644 astrobasis/migrations/0073_astrobasisusercat.py create mode 100644 astrobasis/migrations/0074_magnitude.py create mode 100644 astrobasis/migrations/0075_astrobasisusercat_magnitudes.py create mode 100644 astrobasis/migrations/0076_magnitude_error.py create mode 100644 astrobasis/migrations/0077_alter_magnitude_band.py create mode 100644 astrobasis/migrations/0078_alter_simbad_obj_class.py create mode 100644 astrobasis/migrations/0079_bjfromvizier.py create mode 100644 astrobasis/migrations/0080_auto_20231023_1633.py create mode 100644 astrobasis/migrations/0081_alter_bjfromvizier_error_radius.py create mode 100644 astrobasis/migrations/0082_alter_bjfromvizier_error_radius.py create mode 100644 astrobasis/migrations/0083_remove_bjfromvizier_recno.py create mode 100644 astrobasis/migrations/0084_alter_bjfromvizier_flag.py create mode 100644 astrobasis/migrations/0085_remove_bjfromvizier_flag.py create mode 100644 astrobasis/migrations/0086_gaiadr3fromvizier.py create mode 100644 astrobasis/migrations/0087_auto_20231025_1924.py create mode 100644 astrobasis/migrations/__init__.py create mode 100644 astrobasis/models.py create mode 100644 astrobasis/templatetags/__init__.py create mode 100644 astrobasis/templatetags/simbadescape.py create mode 100644 astrobasis/tests.py create mode 100644 astrobasis/urls.py create mode 100644 astrobasis/utils.py create mode 100644 astrobasis/views.py create mode 100755 genericsource/__init__.py create mode 100755 genericsource/admin.py create mode 100755 genericsource/apps.py create mode 100644 genericsource/migrations/0001_initial.py create mode 100644 genericsource/migrations/0002_auto_20200601_2248.py create mode 100644 genericsource/migrations/0003_auto_20200626_1000.py create mode 100644 genericsource/migrations/0004_auto_20200626_1523.py create mode 100644 genericsource/migrations/0005_genericsource_src_class.py create mode 100644 genericsource/migrations/0006_srcauxdata.py create mode 100644 genericsource/migrations/0007_srcauxdata_specimg.py create mode 100644 genericsource/migrations/0008_genericcatalogsgroup.py create mode 100644 genericsource/migrations/0009_srcauxdata_xcm.py create mode 100644 genericsource/migrations/0010_auto_20201023_1729.py create mode 100644 genericsource/migrations/0011_auto_20201103_1325.py create mode 100644 genericsource/migrations/0012_auto_20201104_1915.py create mode 100644 genericsource/migrations/0013_auto_20201106_1817.py create mode 100644 genericsource/migrations/0014_auto_20201112_1303.py create mode 100644 genericsource/migrations/0015_gsbibliogrpahy.py create mode 100644 genericsource/migrations/0016_auto_20211229_1259.py create mode 100755 genericsource/migrations/__init__.py create mode 100755 genericsource/models.py create mode 100755 genericsource/tests.py create mode 100644 genericsource/urls.py create mode 100755 genericsource/views.py create mode 100644 monthplan/__init__.py create mode 100644 monthplan/admin.py create mode 100644 monthplan/apps.py create mode 100644 monthplan/forms.py create mode 100644 monthplan/management/commands/00_load_surveypath.py create mode 100644 monthplan/management/commands/00_watchdog.py create mode 100644 monthplan/management/commands/01_init_monthplan.py create mode 100644 monthplan/management/commands/02_load_monthplan.py create mode 100644 monthplan/management/commands/03_load_flightplan.py create mode 100644 monthplan/management/commands/04_load_datadump.py create mode 100644 monthplan/management/commands/05_load_survey_track.py create mode 100644 monthplan/management/commands/05_load_survey_track.py.bak create mode 100644 monthplan/management/commands/06_load_scan_track.py create mode 100644 monthplan/management/commands/init_survey_healpix_plates.py create mode 100644 monthplan/management/commands/init_survey_healpix_sub_plates.py create mode 100644 monthplan/management/commands/mark_survey_healpix_plates.py create mode 100644 monthplan/management/commands/real_time_track.py create mode 100644 monthplan/management/commands/show_user_emails.py create mode 100644 monthplan/migrations/0001_initial.py create mode 100644 monthplan/migrations/0002_auto_20191209_2233.py create mode 100644 monthplan/migrations/0003_auto_20200210_1503.py create mode 100644 monthplan/migrations/0004_survey_loaded.py create mode 100644 monthplan/migrations/0005_auto_20200306_1022.py create mode 100644 monthplan/migrations/0006_surveyhealpixsubplate.py create mode 100644 monthplan/migrations/0007_auto_20211229_1259.py create mode 100644 monthplan/migrations/__init__.py create mode 100644 monthplan/models.py create mode 100644 monthplan/templatetags/__init__.py create mode 100644 monthplan/templatetags/monthplan_templates.py create mode 100644 monthplan/tests.py create mode 100644 monthplan/urls.py create mode 100644 monthplan/utils.py create mode 100644 monthplan/views.py create mode 100644 plan/__init__.py create mode 100644 plan/admin.py create mode 100644 plan/apps.py create mode 100644 plan/migrations/0001_initial.py create mode 100644 plan/migrations/0002_auto_20211229_1259.py create mode 100644 plan/migrations/__init__.py create mode 100644 plan/models.py create mode 100644 plan/tests.py create mode 100644 plan/urls.py create mode 100644 plan/views.py create mode 100644 srgcat/__init__.py create mode 100644 srgcat/admin.py create mode 100644 srgcat/apps.py create mode 100644 srgcat/forms.py create mode 100644 srgcat/management/commands/00_init_skymaps.py create mode 100644 srgcat/management/commands/00_load_art_gc.py create mode 100644 srgcat/management/commands/00_load_gyro.py create mode 100644 srgcat/management/commands/01_load_galcenter_for_artsurvey.py create mode 100644 srgcat/management/commands/01_load_skymap_erosita.py create mode 100644 srgcat/management/commands/01_load_skymap_for_artsurvey.py create mode 100644 srgcat/management/commands/01_load_skymap_sources.py create mode 100644 srgcat/management/commands/01_load_skymaps.py create mode 100644 srgcat/management/commands/01_update_allsky_missed.py create mode 100644 srgcat/management/commands/run_find_heasarc_counterparts.py create mode 100644 srgcat/management/commands/run_find_simbad_counterparts.py create mode 100644 srgcat/management/commands/simple_healpix_match.py create mode 100644 srgcat/management/commands/testme_artcat.py create mode 100644 srgcat/migrations/0001_initial.py create mode 100644 srgcat/migrations/0002_skymaps.py create mode 100644 srgcat/migrations/0003_remove_skymaps_healpix_plate.py create mode 100644 srgcat/migrations/0004_skymaps_survey_healpix_plate.py create mode 100644 srgcat/migrations/0005_skymaps_neighbours.py create mode 100644 srgcat/migrations/0006_srgdatadump.py create mode 100644 srgcat/migrations/0007_srgdatadump_version.py create mode 100644 srgcat/migrations/0008_skymapsource.py create mode 100644 srgcat/migrations/0009_skymapsource_simbad.py create mode 100644 srgcat/migrations/0010_skymapsource_gaia.py create mode 100644 srgcat/migrations/0011_skymapsource_dump.py create mode 100644 srgcat/migrations/0012_srgdatadump_catalog_loaded.py create mode 100644 srgcat/migrations/0013_skymapsource_new.py create mode 100644 srgcat/migrations/0014_skymapsource_bad.py create mode 100644 srgcat/migrations/0015_skymapsource_healpix_plate.py create mode 100644 srgcat/migrations/0016_skymapsource_good.py create mode 100644 srgcat/migrations/0017_selectallskymissed.py create mode 100644 srgcat/migrations/0018_selection.py create mode 100644 srgcat/migrations/0019_delete_selection.py create mode 100644 srgcat/migrations/0020_selection.py create mode 100644 srgcat/migrations/0021_auto_20200325_2157.py create mode 100644 srgcat/migrations/0022_skymapsource_catalog.py create mode 100644 srgcat/migrations/0023_erositamatch.py create mode 100644 srgcat/migrations/0024_remove_erositamatch_source.py create mode 100644 srgcat/migrations/0025_delete_erositamatch.py create mode 100644 srgcat/migrations/0026_erositamatch.py create mode 100644 srgcat/migrations/0027_usersessionparameters.py create mode 100644 srgcat/migrations/0028_auto_20200503_1438.py create mode 100644 srgcat/migrations/0029_auto_20200505_0947.py create mode 100644 srgcat/migrations/0030_auto_20200505_1338.py create mode 100644 srgcat/migrations/0031_trackstats.py create mode 100644 srgcat/migrations/0032_auto_20200515_1706.py create mode 100644 srgcat/migrations/0033_trackstats_group.py create mode 100644 srgcat/migrations/0034_skymapsource_simbad_failed.py create mode 100644 srgcat/migrations/0035_auto_20200601_2317.py create mode 100644 srgcat/migrations/0036_skymapsource_simbad_notfound.py create mode 100644 srgcat/migrations/0037_skymapsource_artsurvey_source.py create mode 100644 srgcat/migrations/0038_auto_20210311_1313.py create mode 100644 srgcat/migrations/0039_skymapsource_ext.py create mode 100644 srgcat/migrations/0040_skymapsource_cnts_tot.py create mode 100644 srgcat/migrations/0041_auto_20210311_1338.py create mode 100644 srgcat/migrations/0042_auto_20210311_1340.py create mode 100644 srgcat/migrations/0043_auto_20210311_1342.py create mode 100644 srgcat/migrations/0044_auto_20210315_1542.py create mode 100644 srgcat/migrations/0045_srgdatadump_gyro_loaded.py create mode 100644 srgcat/migrations/0046_gyro.py create mode 100644 srgcat/migrations/0047_auto_20210426_1746.py create mode 100644 srgcat/migrations/0048_auto_20210426_1805.py create mode 100644 srgcat/migrations/0049_gyro_healpix.py create mode 100644 srgcat/migrations/0050_auto_20210426_1836.py create mode 100644 srgcat/migrations/0051_skymapsource_ext_id.py create mode 100644 srgcat/migrations/0052_auto_20210615_0831.py create mode 100644 srgcat/migrations/0053_auto_20210615_0902.py create mode 100644 srgcat/migrations/0054_auto_20211229_1259.py create mode 100644 srgcat/migrations/0055_remove_skymapsource_gaia.py create mode 100644 srgcat/migrations/0056_alter_skymapsource_name_orig.py create mode 100644 srgcat/migrations/0057_auto_20220303_1826.py create mode 100644 srgcat/migrations/0058_alter_skymapsource_ext_id.py create mode 100644 srgcat/migrations/0059_auto_20230516_1536.py create mode 100644 srgcat/migrations/__init__.py create mode 100644 srgcat/models.py create mode 100644 srgcat/templatetags/__init__.py create mode 100644 srgcat/templatetags/srgcat_templates.py create mode 100644 srgcat/tests.py create mode 100644 srgcat/urls.py create mode 100644 srgcat/utils.py create mode 100644 srgcat/views.py create mode 100644 srglib/__init__.py create mode 100644 srglib/admin.py create mode 100644 srglib/apps.py create mode 100644 srglib/check_transient.py create mode 100644 srglib/correlate_utils.py create mode 100644 srglib/genericsourceutils.py create mode 100644 srglib/load_transinets.py create mode 100644 srglib/migrations/__init__.py create mode 100644 srglib/models.py create mode 100644 srglib/tasks.py create mode 100644 srglib/tests.py create mode 100644 srglib/update_erotrans_tables.py create mode 100644 srglib/utils.py create mode 100644 srglib/views.py create mode 100644 srglib/ztf.py diff --git a/artsurvey/__init__.py b/artsurvey/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/artsurvey/admin.py b/artsurvey/admin.py new file mode 100644 index 0000000..039a7df --- /dev/null +++ b/artsurvey/admin.py @@ -0,0 +1,9 @@ +from django.contrib import admin +from artsurvey.models import Category, ArtSurvey, Catalog, MetaComment, EnergyBand +# Register your models here. + +admin.site.register(Catalog) +admin.site.register(Category) +admin.site.register(ArtSurvey) +admin.site.register(MetaComment) +admin.site.register(EnergyBand) diff --git a/artsurvey/apps.py b/artsurvey/apps.py new file mode 100644 index 0000000..2dc0772 --- /dev/null +++ b/artsurvey/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ArtsurveyConfig(AppConfig): + name = 'artsurvey' diff --git a/artsurvey/forms.py b/artsurvey/forms.py new file mode 100644 index 0000000..ceaebac --- /dev/null +++ b/artsurvey/forms.py @@ -0,0 +1,105 @@ +from django import forms +from heasarc.models import HeasarcSimpleClass +from artsurvey.models import ArtSurveyParams +from artsurvey.models import ArtSurveyMatchParams +from artsurvey.models import OpticalCounterpart +from artsurvey.models import Category +from artsurvey.models import ArtSurveySource +from artsurvey.models import OtherName +from artsurvey.models import Upload +from artsurvey.models import ArtSurvey +from artsurvey.models import ArtBasket +from artsurvey.models import MetaComment + +class UploadForm(forms.ModelForm): + class Meta: + model = Upload + fields = ('title', 'notes', 'filefield') + widgets = { + 'title': forms.TextInput(attrs={'style': 'width: 620px;'}), + } + + +class ArtSurveySourceTypeForm(forms.Form): + users = forms.ModelChoiceField(queryset=HeasarcSimpleClass.objects.all().order_by('class_id'),widget=forms.Select(),required=True) + +class NotesForm(forms.Form): + redshift = forms.FloatField(label='Redshift',required=False) + notes = forms.CharField(widget=forms.Textarea(attrs={"rows":10, "cols":60}),required=False) + follow_up = forms.CharField(widget=forms.Textarea(attrs={"rows":10, "cols":60}),required=False) + notes_paper = forms.CharField(widget=forms.Textarea(attrs={"rows":10, "cols":60}),required=False) + bibtex_paper = forms.CharField(widget=forms.Textarea(attrs={"rows":10, "cols":60}),required=False) + +class NotesSurveyForm(forms.Form): + notes = forms.CharField(widget=forms.Textarea(attrs={"rows":10, "cols":60}),required=False) + +class CnameForm(forms.Form): + cname = forms.CharField(label='cname', required=False) + +class ArtSurveyParamsForm(forms.ModelForm): + survey = forms.ModelChoiceField(queryset=ArtSurvey.objects.all().filter(hidden__exact=False),widget=forms.Select(),required=True,help_text='Survey number N=[1..8] of version V=[0..9] is named as SN.V;
The combined survey is labeled as SN<...>N.V;
SGPLLL: 5x5 degrees Survey Galactic Plane at longitude LLL;
Tags: active (*), in work (W), archived (A, archived (H)') + #marshall = forms.ModelChoiceField(queryset=ArtSurveyParams.MARSHALL_SELECT, required=False) + class Meta: + model = ArtSurveyParams + fields = ('sky','survey','band','exclude_survey','exclude_band','exclude_log_nfalse','exclude_log_ml_nfalse','sign_ml_min','sign_ml_max','log_nfalse_min','log_nfalse_max','log_ml_nfalse_min','log_ml_nfalse_max','detlike_min','detlike_max','exposure_min','ext_min','ext_max','class_startswith','cname_contains','category','exclude_category','category_unclassified','marshall','gaia_primary','allwise_primary','turk_possible','glat_min','glat_max','dec_min','dec_max','ecl_lat_min','ecl_lat_max','circle_ra','circle_dec','circle_rmax_deg') + +class ArtSurveyMatchParamsForm(forms.ModelForm): + class Meta: + model = ArtSurveyMatchParams + fields = ('band1','survey1','sign_min1','sign_max1','log_ml_nfalse_min1','log_ml_nfalse_max1', + 'band2','survey2','sign_min2','sign_max2','log_ml_nfalse_min2','log_ml_nfalse_max2', + 'Rmax','maxdist','f2f1_ratio_min','f2f1_ratio_max') + +class OpticalCounterpartForm(forms.ModelForm): + class Meta: + model = OpticalCounterpart + fields = ('name','objid','t_objid','redshift','mag','ra','dec',) + +class OtherNameForm(forms.ModelForm): + class Meta: + model = OtherName + fields = ('name',) + +class CategoryForm(forms.ModelForm): + category = forms.ModelChoiceField(queryset=Category.objects.all(),widget=forms.Select(),required=False) + class Meta: + model = ArtSurveySource + fields = ('category',) + + +class ArtSurveySourceForm(forms.ModelForm): + class Meta: + model = ArtSurveySource + fields = ('name','ra','dec','flux','fluxerr_lo','fluxerr_up','nfalse','sig','ext',) + +class SearchCoordsForm(forms.Form): + ra = forms.FloatField(label='RAJ2000',required=True) + dec = forms.FloatField(label='DEJ2000',required=True) + rmax = forms.FloatField(initial=40, max_value=600, min_value=1, label='Rmax',required=True,help_text="arcsec") + maxdist = forms.FloatField(initial=120, max_value=1200, min_value=60,label='Maxdist',required=True,help_text="arcsec") + +class SearchNameForm(forms.Form): + name = forms.CharField(label='Name', required=True,help_text="Any part of SRGA name") + +class ResolveNameForm(forms.Form): + name = forms.CharField(label='Name', max_length=65) + rmax = forms.FloatField(initial=40, max_value=600, min_value=1, label='Rmax',required=True,help_text="arcsec") + maxdist = forms.FloatField(initial=120, max_value=1200, min_value=60,label='Maxdist',required=True,help_text="arcsec") + #load_surveypath = forms.BooleanField(required=False, initial=False) + +class MetaCommentForm(forms.ModelForm): + body = forms.CharField(label="", help_text="",widget=forms.Textarea(attrs={'rows':'3', 'cols':'60'})) + class Meta: + model = MetaComment + fields = ('body',) + + +class UserArtBasketListForm(forms.Form): + folder = forms.ModelChoiceField(queryset=ArtBasket.objects.all(),widget=forms.Select(),required=True) + + """ + def __init__(self, *args, **kwargs): + qs = kwargs.pop('bars') + super(FooForm, self).__init__(*args, **kwargs) + self.fields['bar'].queryset = qs + """ diff --git a/artsurvey/management/commands/00_artsurvey_addsource.py b/artsurvey/management/commands/00_artsurvey_addsource.py new file mode 100644 index 0000000..5ac47cd --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_addsource.py @@ -0,0 +1,55 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand +from artsurvey.utils import setup_metasource +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise + +def addsource(survey_version, band): + + try: + survey = ArtSurvey.objects.get(version=survey_version) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + try: + band = EnergyBand.objects.get(slug='E0') + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + addsource(12.2,'E0') + + self.stdout.write(self.style.SUCCESS('Not completed!')) + diff --git a/artsurvey/management/commands/00_artsurvey_all.py b/artsurvey/management/commands/00_artsurvey_all.py new file mode 100644 index 0000000..733ff17 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_all.py @@ -0,0 +1,117 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import sys + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_vlass +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise +from srglib.utils import load_simbad_for_skymap_sources +from srglib.utils import load_vizier_bj +from srglib.utils import load_vizier_gaia_dr3 +#from srglib.utils import select_turkpossible + +def do_all(survey_version,survey_name,slug="E0"): + minrad=40 + maxdist=600 + + #bad_category = Category.objects.get(slug='false') + + #minrad=180 + #maxdist=640 + + try: + band = EnergyBand.objects.get(slug=slug) + print("use EnergyBand {}".format(band)) + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return + + try: + survey = ArtSurvey.objects.get(Q(version=survey_version) & Q(name=survey_name)) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + srcs = survey.artsurveysource_set.all().filter(band=band).filter(log_ml_nfalse__lt=-2.3206) + print("Selected {} from {} {}".format(len(srcs),survey,band)) + #sys.exit() + + load_simbad_for_skymap_sources(srcs,minrad=minrad,maxdist=maxdist) + load_vizier_allwise(srcs, minrad=minrad, maxdist=maxdist) + + load_vizier_bj(srcs, minrad=minrad, maxdist=60) + load_vizier_gaia_dr3(srcs, minrad=minrad, maxdist=60) + + load_vizier_nvss(srcs, minrad=minrad, maxdist=maxdist) + load_vizier_first(srcs, minrad=minrad, maxdist=maxdist) + load_vizier_sumss(srcs, minrad=minrad, maxdist=maxdist) + load_vizier_vlass(srcs, minrad=minrad, maxdist=maxdist) + find_counterparts(srcs, GAIADR3.objects.all(), "gaia3", maxdist=maxdist, minrad=minrad) + + """ + for src in srcs: + src.heasarc.clear() + find_counterparts(srcs, HeasarcBase.objects.all(),'heasarc', maxdist=maxdist, minrad=minrad) + """ + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + #do_all(12.5) + #do_all(1.7) + #do_all(2.7) + + #do_all(1234.0) + #do_all(3.0) + #do_all(4.0) + #do_all(34.0,"34") + #do_all(5.0,"5") + #do_all(12345.0,"1-5") + + #do_all(0.12,"L20") + #do_all(0.12,"L20p1") + #do_all(0.12,"L20p2") + #do_all(0.12,"L20p3") + #do_all(0.12,"L20p4") + #do_all(0.13,"L20rb") + #do_all(0.15,"L20rb_ml") + + #do_all(12345.12,"1-5",slug="E5") + #do_all(12345.13,"1-5",slug="E5") + do_all(12345.14,"1-5",slug="E5") + + #names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345'] + + #names=['GP055','GP060','GP065'] + #names=['GP000','GP005','GP010','GP015','GP020','GP025','GP030','GP035','GP040','GP045','GP050','GP055','GP060','GP065','GP070','GP085','GP090', + # 'GP270','GP275','GP280','GP285','GP290','GP295','GP300','GP305','GP310','GP315','GP320','GP325'] + #names=['GP315'] + names=['GP330','GP335','GP340'] + """ + for name in names: + do_all(0.11,name) + """ + + + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_catalog_nfalse.py b/artsurvey/management/commands/00_artsurvey_catalog_nfalse.py new file mode 100644 index 0000000..a8f28ce --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_catalog_nfalse.py @@ -0,0 +1,178 @@ +from django.core.management.base import BaseCommand, CommandError + +import sys +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from django.db.models import Q + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, Catalog, MetaSource, Category +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise + + +def get_survey(version): + try: + survey = ArtSurvey.objects.get(version=version) + print("use ArtSurvey {}".format(survey)) + return survey + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(version)) + return None + +def get_band(slug): + try: + band = EnergyBand.objects.get(slug=slug) + print("use EnergyBand {}".format(band)) + return band + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return None + +def select(catalog,survey,band,log_nfalse,threshold): + srcs = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(Q(log_nfalse__lte=log_nfalse) | Q(ext__gte=1)) + for src in srcs: + src.catalog.add(catalog) + ms = src.metasource + ms.catalog=catalog + ms.save() + src.save() + count=srcs.count() + print("{} {}, log_nfalse {}, selected {}, {:.1f}% false count: {:.1f}".format(survey,band.slug,log_nfalse,count,threshold*100,count*threshold)) + +def select_ml(catalog,survey,band,log_ml_nfalse, threshold): + srcs = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(log_ml_nfalse__lte=log_ml_nfalse) # no extended sources in 2023 + + bad_category = Category.objects.get(slug='false') + + count=0 + for src in srcs: + if(src.category == bad_category): + print(src," --bad") + continue + print(src) + src.catalog.add(catalog) + if not (src.metasource): + raise ValueError("{} has no MetaSource".format(src)) + ms = src.metasource + ms.catalog=catalog + ms.save() + src.save() + count=count+1 + + #count=srcs.count() + print("{} {}, log_ml_nfalse {}, selected {}, {:.1f}% false count: {:.1f}".format(survey,band.slug,log_ml_nfalse,count,threshold*100,count*threshold)) + +def select_catalog(): + minrad=40 + maxdist=120 + + #s1s2=get_survey(12.5) + #s1=get_survey(1.7) + #s2=get_survey(2.7) + + #s15=get_survey(12345.12) + s15=get_survey(12345.14) + + #e0=get_band('E0') + #e1=get_band('E1') + #e2=get_band('E2') + #e3=get_band('E3') + #e4=get_band('E4') + e5=get_band('E5') + + cat1=Catalog.objects.get(year=2021) + cat2=Catalog.objects.get(year=2023) + + """ + RUN METASOURCE FIRST + """ + + """ clean links in MetaSource """ + msrcs = MetaSource.objects.all() + for ms in msrcs: + """ clean Pape II only, if exists """ + if(ms.catalog == cat2): + print("{} Clean catalog {}".format(ms,cat2)) + ms.catalog = None + ms.save() + + + + srcs = ArtSurveySource.objects.all() + + for src in srcs: + for cat in src.catalog.all(): + if(cat == cat1): + print("{} {} -- keep".format(src,cat)) + continue + + if(cat == cat2): + print("{} {} -- clean".format(src,cat)) + #continue + src.catalog.remove(cat2) + src.save() + continue + + + #sys.exit() + + + #select_ml(cat2,s15,e5,-3.1222,0.02)# ML appeared in second catalog + + select_ml(cat2,s15,e5,-3.1264,0.02)# ML appeared in second catalog + + sys.exit() + + #select(cat,s1s2,e0,-2.6751,0.10) + #select(cat,s1s2,e1,-3.2786) + #select(cat,s1s2,e2,-3.3014) + #select(cat,s1s2,e3,-3.9955) + + # false 1% + #select(cat,s1s2,e1,-4.0384,0.01) + #select(cat,s1s2,e2,-4.0799,0.01) + #select(cat,s1s2,e3,-4.5918,0.01) + + # false 5% + #select(cat,s1,e0,-3.2900) + #select(cat,s1,e1,-3.4620 ) + #select(cat,s1,e2,-3.5190 ) + #select(cat,s1,e3,-4.0747 ) + + # false 5% + #select(cat,s2,e0,-3.3065) + #select(cat,s2,e1,-3.5003 ) + #select(cat,s2,e2,-3.5262 ) + #select(cat,s2,e3,-4.1576 ) + + # false 1% + #select(cat,s1,e0,-4.0519,0.01) + + # false 1% + #select(cat,s2,e0,-4.0747,0.01) + + + + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + select_catalog() + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_catalog_sigma.py b/artsurvey/management/commands/00_artsurvey_catalog_sigma.py new file mode 100644 index 0000000..e15df49 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_catalog_sigma.py @@ -0,0 +1,93 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, Catalog, MetaSource +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise + + +def get_survey(version): + try: + survey = ArtSurvey.objects.get(version=version) + print("use ArtSurvey {}".format(survey)) + return survey + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(version)) + return None + +def get_band(slug): + try: + band = EnergyBand.objects.get(slug=slug) + print("use EnergyBand {}".format(band)) + return band + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return None + +def select(catalog,survey,band,threshold): + srcs = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(Q(sig__gte=threshold) | Q(ext__gte=1)) + + for src in srcs: + src.catalog.add(catalog) + ms = src.metasource + ms.catalog=catalog + ms.save() + src.save() + count=srcs.count() + print("{} {}, sign. threshold {}, selected {}".format(survey,band.slug,threshold,count)) + +def select_catalog(): + minrad=40 + maxdist=120 + + s1s2=get_survey(12.5) + s1=get_survey(1.7) + s2=get_survey(2.7) + + e0=get_band('E0') + e1=get_band('E1') + e2=get_band('E2') + e3=get_band('E3') + + cat=Catalog.objects.get(year=2021) + + """ + RUN METASOURCE FIRST + """ + + """ clean links in MetaSource """ + msrcs = MetaSource.objects.all() + for ms in msrcs: + ms.catalog = None + ms.save() + + srcs = ArtSurveySource.objects.all() + for src in srcs: + src.catalog.clear() + + select(cat,s1s2,e0,4.818) + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + select_catalog() + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_cds.py b/artsurvey/management/commands/00_artsurvey_cds.py new file mode 100644 index 0000000..a9722d0 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_cds.py @@ -0,0 +1,21 @@ +from django.core.management.base import BaseCommand, CommandError + +import astropy + +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + filename='/export/django/srg/data/artsurvey/catalog.csv' + data = astropy.table.Table.read(filename,format='ascii.csv',delimiter=',') + print(data.info) + + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_crossmatch.py b/artsurvey/management/commands/00_artsurvey_crossmatch.py new file mode 100644 index 0000000..c7a685c --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_crossmatch.py @@ -0,0 +1,80 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import F, Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +import numpy as np +from sqlalchemy import create_engine +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand +from astrobasis.models import GAIADR2 +from srglib.utils import find_counterparts +from srglib.utils import crossmatch +from srglib.utils import transfer +from heasarc.models import NSIDE_SOURCES, ORDER + + + + + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ arcsecond """ + minrad=40.0 + maxdist=120.0 + survey1=1.1 + survey2=12.1 + sig_min=0.0 + sig_max=1000.0 + print('***') + print("*** Filter: sig_min={} sig_max={} minrad={:.1f}'' maxdist={}'' ***".format(sig_min,sig_max,minrad,maxdist)) + print('***') + + try: + band = EnergyBand.objects.get(slug='E0') + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + print("use EnergyBand {}".format(band)) + + + + try: + s1 = ArtSurvey.objects.get(version=survey1) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey1)) + return + + try: + s2 = ArtSurvey.objects.get(version=survey2) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey2)) + return + + srcs1=s1.artsurveysource_set.all().filter(band=band).filter(Q(sig__gt=sig_min) & Q(sig__lt=sig_max)) + srcs2=s2.artsurveysource_set.all().filter(band=band).filter(Q(sig__gt=sig_min) & Q(sig__lt=sig_max)) + + print("use ArtSurvey {} nrows={}".format(s1, srcs1.count())) + print("use ArtSurvey {} nrows={}".format(s2, srcs2.count())) + match_count=transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist) + print("Total matched {}".format(match_count)) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_find_orig.py b/artsurvey/management/commands/00_artsurvey_find_orig.py new file mode 100644 index 0000000..df47c12 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_find_orig.py @@ -0,0 +1,70 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand +from srglib.utils import find_counterparts +from srgcat.models import SrgDataDump + +def find_orig_survey(init_version,survey_name,dump_name,slug): + name=dump_name+'.'+slug + try: + survey = ArtSurvey.objects.get(version=init_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey not found") + #survey = ArtSurvey(version=init_version, name=survey_name) + #survey.save() + return + + print("use ArtSurvey {}".format(survey)) + + + try: + band = EnergyBand.objects.get(slug=slug) + print("use EnergyBand {}".format(band)) + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return + + + """ copy catalog from this dump """ + dump = SrgDataDump.objects.get(name=name) + srcs = dump.skymapsource_set.all() + for src in srcs: + #if not (src.artsurvey_source): + + try: + source = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(name=src.name) + except Exception as e: + print("{} {}".format(src,e)) + continue + if (source.count() > 1): + for s in source: + print("{} --> {}".format(src,s)) + continue + if(source.count() == 0): + print("{} --> not found".format(src)) + continue + print("{} {} {}".format(slug,src.name,src.artsurvey_source)) + + # artsource = ArtSurveySource.objects.create_source(src,survey,band) + # artsource.save() + + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + find_orig_survey(1234.0,'1-4','S1-4','E0') + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_find_skymaps.py b/artsurvey/management/commands/00_artsurvey_find_skymaps.py new file mode 100644 index 0000000..7dd62f5 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_find_skymaps.py @@ -0,0 +1,33 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from django.db.models import Q + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from artsurvey.models import EnergyBand + +from artsurvey.utils import find_skymaps + + + + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + find_skymaps() + + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_heasarc.py b/artsurvey/management/commands/00_artsurvey_heasarc.py new file mode 100644 index 0000000..c6af5a7 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_heasarc.py @@ -0,0 +1,133 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime, os, sys +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from math import isfinite, sqrt + +from heasarc.models import HeasarcBase +from heasarc.models import Heasarc4XMMDR12 + +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, Category +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise +from srglib.utils import load_simbad_for_skymap_sources +from artsurvey.utils import find_heasarc + +from heasarc.models import HeasarcBase +from heasarc.models import HeasarcSwiftBAT105m +from heasarc.models import HeasarcIntRefCat +from heasarc.models import HeasarcXTEASSCAT +from heasarc.models import HeasarcXMMSL2 +from heasarc.models import Heasarc4FGL +from heasarc.models import HeasarcIntegral2020 +from heasarc.models import HeasarcMAXI7YR +from heasarc.models import HeasarcRASS2RXS + +def do_heasarc(survey_version,slug,log_ml_nfalse=None): + bad_category = Category.objects.get(slug='false') + minrad=40 + maxdist=600 + try: + band = EnergyBand.objects.get(slug=slug) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + + try: + survey = ArtSurvey.objects.get(version=survey_version) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + srcs = survey.artsurveysource_set.all().filter(band=band).filter(log_ml_nfalse__lte=log_ml_nfalse).exclude(category__exact=bad_category) + print("Selected {}".format(srcs.count())) + + # normal run + find_heasarc(srcs, maxdist=maxdist, minrad=minrad, offset=False, syserr=10.0) + return + + + # make offset + + """ + # script to read this file: + import pandas as pd + df=pd.read_csv('offset.csv') + + for k in df.keys(): + print("{} {:.1f} +/- {:.1f} std={:.1f}".format(k,df[k].mean(),df[k].sem(),df[k].std())) + """ + + fout='offset.csv' + try: + os.remove(fout) + except OSError: + pass + with open(fout, "a") as myfile: + myfile.write("RASS2RXS,XMMSL2,MAXI7YR,SwiftBAT105m,Integral2020,4FGL\n") + + for x in range(100): + find_heasarc(srcs, maxdist=maxdist, minrad=minrad, offset=True, syserr=10.0) + + SwiftBAT105m=0 + XMMSL2=0 + FGL=0 + Integral2020=0 + MAXI7YR=0 + RASS2RXS=0 + + for src in srcs: + heasarcs = src.heasarc.all() + if not (heasarcs.count()): + continue + if (heasarcs.filter(instance_of=HeasarcSwiftBAT105m).count()): + SwiftBAT105m+=1 + if (heasarcs.filter(instance_of=HeasarcXMMSL2).count()): + XMMSL2+=1 + if (heasarcs.filter(instance_of=Heasarc4FGL).count()): + FGL+=1 + if (heasarcs.filter(instance_of=HeasarcIntegral2020).count()): + Integral2020+=1 + if (heasarcs.filter(instance_of=HeasarcMAXI7YR).count()): + MAXI7YR+=1 + if (heasarcs.filter(instance_of=HeasarcRASS2RXS).count()): + RASS2RXS+=1 + with open(fout, "a") as myfile: + myfile.write("{}, {}, {}, {}, {}, {}\n".format(RASS2RXS,XMMSL2,MAXI7YR,SwiftBAT105m,Integral2020,FGL)) + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + #all=HeasarcBase.objects.all() + #print(all.count()) + + #do_heasarc(12345.12,'E5',log_ml_nfalse=-3.1222) + #do_heasarc(12345.13,'E5',log_ml_nfalse=-3.1264) + do_heasarc(12345.14,'E5',log_ml_nfalse=-3.1264) + + """ + srcs = ArtSurveySource.objects.all().filter(heasarc__heasarc4xmmdr12__isnull=False)[:10] + print("Selected {} in total".format(srcs.count())) + for src in srcs: + print(src.get_absolute_url()) + """ + + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_init.py b/artsurvey/management/commands/00_artsurvey_init.py new file mode 100644 index 0000000..4a468ab --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_init.py @@ -0,0 +1,105 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand +from srglib.utils import find_counterparts +from srgcat.models import SrgDataDump + +def init_survey(init_version,survey_name,dump_name,slug): + name=dump_name+'.'+slug + try: + survey = ArtSurvey.objects.get(Q(version=init_version) & Q(name=survey_name)) + except ArtSurvey.DoesNotExist: + print("create ArtSurvey") + survey = ArtSurvey(version=init_version, name=survey_name) + survey.save() + + print("use ArtSurvey {}".format(survey)) + + try: + band = EnergyBand.objects.get(slug=slug) + print("use EnergyBand {}".format(band)) + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return + + + + """ copy catalog from this dump """ + dump = SrgDataDump.objects.get(name=name) + print(dump) + + srcs = dump.skymapsource_set.all() + for src in srcs: + if not (src.artsurvey_source): + print("{} {}".format(slug,src)) + artsource = ArtSurveySource.objects.create_source(src,survey,band) + artsource.save() + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + """ + bands = ['E0','E1','E2','E3','E4'] + for slug in bands: + init_survey(12.5,'S12',slug) + """ + + #init_survey(123.0,'S123','E0') + #init_survey(1234.0,'1-4','S1-4','E0') + #init_survey(3.0,'3','S3','E0') + + + #version=0.1 + #init_survey(version,'L20','L20','E0') + + + #init_survey(0.13,'L20rb','L20rb','E0') + #init_survey(0.15,'L20rb_ml','L20rb_ml','E0') + #init_survey(12345.12,'1-5', '22y','E0') + #init_survey(12345.12,'1-5', '22y','E5') + #init_survey(12345.13,'1-5', '22y','E5') + init_survey(12345.14,'1-5', '22y','E5') + + #init_survey(version,'GC2019','GC','E0') + #init_survey(34.0,'34','S34','E0') + #init_survey(5.0,'5','S5','E0') + #init_survey(12345.0,'1-5','S1-5','E0') + + #init_survey(0.12,'L20','L20','E0') + #init_survey(0.12,'L20p1','L20p1','E0') + #init_survey(0.12,'L20p2','L20p2','E0') + #init_survey(0.12,'L20p3','L20p3','E0') + #init_survey(0.12,'L20p4','L20p4','E0') + + + version=0.11 + #fields=[50,45,40,35,30,'025','020','015','010','005','000','355','350','345',] + #fields=['070','075','080','085','090'] + #fields=['270','275','280','285','290','295','300','305','310','215','320','325'] + #fields=['315'] + + fields=['330','335','340'] + """ + for field in fields: + init_survey(version,"GP{}".format(field),"GP{}".format(field),'E0') + init_survey(version,"GP{}".format(field),"GP{}".format(field),'E1') + init_survey(version,"GP{}".format(field),"GP{}".format(field),'E2') + init_survey(version,"GP{}".format(field),"GP{}".format(field),'E3') + init_survey(version,"GP{}".format(field),"GP{}".format(field),'E4') + """ + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_mark_turkpossible.py b/artsurvey/management/commands/00_artsurvey_mark_turkpossible.py new file mode 100644 index 0000000..2fa7ac1 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_mark_turkpossible.py @@ -0,0 +1,33 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from django.db.models import Q + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from artsurvey.models import EnergyBand + +from artsurvey.utils import select_turkpossible + + + + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + select_turkpossible(12345.11,'E0') + + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_mark_xrays.py b/artsurvey/management/commands/00_artsurvey_mark_xrays.py new file mode 100644 index 0000000..487d00e --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_mark_xrays.py @@ -0,0 +1,176 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from django.db.models import Q + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from artsurvey.models import EnergyBand + +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise +from srglib.utils import load_simbad_for_skymap_sources + +from heasarc.models import HeasarcSwiftBAT105m +from heasarc.models import HeasarcXMMSL2 +from heasarc.models import Heasarc4FGL +from heasarc.models import HeasarcIntegral2020, HeasarcMAXI7YR, HeasarcRASS2RXS + +def do_all(survey_version,slug, log_ml_nfalse=None): + minrad=40 + maxdist=600 + + #minrad=180 + #maxdist=640 + + try: + band = EnergyBand.objects.get(slug=slug) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + + try: + survey = ArtSurvey.objects.get(version=survey_version) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + + srcs = survey.artsurveysource_set.all().filter(band=band).filter(log_ml_nfalse__lte=log_ml_nfalse) + + for src in srcs: + src.xrays=False + src.save() + + SwiftBAT105m=0 + XMMSL2=0 + FGL=0 + Integral2020=0 + MAXI7YR=0 + RASS2RXS=0 + for src in srcs: + heasarcs = src.heasarc.all() + #if not (heasarcs.count()): + # continue + + if (heasarcs.filter(instance_of=HeasarcSwiftBAT105m).count()): + SwiftBAT105m+=1 + src.xrays=True + src.save() + + if (heasarcs.filter(instance_of=HeasarcXMMSL2).count()): + XMMSL2+=1 + src.xrays=True + src.save() + + if (heasarcs.filter(instance_of=Heasarc4FGL).count()): + FGL+=1 + src.xrays=True + src.save() + + if (heasarcs.filter(instance_of=HeasarcIntegral2020).count()): + Integral2020+=1 + src.xrays=True + src.save() + + if (heasarcs.filter(instance_of=HeasarcMAXI7YR).count()): + MAXI7YR+=1 + src.xrays=True + src.save() + + if (heasarcs.filter(instance_of=HeasarcRASS2RXS).count()): + RASS2RXS+=1 + src.xrays=True + src.save() + + print("{} {}".format(src,heasarcs.count())) + + selection=srcs.filter(xrays__exact=True) + print("Xrays=True for {} from {}".format(selection.count(),srcs.count())) + + print("SwiftBAT105m={}".format(SwiftBAT105m)) + print("XMMSL2={}".format(XMMSL2)) + print("4FGL={}".format(FGL)) + print("Integral2020={}".format(Integral2020)) + print("MAXI7YR={}".format(MAXI7YR)) + print("RASS2RXS={}".format(RASS2RXS)) + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + #do_all(12.5,'E0') + #do_all(1.7) + #do_all(2.7) + + #do_all(12345.13,'E5', log_ml_nfalse=-3.1264) + do_all(12345.14,'E5', log_ml_nfalse=-3.1264) + + self.stdout.write(self.style.SUCCESS('Done')) + +""" + +SwiftBAT105m=7 +XMMSL2=2 +4FGL=26 +Integral2020=8 +MAXI7YR=7 +RASS2RXS=6 + +SwiftBAT105m=10 +XMMSL2=1 +4FGL=17 +Integral2020=7 +MAXI7YR=1 +RASS2RXS=4 + +Xrays=True for 41 from 1541 +SwiftBAT105m=5 +XMMSL2=2 +4FGL=20 +Integral2020=6 +MAXI7YR=3 +RASS2RXS=6 + +Xrays=True for 30 from 1541 +SwiftBAT105m=7 +XMMSL2=1 +4FGL=13 +Integral2020=3 +MAXI7YR=7 +RASS2RXS=4 + +Xrays=True for 34 from 1541 +SwiftBAT105m=5 +XMMSL2=1 +4FGL=18 +Integral2020=2 +MAXI7YR=5 +RASS2RXS=4 + +Xrays=True for 44 from 1541 +SwiftBAT105m=5 +XMMSL2=7 +4FGL=17 +Integral2020=9 +MAXI7YR=9 +RASS2RXS=4 + + +""" diff --git a/artsurvey/management/commands/00_artsurvey_metasource_check.py b/artsurvey/management/commands/00_artsurvey_metasource_check.py new file mode 100644 index 0000000..74d8655 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_metasource_check.py @@ -0,0 +1,60 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import F, Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +import numpy as np +from sqlalchemy import create_engine +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource, Catalog +from astrobasis.models import GAIADR2 +from srglib.utils import find_counterparts +from srglib.utils import crossmatch +from srglib.utils import transfer +from heasarc.models import NSIDE_SOURCES, ORDER + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + srcs = MetaSource.objects.all() + print("Check for empty MetaSource") + for src in srcs: + count = src.artsurveysource_set.all().count() + if (count==0): + print("Empty MetaSource {} paper={}".format(src,src.catalog)) + + self.stdout.write(self.style.SUCCESS('Done')) + + print("Check for empty Paper reference") + + for src in srcs: + if not src.catalog: + continue + + asrcs = src.artsurveysource_set.all() + count=0 + for a in asrcs: + if not (a.catalog.all().count()): + continue + if(src.catalog in a.catalog.all()): + count=count+1 + if (count==0): + print("Empty MetaSource {} paper={}".format(src,src.catalog)) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_metasource_manual_setup.py b/artsurvey/management/commands/00_artsurvey_metasource_manual_setup.py new file mode 100644 index 0000000..b87f0aa --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_metasource_manual_setup.py @@ -0,0 +1,98 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from artsurvey.utils import setup_metasource +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise + +def do_all(survey_version): + minrad=40 + maxdist=120 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + srcs = survey.artsurveysource_set.all() + print("Selected {} sources".format(srcs.count())) + for src in srcs: + if(src.skymapsource_set.all().count()): + orig=src.skymapsource_set.all()[0] + """ + src.nfalse=orig.nfalse + src.save() + print("{} <-- {}".format(src.nfalse,orig.nfalse)) + """ + if not orig.ext_id == '': + #src.ext_id=orig.ext_id + #src.save() + print("{} {} <-- {}".format(src,src.ext_id,orig.ext_id)) + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ + setup_metasource('J041324.8+102752','J041326.3+102725') + setup_metasource('J151003.9+333057','J151014.3+332942') + setup_metasource('J210336.2+454507','J210331.8+454453') + setup_metasource('J142557.4+374954','J142604.6+374930') + setup_metasource('J171226.3-232138','J171223.5-232048') + + setup_metasource('J151350.7-590814','J151355.6-590806') + setup_metasource('J151350.7-590814','J151355.7-590806') + setup_metasource('J151350.7-590814','J151355.6-590802') + setup_metasource('J151350.7-590814','J151354.2-590805') + setup_metasource('J151350.7-590814','J151354.1-590805') + setup_metasource('J151350.7-590814','J151354.3-590805') + setup_metasource('J151350.7-590814','J151355.4-590802') + setup_metasource('J151350.7-590814','J151354.5-590806') + # "Empty" MetaSource SRGA J151354.5-590806 has been deleted by hand + """ + + """ + setup_metasource('J151054.2+054422','J151054.9+054357') + setup_metasource('J151054.2+054422','J151053.6+054345') + setup_metasource('J151054.2+054422','J151053.6+054344') + """ + setup_metasource('J210336.2+454507','J210331.8+454453') + setup_metasource('J151054.2+054422','J151054.9+054357') + setup_metasource('J151054.2+054422','J151053.6+054344') + setup_metasource('J151054.2+054422','J151053.6+054345') + setup_metasource('J142557.4+374954','J142604.6+374930') + + setup_metasource('J121033.1+392421','J121030.2+392422') + setup_metasource('J123855.1-271830','J123823.9-253148') + setup_metasource('J174155.6-121207','J174154.0-121231') + setup_metasource('J184817.8-031020','J184816.1-030956') + setup_metasource('J092234.6-631738','J092239.7-631738') + #setup_metasource('','') + + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_radio.py b/artsurvey/management/commands/00_artsurvey_radio.py new file mode 100644 index 0000000..47df684 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_radio.py @@ -0,0 +1,41 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=2.3 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + + load_vizier_nvss(srcs, minrad=40, maxdist=120) + load_vizier_first(srcs, minrad=40, maxdist=120) + load_vizier_sumss(srcs, minrad=40, maxdist=120) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_replica.py b/artsurvey/management/commands/00_artsurvey_replica.py new file mode 100644 index 0000000..80fc48b --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_replica.py @@ -0,0 +1,125 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from django.db.models import Q + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand +from srglib.utils import find_counterparts +from srgcat.models import SrgDataDump + +def make_survey_replica(version,slug): + """ note that we don't copy catalogs, i.e. papers reletions """ + + + """ delete all replicas of this survey """ + try: + replicas = ArtSurvey.objects.filter(Q(version=version) & Q(parent__isnull=False)) + for replica in replicas: + print("Delete Replica ArtSurvey {} --> {}".format(replica,replica.parent)) + replicas.delete() + except ArtSurvey.DoesNotExist: + pass + + + + """ For cloning select only original surveys, i.e. without parents """ + try: + survey = ArtSurvey.objects.get(Q(version=version) & Q(parent=None)) + except ArtSurvey.DoesNotExist: + print("For cloning select only original surveys, i.e. without parents") + print("Survey {} with parent=None not found".format(version)) + return + + + + print("use ArtSurvey {}".format(survey)) + + try: + band = EnergyBand.objects.get(slug=slug) + print("use EnergyBand {}".format(band)) + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return + + child = ArtSurvey(version=version,parent=survey) + child.save() + + srcs = survey.artsurveysource_set.all().filter(band=band) + print("Selected {}".format(srcs.count())) + + for src in srcs: + print("Cloning {}".format(src)) + self_pk = src.pk + onames = src.othername_set.all() + for o in onames: + print("{} OtherNames {}".format(src,o)) + ads = src.ads.all() + simbad = src.simbad.all() + nvss = src.nvss.all() + first = src.first.all() + sumss = src.sumss.all() + heasarc = src.heasarc.all() + allwise = src.allwise.all() + gaia3 = src.gaia3.all() + gaia3_primary = src.gaia3_primary + + clone = src + clone.pk = None + clone.id = None + clone._state.adding = True + clone.survey = child + clone.save() + try: + parent = ArtSurveySource.objects.get(pk=self_pk) + except ArtSurveySource.DoesNotExist: + print("Parent source not found for {}".format(clone)) + return + + clone.parent = parent + clone.ads.set(ads) + clone.simbad.set(simbad) + clone.nvss.set(nvss) + clone.first.set(first) + clone.sumss.set(sumss) + clone.heasarc.set(heasarc) + clone.allwise.set(allwise) + clone.gaia3.set(gaia3) + + """ clone OtherName """ + for o in onames: + print("clone OtherName {}".format(o)) + o.source=clone + o.pk=None + o.id=None + o._state.adding = True + o.save() + + clone.save() + + + + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + """ + bands = ['E0','E1','E2','E3','E4'] + for slug in bands: + init_survey(12.5,'S12',slug) + """ + + make_survey_replica(12.5,'E0') + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_subquery.py b/artsurvey/management/commands/00_artsurvey_subquery.py new file mode 100644 index 0000000..6cace7a --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_subquery.py @@ -0,0 +1,77 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import Count +from django.db.models import OuterRef, Subquery, Exists + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource +from srglib.utils import find_counterparts +from srgcat.models import SrgDataDump + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + #srcs = ArtSurveySource.objects.all() + + """ Find first survey """ + version=12.2 + + try: + survey = ArtSurvey.objects.get(version=version) + except ArtSurvey.DoesNotExist: + return + + print("use ArtSurvey {}".format(survey)) + + + try: + exclude_survey = ArtSurvey.objects.get(version=2.4) + except ArtSurvey.DoesNotExist: + return + + print("exclude ArtSurvey {}".format(exclude_survey)) + + + try: + band = EnergyBand.objects.get(slug='E0') + print("use EnergyBand {}".format(band)) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + try: + exclude_band = EnergyBand.objects.get(slug='E0') + print("use exclude EnergyBand {}".format(exclude_band)) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + + srcs = ArtSurveySource.objects.all().filter(survey=survey) + srcs=srcs.filter(band=band) + + subquery=MetaSource.objects.filter(artsurveysource=OuterRef("pk")).filter(artsurveysource__survey=exclude_survey).filter(artsurveysource__band=exclude_band) + #srcs=srcs.annotate(exclude=Subquery(subquery.values("name")[:1])) + srcs=srcs.annotate(exclude=Exists(subquery)) + + for s in srcs: + print(s,s.exclude) + + print(srcs.count()) + + """ + msrcs=MetaSource.objects.annotate(num_sources=Count("artsurveysource")) + for m in msrcs: + print(m, m.num_sources) + """ + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_sync_notes.py b/artsurvey/management/commands/00_artsurvey_sync_notes.py new file mode 100644 index 0000000..3bd523e --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_sync_notes.py @@ -0,0 +1,57 @@ +from django.core.management.base import BaseCommand, CommandError + +import astropy + +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + + try: + band = EnergyBand.objects.get(slug='E0') + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + + try: + survey = ArtSurvey.objects.get(version=12.5) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + + filename='/export/django/srg/data/artsurvey_notes_revised_paper1.tex' + data = astropy.table.Table.read(filename,format='ascii.csv',delimiter='|') + print(data.info) + + + + for item in data: + name=item[0] + notes=item[1] + print() + print("[new] {} --> {}".format(item[0],item[1])) + try: + srcs=ArtSurveySource.objects.filter(name__exact=name).filter(survey=survey).filter(band=band) + except Exception as e: + print(e) + continue + if(srcs.count()>1): + print("More than one source found!") + return + for src in srcs: + print("[old] {} --> {}".format(src.name,src.notes_paper)) + #src.notes_paper=notes + #src.save() + + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_transfer.old b/artsurvey/management/commands/00_artsurvey_transfer.old new file mode 100644 index 0000000..d1a4baf --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_transfer.old @@ -0,0 +1,175 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from astrobasis.models import GAIADR2 +from srglib.utils import find_counterparts +from heasarc.models import NSIDE_SOURCES, ORDER + +def transfer_data(psrc,child): + print("--> Transfer data from {} (S{} E{}) to {} (S{} E{})".format(psrc,psrc.survey, psrc.band, child, child.survey, child.band)) + + child.owner=psrc.owner + child.notes=psrc.notes + child.redshift=psrc.redshift + child.class_id=psrc.class_id + child.object_class=psrc.object_class + child.cname=psrc.cname + child.refid=psrc.refid + child.erosita_data=psrc.erosita_data + child.gaia_primary=psrc.gaia_primary + child.class_tentative=psrc.class_tentative + child.category=psrc.category + + for a in psrc.ads.all(): + child.ads.add(a) + + """ clone OtherName """ + for o in psrc.othername_set.all(): + o.source=child + o.pk=None + o.save() + + child.save() + + + +class Command(BaseCommand): + help = 'DEPRECATED' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ arcsecond """ + tolerance=10 + parent_version=1.1 + children_version=1.2 + + try: + parent = ArtSurvey.objects.get(version=parent_version) + except ArtSurvey.DoesNotExist: + print("parent ArtSurvey {} not found".format(parent_version)) + return + + try: + children = ArtSurvey.objects.get(version=children_version) + except ArtSurvey.DoesNotExist: + print("children ArtSurvey {} not found".format(children_version)) + return + + print("use parent ArtSurvey {}".format(parent)) + print("use children ArtSurvey {}".format(children)) + + """ Reset parent in children's relations """ + + for child in children.artsurveysource_set.all(): + child.parent=None + child.owner=None + child.notes='' + child.redshift=0.0 + child.class_id=0 + child.object_class=None + child.cname='' + child.refid=0 + child.erosita_data=False + child.gaia_primary=None + child.class_tentative=False + child.category=None + child.ads.clear() + for o in child.othername_set.all(): + print("Delete {} in {}".format(o,child)) + o.delete() + child.save() + + + for psrc in parent.artsurveysource_set.all(): + c = SkyCoord(psrc.ra, psrc.dec, frame="fk5", unit="deg") + + #print("{} {}".format(psrc,psrc.healpix)) + try: + csrcs=children.artsurveysource_set.all().filter(healpix=psrc.healpix) + except: + #self.stdout.write(self.style.ERROR("Missed childrens for {}".format(psrc))) + print("[cent] Missed childrens for {}".format(psrc)) + continue + #print("{} {} --> {} {}".format(psrc,psrc.healpix,csrc, csrc.healpix)) + for csrc in csrcs: + c0 = SkyCoord(csrc.ra, csrc.dec, frame="fk5", unit="deg") + sep=c.separation(c0).arcsecond + if(sep <= tolerance): + print("[cent] Found children {} (S{} E{}) for parent {} (S{} E{}) at {:.1f}''".format(csrc,csrc.survey,csrc.band,psrc,psrc.survey,psrc.band,sep)) + csrc.parent=psrc + csrc.save() + transfer_data(psrc,csrc) + else: + #print("[cent] Possible children {} for parent {} is too far away {:.1f}'' (tol {}'')".format(csrc,psrc,sep,tolerance)) + pass + + neib = neighbours(psrc.healpix, NSIDE_SOURCES, order=ORDER) + for heal in neib: + try: + csrcs=children.artsurveysource_set.all().filter(healpix=heal) + except: + #print("neib: Missed children for {}".format(psrc)) + continue + for csrc in csrcs: + c0 = SkyCoord(csrc.ra, csrc.dec, frame="fk5", unit="deg") + sep=c.separation(c0).arcsecond + if(sep <= tolerance): + self.stdout.write(self.style.SUCCESS("[neib] Found children {} (S{} E{}) for parent {} (S{} E{}) at {:1f}''".format(csrc,csrc.survey,csrc.band,psrc,psrc.survey,psrc.band,sep))) + csrc.parent=psrc + csrc.save() + transfer_data(psrc,csrc) + else: + print("[neib] Possible children {} for parent {} is too far away {:.1f}'' (tol {}'')".format(csrc,psrc,sep,tolerance)) + + self.stdout.write(self.style.SUCCESS('Done')) + return + """ dalshe ne rabotaet """ + """ print warning for missed sources """ + for psrc in parent.artsurveysource_set.all(): + if (psrc.children.all().count() == 0): + self.stdout.write(self.style.ERROR("Missed children for {}".format(psrc))) + if (psrc.children.all().count() > 1): + self.stdout.write(self.style.ERROR("Too many ({}) children for {}".format(psrc.children.all().count(), psrc))) + if (psrc.children.all().count() == 1): + + """ transfer catalog information """ + child = psrc.children.all()[0] + print("Transfer data from {} (S{} E{}) to {} (S{} E{})".format(psrc,psrc.survey, psrc.band, child, child.survey, child.band)) + child.owner=psrc.owner + child.notes=psrc.notes + child.redshift=psrc.redshift + child.class_id=psrc.class_id + child.object_class=psrc.object_class + child.cname=psrc.cname + child.refid=psrc.refid + child.erosita_data=psrc.erosita_data + child.gaia_primary=psrc.gaia_primary + child.class_tentative=psrc.class_tentative + child.category=psrc.category + for a in psrc.ads.all(): + child.ads.add(a) + """ clone OtherName """ + for o in psrc.othername_set.all(): + o.source=child + o.pk=None + o.save() + child.save() + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_transfer.py b/artsurvey/management/commands/00_artsurvey_transfer.py new file mode 100644 index 0000000..c82fb6e --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_transfer.py @@ -0,0 +1,188 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import F, Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +import numpy as np +from sqlalchemy import create_engine +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand +from astrobasis.models import GAIADR2 +from srglib.utils import find_counterparts +from srglib.utils import crossmatch +from srglib.utils import transfer +from heasarc.models import NSIDE_SOURCES, ORDER + + +def do_transfer(parent, child_version, child_name, slug, minrad=30, maxdist=500): + """ the same band assumed for parent and child """ + try: + band = EnergyBand.objects.get(slug=slug) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + print("use EnergyBand {}".format(band)) + + + + try: + s1 = ArtSurvey.objects.get(Q(version=parent)) # & Q(archived=True)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(parent)) + return + + try: + s2 = ArtSurvey.objects.get(Q(version=child_version) & Q(name=child_name)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(child_version)) + return + + srcs1=s1.artsurveysource_set.all().filter(band=band) + srcs2=s2.artsurveysource_set.all().filter(band=band) + + print("[{}] use parent ArtSurvey {} nrows={}".format(slug, s1, srcs1.count())) + print("[{}] use child ArtSurvey {} nrows={}".format(slug, s2, srcs2.count())) + match_count = transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist) + print("Total matched {}".format(match_count)) + +def do_transfer_band(parent, parent_band, child_version, child_name, child_band, minrad=30, maxdist=500): + """ the same band assumed for parent and child """ + + try: + band1 = EnergyBand.objects.get(slug=parent_band) + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return + + try: + band2 = EnergyBand.objects.get(slug=child_band) + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return + + print("use EnergyBand {} --> {}".format(band1, band2)) + + try: + s1 = ArtSurvey.objects.get(Q(version=parent)) # & Q(archived=True)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(parent)) + return + + try: + s2 = ArtSurvey.objects.get(Q(version=child_version) & Q(name=child_name)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(child_version)) + return + + srcs1=s1.artsurveysource_set.all().filter(band=band1) + srcs2=s2.artsurveysource_set.all().filter(band=band2) + + print("[{}] use parent ArtSurvey {} nrows={}".format(child_band, s1, srcs1.count())) + print("[{}] use child ArtSurvey {} nrows={}".format(child_band, s2, srcs2.count())) + match_count = transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist) + print("Total matched {}".format(match_count)) + +def do_transfer_by_name(parent_version, parent_name, child_version, child_name, slug, minrad=30, maxdist=500): + try: + band = EnergyBand.objects.get(slug=slug) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + print("use EnergyBand {}".format(band)) + + + + try: + s1 = ArtSurvey.objects.get(Q(version=parent_version) & Q(name=parent_name)) + except ArtSurvey.DoesNotExist: + print("*** parent ArtSurvey {} {} not found".format(parent_name,parent_version)) + return + + try: + s2 = ArtSurvey.objects.get(Q(version=child_version) & Q(name=child_name)) + except ArtSurvey.DoesNotExist: + print("*** child ArtSurvey {} not found".format(child_version)) + return + + srcs1=s1.artsurveysource_set.all().filter(band=band) + srcs2=s2.artsurveysource_set.all().filter(band=band) + + print("[{}] use parent ArtSurvey {} nrows={}".format(slug, s1, srcs1.count())) + print("[{}] use child ArtSurvey {} nrows={}".format(slug, s2, srcs2.count())) + match_count = transfer(srcs1, srcs2, minrad=minrad, maxdist=maxdist) + print("Total matched {}".format(match_count)) + + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ arcsecond """ + minrad=30.0 + maxdist=500.0 + + + """ + bands = ['E0','E1','E2','E3'] + for slug in bands: + print (slug) + do_transfer(12.4, 12.5, slug, minrad=minrad, maxdist=maxdist) + + do_transfer(1.6, 1.7, 'E0', minrad=minrad, maxdist=maxdist) + do_transfer(2.6, 2.7, 'E0', minrad=minrad, maxdist=maxdist) + """ + + #do_transfer(12.5, 1234.0, 'E0', minrad=minrad, maxdist=maxdist) + + #do_transfer(12.5, 3.0, 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer(12.5, 34.0,"34", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer(12.5, 5.0,"5", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer(1234.0, 12345.0,"1-5", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer(12345.0, 12345.1,"1-5", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer(12345.0, 12345.11,"1-5", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer(12345.11, 12345.12,"1-5", 'E0', minrad=minrad, maxdist=maxdist) + + #do_transfer_band(12345.11, "E0", 12345.12,"1-5", 'E5', minrad=minrad, maxdist=maxdist) + #do_transfer_band(12345.12, "E5", 12345.13,"1-5", 'E5', minrad=minrad, maxdist=maxdist) + do_transfer_band(12345.13, "E5", 12345.14,"1-5", 'E5', minrad=minrad, maxdist=maxdist) + + + + #names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345',] + #names=['GP055','GP060','GP065'] + names=['GP070','GP075','GP080','GP085','GP090', 'GP270','GP275','GP280','GP285','GP290','GP295','GP300','GP305','GP310','GP215','GP320','GP325'] + """ + for name in names: + do_transfer(12.5, 0.11, name, 'E0', minrad=minrad, maxdist=maxdist) + """ + + """ + names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345',] + for name in names: + do_transfer_by_name(0.10, name, 0.11, name, 'E0', minrad=minrad, maxdist=maxdist) + """ + + #do_transfer_by_name(0.10, "L20", 0.12, "L20", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer_by_name(0.10, "L20", 0.12, "L20p1", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer_by_name(0.10, "L20", 0.12, "L20p2", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer_by_name(0.10, "L20", 0.12, "L20p3", 'E0', minrad=minrad, maxdist=maxdist) + #do_transfer_by_name(0.10, "L20", 0.12, "L20p4", 'E0', minrad=minrad, maxdist=maxdist) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_update.py b/artsurvey/management/commands/00_artsurvey_update.py new file mode 100644 index 0000000..ca1265a --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_update.py @@ -0,0 +1,76 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import load_vizier_nvss +from srglib.utils import load_vizier_first +from srglib.utils import load_vizier_sumss +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts +from srglib.utils import load_vizier_allwise + +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import BarycentricTrueEcliptic +from astropy.coordinates import HeliocentricTrueEcliptic +from astropy.coordinates import GeocentricTrueEcliptic + + +def do_all(survey_version): + minrad=40 + maxdist=120 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + srcs = survey.artsurveysource_set.all() + print("Selected {} sources".format(srcs.count())) + + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + src.ecl_l=crd.transform_to(BarycentricTrueEcliptic).lon.value + src.ecl_b=crd.transform_to(BarycentricTrueEcliptic).lat.value + src.save() + + return + + + for src in srcs: + if(src.skymapsource_set.all().count()): + orig=src.skymapsource_set.all()[0] + """ + src.nfalse=orig.nfalse + src.save() + print("{} <-- {}".format(src.nfalse,orig.nfalse)) + """ + if not orig.ext_id == '': + src.ext_id=orig.ext_id + src.save() + print("{} {} <-- {}".format(src,src.ext_id,orig.ext_id)) + + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + do_all(12345.0) + self.stdout.write(self.style.SUCCESS('Done')) + diff --git a/artsurvey/management/commands/00_artsurvey_update_marshall.py b/artsurvey/management/commands/00_artsurvey_update_marshall.py new file mode 100644 index 0000000..bdd99e3 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_update_marshall.py @@ -0,0 +1,67 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import F, Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +import numpy as np +from sqlalchemy import create_engine +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource +from artsurvey.utils import update_marshall + +from astrobasis.models import GAIADR2 +from srglib.utils import find_counterparts +from srglib.utils import crossmatch +from srglib.utils import transfer +from heasarc.models import NSIDE_SOURCES, ORDER + + + +def update_local(survey_version,survey_name,en='E0',minrad=15,maxdist=120): + try: + s = ArtSurvey.objects.get(Q(version=survey_version) & Q(name=survey_name)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} {} not found".format(survey_name,survey_version)) + return + + try: + band = EnergyBand.objects.get(slug=en) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + srcs=s.artsurveysource_set.all().filter(band=band) + count = update_marshall(srcs) + print("use ArtSurvey {} band={} nrows={} , marshall={}".format(s, band, srcs.count(), count)) + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ arcsecond """ + minrad=30.0 + maxdist=500.0 + + """ Detele MetaSource """ + #msrcs=MetaSource.objects.all() + #print("Delete MetaSource nrows={}".format(msrcs.count())) + #msrcs.delete() + + + update_local(12345.14, "1-5", en='E5') + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_artsurvey_update_metasource.py b/artsurvey/management/commands/00_artsurvey_update_metasource.py new file mode 100644 index 0000000..3d3ba98 --- /dev/null +++ b/artsurvey/management/commands/00_artsurvey_update_metasource.py @@ -0,0 +1,137 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import F, Q + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +import numpy as np +from sqlalchemy import create_engine +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey, EnergyBand, MetaSource +from artsurvey.utils import update_metasource + +from astrobasis.models import GAIADR2 +from srglib.utils import find_counterparts +from srglib.utils import crossmatch +from srglib.utils import transfer +from heasarc.models import NSIDE_SOURCES, ORDER + + + +def update_meta(survey_version,survey_name,en='E0',minrad=15,maxdist=120): + try: + s = ArtSurvey.objects.get(Q(version=survey_version) & Q(name=survey_name)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} {} not found".format(survey_name,survey_version)) + return + + try: + band = EnergyBand.objects.get(slug=en) + except EnergyBand.DoesNotExist: + print("EnergyBand bot found") + return + + + srcs=s.artsurveysource_set.all().filter(band=band) + count = update_metasource(srcs, minrad=minrad, maxdist=maxdist) + print("use ArtSurvey {} band={} nrows={} , meta={}".format(s, band, srcs.count(), count)) + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ arcsecond """ + minrad=30.0 + maxdist=500.0 + + """ Detele MetaSource """ + #msrcs=MetaSource.objects.all() + #print("Delete MetaSource nrows={}".format(msrcs.count())) + #msrcs.delete() + + + + """ + Start filling MetaSource with the most deep survey + + Attention! Start with S12.E0 because only it contains extended objects + + """ + + """ + update_meta(survey=12.5, en='E0', minrad=minrad, maxdist=maxdist) + update_meta(survey=12.5, en='E1', minrad=minrad, maxdist=maxdist) + update_meta(survey=12.5, en='E2', minrad=minrad, maxdist=maxdist) + update_meta(survey=12.5, en='E3', minrad=minrad, maxdist=maxdist) + update_meta(survey=12.5, en='E4', minrad=minrad, maxdist=maxdist) + + + update_meta(survey=1.7, en='E0', minrad=minrad, maxdist=maxdist) + update_meta(survey=1.7, en='E1', minrad=minrad, maxdist=maxdist) + update_meta(survey=1.7, en='E2', minrad=minrad, maxdist=maxdist) + update_meta(survey=1.7, en='E3', minrad=minrad, maxdist=maxdist) + update_meta(survey=1.7, en='E4', minrad=minrad, maxdist=maxdist) + + update_meta(survey=2.7, en='E0', minrad=minrad, maxdist=maxdist) + update_meta(survey=2.7, en='E1', minrad=minrad, maxdist=maxdist) + update_meta(survey=2.7, en='E2', minrad=minrad, maxdist=maxdist) + update_meta(survey=2.7, en='E3', minrad=minrad, maxdist=maxdist) + update_meta(survey=2.7, en='E4', minrad=minrad, maxdist=maxdist) + """ + + + #update_meta(survey=0.0, en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(survey=0.0, en='E1', minrad=minrad, maxdist=maxdist) + #update_meta(survey=0.0, en='E2', minrad=minrad, maxdist=maxdist) + + #update_meta(survey=123.0, en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(survey=1234.0, en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(survey=3.0, en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(survey=4.0, en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(34.0, "34", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(5.0, "5", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(12345.0, "1-5", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(12345.1, "1-5", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(12345.11, "1-5", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(12345.12, "1-5", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(12345.12, "1-5", en='E5', minrad=minrad, maxdist=maxdist) + #update_meta(12345.13, "1-5", en='E5', minrad=minrad, maxdist=maxdist) + update_meta(12345.14, "1-5", en='E5', minrad=minrad, maxdist=maxdist) + + #update_meta(0.12, "L20", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(0.12, "L20p1", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(0.12, "L20p2", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(0.12, "L20p3", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(0.12, "L20p4", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(0.13, "L20rb", en='E0', minrad=minrad, maxdist=maxdist) + #update_meta(0.15, "L20rb_ml", en='E0', minrad=minrad, maxdist=maxdist) + + version=0.11 + + #names=['GP025','GP020','GP015','GP010','GP005','GP000','GP355','GP350','GP345',] + #names=['GP055','GP060','GP065'] + #names=['GP070','GP075','GP080','GP085','GP090', 'GP270','GP275','GP280','GP285','GP290','GP295','GP300','GP305','GP310','GP215','GP320','GP325'] + #names=['GP315'] + names=['GP330','GP335','GP340'] + """ + for name in names: + update_meta(version, name, en='E0', minrad=minrad, maxdist=maxdist) + update_meta(version, name, en='E1', minrad=minrad, maxdist=maxdist) + update_meta(version, name, en='E2', minrad=minrad, maxdist=maxdist) + update_meta(version, name, en='E3', minrad=minrad, maxdist=maxdist) + update_meta(version, name, en='E4', minrad=minrad, maxdist=maxdist) + """ + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/00_init_energy_bands.py b/artsurvey/management/commands/00_init_energy_bands.py new file mode 100644 index 0000000..f94717f --- /dev/null +++ b/artsurvey/management/commands/00_init_energy_bands.py @@ -0,0 +1,49 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import EnergyBand, ArtSurveySource, ArtSurvey +from srglib.utils import find_counterparts +from srgcat.models import SrgDataDump + +""" +0 4.0 12.0 +1 4.0 7.0 +2 7.0 12.0 +3 12.0 20.0 +4 20.0 30.0 +""" + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + bands = EnergyBand.objects.all() + bands.delete() + + e0=EnergyBand(e1=4.0, e2=12.0, slug='E0') + e0.save() + + e1=EnergyBand(e1=4.0, e2=7.0, slug='E1') + e1.save() + + e2=EnergyBand(e1=7.0, e2=12.0, slug='E2') + e2.save() + + e3=EnergyBand(e1=12.0, e2=20.0, slug='E3') + e3.save() + + e4=EnergyBand(e1=20.0, e2=30.0, slug='E4') + e4.save() + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/01_artsurvey_heasarc.py b/artsurvey/management/commands/01_artsurvey_heasarc.py new file mode 100644 index 0000000..1d4cb3d --- /dev/null +++ b/artsurvey/management/commands/01_artsurvey_heasarc.py @@ -0,0 +1,36 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import find_counterparts + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=2.3 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + find_counterparts(srcs, HeasarcBase.objects.all(),'heasarc',minrad=40,verbose=True) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/01_artsurvey_load_erositamatch.py b/artsurvey/management/commands/01_artsurvey_load_erositamatch.py new file mode 100644 index 0000000..96022af --- /dev/null +++ b/artsurvey/management/commands/01_artsurvey_load_erositamatch.py @@ -0,0 +1,199 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import logging +import os +from datetime import datetime +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurvey, ArtSurveySource, eRositaMatch +from srglib.utils import find_counterparts + +import numpy.ma as ma +from django.core.exceptions import ObjectDoesNotExist + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +def load_skymap_sources_erosita(filename, survey): + """ Loads sources cross-matched with eRosita + + Parameters + ---------- + filename : str + Absolute path to the file + + + :Author: + Roman Krivonos + """ + logger = logging.getLogger('django') + + if not os.path.isfile(filename): + logger.error("File {} does not exist".format(filename)) + return + + logger.info("Loading sources from {}".format(filename)) + + name = os.path.basename(filename) + catalog_name = os.path.splitext(name)[0] + logger.info("Filename {} catalog {}".format(name, catalog_name)) + + format_string='srg_%Y%m%d_%H%M%S' + dtime = datetime.strptime(catalog_name[:-4], format_string) + tm = Time(dtime, format='datetime', scale='utc') + dt = tm.to_datetime(timezone=TZ_MSK) + + data = astropy.table.Table.read(filename,format='ascii.csv') + print(data.info) + + """ + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + """ + + for item in data: + + try: + #srcs = ArtSurveySource.objects.filter(name_orig=item['NAME']) + src = survey.artsurveysource_set.get(name_orig=item['NAME']) + except Exception as e: + print("{} failed to load {}".format(item['NAME'],e)) + print("Missed {} cnts {}, flux {}, exptime {}".format(item['NAME'],item['CNTS'],item['FLUX'],item['EXPTIME'],)) + continue + + + match_soft = True if item['match_soft'] == 'True' else False + match_med = True if item['match_med'] == 'True' else False + match_hard = True if item['match_hard'] == 'True' else False + #print("{}".format(src)) + + src.erosita_data=True + src.save() + + if not (match_soft or match_med or match_hard): + print("3 match failed for {}".format(item['NAME'])) + continue + + hpidx_soft = item['hpidx_soft'] if not ma.is_masked(item['hpidx_soft']) else None + hpidx_med = item['hpidx_med'] if not ma.is_masked(item['hpidx_med']) else None + hpidx_hard = item['hpidx_hard'] if not ma.is_masked(item['hpidx_hard']) else None + + ero = eRositaMatch(source=src, + ero_exp = item['ero_exp'], + lkh = item['lkh'], + sep_soft = item['sep_soft'], + match_soft = item['match_soft'], + srcname_soft = item['srcname_soft'], + hpidx_soft = hpidx_soft, + RA_fin_soft = item['RA_fin_soft'], + DEC_fin_soft = item['DEC_fin_soft'], + RA_soft = item['RA_soft'], + DEC_soft = item['DEC_soft'], + RADEC_ERR_soft = item['RADEC_ERR_soft'], + DET_LIKE_0_soft = item['DET_LIKE_0_soft'], + ML_FLUX_0_soft = item['ML_FLUX_0_soft'], + ML_FLUX_ERR_0_soft = item['ML_FLUX_ERR_0_soft'], + ML_CTS_0_soft = item['ML_CTS_0_soft'], + ML_CTS_ERR_0_soft = item['ML_CTS_ERR_0_soft'], + ML_BKG_0_soft = item['ML_BKG_0_soft'], + EXT_soft = item['EXT_soft'], + EXT_LIKE_soft = item['EXT_LIKE_soft'], + EXT_ERR_soft = item['EXT_ERR_soft'], + ML_EXP_1_soft = item['ML_EXP_1_soft'], + sep_med = item['sep_med'], + match_med = item['match_med'], + srcname_med = item['srcname_med'], + hpidx_med = hpidx_med, + RA_fin_med = item['RA_fin_med'], + DEC_fin_med = item['DEC_fin_med'], + RA_med = item['RA_med'], + DEC_med = item['DEC_med'], + RADEC_ERR_med = item['RADEC_ERR_med'], + DET_LIKE_0_med = item['DET_LIKE_0_med'], + ML_FLUX_0_med = item['ML_FLUX_0_med'], + ML_FLUX_ERR_0_med = item['ML_FLUX_ERR_0_med'], + ML_CTS_0_med = item['ML_CTS_0_med'], + ML_CTS_ERR_0_med = item['ML_CTS_ERR_0_med'], + ML_BKG_0_med = item['ML_BKG_0_med'], + EXT_med = item['EXT_med'], + EXT_LIKE_med = item['EXT_LIKE_med'], + EXT_ERR_med = item['EXT_ERR_med'], + ML_EXP_1_med = item['ML_EXP_1_med'], + sep_hard = item['sep_hard'], + match_hard = item['match_hard'], + srcname_hard = item['srcname_hard'], + hpidx_hard = hpidx_hard, + RA_fin_hard = item['RA_fin_hard'], + DEC_fin_hard = item['DEC_fin_hard'], + RA_hard = item['RA_hard'], + DEC_hard = item['DEC_hard'], + RADEC_ERR_hard = item['RADEC_ERR_hard'], + DET_LIKE_0_hard = item['DET_LIKE_0_hard'], + ML_FLUX_0_hard = item['ML_FLUX_0_hard'], + ML_FLUX_ERR_0_hard = item['ML_FLUX_ERR_0_hard'], + ML_CTS_0_hard = item['ML_CTS_0_hard'], + ML_CTS_ERR_0_hard = item['ML_CTS_ERR_0_hard'], + ML_BKG_0_hard = item['ML_BKG_0_hard'], + EXT_hard = item['EXT_hard'], + EXT_LIKE_hard = item['EXT_LIKE_hard'], + EXT_ERR_hard = item['EXT_ERR_hard'], + ML_EXP_1_hard = item['ML_EXP_1_hard'], + sep_simb = item['sep_simb'], + match_simb = item['match_simb'], + ra_match = item['ra_match'], + dec_match = item['dec_match'], + RAJ2000_simb = item['RAJ2000_simb'], + DEJ2000_simb = item['DEJ2000_simb'], + z_simb = item['z_simb'], + otype_simb = item['otype_simb']) + ero.save() + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + survey_version=1.1 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + for src in srcs: + try: + src.erositamatch + except ObjectDoesNotExist: + continue + print(src.erositamatch) + src.erositamatch.delete() + src.erosita_data=False + src.save() + + load_skymap_sources_erosita("/data/artxc/erosita/srg_20200610_000000_000.csv", survey) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/02_artsurvey_load_wise.py b/artsurvey/management/commands/02_artsurvey_load_wise.py new file mode 100644 index 0000000..82ec8af --- /dev/null +++ b/artsurvey/management/commands/02_artsurvey_load_wise.py @@ -0,0 +1,196 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import logging +import os +from datetime import datetime +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch +from astropy.io import fits + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +from astropy_healpix import HEALPix, neighbours + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurvey, ArtSurveySource, WiseCounterpart +from srglib.utils import find_counterparts + +import numpy.ma as ma +from django.core.exceptions import ObjectDoesNotExist + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +def load_skymap_sources_wise(filename, survey): + """ Loads sources cross-matched with eRosita + + Parameters + ---------- + filename : str + Absolute path to the file + + + :Author: + Roman Krivonos + """ + logger = logging.getLogger('django') + + if not os.path.isfile(filename): + logger.error("File {} does not exist".format(filename)) + return + + logger.info("Loading sources from {}".format(filename)) + + name = os.path.basename(filename) + catalog_name = os.path.splitext(name)[0] + logger.info("Filename {} catalog {}".format(name, catalog_name)) + + + + format_string='srg_%Y%m%d_%H%M%S_000_ps2wf' + dtime = datetime.strptime(catalog_name[:-4], format_string) + tm = Time(dtime, format='datetime', scale='utc') + dt = tm.to_datetime(timezone=TZ_MSK) + + hdul = fits.open(filename) + data = hdul[1].data + hdul.info() + print(hdul[1].columns) + + for item in data: + + try: + #srcs = ArtSurveySource.objects.filter(name_orig=item['NAME']) + src = survey.artsurveysource_set.get(name_orig=item['NAME_cl']) + except Exception as e: + print("{} failed to load {}".format(item['NAME_cl'],e)) + print("Missed {} cnts {}, flux {}, exptime {}".format(item['NAME_cl'],item['CNTS_cl'],item['FLUX_cl'],item['EXPTIME_cl'],)) + continue + + + src.wise_data=True + src.save() + + w1mag = item['w1mag'] if not (ma.is_masked(item['w1mag']) or item['w1mag'] < 0.0) else None + w2mag = item['w2mag'] if not (ma.is_masked(item['w2mag']) or item['w2mag'] < 0.0) else None + dw1mag = item['dw1mag'] if not (ma.is_masked(item['dw1mag']) or item['dw1mag'] < 0.0) else None + dw2mag = item['dw2mag'] if not (ma.is_masked(item['dw2mag']) or item['dw2mag'] < 0.0) else None + w1flux = item['w1flux'] if not (ma.is_masked(item['w1flux']) or item['w1flux'] < 0.0) else None + w2flux = item['w2flux'] if not (ma.is_masked(item['w2flux']) or item['w2flux'] < 0.0) else None + dw1flux = item['dw1flux'] if not (ma.is_masked(item['dw1flux']) or item['dw1flux'] < 0.0) else None + dw2flux = item['dw2flux'] if not (ma.is_masked(item['dw2flux']) or item['dw2flux'] < 0.0) else None + gPSFMag = item['gPSFMag'] if not (ma.is_masked(item['gPSFMag']) or item['gPSFMag'] < 0.0) else None + iPSFMag = item['iPSFMag'] if not (ma.is_masked(item['iPSFMag']) or item['iPSFMag'] < 0.0) else None + rPSFMag = item['rPSFMag'] if not (ma.is_masked(item['rPSFMag']) or item['rPSFMag'] < 0.0) else None + yPSFMag = item['yPSFMag'] if not (ma.is_masked(item['yPSFMag']) or item['yPSFMag'] < 0.0) else None + zPSFMag = item['zPSFMag'] if not (ma.is_masked(item['zPSFMag']) or item['zPSFMag'] < 0.0) else None + primaryDetection = True if not (ma.is_masked(item['primaryDetection']) or int(item['primaryDetection']) == 0) else False + bestDetection = True if not (ma.is_masked(item['bestDetection']) or int(item['bestDetection']) == 0) else False + + wise = WiseCounterpart(source=src, + sep=item['sep_cl'], + objID=item['objID'], + raStack=item['raStack'], + decStack=item['decStack'], + raStackErr=item['raStackErr'], + decStackErr=item['decStackErr'], + raMean=item['raMean'], + decMean=item['decMean'], + raMeanErr=item['raMeanErr'], + decMeanErr=item['decMeanErr'], + objInfoFlag=item['objInfoFlag'], + qualityFlag=item['qualityFlag'], + primaryDetection=primaryDetection, + bestDetection=bestDetection, + gPSFMag=gPSFMag, + gPSFMagErr=item['gPSFMagErr'], + gApMag=item['gApMag'], + gApMagErr=item['gApMagErr'], + gKronMag=item['gKronMag'], + gKronMagErr=item['gKronMagErr'], + rPSFMag=rPSFMag, + rPSFMagErr=item['rPSFMagErr'], + rApMag=item['rApMag'], + rApMagErr=item['rApMagErr'], + rKronMag=item['rKronMag'], + rKronMagErr=item['rKronMagErr'], + iPSFMag=iPSFMag, + iPSFMagErr=item['iPSFMagErr'], + iApMag=item['iApMag'], + iApMagErr=item['iApMagErr'], + iKronMag=item['iKronMag'], + iKronMagErr=item['iKronMagErr'], + zPSFMag=zPSFMag, + zPSFMagErr=item['zPSFMagErr'], + zApMag=item['zApMag'], + zApMagErr=item['zApMagErr'], + zKronMag=item['zKronMag'], + zKronMagErr=item['zKronMagErr'], + yPSFMag=yPSFMag, + yPSFMagErr=item['yPSFMagErr'], + yApMag=item['yApMag'], + yApMagErr=item['yApMagErr'], + yKronMag=item['yKronMag'], + yKronMagErr=item['yKronMagErr'], + wisefield=item['wisefield'], + fitext=item['fitext'], + devaucou=item['devaucou'], + star=item['star'], + w1flux=w1flux, + dw1flux=dw1flux, + w1mag=w1mag, + dw1mag=dw1mag, + w1reff=item['w1reff'], + w2flux=w2flux, + dw2flux=dw2flux, + w2mag=w2mag, + dw2mag=dw2mag, + w2reff=item['w2reff']) + + wise.save() + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + survey_version=1.1 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + """ + srcs = survey.artsurveysource_set.all() + for src in srcs: + try: + src.erositamatch + except ObjectDoesNotExist: + continue + print(src.erositamatch) + src.erositamatch.delete() + src.erosita_data=False + src.save() + """ + w = WiseCounterpart.objects.all() + w.delete() + + load_skymap_sources_wise("/data/artxc/wise/srg_20200610_191805_000_ps2wf_r40.fits", survey) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/02_artsurvey_simbad.py b/artsurvey/management/commands/02_artsurvey_simbad.py new file mode 100644 index 0000000..b6f3913 --- /dev/null +++ b/artsurvey/management/commands/02_artsurvey_simbad.py @@ -0,0 +1,37 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import load_simbad_for_skymap_sources + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=12.1 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + #srcs = ArtSurveySource.objects.all() + load_simbad_for_skymap_sources(srcs,minrad=40,maxdist=120) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/03_arsurvey_allwise.py b/artsurvey/management/commands/03_arsurvey_allwise.py new file mode 100644 index 0000000..3d2fd43 --- /dev/null +++ b/artsurvey/management/commands/03_arsurvey_allwise.py @@ -0,0 +1,37 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import load_vizier_allwise + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=12.1 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + + load_vizier_allwise(srcs, minrad=40, maxdist=120) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/03_artsurvey_gaia.py b/artsurvey/management/commands/03_artsurvey_gaia.py new file mode 100644 index 0000000..2a87004 --- /dev/null +++ b/artsurvey/management/commands/03_artsurvey_gaia.py @@ -0,0 +1,42 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from astrobasis.models import GAIADR3 +from srglib.utils import find_counterparts + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=1.3 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + + + + #srcs = ArtSurveySource.objects.all() + find_counterparts(srcs, GAIADR3.objects.all(), "gaia3", maxdist=60., minrad=40.) + + diff --git a/artsurvey/management/commands/04_artsurvey_nvss.py b/artsurvey/management/commands/04_artsurvey_nvss.py new file mode 100644 index 0000000..08f6957 --- /dev/null +++ b/artsurvey/management/commands/04_artsurvey_nvss.py @@ -0,0 +1,37 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import load_vizier_nvss + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=12.1 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + + load_vizier_nvss(srcs, minrad=40, maxdist=120) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/05_artsurvey_first.py b/artsurvey/management/commands/05_artsurvey_first.py new file mode 100644 index 0000000..0750150 --- /dev/null +++ b/artsurvey/management/commands/05_artsurvey_first.py @@ -0,0 +1,37 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import load_vizier_first + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=12.0 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + + load_vizier_first(srcs, minrad=40, maxdist=120) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/06_artsurvey_sumss.py b/artsurvey/management/commands/06_artsurvey_sumss.py new file mode 100644 index 0000000..9cf8ed8 --- /dev/null +++ b/artsurvey/management/commands/06_artsurvey_sumss.py @@ -0,0 +1,37 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.models import HeasarcBase +from artsurvey.models import ArtSurveySource, ArtSurvey +from srglib.utils import load_vizier_sumss + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + survey_version=12.0 + + try: + survey = ArtSurvey.objects.get(version=survey_version) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + + print("use ArtSurvey {}".format(survey)) + + srcs = survey.artsurveysource_set.all() + + load_vizier_sumss(srcs, minrad=40, maxdist=120) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/artsurvey/management/commands/` b/artsurvey/management/commands/` new file mode 100644 index 0000000..bbeba64 --- /dev/null +++ b/artsurvey/management/commands/` @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/artsurvey/migrations/0001_initial.py b/artsurvey/migrations/0001_initial.py new file mode 100644 index 0000000..a022bb9 --- /dev/null +++ b/artsurvey/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 2.2.6 on 2020-06-19 14:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('heasarc', '0020_rosatexpmap_detlimflux'), + ('astrobasis', '0035_auto_20200225_1625'), + ] + + operations = [ + migrations.CreateModel( + name='ArtSurveySource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('simbad_failed', models.BooleanField(default=False, null=True)), + ('simbad_notfound', models.BooleanField(default=False, null=True)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('healpix_plate', models.IntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('lii', models.FloatField(default=0.0)), + ('bii', models.FloatField(default=0.0)), + ('radec_error', models.FloatField(default=0.0)), + ('name', models.CharField(blank=True, default='', max_length=36)), + ('cnts', models.FloatField(default=0.0)), + ('cnts_err', models.FloatField(default=0.0)), + ('cnts_bg', models.FloatField(default=0.0)), + ('exptime', models.FloatField(default=0.0)), + ('rate', models.FloatField(default=0.0)), + ('rate_err', models.FloatField(default=0.0)), + ('flux', models.FloatField(default=0.0)), + ('flux_err', models.FloatField(default=0.0)), + ('sig', models.FloatField(default=0.0)), + ('nfalse', models.FloatField(default=0.0)), + ('created', models.DateTimeField(auto_now_add=True)), + ('heasarc', models.ManyToManyField(to='heasarc.HeasarcBase')), + ('simbad', models.ManyToManyField(to='astrobasis.Simbad')), + ], + ), + ] diff --git a/artsurvey/migrations/0002_artsurveysource_cname.py b/artsurvey/migrations/0002_artsurveysource_cname.py new file mode 100644 index 0000000..1a39f03 --- /dev/null +++ b/artsurvey/migrations/0002_artsurveysource_cname.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-19 15:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='cname', + field=models.CharField(blank=True, default='', max_length=36), + ), + ] diff --git a/artsurvey/migrations/0003_auto_20200619_1818.py b/artsurvey/migrations/0003_auto_20200619_1818.py new file mode 100644 index 0000000..bf7e9d4 --- /dev/null +++ b/artsurvey/migrations/0003_auto_20200619_1818.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.6 on 2020-06-19 15:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('heasarc', '0020_rosatexpmap_detlimflux'), + ('artsurvey', '0002_artsurveysource_cname'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='class_id', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='artsurveysource', + name='object_class', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='heasarc.HeasarcObjectClass'), + ), + ] diff --git a/artsurvey/migrations/0004_artsurveysource_ads.py b/artsurvey/migrations/0004_artsurveysource_ads.py new file mode 100644 index 0000000..ea6439a --- /dev/null +++ b/artsurvey/migrations/0004_artsurveysource_ads.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-06-19 16:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nasa', '0002_auto_20200619_1942'), + ('artsurvey', '0003_auto_20200619_1818'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='ads', + field=models.ManyToManyField(to='nasa.Ads'), + ), + ] diff --git a/artsurvey/migrations/0005_artsurveysource_refid.py b/artsurvey/migrations/0005_artsurveysource_refid.py new file mode 100644 index 0000000..9a385d2 --- /dev/null +++ b/artsurvey/migrations/0005_artsurveysource_refid.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-19 16:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0004_artsurveysource_ads'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='refid', + field=models.BigIntegerField(default=0), + ), + ] diff --git a/artsurvey/migrations/0006_remove_artsurveysource_object_class.py b/artsurvey/migrations/0006_remove_artsurveysource_object_class.py new file mode 100644 index 0000000..8a175c8 --- /dev/null +++ b/artsurvey/migrations/0006_remove_artsurveysource_object_class.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.6 on 2020-06-19 19:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0005_artsurveysource_refid'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='object_class', + ), + ] diff --git a/artsurvey/migrations/0007_artsurveysource_object_class.py b/artsurvey/migrations/0007_artsurveysource_object_class.py new file mode 100644 index 0000000..9d9dfbc --- /dev/null +++ b/artsurvey/migrations/0007_artsurveysource_object_class.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.6 on 2020-06-19 19:22 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('heasarc', '0021_heasarcsimpleclass'), + ('artsurvey', '0006_remove_artsurveysource_object_class'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='object_class', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='heasarc.HeasarcSimpleClass'), + ), + ] diff --git a/artsurvey/migrations/0008_artsurveysource_owner.py b/artsurvey/migrations/0008_artsurveysource_owner.py new file mode 100644 index 0000000..ac57783 --- /dev/null +++ b/artsurvey/migrations/0008_artsurveysource_owner.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.6 on 2020-06-23 08:56 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0007_artsurveysource_object_class'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/artsurvey/migrations/0009_artsurveysource_notes.py b/artsurvey/migrations/0009_artsurveysource_notes.py new file mode 100644 index 0000000..843f8cb --- /dev/null +++ b/artsurvey/migrations/0009_artsurveysource_notes.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-23 13:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0008_artsurveysource_owner'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='notes', + field=models.TextField(blank=True, max_length=2000, null=True, verbose_name='notes'), + ), + ] diff --git a/artsurvey/migrations/0010_artsurveyparams.py b/artsurvey/migrations/0010_artsurveyparams.py new file mode 100644 index 0000000..5e328c7 --- /dev/null +++ b/artsurvey/migrations/0010_artsurveyparams.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.6 on 2020-06-24 12:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0009_artsurveysource_notes'), + ] + + operations = [ + migrations.CreateModel( + name='ArtSurveyParams', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sign_threshold', models.FloatField(blank=True, default=0.0, null=True)), + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='artsurveyparams', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/artsurvey/migrations/0011_artsurveyparams_class_startswith.py b/artsurvey/migrations/0011_artsurveyparams_class_startswith.py new file mode 100644 index 0000000..b704c5f --- /dev/null +++ b/artsurvey/migrations/0011_artsurveyparams_class_startswith.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-24 13:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0010_artsurveyparams'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='class_startswith', + field=models.CharField(blank=True, default='', max_length=36, null=True), + ), + ] diff --git a/artsurvey/migrations/0012_auto_20200624_1800.py b/artsurvey/migrations/0012_auto_20200624_1800.py new file mode 100644 index 0000000..fe088cf --- /dev/null +++ b/artsurvey/migrations/0012_auto_20200624_1800.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-24 15:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0011_artsurveyparams_class_startswith'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveysource', + name='cname', + field=models.CharField(blank=True, default='', max_length=36, null=True), + ), + ] diff --git a/artsurvey/migrations/0013_erositamatch.py b/artsurvey/migrations/0013_erositamatch.py new file mode 100644 index 0000000..cd8bfc0 --- /dev/null +++ b/artsurvey/migrations/0013_erositamatch.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.6 on 2020-06-25 06:02 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0012_auto_20200624_1800'), + ] + + operations = [ + migrations.CreateModel( + name='eRositaMatch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('exp', models.FloatField(default=0.0)), + ('sep', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('lkh', models.FloatField(default=0.0)), + ('cts', models.FloatField(default=0.0)), + ('flux', models.FloatField(default=0.0)), + ('source', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='artsurvey.ArtSurveySource')), + ], + ), + ] diff --git a/artsurvey/migrations/0014_auto_20200625_0914.py b/artsurvey/migrations/0014_auto_20200625_0914.py new file mode 100644 index 0000000..97d3c7f --- /dev/null +++ b/artsurvey/migrations/0014_auto_20200625_0914.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-06-25 06:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0013_erositamatch'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='field', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='artsurveysource', + name='field_ind', + field=models.IntegerField(default=0), + ), + ] diff --git a/artsurvey/migrations/0015_artsurveysource_name_orig.py b/artsurvey/migrations/0015_artsurveysource_name_orig.py new file mode 100644 index 0000000..15f5c89 --- /dev/null +++ b/artsurvey/migrations/0015_artsurveysource_name_orig.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-25 06:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0014_auto_20200625_0914'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='name_orig', + field=models.CharField(max_length=20, null=True), + ), + ] diff --git a/artsurvey/migrations/0016_artsurvey.py b/artsurvey/migrations/0016_artsurvey.py new file mode 100644 index 0000000..8ad39c6 --- /dev/null +++ b/artsurvey/migrations/0016_artsurvey.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.6 on 2020-06-25 06:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0015_artsurveysource_name_orig'), + ] + + operations = [ + migrations.CreateModel( + name='ArtSurvey', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('number', models.IntegerField(default=0)), + ], + ), + ] diff --git a/artsurvey/migrations/0017_artsurveysource_survey.py b/artsurvey/migrations/0017_artsurveysource_survey.py new file mode 100644 index 0000000..d73a1ff --- /dev/null +++ b/artsurvey/migrations/0017_artsurveysource_survey.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-06-25 06:54 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0016_artsurvey'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='survey', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0018_auto_20200625_0957.py b/artsurvey/migrations/0018_auto_20200625_0957.py new file mode 100644 index 0000000..51a38cc --- /dev/null +++ b/artsurvey/migrations/0018_auto_20200625_0957.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-06-25 06:57 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0017_artsurveysource_survey'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveysource', + name='survey', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0019_auto_20200625_0959.py b/artsurvey/migrations/0019_auto_20200625_0959.py new file mode 100644 index 0000000..aefb612 --- /dev/null +++ b/artsurvey/migrations/0019_auto_20200625_0959.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-25 06:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0018_auto_20200625_0957'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurvey', + name='number', + field=models.IntegerField(default=0, unique=True), + ), + ] diff --git a/artsurvey/migrations/0020_auto_20200625_1146.py b/artsurvey/migrations/0020_auto_20200625_1146.py new file mode 100644 index 0000000..cd459a8 --- /dev/null +++ b/artsurvey/migrations/0020_auto_20200625_1146.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-06-25 08:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0019_auto_20200625_0959'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='ecl_b', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='ecl_l', + field=models.FloatField(default=0.0), + ), + ] diff --git a/artsurvey/migrations/0021_auto_20200625_1148.py b/artsurvey/migrations/0021_auto_20200625_1148.py new file mode 100644 index 0000000..b70300e --- /dev/null +++ b/artsurvey/migrations/0021_auto_20200625_1148.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.6 on 2020-06-25 08:48 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0020_auto_20200625_1146'), + ] + + operations = [ + migrations.RemoveField( + model_name='erositamatch', + name='cts', + ), + migrations.RemoveField( + model_name='erositamatch', + name='dec', + ), + migrations.RemoveField( + model_name='erositamatch', + name='exp', + ), + migrations.RemoveField( + model_name='erositamatch', + name='flux', + ), + migrations.RemoveField( + model_name='erositamatch', + name='lkh', + ), + migrations.RemoveField( + model_name='erositamatch', + name='ra', + ), + migrations.RemoveField( + model_name='erositamatch', + name='sep', + ), + ] diff --git a/artsurvey/migrations/0022_auto_20200625_1159.py b/artsurvey/migrations/0022_auto_20200625_1159.py new file mode 100644 index 0000000..70dd081 --- /dev/null +++ b/artsurvey/migrations/0022_auto_20200625_1159.py @@ -0,0 +1,348 @@ +# Generated by Django 2.2.6 on 2020-06-25 08:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0021_auto_20200625_1148'), + ] + + operations = [ + migrations.AddField( + model_name='erositamatch', + name='DEC_fin_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DEC_fin_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DEC_fin_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DEC_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DEC_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DEC_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DEJ2000_simb', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DET_LIKE_0_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DET_LIKE_0_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='DET_LIKE_0_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_ERR_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_ERR_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_ERR_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_LIKE_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_LIKE_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_LIKE_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='EXT_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_BKG_0_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_BKG_0_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_BKG_0_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_CTS_0_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_CTS_0_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_CTS_0_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_CTS_ERR_0_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_CTS_ERR_0_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_CTS_ERR_0_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_EXP_1_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_EXP_1_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_EXP_1_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_FLUX_0_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_FLUX_0_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_FLUX_0_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_FLUX_ERR_0_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_FLUX_ERR_0_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ML_FLUX_ERR_0_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RADEC_ERR_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RADEC_ERR_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RADEC_ERR_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RAJ2000_simb', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RA_fin_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RA_fin_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RA_fin_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RA_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RA_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='RA_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='dec_match', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ero_exp', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='erositamatch', + name='hpidx_hard', + field=models.IntegerField(default=0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='hpidx_med', + field=models.IntegerField(default=0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='hpidx_soft', + field=models.IntegerField(default=0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='lkh', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='erositamatch', + name='match_hard', + field=models.CharField(blank=True, default='', max_length=5, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='match_med', + field=models.CharField(blank=True, default='', max_length=5, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='match_simb', + field=models.CharField(blank=True, default='', max_length=5, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='match_soft', + field=models.CharField(blank=True, default='', max_length=5, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='otype_simb', + field=models.CharField(blank=True, default='', max_length=17, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='ra_match', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='sep_hard', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='sep_med', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='sep_simb', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='sep_soft', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='srcname_hard', + field=models.CharField(blank=True, default='', max_length=21, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='srcname_med', + field=models.CharField(blank=True, default='', max_length=21, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='srcname_soft', + field=models.CharField(blank=True, default='', max_length=21, null=True), + ), + migrations.AddField( + model_name='erositamatch', + name='z_simb', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0023_auto_20200625_1251.py b/artsurvey/migrations/0023_auto_20200625_1251.py new file mode 100644 index 0000000..9896892 --- /dev/null +++ b/artsurvey/migrations/0023_auto_20200625_1251.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-25 09:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0022_auto_20200625_1159'), + ] + + operations = [ + migrations.AlterField( + model_name='erositamatch', + name='hpidx_soft', + field=models.BigIntegerField(default=0, null=True), + ), + ] diff --git a/artsurvey/migrations/0024_auto_20200625_1252.py b/artsurvey/migrations/0024_auto_20200625_1252.py new file mode 100644 index 0000000..a5ab8f0 --- /dev/null +++ b/artsurvey/migrations/0024_auto_20200625_1252.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-06-25 09:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0023_auto_20200625_1251'), + ] + + operations = [ + migrations.AlterField( + model_name='erositamatch', + name='hpidx_hard', + field=models.BigIntegerField(default=0, null=True), + ), + migrations.AlterField( + model_name='erositamatch', + name='hpidx_med', + field=models.BigIntegerField(default=0, null=True), + ), + ] diff --git a/artsurvey/migrations/0025_auto_20200625_1406.py b/artsurvey/migrations/0025_auto_20200625_1406.py new file mode 100644 index 0000000..d7179f0 --- /dev/null +++ b/artsurvey/migrations/0025_auto_20200625_1406.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.6 on 2020-06-25 11:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0024_auto_20200625_1252'), + ] + + operations = [ + migrations.AlterField( + model_name='erositamatch', + name='match_hard', + field=models.BooleanField(default=False, null=True), + ), + migrations.AlterField( + model_name='erositamatch', + name='match_med', + field=models.BooleanField(default=False, null=True), + ), + migrations.AlterField( + model_name='erositamatch', + name='match_simb', + field=models.BooleanField(default=False, null=True), + ), + migrations.AlterField( + model_name='erositamatch', + name='match_soft', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0026_opticalcounterpart.py b/artsurvey/migrations/0026_opticalcounterpart.py new file mode 100644 index 0000000..b6a7250 --- /dev/null +++ b/artsurvey/migrations/0026_opticalcounterpart.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.6 on 2020-06-25 16:04 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0025_auto_20200625_1406'), + ] + + operations = [ + migrations.CreateModel( + name='OpticalCounterpart', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='artsurvey.ArtSurveySource')), + ], + ), + ] diff --git a/artsurvey/migrations/0027_auto_20200625_1908.py b/artsurvey/migrations/0027_auto_20200625_1908.py new file mode 100644 index 0000000..76f5919 --- /dev/null +++ b/artsurvey/migrations/0027_auto_20200625_1908.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.6 on 2020-06-25 16:08 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0026_opticalcounterpart'), + ] + + operations = [ + migrations.AddField( + model_name='opticalcounterpart', + name='dec', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='opticalcounterpart', + name='mag', + field=models.FloatField(blank=True, default=0, null=True, validators=[django.core.validators.MinValueValidator(0.0)]), + ), + migrations.AddField( + model_name='opticalcounterpart', + name='name', + field=models.CharField(default='', max_length=60), + ), + migrations.AddField( + model_name='opticalcounterpart', + name='objid', + field=models.BigIntegerField(default=0), + ), + migrations.AddField( + model_name='opticalcounterpart', + name='ra', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='opticalcounterpart', + name='redshift', + field=models.FloatField(default=0, validators=[django.core.validators.MinValueValidator(0.0)]), + ), + migrations.AddField( + model_name='opticalcounterpart', + name='t_objid', + field=models.CharField(default='', max_length=15), + ), + ] diff --git a/artsurvey/migrations/0028_artsurveysource_gaia.py b/artsurvey/migrations/0028_artsurveysource_gaia.py new file mode 100644 index 0000000..81feaad --- /dev/null +++ b/artsurvey/migrations/0028_artsurveysource_gaia.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-06-25 19:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0035_auto_20200225_1625'), + ('artsurvey', '0027_auto_20200625_1908'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='gaia', + field=models.ManyToManyField(to='astrobasis.GAIADR2'), + ), + ] diff --git a/artsurvey/migrations/0029_category.py b/artsurvey/migrations/0029_category.py new file mode 100644 index 0000000..4bfb4a9 --- /dev/null +++ b/artsurvey/migrations/0029_category.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.6 on 2020-06-26 10:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0028_artsurveysource_gaia'), + ] + + operations = [ + migrations.CreateModel( + name='Category', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200)), + ('slug', models.SlugField()), + ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='artsurvey.Category')), + ], + options={ + 'verbose_name_plural': 'categories', + 'unique_together': {('slug', 'parent')}, + }, + ), + ] diff --git a/artsurvey/migrations/0030_artsurveysource_category.py b/artsurvey/migrations/0030_artsurveysource_category.py new file mode 100644 index 0000000..8c84d2a --- /dev/null +++ b/artsurvey/migrations/0030_artsurveysource_category.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-06-26 10:41 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0029_category'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='category', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.Category'), + ), + ] diff --git a/artsurvey/migrations/0031_auto_20200626_1427.py b/artsurvey/migrations/0031_auto_20200626_1427.py new file mode 100644 index 0000000..bbfd3d5 --- /dev/null +++ b/artsurvey/migrations/0031_auto_20200626_1427.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.6 on 2020-06-26 11:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0030_artsurveysource_category'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurvey', + name='number', + ), + migrations.AddField( + model_name='artsurvey', + name='version', + field=models.FloatField(default=0.0, unique=True), + ), + ] diff --git a/artsurvey/migrations/0032_artsurveysource_redshift.py b/artsurvey/migrations/0032_artsurveysource_redshift.py new file mode 100644 index 0000000..0d9f570 --- /dev/null +++ b/artsurvey/migrations/0032_artsurveysource_redshift.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-30 12:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0031_auto_20200626_1427'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='redshift', + field=models.FloatField(default=0.0), + ), + ] diff --git a/artsurvey/migrations/0033_auto_20200630_1521.py b/artsurvey/migrations/0033_auto_20200630_1521.py new file mode 100644 index 0000000..0648dbd --- /dev/null +++ b/artsurvey/migrations/0033_auto_20200630_1521.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-30 12:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0032_artsurveysource_redshift'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveysource', + name='redshift', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0034_artsurveysource_parent.py b/artsurvey/migrations/0034_artsurveysource_parent.py new file mode 100644 index 0000000..8395152 --- /dev/null +++ b/artsurvey/migrations/0034_artsurveysource_parent.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-06-30 17:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0033_auto_20200630_1521'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='artsurvey.ArtSurveySource'), + ), + ] diff --git a/artsurvey/migrations/0035_artsurveyparams_survey.py b/artsurvey/migrations/0035_artsurveyparams_survey.py new file mode 100644 index 0000000..8c47f77 --- /dev/null +++ b/artsurvey/migrations/0035_artsurveyparams_survey.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-07-01 11:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0034_artsurveysource_parent'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='survey', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0036_artsurveysource_gaia_primary.py b/artsurvey/migrations/0036_artsurveysource_gaia_primary.py new file mode 100644 index 0000000..fe75e3f --- /dev/null +++ b/artsurvey/migrations/0036_artsurveysource_gaia_primary.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.8 on 2020-07-10 08:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0035_auto_20200225_1625'), + ('artsurvey', '0035_artsurveyparams_survey'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='gaia_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.GAIADR2'), + ), + ] diff --git a/artsurvey/migrations/0037_artsurveyparams_sky.py b/artsurvey/migrations/0037_artsurveyparams_sky.py new file mode 100644 index 0000000..2e18e4a --- /dev/null +++ b/artsurvey/migrations/0037_artsurveyparams_sky.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-07-11 13:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0036_artsurveysource_gaia_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='sky', + field=models.CharField(choices=[('allsky', 'All-sky'), ('ru', 'RU-sky'), ('de', 'DE-sky'), ('ru,de', 'RU,DE')], default='allsky', max_length=6), + ), + ] diff --git a/artsurvey/migrations/0038_auto_20200711_1750.py b/artsurvey/migrations/0038_auto_20200711_1750.py new file mode 100644 index 0000000..0877e05 --- /dev/null +++ b/artsurvey/migrations/0038_auto_20200711_1750.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2020-07-11 14:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0037_artsurveyparams_sky'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='erosita_data', + field=models.BooleanField(default=False, null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='sky', + field=models.CharField(choices=[('allsky', 'All-sky'), ('ru', 'RU-sky'), ('de', 'DE-sky')], default='allsky', max_length=6), + ), + ] diff --git a/artsurvey/migrations/0039_wisecounterpart.py b/artsurvey/migrations/0039_wisecounterpart.py new file mode 100644 index 0000000..b2abe65 --- /dev/null +++ b/artsurvey/migrations/0039_wisecounterpart.py @@ -0,0 +1,80 @@ +# Generated by Django 3.0.8 on 2020-07-22 12:20 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0038_auto_20200711_1750'), + ] + + operations = [ + migrations.CreateModel( + name='WiseCounterpart', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sep', models.FloatField(default=0, validators=[django.core.validators.MinValueValidator(0.0)])), + ('objID', models.BigIntegerField(default=0)), + ('raStack', models.FloatField(default=0.0)), + ('decStack', models.FloatField(default=0.0)), + ('raStackErr', models.FloatField(default=0.0)), + ('decStackErr', models.FloatField(default=0.0)), + ('raMean', models.FloatField(default=0.0)), + ('decMean', models.FloatField(default=0.0)), + ('raMeanErr', models.FloatField(default=0.0)), + ('decMeanErr', models.FloatField(default=0.0)), + ('objInfoFlag', models.BigIntegerField(default=0)), + ('qualityFlag', models.IntegerField(default=0)), + ('primaryDetection', models.IntegerField(default=0)), + ('bestDetection', models.IntegerField(default=0)), + ('gPSFMag', models.FloatField(default=0.0)), + ('gPSFMagErr', models.FloatField(default=0.0)), + ('gApMag', models.FloatField(default=0.0)), + ('gApMagErr', models.FloatField(default=0.0)), + ('gKronMag', models.FloatField(default=0.0)), + ('gKronMagErr', models.FloatField(default=0.0)), + ('rPSFMag', models.FloatField(default=0.0)), + ('rPSFMagErr', models.FloatField(default=0.0)), + ('rApMag', models.FloatField(default=0.0)), + ('rApMagErr', models.FloatField(default=0.0)), + ('rKronMag', models.FloatField(default=0.0)), + ('rKronMagErr', models.FloatField(default=0.0)), + ('iPSFMag', models.FloatField(default=0.0)), + ('iPSFMagErr', models.FloatField(default=0.0)), + ('iApMag', models.FloatField(default=0.0)), + ('iApMagErr', models.FloatField(default=0.0)), + ('iKronMag', models.FloatField(default=0.0)), + ('iKronMagErr', models.FloatField(default=0.0)), + ('zPSFMag', models.FloatField(default=0.0)), + ('zPSFMagErr', models.FloatField(default=0.0)), + ('zApMag', models.FloatField(default=0.0)), + ('zApMagErr', models.FloatField(default=0.0)), + ('zKronMag', models.FloatField(default=0.0)), + ('zKronMagErr', models.FloatField(default=0.0)), + ('yPSFMag', models.FloatField(default=0.0)), + ('yPSFMagErr', models.FloatField(default=0.0)), + ('yApMag', models.FloatField(default=0.0)), + ('yApMagErr', models.FloatField(default=0.0)), + ('yKronMag', models.FloatField(default=0.0)), + ('yKronMagErr', models.FloatField(default=0.0)), + ('wisefield', models.CharField(default='', max_length=8)), + ('fitext', models.BooleanField(default=False, null=True)), + ('devaucou', models.BooleanField(default=False, null=True)), + ('star', models.BooleanField(default=False, null=True)), + ('w1flux', models.FloatField(default=0.0)), + ('dw1flux', models.FloatField(default=0.0)), + ('w1mag', models.FloatField(default=0.0)), + ('dw1mag', models.FloatField(default=0.0)), + ('w1reff', models.FloatField(default=0.0)), + ('w2flux', models.FloatField(default=0.0)), + ('dw2flux', models.FloatField(default=0.0)), + ('w2mag', models.FloatField(default=0.0)), + ('dw2mag', models.FloatField(default=0.0)), + ('w2reff', models.FloatField(default=0.0)), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='artsurvey.ArtSurveySource')), + ], + ), + ] diff --git a/artsurvey/migrations/0040_artsurveysource_wise_data.py b/artsurvey/migrations/0040_artsurveysource_wise_data.py new file mode 100644 index 0000000..ebc70be --- /dev/null +++ b/artsurvey/migrations/0040_artsurveysource_wise_data.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-07-22 13:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0039_wisecounterpart'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='wise_data', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0041_delete_wisecounterpart.py b/artsurvey/migrations/0041_delete_wisecounterpart.py new file mode 100644 index 0000000..33ec1e8 --- /dev/null +++ b/artsurvey/migrations/0041_delete_wisecounterpart.py @@ -0,0 +1,16 @@ +# Generated by Django 3.0.8 on 2020-07-22 14:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0040_artsurveysource_wise_data'), + ] + + operations = [ + migrations.DeleteModel( + name='WiseCounterpart', + ), + ] diff --git a/artsurvey/migrations/0042_wisecounterpart.py b/artsurvey/migrations/0042_wisecounterpart.py new file mode 100644 index 0000000..6359109 --- /dev/null +++ b/artsurvey/migrations/0042_wisecounterpart.py @@ -0,0 +1,81 @@ +# Generated by Django 3.0.8 on 2020-07-22 14:33 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0041_delete_wisecounterpart'), + ] + + operations = [ + migrations.CreateModel( + name='WiseCounterpart', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sep', models.FloatField(default=0, validators=[django.core.validators.MinValueValidator(0.0)])), + ('objID', models.BigIntegerField(default=0)), + ('raStack', models.FloatField(default=0.0)), + ('decStack', models.FloatField(default=0.0)), + ('raStackErr', models.FloatField(default=0.0)), + ('decStackErr', models.FloatField(default=0.0)), + ('raMean', models.FloatField(default=0.0)), + ('decMean', models.FloatField(default=0.0)), + ('raMeanErr', models.FloatField(default=0.0)), + ('decMeanErr', models.FloatField(default=0.0)), + ('objInfoFlag', models.BigIntegerField(default=0)), + ('qualityFlag', models.IntegerField(default=0)), + ('primaryDetection', models.IntegerField(default=0)), + ('bestDetection', models.IntegerField(default=0)), + ('gPSFMag', models.FloatField(default=0.0)), + ('gPSFMagErr', models.FloatField(default=0.0)), + ('gApMag', models.FloatField(default=0.0)), + ('gApMagErr', models.FloatField(default=0.0)), + ('gKronMag', models.FloatField(default=0.0)), + ('gKronMagErr', models.FloatField(default=0.0)), + ('rPSFMag', models.FloatField(default=0.0)), + ('rPSFMagErr', models.FloatField(default=0.0)), + ('rApMag', models.FloatField(default=0.0)), + ('rApMagErr', models.FloatField(default=0.0)), + ('rKronMag', models.FloatField(default=0.0)), + ('rKronMagErr', models.FloatField(default=0.0)), + ('iPSFMag', models.FloatField(default=0.0)), + ('iPSFMagErr', models.FloatField(default=0.0)), + ('iApMag', models.FloatField(default=0.0)), + ('iApMagErr', models.FloatField(default=0.0)), + ('iKronMag', models.FloatField(default=0.0)), + ('iKronMagErr', models.FloatField(default=0.0)), + ('zPSFMag', models.FloatField(default=0.0)), + ('zPSFMagErr', models.FloatField(default=0.0)), + ('zApMag', models.FloatField(default=0.0)), + ('zApMagErr', models.FloatField(default=0.0)), + ('zKronMag', models.FloatField(default=0.0)), + ('zKronMagErr', models.FloatField(default=0.0)), + ('yPSFMag', models.FloatField(default=0.0)), + ('yPSFMagErr', models.FloatField(default=0.0)), + ('yApMag', models.FloatField(default=0.0)), + ('yApMagErr', models.FloatField(default=0.0)), + ('yKronMag', models.FloatField(default=0.0)), + ('yKronMagErr', models.FloatField(default=0.0)), + ('wisefield', models.CharField(default='', max_length=8)), + ('fitext', models.BooleanField(default=False, null=True)), + ('devaucou', models.BooleanField(default=False, null=True)), + ('star', models.BooleanField(default=False, null=True)), + ('w1flux', models.FloatField(default=0.0)), + ('dw1flux', models.FloatField(default=0.0)), + ('w1mag', models.FloatField(default=0.0)), + ('dw1mag', models.FloatField(default=0.0)), + ('w1reff', models.FloatField(default=0.0)), + ('w2flux', models.FloatField(default=0.0)), + ('dw2flux', models.FloatField(default=0.0)), + ('w2mag', models.FloatField(default=0.0)), + ('dw2mag', models.FloatField(default=0.0)), + ('w2reff', models.FloatField(default=0.0)), + ('primary', models.BooleanField(default=False, null=True)), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='artsurvey.ArtSurveySource')), + ], + ), + ] diff --git a/artsurvey/migrations/0043_auto_20200722_1737.py b/artsurvey/migrations/0043_auto_20200722_1737.py new file mode 100644 index 0000000..ce2f406 --- /dev/null +++ b/artsurvey/migrations/0043_auto_20200722_1737.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2020-07-22 14:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0042_wisecounterpart'), + ] + + operations = [ + migrations.AlterField( + model_name='wisecounterpart', + name='w1mag', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='w2mag', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0044_auto_20200722_1747.py b/artsurvey/migrations/0044_auto_20200722_1747.py new file mode 100644 index 0000000..2401d02 --- /dev/null +++ b/artsurvey/migrations/0044_auto_20200722_1747.py @@ -0,0 +1,57 @@ +# Generated by Django 3.0.8 on 2020-07-22 14:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0043_auto_20200722_1737'), + ] + + operations = [ + migrations.RemoveField( + model_name='wisecounterpart', + name='primaryDetection', + ), + migrations.AlterField( + model_name='wisecounterpart', + name='dw1flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='dw1mag', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='dw2flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='dw2mag', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='w1flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='w1reff', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='w2flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='w2reff', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0045_wisecounterpart_primarydetection.py b/artsurvey/migrations/0045_wisecounterpart_primarydetection.py new file mode 100644 index 0000000..c3b16bb --- /dev/null +++ b/artsurvey/migrations/0045_wisecounterpart_primarydetection.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-07-22 14:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0044_auto_20200722_1747'), + ] + + operations = [ + migrations.AddField( + model_name='wisecounterpart', + name='primaryDetection', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0046_auto_20200723_0856.py b/artsurvey/migrations/0046_auto_20200723_0856.py new file mode 100644 index 0000000..50e54ee --- /dev/null +++ b/artsurvey/migrations/0046_auto_20200723_0856.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-07-23 05:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0045_wisecounterpart_primarydetection'), + ] + + operations = [ + migrations.AlterField( + model_name='wisecounterpart', + name='bestDetection', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0047_auto_20200723_0907.py b/artsurvey/migrations/0047_auto_20200723_0907.py new file mode 100644 index 0000000..d09e36b --- /dev/null +++ b/artsurvey/migrations/0047_auto_20200723_0907.py @@ -0,0 +1,38 @@ +# Generated by Django 3.0.8 on 2020-07-23 06:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0046_auto_20200723_0856'), + ] + + operations = [ + migrations.AlterField( + model_name='wisecounterpart', + name='gPSFMag', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='iPSFMag', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='rPSFMag', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='yPSFMag', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='zPSFMag', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0048_artsurveysource_allwise.py b/artsurvey/migrations/0048_artsurveysource_allwise.py new file mode 100644 index 0000000..a801cd7 --- /dev/null +++ b/artsurvey/migrations/0048_artsurveysource_allwise.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2020-08-04 07:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0039_remove_allwise_image_url'), + ('artsurvey', '0047_auto_20200723_0907'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='allwise', + field=models.ManyToManyField(to='astrobasis.AllWise'), + ), + ] diff --git a/artsurvey/migrations/0049_artsurveysource_allwise_primary.py b/artsurvey/migrations/0049_artsurveysource_allwise_primary.py new file mode 100644 index 0000000..04aa571 --- /dev/null +++ b/artsurvey/migrations/0049_artsurveysource_allwise_primary.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.8 on 2020-08-04 08:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0039_remove_allwise_image_url'), + ('artsurvey', '0048_artsurveysource_allwise'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='allwise_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.AllWise'), + ), + ] diff --git a/artsurvey/migrations/0050_artsurveysource_nvss.py b/artsurvey/migrations/0050_artsurveysource_nvss.py new file mode 100644 index 0000000..495f31a --- /dev/null +++ b/artsurvey/migrations/0050_artsurveysource_nvss.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2020-08-04 10:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0040_nvss'), + ('artsurvey', '0049_artsurveysource_allwise_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='nvss', + field=models.ManyToManyField(to='astrobasis.NVSS'), + ), + ] diff --git a/artsurvey/migrations/0051_artsurveysource_first.py b/artsurvey/migrations/0051_artsurveysource_first.py new file mode 100644 index 0000000..f6c05b8 --- /dev/null +++ b/artsurvey/migrations/0051_artsurveysource_first.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2020-08-04 12:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0042_first'), + ('artsurvey', '0050_artsurveysource_nvss'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='first', + field=models.ManyToManyField(to='astrobasis.FIRST'), + ), + ] diff --git a/artsurvey/migrations/0052_artsurveysource_sumss.py b/artsurvey/migrations/0052_artsurveysource_sumss.py new file mode 100644 index 0000000..6ea6a48 --- /dev/null +++ b/artsurvey/migrations/0052_artsurveysource_sumss.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2020-08-04 13:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0043_sumss'), + ('artsurvey', '0051_artsurveysource_first'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='sumss', + field=models.ManyToManyField(to='astrobasis.SUMSS'), + ), + ] diff --git a/artsurvey/migrations/0053_artsurveysource_nvss_primary.py b/artsurvey/migrations/0053_artsurveysource_nvss_primary.py new file mode 100644 index 0000000..839504a --- /dev/null +++ b/artsurvey/migrations/0053_artsurveysource_nvss_primary.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.8 on 2020-08-06 17:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0045_auto_20200804_1707'), + ('artsurvey', '0052_artsurveysource_sumss'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='nvss_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.NVSS'), + ), + ] diff --git a/artsurvey/migrations/0054_auto_20200806_2017.py b/artsurvey/migrations/0054_auto_20200806_2017.py new file mode 100644 index 0000000..b5228a9 --- /dev/null +++ b/artsurvey/migrations/0054_auto_20200806_2017.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.8 on 2020-08-06 17:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0045_auto_20200804_1707'), + ('artsurvey', '0053_artsurveysource_nvss_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='first_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.FIRST'), + ), + migrations.AddField( + model_name='artsurveysource', + name='sumss_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.SUMSS'), + ), + ] diff --git a/artsurvey/migrations/0055_othername.py b/artsurvey/migrations/0055_othername.py new file mode 100644 index 0000000..ffd34fe --- /dev/null +++ b/artsurvey/migrations/0055_othername.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.8 on 2020-08-11 06:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0054_auto_20200806_2017'), + ] + + operations = [ + migrations.CreateModel( + name='OtherName', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(default='', max_length=60)), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='artsurvey.ArtSurveySource')), + ], + ), + ] diff --git a/artsurvey/migrations/0056_othername_owner.py b/artsurvey/migrations/0056_othername_owner.py new file mode 100644 index 0000000..e65c5d1 --- /dev/null +++ b/artsurvey/migrations/0056_othername_owner.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.8 on 2020-08-11 06:39 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0055_othername'), + ] + + operations = [ + migrations.AddField( + model_name='othername', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/artsurvey/migrations/0057_artsurveysource_class_tentative.py b/artsurvey/migrations/0057_artsurveysource_class_tentative.py new file mode 100644 index 0000000..9a4154a --- /dev/null +++ b/artsurvey/migrations/0057_artsurveysource_class_tentative.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-09-26 09:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0056_othername_owner'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='class_tentative', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0058_artsurveyparams_band.py b/artsurvey/migrations/0058_artsurveyparams_band.py new file mode 100644 index 0000000..e46168e --- /dev/null +++ b/artsurvey/migrations/0058_artsurveyparams_band.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-11 08:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0057_artsurveysource_class_tentative'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV')], default='E0', max_length=10), + ), + ] diff --git a/artsurvey/migrations/0059_energyband.py b/artsurvey/migrations/0059_energyband.py new file mode 100644 index 0000000..748e356 --- /dev/null +++ b/artsurvey/migrations/0059_energyband.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.8 on 2021-03-11 08:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0058_artsurveyparams_band'), + ] + + operations = [ + migrations.CreateModel( + name='EnergyBand', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('e1', models.FloatField(default=0.0)), + ('e2', models.FloatField(default=0.0)), + ('slug', models.SlugField()), + ], + ), + ] diff --git a/artsurvey/migrations/0060_artsurveysource_band.py b/artsurvey/migrations/0060_artsurveysource_band.py new file mode 100644 index 0000000..865e5b3 --- /dev/null +++ b/artsurvey/migrations/0060_artsurveysource_band.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-03-11 08:43 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0059_energyband'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='band', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.EnergyBand'), + ), + ] diff --git a/artsurvey/migrations/0061_artsurvey_active.py b/artsurvey/migrations/0061_artsurvey_active.py new file mode 100644 index 0000000..2b9f93a --- /dev/null +++ b/artsurvey/migrations/0061_artsurvey_active.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-11 09:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0060_artsurveysource_band'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='active', + field=models.BooleanField(default=False), + ), + ] diff --git a/artsurvey/migrations/0062_artsurveyparams_category.py b/artsurvey/migrations/0062_artsurveyparams_category.py new file mode 100644 index 0000000..87c4d9a --- /dev/null +++ b/artsurvey/migrations/0062_artsurveyparams_category.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-03-11 09:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0061_artsurvey_active'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='category', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.Category'), + ), + ] diff --git a/artsurvey/migrations/0063_auto_20210311_1458.py b/artsurvey/migrations/0063_auto_20210311_1458.py new file mode 100644 index 0000000..799ee73 --- /dev/null +++ b/artsurvey/migrations/0063_auto_20210311_1458.py @@ -0,0 +1,188 @@ +# Generated by Django 3.0.8 on 2021-03-11 11:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0062_artsurveyparams_category'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='cnts0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_bg0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_bg1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_bg2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_bg3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_bg4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_er0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_er1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_er2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_er3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_er4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_t0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_t1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_t2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_t3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='cnts_t4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='exptime0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='exptime1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='exptime2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='exptime3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='exptime4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux_er0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux_er1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux_er2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux_er3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux_er4', + field=models.FloatField(default=0.0), + ), + ] diff --git a/artsurvey/migrations/0064_auto_20210312_1440.py b/artsurvey/migrations/0064_auto_20210312_1440.py new file mode 100644 index 0000000..1138c7f --- /dev/null +++ b/artsurvey/migrations/0064_auto_20210312_1440.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.8 on 2021-03-12 11:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0063_auto_20210311_1458'), + ] + + operations = [ + migrations.AlterModelOptions( + name='artsurvey', + options={'ordering': ['version']}, + ), + migrations.AddField( + model_name='artsurvey', + name='archived', + field=models.BooleanField(default=False), + ), + ] diff --git a/artsurvey/migrations/0065_artsurvey_inwork.py b/artsurvey/migrations/0065_artsurvey_inwork.py new file mode 100644 index 0000000..ed15392 --- /dev/null +++ b/artsurvey/migrations/0065_artsurvey_inwork.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-12 11:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0064_auto_20210312_1440'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='inwork', + field=models.BooleanField(default=False), + ), + ] diff --git a/artsurvey/migrations/0066_artsurveysource_gaia3.py b/artsurvey/migrations/0066_artsurveysource_gaia3.py new file mode 100644 index 0000000..30631bc --- /dev/null +++ b/artsurvey/migrations/0066_artsurveysource_gaia3.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-03-12 14:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0053_remove_gaiadr3_ref_epoch'), + ('artsurvey', '0065_artsurvey_inwork'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='gaia3', + field=models.ManyToManyField(to='astrobasis.GAIADR3'), + ), + ] diff --git a/artsurvey/migrations/0067_artsurveysource_gaia3_primary.py b/artsurvey/migrations/0067_artsurveysource_gaia3_primary.py new file mode 100644 index 0000000..84dd9ac --- /dev/null +++ b/artsurvey/migrations/0067_artsurveysource_gaia3_primary.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.8 on 2021-03-12 14:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0053_remove_gaiadr3_ref_epoch'), + ('artsurvey', '0066_artsurveysource_gaia3'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='gaia3_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.GAIADR3'), + ), + ] diff --git a/artsurvey/migrations/0068_artsurveymatchparams.py b/artsurvey/migrations/0068_artsurveymatchparams.py new file mode 100644 index 0000000..814c040 --- /dev/null +++ b/artsurvey/migrations/0068_artsurveymatchparams.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.8 on 2021-03-12 15:25 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0067_artsurveysource_gaia3_primary'), + ] + + operations = [ + migrations.CreateModel( + name='ArtSurveyMatchParams', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('band1', models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV')], default='E0', max_length=10)), + ('band2', models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV')], default='E0', max_length=10)), + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='artsurveymatchparams', to=settings.AUTH_USER_MODEL)), + ('survey1', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='matchsurvey1', to='artsurvey.ArtSurvey')), + ('survey2', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='matchsurvey2', to='artsurvey.ArtSurvey')), + ], + ), + ] diff --git a/artsurvey/migrations/0069_auto_20210312_1841.py b/artsurvey/migrations/0069_auto_20210312_1841.py new file mode 100644 index 0000000..a784b0b --- /dev/null +++ b/artsurvey/migrations/0069_auto_20210312_1841.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.8 on 2021-03-12 15:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0068_artsurveymatchparams'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveymatchparams', + name='maxdist', + field=models.FloatField(blank=True, default=120.0, null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='minrad', + field=models.FloatField(blank=True, default=40.0, null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='sign_max', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='sign_min', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0070_auto_20210312_1848.py b/artsurvey/migrations/0070_auto_20210312_1848.py new file mode 100644 index 0000000..13e4b9a --- /dev/null +++ b/artsurvey/migrations/0070_auto_20210312_1848.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-12 15:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0069_auto_20210312_1841'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveymatchparams', + name='sign_max', + field=models.FloatField(blank=True, default=1000.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0071_auto_20210312_1924.py b/artsurvey/migrations/0071_auto_20210312_1924.py new file mode 100644 index 0000000..da39953 --- /dev/null +++ b/artsurvey/migrations/0071_auto_20210312_1924.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-12 16:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0070_auto_20210312_1848'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveymatchparams', + name='sign_max', + field=models.FloatField(blank=True, default=None, null=True), + ), + ] diff --git a/artsurvey/migrations/0072_auto_20210312_1924.py b/artsurvey/migrations/0072_auto_20210312_1924.py new file mode 100644 index 0000000..fd71569 --- /dev/null +++ b/artsurvey/migrations/0072_auto_20210312_1924.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-12 16:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0071_auto_20210312_1924'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveymatchparams', + name='sign_max', + field=models.FloatField(blank=True, default=1000.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0073_auto_20210312_1927.py b/artsurvey/migrations/0073_auto_20210312_1927.py new file mode 100644 index 0000000..8dd6577 --- /dev/null +++ b/artsurvey/migrations/0073_auto_20210312_1927.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-03-12 16:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0072_auto_20210312_1924'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveymatchparams', + name='f1f2_ratio_max', + field=models.FloatField(blank=True, default=1000.0, null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='f1f2_ratio_min', + field=models.FloatField(blank=True, default=0.001, null=True), + ), + ] diff --git a/artsurvey/migrations/0074_auto_20210312_1945.py b/artsurvey/migrations/0074_auto_20210312_1945.py new file mode 100644 index 0000000..208c051 --- /dev/null +++ b/artsurvey/migrations/0074_auto_20210312_1945.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.8 on 2021-03-12 16:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0073_auto_20210312_1927'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveymatchparams', + name='f1f2_ratio_max', + ), + migrations.RemoveField( + model_name='artsurveymatchparams', + name='f1f2_ratio_min', + ), + ] diff --git a/artsurvey/migrations/0075_auto_20210312_1946.py b/artsurvey/migrations/0075_auto_20210312_1946.py new file mode 100644 index 0000000..be61281 --- /dev/null +++ b/artsurvey/migrations/0075_auto_20210312_1946.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-03-12 16:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0074_auto_20210312_1945'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveymatchparams', + name='f2f1_ratio_max', + field=models.FloatField(blank=True, default=1000.0, null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='f2f1_ratio_min', + field=models.FloatField(blank=True, default=0.001, null=True), + ), + ] diff --git a/artsurvey/migrations/0076_metasource.py b/artsurvey/migrations/0076_metasource.py new file mode 100644 index 0000000..da68be1 --- /dev/null +++ b/artsurvey/migrations/0076_metasource.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.8 on 2021-03-15 07:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0075_auto_20210312_1946'), + ] + + operations = [ + migrations.CreateModel( + name='MetaSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=36)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('glon', models.FloatField(default=0.0)), + ('glat', models.FloatField(default=0.0)), + ('elon', models.FloatField(default=0.0)), + ('elat', models.FloatField(default=0.0)), + ('radec_error', models.FloatField(default=0.0)), + ], + options={ + 'ordering': ['-ra'], + }, + ), + ] diff --git a/artsurvey/migrations/0077_artsurveysource_metasource.py b/artsurvey/migrations/0077_artsurveysource_metasource.py new file mode 100644 index 0000000..0701b07 --- /dev/null +++ b/artsurvey/migrations/0077_artsurveysource_metasource.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-03-15 07:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0076_metasource'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='metasource', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.MetaSource'), + ), + ] diff --git a/artsurvey/migrations/0078_artsurveysource_metasource_sep.py b/artsurvey/migrations/0078_artsurveysource_metasource_sep.py new file mode 100644 index 0000000..0592ccb --- /dev/null +++ b/artsurvey/migrations/0078_artsurveysource_metasource_sep.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-15 09:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0077_artsurveysource_metasource'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='metasource_sep', + field=models.FloatField(default=0.0), + ), + ] diff --git a/artsurvey/migrations/0079_artsurvey_label.py b/artsurvey/migrations/0079_artsurvey_label.py new file mode 100644 index 0000000..3c4144a --- /dev/null +++ b/artsurvey/migrations/0079_artsurvey_label.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-15 14:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0078_artsurveysource_metasource_sep'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='label', + field=models.CharField(blank=True, default='', max_length=12), + ), + ] diff --git a/artsurvey/migrations/0080_auto_20210317_1721.py b/artsurvey/migrations/0080_auto_20210317_1721.py new file mode 100644 index 0000000..487ada3 --- /dev/null +++ b/artsurvey/migrations/0080_auto_20210317_1721.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-17 14:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0079_artsurvey_label'), + ] + + operations = [ + migrations.RenameField( + model_name='artsurveymatchparams', + old_name='minrad', + new_name='Rmax', + ), + ] diff --git a/artsurvey/migrations/0081_auto_20210317_1726.py b/artsurvey/migrations/0081_auto_20210317_1726.py new file mode 100644 index 0000000..b39d48a --- /dev/null +++ b/artsurvey/migrations/0081_auto_20210317_1726.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-03-17 14:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0080_auto_20210317_1721'), + ] + + operations = [ + migrations.RenameField( + model_name='artsurveymatchparams', + old_name='sign_max', + new_name='sign_max1', + ), + migrations.RenameField( + model_name='artsurveymatchparams', + old_name='sign_min', + new_name='sign_min1', + ), + ] diff --git a/artsurvey/migrations/0082_auto_20210317_1726.py b/artsurvey/migrations/0082_auto_20210317_1726.py new file mode 100644 index 0000000..fc5eb8b --- /dev/null +++ b/artsurvey/migrations/0082_auto_20210317_1726.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-03-17 14:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0081_auto_20210317_1726'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveymatchparams', + name='sign_max2', + field=models.FloatField(blank=True, default=1000.0, null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='sign_min2', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0083_auto_20210319_1225.py b/artsurvey/migrations/0083_auto_20210319_1225.py new file mode 100644 index 0000000..fba503e --- /dev/null +++ b/artsurvey/migrations/0083_auto_20210319_1225.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-03-19 09:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0082_auto_20210317_1726'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='sign_max', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV')], default='E0', max_length=10), + ), + ] diff --git a/artsurvey/migrations/0084_auto_20210319_1225.py b/artsurvey/migrations/0084_auto_20210319_1225.py new file mode 100644 index 0000000..7885270 --- /dev/null +++ b/artsurvey/migrations/0084_auto_20210319_1225.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-19 09:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0083_auto_20210319_1225'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='sign_max', + field=models.FloatField(blank=True, default=1000.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0085_auto_20210319_1231.py b/artsurvey/migrations/0085_auto_20210319_1231.py new file mode 100644 index 0000000..a1f1f04 --- /dev/null +++ b/artsurvey/migrations/0085_auto_20210319_1231.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-19 09:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0084_auto_20210319_1225'), + ] + + operations = [ + migrations.RenameField( + model_name='artsurveyparams', + old_name='sign_threshold', + new_name='sign_min', + ), + ] diff --git a/artsurvey/migrations/0086_auto_20210415_1544.py b/artsurvey/migrations/0086_auto_20210415_1544.py new file mode 100644 index 0000000..ca9ee90 --- /dev/null +++ b/artsurvey/migrations/0086_auto_20210415_1544.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-04-15 12:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0085_auto_20210319_1231'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='ext', + field=models.FloatField(default=0.0), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV')], default='E0', max_length=10), + ), + ] diff --git a/artsurvey/migrations/0087_auto_20210415_1607.py b/artsurvey/migrations/0087_auto_20210415_1607.py new file mode 100644 index 0000000..ca2acc0 --- /dev/null +++ b/artsurvey/migrations/0087_auto_20210415_1607.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-04-15 13:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0086_auto_20210415_1544'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='ext_max', + field=models.FloatField(blank=True, default=1000.0, null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='ext_min', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0088_artsurveysource_exti.py b/artsurvey/migrations/0088_artsurveysource_exti.py new file mode 100644 index 0000000..f0ffdf9 --- /dev/null +++ b/artsurvey/migrations/0088_artsurveysource_exti.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-04-16 10:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0087_auto_20210415_1607'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='exti', + field=models.IntegerField(default=0), + ), + ] diff --git a/artsurvey/migrations/0089_remove_artsurveysource_ext.py b/artsurvey/migrations/0089_remove_artsurveysource_ext.py new file mode 100644 index 0000000..62abd67 --- /dev/null +++ b/artsurvey/migrations/0089_remove_artsurveysource_ext.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2021-04-16 10:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0088_artsurveysource_exti'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='ext', + ), + ] diff --git a/artsurvey/migrations/0090_auto_20210416_1322.py b/artsurvey/migrations/0090_auto_20210416_1322.py new file mode 100644 index 0000000..55b4a6c --- /dev/null +++ b/artsurvey/migrations/0090_auto_20210416_1322.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-04-16 10:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0089_remove_artsurveysource_ext'), + ] + + operations = [ + migrations.RenameField( + model_name='artsurveysource', + old_name='exti', + new_name='ext', + ), + ] diff --git a/artsurvey/migrations/0091_auto_20210416_1322.py b/artsurvey/migrations/0091_auto_20210416_1322.py new file mode 100644 index 0000000..5f67c48 --- /dev/null +++ b/artsurvey/migrations/0091_auto_20210416_1322.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-04-16 10:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0090_auto_20210416_1322'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='ext_max', + field=models.IntegerField(blank=True, default=1000.0, null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='ext_min', + field=models.IntegerField(blank=True, default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0092_artsurveyparams_history.py b/artsurvey/migrations/0092_artsurveyparams_history.py new file mode 100644 index 0000000..b8935cc --- /dev/null +++ b/artsurvey/migrations/0092_artsurveyparams_history.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-04-22 12:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0091_auto_20210416_1322'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='history', + field=models.TextField(default='', editable=False), + ), + ] diff --git a/artsurvey/migrations/0093_artsurveysource_smapnr.py b/artsurvey/migrations/0093_artsurveysource_smapnr.py new file mode 100644 index 0000000..e3189b6 --- /dev/null +++ b/artsurvey/migrations/0093_artsurveysource_smapnr.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-04-28 17:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0092_artsurveyparams_history'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='SMAPNR', + field=models.PositiveIntegerField(db_index=True, default=0), + ), + ] diff --git a/artsurvey/migrations/0094_auto_20210430_1033.py b/artsurvey/migrations/0094_auto_20210430_1033.py new file mode 100644 index 0000000..612689a --- /dev/null +++ b/artsurvey/migrations/0094_auto_20210430_1033.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.8 on 2021-04-30 07:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0093_artsurveysource_smapnr'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='circle_dec', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='circle_ra', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='circle_rmax_deg', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0095_auto_20210430_1036.py b/artsurvey/migrations/0095_auto_20210430_1036.py new file mode 100644 index 0000000..0caed88 --- /dev/null +++ b/artsurvey/migrations/0095_auto_20210430_1036.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.8 on 2021-04-30 07:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0094_auto_20210430_1033'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='circle_dec', + field=models.FloatField(blank=True, default=0.0, help_text='Selection circle Dec', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='circle_ra', + field=models.FloatField(blank=True, default=0.0, help_text='Selection circle RA', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='circle_rmax_deg', + field=models.FloatField(blank=True, default=0.0, help_text='Selection circle Rmax in degrees', null=True), + ), + ] diff --git a/artsurvey/migrations/0096_artsurveyparams_exclude_survey.py b/artsurvey/migrations/0096_artsurveyparams_exclude_survey.py new file mode 100644 index 0000000..e44c374 --- /dev/null +++ b/artsurvey/migrations/0096_artsurveyparams_exclude_survey.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-04-30 09:04 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0095_auto_20210430_1036'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='exclude_survey', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='exclude_survey', to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0097_artsurveyparams_exclude_band.py b/artsurvey/migrations/0097_artsurveyparams_exclude_band.py new file mode 100644 index 0000000..5f179d4 --- /dev/null +++ b/artsurvey/migrations/0097_artsurveyparams_exclude_band.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-04-30 09:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0096_artsurveyparams_exclude_survey'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='exclude_band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV')], default='E0', max_length=10), + ), + ] diff --git a/artsurvey/migrations/0098_artsurveysource_turkish.py b/artsurvey/migrations/0098_artsurveysource_turkish.py new file mode 100644 index 0000000..4e28b87 --- /dev/null +++ b/artsurvey/migrations/0098_artsurveysource_turkish.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-05 18:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0097_artsurveyparams_exclude_band'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='turkish', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0099_artsurveysource_follow_up.py b/artsurvey/migrations/0099_artsurveysource_follow_up.py new file mode 100644 index 0000000..4b1b17e --- /dev/null +++ b/artsurvey/migrations/0099_artsurveysource_follow_up.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-05 18:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0098_artsurveysource_turkish'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='follow_up', + field=models.TextField(blank=True, max_length=2000, null=True, verbose_name='follow-up'), + ), + ] diff --git a/artsurvey/migrations/0100_artsurveysource_ext_id.py b/artsurvey/migrations/0100_artsurveysource_ext_id.py new file mode 100644 index 0000000..ae40856 --- /dev/null +++ b/artsurvey/migrations/0100_artsurveysource_ext_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-24 10:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0099_artsurveysource_follow_up'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='ext_id', + field=models.CharField(blank=True, default='', max_length=100), + ), + ] diff --git a/artsurvey/migrations/0101_auto_20210526_1619.py b/artsurvey/migrations/0101_auto_20210526_1619.py new file mode 100644 index 0000000..5dbd5e5 --- /dev/null +++ b/artsurvey/migrations/0101_auto_20210526_1619.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-05-26 13:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0100_artsurveysource_ext_id'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='log_nfalse_max', + field=models.FloatField(blank=True, default=-100.0, null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='log_nfalse_min', + field=models.FloatField(blank=True, default=-1.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0102_auto_20210526_1621.py b/artsurvey/migrations/0102_auto_20210526_1621.py new file mode 100644 index 0000000..0460d45 --- /dev/null +++ b/artsurvey/migrations/0102_auto_20210526_1621.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-26 13:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0101_auto_20210526_1619'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='log_nfalse_max', + field=models.FloatField(blank=True, default=-999.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0103_artsurveysource_log_nfalse.py b/artsurvey/migrations/0103_artsurveysource_log_nfalse.py new file mode 100644 index 0000000..7a83c73 --- /dev/null +++ b/artsurvey/migrations/0103_artsurveysource_log_nfalse.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-26 15:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0102_auto_20210526_1621'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='log_nfalse', + field=models.FloatField(default=0.0), + ), + ] diff --git a/artsurvey/migrations/0104_artsurveyparams_gaia_primary.py b/artsurvey/migrations/0104_artsurveyparams_gaia_primary.py new file mode 100644 index 0000000..c658a85 --- /dev/null +++ b/artsurvey/migrations/0104_artsurveyparams_gaia_primary.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-27 13:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0103_artsurveysource_log_nfalse'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='gaia_primary', + field=models.BooleanField(default=False, help_text='Select sources with Gaia counterpart'), + ), + ] diff --git a/artsurvey/migrations/0105_artsurveyparams_allwise_primary.py b/artsurvey/migrations/0105_artsurveyparams_allwise_primary.py new file mode 100644 index 0000000..0503ff2 --- /dev/null +++ b/artsurvey/migrations/0105_artsurveyparams_allwise_primary.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-27 13:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0104_artsurveyparams_gaia_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='allwise_primary', + field=models.BooleanField(default=False, help_text='Select sources with AllWISE counterpart'), + ), + ] diff --git a/artsurvey/migrations/0106_artsurveysource_catalog_version.py b/artsurvey/migrations/0106_artsurveysource_catalog_version.py new file mode 100644 index 0000000..09075bb --- /dev/null +++ b/artsurvey/migrations/0106_artsurveysource_catalog_version.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-27 15:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0105_artsurveyparams_allwise_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='catalog_version', + field=models.PositiveIntegerField(default=0), + ), + ] diff --git a/artsurvey/migrations/0107_catalog.py b/artsurvey/migrations/0107_catalog.py new file mode 100644 index 0000000..62e07ca --- /dev/null +++ b/artsurvey/migrations/0107_catalog.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.8 on 2021-05-28 09:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0106_artsurveysource_catalog_version'), + ] + + operations = [ + migrations.CreateModel( + name='Catalog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=36)), + ('year', models.PositiveIntegerField(default=0)), + ], + ), + ] diff --git a/artsurvey/migrations/0108_artsurveysource_catalog.py b/artsurvey/migrations/0108_artsurveysource_catalog.py new file mode 100644 index 0000000..7afae78 --- /dev/null +++ b/artsurvey/migrations/0108_artsurveysource_catalog.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-28 09:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0107_catalog'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='catalog', + field=models.ManyToManyField(to='artsurvey.Catalog'), + ), + ] diff --git a/artsurvey/migrations/0109_remove_artsurveysource_catalog_version.py b/artsurvey/migrations/0109_remove_artsurveysource_catalog_version.py new file mode 100644 index 0000000..b620537 --- /dev/null +++ b/artsurvey/migrations/0109_remove_artsurveysource_catalog_version.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2021-05-28 09:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0108_artsurveysource_catalog'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='catalog_version', + ), + ] diff --git a/artsurvey/migrations/0110_metasource_catalog.py b/artsurvey/migrations/0110_metasource_catalog.py new file mode 100644 index 0000000..1898058 --- /dev/null +++ b/artsurvey/migrations/0110_metasource_catalog.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-05-28 09:52 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0109_remove_artsurveysource_catalog_version'), + ] + + operations = [ + migrations.AddField( + model_name='metasource', + name='catalog', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.Catalog'), + ), + ] diff --git a/artsurvey/migrations/0111_artsurveyparams_exclude_log_nfalse.py b/artsurvey/migrations/0111_artsurveyparams_exclude_log_nfalse.py new file mode 100644 index 0000000..7d5d757 --- /dev/null +++ b/artsurvey/migrations/0111_artsurveyparams_exclude_log_nfalse.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-28 16:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0110_metasource_catalog'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='exclude_log_nfalse', + field=models.FloatField(blank=True, default=-1.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0112_auto_20210604_1059.py b/artsurvey/migrations/0112_auto_20210604_1059.py new file mode 100644 index 0000000..a91030d --- /dev/null +++ b/artsurvey/migrations/0112_auto_20210604_1059.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.8 on 2021-06-04 07:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0111_artsurveyparams_exclude_log_nfalse'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='notes_public', + field=models.BooleanField(default=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('ALL', 'All bands')], default='E0', max_length=10), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='exclude_band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('ALL', 'All bands')], default='E0', max_length=10), + ), + ] diff --git a/artsurvey/migrations/0113_auto_20210604_1143.py b/artsurvey/migrations/0113_auto_20210604_1143.py new file mode 100644 index 0000000..c39a3ce --- /dev/null +++ b/artsurvey/migrations/0113_auto_20210604_1143.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.8 on 2021-06-04 08:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0112_auto_20210604_1059'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='notes_public', + ), + migrations.AddField( + model_name='artsurveysource', + name='notes_paper', + field=models.TextField(blank=True, max_length=2000, null=True, verbose_name='notes paper'), + ), + ] diff --git a/artsurvey/migrations/0114_auto_20210607_1736.py b/artsurvey/migrations/0114_auto_20210607_1736.py new file mode 100644 index 0000000..4771fc3 --- /dev/null +++ b/artsurvey/migrations/0114_auto_20210607_1736.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:36 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0113_auto_20210604_1143'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='survey', + field=models.ForeignKey(blank=True, help_text='Survey number N version V is desinighed as SN.V', null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0115_auto_20210607_1739.py b/artsurvey/migrations/0115_auto_20210607_1739.py new file mode 100644 index 0000000..b67faec --- /dev/null +++ b/artsurvey/migrations/0115_auto_20210607_1739.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0114_auto_20210607_1736'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='survey', + field=models.ForeignKey(blank=True, help_text='Survey number N=1..8 of version V=0..9 is desinighed as SN.V; The combined survey is labeled as SNN.V;', null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0116_auto_20210607_1740.py b/artsurvey/migrations/0116_auto_20210607_1740.py new file mode 100644 index 0000000..f2a785f --- /dev/null +++ b/artsurvey/migrations/0116_auto_20210607_1740.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0115_auto_20210607_1739'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='survey', + field=models.ForeignKey(blank=True, help_text='Survey number N=1..8 of version V=0..9 is desinighed as SN.V;
The combined survey is labeled as SNN.V;', null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0117_auto_20210607_1741.py b/artsurvey/migrations/0117_auto_20210607_1741.py new file mode 100644 index 0000000..08ff294 --- /dev/null +++ b/artsurvey/migrations/0117_auto_20210607_1741.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:41 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0116_auto_20210607_1740'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='survey', + field=models.ForeignKey(blank=True, help_text='Survey number N=1..8 of version V=0..9 is named as SN.V;
The combined survey is labeled as SN<...>N.V;', null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0118_auto_20210607_1743.py b/artsurvey/migrations/0118_auto_20210607_1743.py new file mode 100644 index 0000000..7893d7d --- /dev/null +++ b/artsurvey/migrations/0118_auto_20210607_1743.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:43 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0117_auto_20210607_1741'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='survey', + field=models.ForeignKey(blank=True, help_text='Survey number N=[1..8] of version V=[0..9] is named as SN.V;
The combined survey is labeled as SN<...>N.V;', null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0119_auto_20210607_1746.py b/artsurvey/migrations/0119_auto_20210607_1746.py new file mode 100644 index 0000000..9eae14e --- /dev/null +++ b/artsurvey/migrations/0119_auto_20210607_1746.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0118_auto_20210607_1743'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='survey', + field=models.ForeignKey(blank=True, help_text='Survey number N=[1..8] of version V=[0..9] is named as SN.V;
The combined survey is labeled as SN<...>N.V;
Tags: active (*), in work (W), archived (A)', null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0120_auto_20210607_1748.py b/artsurvey/migrations/0120_auto_20210607_1748.py new file mode 100644 index 0000000..5403ec4 --- /dev/null +++ b/artsurvey/migrations/0120_auto_20210607_1748.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0119_auto_20210607_1746'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('ALL', 'All bands')], default='E0', help_text='E0: 4-12 keV, E1: 4-7 keV, E2: 7-12 keV, E3: 12-20 keV, E4: 20-30 keV', max_length=10), + ), + ] diff --git a/artsurvey/migrations/0121_auto_20210607_1750.py b/artsurvey/migrations/0121_auto_20210607_1750.py new file mode 100644 index 0000000..6f1d86a --- /dev/null +++ b/artsurvey/migrations/0121_auto_20210607_1750.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-06-07 14:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0120_auto_20210607_1748'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('ALL', 'All bands')], default='E0', help_text='E0: 4-12 keV, E1: 4-7 keV, E2: 7-12 keV, E3: 12-20 keV, E4: 20-30 keV', max_length=10), + ), + ] diff --git a/artsurvey/migrations/0122_auto_20210615_0915.py b/artsurvey/migrations/0122_auto_20210615_0915.py new file mode 100644 index 0000000..3a61e69 --- /dev/null +++ b/artsurvey/migrations/0122_auto_20210615_0915.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.8 on 2021-06-15 06:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0121_auto_20210607_1750'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='detlike', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='flux_uplim', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='fluxerr_lo', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='artsurveysource', + name='fluxerr_up', + field=models.FloatField(default=0.0), + ), + ] diff --git a/artsurvey/migrations/0123_auto_20210701_1724.py b/artsurvey/migrations/0123_auto_20210701_1724.py new file mode 100644 index 0000000..4c0b882 --- /dev/null +++ b/artsurvey/migrations/0123_auto_20210701_1724.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.8 on 2021-07-01 14:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0122_auto_20210615_0915'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='log_nfalse_max', + field=models.FloatField(blank=True, default=-999.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='log_nfalse_min', + field=models.FloatField(blank=True, default=-1.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='sign_max', + field=models.FloatField(blank=True, default=1000.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='sign_min', + field=models.FloatField(blank=True, default=0.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0124_auto_20210706_1008.py b/artsurvey/migrations/0124_auto_20210706_1008.py new file mode 100644 index 0000000..04b30b3 --- /dev/null +++ b/artsurvey/migrations/0124_auto_20210706_1008.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.8 on 2021-07-06 07:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0123_auto_20210701_1724'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='ecl_lat_min', + field=models.FloatField(blank=True, default=None, help_text='Ecliptic latitude max', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='circle_dec', + field=models.FloatField(blank=True, default=0.0, help_text='Selection circle Dec (not implemented yet)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='circle_ra', + field=models.FloatField(blank=True, default=0.0, help_text='Selection circle RA (not implemented yet)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='circle_rmax_deg', + field=models.FloatField(blank=True, default=0.0, help_text='Selection circle Rmax in degrees (not implemented yet)', null=True), + ), + ] diff --git a/artsurvey/migrations/0125_remove_artsurveyparams_ecl_lat_min.py b/artsurvey/migrations/0125_remove_artsurveyparams_ecl_lat_min.py new file mode 100644 index 0000000..202c0d0 --- /dev/null +++ b/artsurvey/migrations/0125_remove_artsurveyparams_ecl_lat_min.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2021-07-06 07:09 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0124_auto_20210706_1008'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveyparams', + name='ecl_lat_min', + ), + ] diff --git a/artsurvey/migrations/0126_auto_20210706_1010.py b/artsurvey/migrations/0126_auto_20210706_1010.py new file mode 100644 index 0000000..e0f94ff --- /dev/null +++ b/artsurvey/migrations/0126_auto_20210706_1010.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-07-06 07:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0125_remove_artsurveyparams_ecl_lat_min'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='ecl_lat_max', + field=models.FloatField(blank=True, default=None, help_text='Ecliptic latitude max', null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='ecl_lat_min', + field=models.FloatField(blank=True, default=None, help_text='Ecliptic latitude min', null=True), + ), + ] diff --git a/artsurvey/migrations/0127_artsurveysource_xrays.py b/artsurvey/migrations/0127_artsurveysource_xrays.py new file mode 100644 index 0000000..d397772 --- /dev/null +++ b/artsurvey/migrations/0127_artsurveysource_xrays.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-08-21 18:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0126_auto_20210706_1010'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='xrays', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0128_auto_20210826_1325.py b/artsurvey/migrations/0128_auto_20210826_1325.py new file mode 100644 index 0000000..d079a2b --- /dev/null +++ b/artsurvey/migrations/0128_auto_20210826_1325.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-08-26 10:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0127_artsurveysource_xrays'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='ext_max', + field=models.IntegerField(blank=True, default=1000, help_text='Maximum extention in arcseconds', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='ext_min', + field=models.IntegerField(blank=True, default=0, help_text='Minimal extention in arcseconds', null=True), + ), + ] diff --git a/artsurvey/migrations/0129_catalog_test.py b/artsurvey/migrations/0129_catalog_test.py new file mode 100644 index 0000000..2681e3b --- /dev/null +++ b/artsurvey/migrations/0129_catalog_test.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-11-10 09:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0128_auto_20210826_1325'), + ] + + operations = [ + migrations.AddField( + model_name='catalog', + name='test', + field=models.CharField(default='', max_length=36), + ), + ] diff --git a/artsurvey/migrations/0130_remove_catalog_test.py b/artsurvey/migrations/0130_remove_catalog_test.py new file mode 100644 index 0000000..5e702c4 --- /dev/null +++ b/artsurvey/migrations/0130_remove_catalog_test.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2021-11-10 09:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0129_catalog_test'), + ] + + operations = [ + migrations.RemoveField( + model_name='catalog', + name='test', + ), + ] diff --git a/artsurvey/migrations/0131_auto_20211228_1925.py b/artsurvey/migrations/0131_auto_20211228_1925.py new file mode 100644 index 0000000..54fc203 --- /dev/null +++ b/artsurvey/migrations/0131_auto_20211228_1925.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-12-28 16:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0130_remove_catalog_test'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurvey', + name='version', + field=models.FloatField(default=0.0), + ), + ] diff --git a/artsurvey/migrations/0132_artsurvey_parent.py b/artsurvey/migrations/0132_artsurvey_parent.py new file mode 100644 index 0000000..12bc6d9 --- /dev/null +++ b/artsurvey/migrations/0132_artsurvey_parent.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-12-28 16:32 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0131_auto_20211228_1925'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='artsurvey.ArtSurvey'), + ), + ] diff --git a/artsurvey/migrations/0133_auto_20211229_1259.py b/artsurvey/migrations/0133_auto_20211229_1259.py new file mode 100644 index 0000000..ab83f61 --- /dev/null +++ b/artsurvey/migrations/0133_auto_20211229_1259.py @@ -0,0 +1,73 @@ +# Generated by Django 3.2.10 on 2021-12-29 09:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0132_artsurvey_parent'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurvey', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='artsurveymatchparams', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='artsurveysource', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='catalog', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='category', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='energyband', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='erositamatch', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='metasource', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='opticalcounterpart', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='othername', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='wisecounterpart', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/artsurvey/migrations/0134_auto_20220112_1202.py b/artsurvey/migrations/0134_auto_20220112_1202.py new file mode 100644 index 0000000..f1c70d2 --- /dev/null +++ b/artsurvey/migrations/0134_auto_20220112_1202.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.10 on 2022-01-12 09:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0133_auto_20211229_1259'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='gaia', + ), + migrations.RemoveField( + model_name='artsurveysource', + name='gaia_primary', + ), + ] diff --git a/artsurvey/migrations/0135_artsurvey_name.py b/artsurvey/migrations/0135_artsurvey_name.py new file mode 100644 index 0000000..93e3b3a --- /dev/null +++ b/artsurvey/migrations/0135_artsurvey_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.10 on 2022-01-26 13:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0134_auto_20220112_1202'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='name', + field=models.CharField(blank=True, default='', max_length=12), + ), + ] diff --git a/artsurvey/migrations/0136_artsurveysource_turkpossible.py b/artsurvey/migrations/0136_artsurveysource_turkpossible.py new file mode 100644 index 0000000..02c01fd --- /dev/null +++ b/artsurvey/migrations/0136_artsurveysource_turkpossible.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.10 on 2022-02-16 14:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0135_artsurvey_name'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='turkpossible', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0137_artsurveyparams_turkpossible.py b/artsurvey/migrations/0137_artsurveyparams_turkpossible.py new file mode 100644 index 0000000..effc57e --- /dev/null +++ b/artsurvey/migrations/0137_artsurveyparams_turkpossible.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.10 on 2022-02-16 14:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0136_artsurveysource_turkpossible'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='turkpossible', + field=models.BooleanField(default=False, help_text='Select sources suitable for TUBITAK (exclude Swift/BAT 105m (2018), INTEGRAL Reference Catalog (last updated on 16 June 2019) and RXTE all-sky survey'), + ), + ] diff --git a/artsurvey/migrations/0138_alter_artsurveyparams_turkpossible.py b/artsurvey/migrations/0138_alter_artsurveyparams_turkpossible.py new file mode 100644 index 0000000..d4b2bc9 --- /dev/null +++ b/artsurvey/migrations/0138_alter_artsurveyparams_turkpossible.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.10 on 2022-02-16 14:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0137_artsurveyparams_turkpossible'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='turkpossible', + field=models.BooleanField(default=False, help_text='Select sources suitable for TUBITAK'), + ), + ] diff --git a/artsurvey/migrations/0139_auto_20220216_1735.py b/artsurvey/migrations/0139_auto_20220216_1735.py new file mode 100644 index 0000000..4a86647 --- /dev/null +++ b/artsurvey/migrations/0139_auto_20220216_1735.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.10 on 2022-02-16 14:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0138_alter_artsurveyparams_turkpossible'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveyparams', + name='turkpossible', + ), + migrations.AddField( + model_name='artsurveyparams', + name='turk_possible', + field=models.BooleanField(default=False, help_text='Sources suitable for TUBITAK'), + ), + ] diff --git a/artsurvey/migrations/0140_alter_artsurveyparams_turk_possible.py b/artsurvey/migrations/0140_alter_artsurveyparams_turk_possible.py new file mode 100644 index 0000000..8b8d3bb --- /dev/null +++ b/artsurvey/migrations/0140_alter_artsurveyparams_turk_possible.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-02 16:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0139_auto_20220216_1735'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='turk_possible', + field=models.BooleanField(default=False, help_text='Sources suitable for TUBITAK*'), + ), + ] diff --git a/artsurvey/migrations/0141_alter_artsurveysource_name_orig.py b/artsurvey/migrations/0141_alter_artsurveysource_name_orig.py new file mode 100644 index 0000000..ee037b9 --- /dev/null +++ b/artsurvey/migrations/0141_alter_artsurveysource_name_orig.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-03 15:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0140_alter_artsurveyparams_turk_possible'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveysource', + name='name_orig', + field=models.CharField(max_length=22, null=True), + ), + ] diff --git a/artsurvey/migrations/0142_auto_20220303_1837.py b/artsurvey/migrations/0142_auto_20220303_1837.py new file mode 100644 index 0000000..835a6fe --- /dev/null +++ b/artsurvey/migrations/0142_auto_20220303_1837.py @@ -0,0 +1,273 @@ +# Generated by Django 3.2.12 on 2022-03-03 15:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0141_alter_artsurveysource_name_orig'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveysource', + name='cnts', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_bg', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_bg0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_bg1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_bg2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_bg3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_bg4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_er0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_er1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_er2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_er3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_er4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_t0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_t1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_t2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_t3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='cnts_t4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='detlike', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='exptime', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='exptime0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='exptime1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='exptime2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='exptime3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='exptime4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='ext', + field=models.IntegerField(default=0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='ext_id', + field=models.CharField(blank=True, default='', max_length=100, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux_er0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux_er1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux_er2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux_er3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux_er4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='flux_uplim', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='fluxerr_lo', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='fluxerr_up', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='log_nfalse', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='nfalse', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='rate', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='rate_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='artsurveysource', + name='sig', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0143_auto_20220404_1254.py b/artsurvey/migrations/0143_auto_20220404_1254.py new file mode 100644 index 0000000..7be1bfc --- /dev/null +++ b/artsurvey/migrations/0143_auto_20220404_1254.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.12 on 2022-04-04 09:54 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0066_alter_twomass_name'), + ('artsurvey', '0142_auto_20220303_1837'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='vlass', + field=models.ManyToManyField(to='astrobasis.VLASS'), + ), + migrations.AddField( + model_name='artsurveysource', + name='vlass_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.vlass'), + ), + ] diff --git a/artsurvey/migrations/0144_auto_20220404_1642.py b/artsurvey/migrations/0144_auto_20220404_1642.py new file mode 100644 index 0000000..06279fb --- /dev/null +++ b/artsurvey/migrations/0144_auto_20220404_1642.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.12 on 2022-04-04 13:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0143_auto_20220404_1254'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='vlass', + ), + migrations.RemoveField( + model_name='artsurveysource', + name='vlass_primary', + ), + ] diff --git a/artsurvey/migrations/0145_auto_20220404_1643.py b/artsurvey/migrations/0145_auto_20220404_1643.py new file mode 100644 index 0000000..2befba6 --- /dev/null +++ b/artsurvey/migrations/0145_auto_20220404_1643.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.12 on 2022-04-04 13:43 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0068_auto_20220404_1639'), + ('artsurvey', '0144_auto_20220404_1642'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='vlass', + field=models.ManyToManyField(to='astrobasis.VLASSfromVizieR'), + ), + migrations.AddField( + model_name='artsurveysource', + name='vlass_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.vlassfromvizier'), + ), + ] diff --git a/artsurvey/migrations/0146_artsurveysource_heasarc_linked.py b/artsurvey/migrations/0146_artsurveysource_heasarc_linked.py new file mode 100644 index 0000000..edfba1f --- /dev/null +++ b/artsurvey/migrations/0146_artsurveysource_heasarc_linked.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.12 on 2022-04-05 07:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('heasarc', '0045_auto_20220404_1109'), + ('artsurvey', '0145_auto_20220404_1643'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='heasarc_linked', + field=models.ManyToManyField(related_name='artsurveysource_linked', to='heasarc.HeasarcBase'), + ), + ] diff --git a/artsurvey/migrations/0147_upload.py b/artsurvey/migrations/0147_upload.py new file mode 100644 index 0000000..5ed798b --- /dev/null +++ b/artsurvey/migrations/0147_upload.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.12 on 2022-04-11 12:57 + +import artsurvey.models +from django.conf import settings +import django.core.files.storage +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0146_artsurveysource_heasarc_linked'), + ] + + operations = [ + migrations.CreateModel( + name='Upload', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=128)), + ('filename', models.CharField(max_length=256)), + ('filefield', models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/export/django/srg/srg/static/uploads/artsurvey'), upload_to=artsurvey.models.upload_filename, validators=[artsurvey.models.validate_upload])), + ('uploaded', models.DateTimeField(auto_now_add=True)), + ('owner', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='uploads', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/artsurvey/migrations/0148_upload_artsources.py b/artsurvey/migrations/0148_upload_artsources.py new file mode 100644 index 0000000..82de706 --- /dev/null +++ b/artsurvey/migrations/0148_upload_artsources.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-04-11 13:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0147_upload'), + ] + + operations = [ + migrations.AddField( + model_name='upload', + name='artsources', + field=models.ManyToManyField(blank=True, to='artsurvey.ArtSurveySource'), + ), + ] diff --git a/artsurvey/migrations/0149_alter_upload_artsources.py b/artsurvey/migrations/0149_alter_upload_artsources.py new file mode 100644 index 0000000..e98d2dc --- /dev/null +++ b/artsurvey/migrations/0149_alter_upload_artsources.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-04-11 16:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0148_upload_artsources'), + ] + + operations = [ + migrations.AlterField( + model_name='upload', + name='artsources', + field=models.ManyToManyField(blank=True, related_name='uploads', to='artsurvey.ArtSurveySource'), + ), + ] diff --git a/artsurvey/migrations/0150_alter_upload_filefield.py b/artsurvey/migrations/0150_alter_upload_filefield.py new file mode 100644 index 0000000..b244ee6 --- /dev/null +++ b/artsurvey/migrations/0150_alter_upload_filefield.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.12 on 2022-04-13 14:44 + +import artsurvey.models +import django.core.files.storage +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0149_alter_upload_artsources'), + ] + + operations = [ + migrations.AlterField( + model_name='upload', + name='filefield', + field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/export/django/Uploads/artsurvey'), upload_to=artsurvey.models.upload_filename, validators=[artsurvey.models.validate_upload]), + ), + ] diff --git a/artsurvey/migrations/0151_upload_notes.py b/artsurvey/migrations/0151_upload_notes.py new file mode 100644 index 0000000..0a76a19 --- /dev/null +++ b/artsurvey/migrations/0151_upload_notes.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-15 08:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0150_alter_upload_filefield'), + ] + + operations = [ + migrations.AddField( + model_name='upload', + name='notes', + field=models.TextField(blank=True, max_length=2000, null=True, verbose_name='notes'), + ), + ] diff --git a/artsurvey/migrations/0152_artselection.py b/artsurvey/migrations/0152_artselection.py new file mode 100644 index 0000000..7788dc8 --- /dev/null +++ b/artsurvey/migrations/0152_artselection.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.13 on 2022-04-15 09:05 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0151_upload_notes'), + ] + + operations = [ + migrations.CreateModel( + name='ArtSelection', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('sources', models.ManyToManyField(blank=True, to='artsurvey.ArtSurveySource')), + ], + ), + ] diff --git a/artsurvey/migrations/0153_alter_artselection_owner.py b/artsurvey/migrations/0153_alter_artselection_owner.py new file mode 100644 index 0000000..26dd6a7 --- /dev/null +++ b/artsurvey/migrations/0153_alter_artselection_owner.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.13 on 2022-04-15 09:47 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0152_artselection'), + ] + + operations = [ + migrations.AlterField( + model_name='artselection', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, unique=True), + ), + ] diff --git a/artsurvey/migrations/0154_remove_artselection_owner.py b/artsurvey/migrations/0154_remove_artselection_owner.py new file mode 100644 index 0000000..2592cec --- /dev/null +++ b/artsurvey/migrations/0154_remove_artselection_owner.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2022-04-15 09:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0153_alter_artselection_owner'), + ] + + operations = [ + migrations.RemoveField( + model_name='artselection', + name='owner', + ), + ] diff --git a/artsurvey/migrations/0155_artselection_owner.py b/artsurvey/migrations/0155_artselection_owner.py new file mode 100644 index 0000000..b898008 --- /dev/null +++ b/artsurvey/migrations/0155_artselection_owner.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.13 on 2022-04-15 09:50 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0154_remove_artselection_owner'), + ] + + operations = [ + migrations.AddField( + model_name='artselection', + name='owner', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/artsurvey/migrations/0156_auto_20220415_1934.py b/artsurvey/migrations/0156_auto_20220415_1934.py new file mode 100644 index 0000000..07b2148 --- /dev/null +++ b/artsurvey/migrations/0156_auto_20220415_1934.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-04-15 16:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0155_artselection_owner'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='sign_max', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='sign_min', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0157_auto_20220415_1938.py b/artsurvey/migrations/0157_auto_20220415_1938.py new file mode 100644 index 0000000..f5b58d3 --- /dev/null +++ b/artsurvey/migrations/0157_auto_20220415_1938.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.13 on 2022-04-15 16:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0156_auto_20220415_1934'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='ext_max', + field=models.IntegerField(blank=True, default=None, help_text='Maximum extention in arcseconds', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='ext_min', + field=models.IntegerField(blank=True, default=None, help_text='Minimal extention in arcseconds', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='log_nfalse_max', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='log_nfalse_min', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0158_upload_metasource.py b/artsurvey/migrations/0158_upload_metasource.py new file mode 100644 index 0000000..f2ede82 --- /dev/null +++ b/artsurvey/migrations/0158_upload_metasource.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-04-22 13:31 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0157_auto_20220415_1938'), + ] + + operations = [ + migrations.AddField( + model_name='upload', + name='metasource', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='uploads', to='artsurvey.metasource'), + ), + ] diff --git a/artsurvey/migrations/0159_artsurvey_hidden.py b/artsurvey/migrations/0159_artsurvey_hidden.py new file mode 100644 index 0000000..aa6479a --- /dev/null +++ b/artsurvey/migrations/0159_artsurvey_hidden.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-25 10:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0158_upload_metasource'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='hidden', + field=models.BooleanField(default=False), + ), + ] diff --git a/artsurvey/migrations/0160_alter_artsurveysource_options.py b/artsurvey/migrations/0160_alter_artsurveysource_options.py new file mode 100644 index 0000000..c06ba2b --- /dev/null +++ b/artsurvey/migrations/0160_alter_artsurveysource_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2022-04-26 11:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0159_artsurvey_hidden'), + ] + + operations = [ + migrations.AlterModelOptions( + name='artsurveysource', + options={'get_latest_by': 'created'}, + ), + ] diff --git a/artsurvey/migrations/0161_metacomment.py b/artsurvey/migrations/0161_metacomment.py new file mode 100644 index 0000000..7ed5f09 --- /dev/null +++ b/artsurvey/migrations/0161_metacomment.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.13 on 2022-04-28 08:19 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0160_alter_artsurveysource_options'), + ] + + operations = [ + migrations.CreateModel( + name='MetaComment', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('body', models.TextField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('metasource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='artsurvey.metasource')), + ('owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['created'], + }, + ), + ] diff --git a/artsurvey/migrations/0162_alter_metacomment_body.py b/artsurvey/migrations/0162_alter_metacomment_body.py new file mode 100644 index 0000000..cbd1a79 --- /dev/null +++ b/artsurvey/migrations/0162_alter_metacomment_body.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-28 09:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0161_metacomment'), + ] + + operations = [ + migrations.AlterField( + model_name='metacomment', + name='body', + field=models.TextField(verbose_name='E-Mail Address'), + ), + ] diff --git a/artsurvey/migrations/0163_alter_metacomment_body.py b/artsurvey/migrations/0163_alter_metacomment_body.py new file mode 100644 index 0000000..aeca9ba --- /dev/null +++ b/artsurvey/migrations/0163_alter_metacomment_body.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-28 09:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0162_alter_metacomment_body'), + ] + + operations = [ + migrations.AlterField( + model_name='metacomment', + name='body', + field=models.TextField(), + ), + ] diff --git a/artsurvey/migrations/0164_alter_artsurveyparams_log_nfalse_min.py b/artsurvey/migrations/0164_alter_artsurveyparams_log_nfalse_min.py new file mode 100644 index 0000000..3d07769 --- /dev/null +++ b/artsurvey/migrations/0164_alter_artsurveyparams_log_nfalse_min.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-04-28 15:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0163_alter_metacomment_body'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='log_nfalse_min', + field=models.FloatField(blank=True, default=-1.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0165_artsurveyparams_exclude_category.py b/artsurvey/migrations/0165_artsurveyparams_exclude_category.py new file mode 100644 index 0000000..8786db7 --- /dev/null +++ b/artsurvey/migrations/0165_artsurveyparams_exclude_category.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-06-01 12:58 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0164_alter_artsurveyparams_log_nfalse_min'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='exclude_category', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='exclude_category', to='artsurvey.category'), + ), + ] diff --git a/artsurvey/migrations/0166_alter_artsurveyparams_exclude_category.py b/artsurvey/migrations/0166_alter_artsurveyparams_exclude_category.py new file mode 100644 index 0000000..36842d0 --- /dev/null +++ b/artsurvey/migrations/0166_alter_artsurveyparams_exclude_category.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-06-01 13:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0165_artsurveyparams_exclude_category'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='exclude_category', + field=models.ForeignKey(blank=True, help_text='Hide sources with this Category', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='exclude_category', to='artsurvey.category'), + ), + ] diff --git a/artsurvey/migrations/0167_artsurvey_notes.py b/artsurvey/migrations/0167_artsurvey_notes.py new file mode 100644 index 0000000..32636bd --- /dev/null +++ b/artsurvey/migrations/0167_artsurvey_notes.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-06 16:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0166_alter_artsurveyparams_exclude_category'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='notes', + field=models.TextField(blank=True, max_length=3000, null=True, verbose_name='notes'), + ), + ] diff --git a/artsurvey/migrations/0168_artsurveysource_metasource_primary.py b/artsurvey/migrations/0168_artsurveysource_metasource_primary.py new file mode 100644 index 0000000..f1aa40c --- /dev/null +++ b/artsurvey/migrations/0168_artsurveysource_metasource_primary.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-06-07 11:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0167_artsurvey_notes'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='metasource_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='artsurvey.metasource'), + ), + ] diff --git a/artsurvey/migrations/0169_remove_artsurveysource_metasource_primary.py b/artsurvey/migrations/0169_remove_artsurveysource_metasource_primary.py new file mode 100644 index 0000000..c390d19 --- /dev/null +++ b/artsurvey/migrations/0169_remove_artsurveysource_metasource_primary.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2022-06-07 11:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0168_artsurveysource_metasource_primary'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='metasource_primary', + ), + ] diff --git a/artsurvey/migrations/0170_artsurveysource_metasource_primary.py b/artsurvey/migrations/0170_artsurveysource_metasource_primary.py new file mode 100644 index 0000000..afc7684 --- /dev/null +++ b/artsurvey/migrations/0170_artsurveysource_metasource_primary.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-06-07 11:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0169_remove_artsurveysource_metasource_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='metasource_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='artsurvey.metasource'), + ), + ] diff --git a/artsurvey/migrations/0171_artsurveysource_lastvisited.py b/artsurvey/migrations/0171_artsurveysource_lastvisited.py new file mode 100644 index 0000000..ee2215b --- /dev/null +++ b/artsurvey/migrations/0171_artsurveysource_lastvisited.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.13 on 2022-06-09 14:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('heasarc', '0050_auto_20220428_1809'), + ('artsurvey', '0170_artsurveysource_metasource_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='lastvisited', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_referrer', to='heasarc.heasarcbase'), + ), + ] diff --git a/artsurvey/migrations/0172_remove_artsurveysource_lastvisited.py b/artsurvey/migrations/0172_remove_artsurveysource_lastvisited.py new file mode 100644 index 0000000..8b7f07e --- /dev/null +++ b/artsurvey/migrations/0172_remove_artsurveysource_lastvisited.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2022-06-09 14:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0171_artsurveysource_lastvisited'), + ] + + operations = [ + migrations.RemoveField( + model_name='artsurveysource', + name='lastvisited', + ), + ] diff --git a/artsurvey/migrations/0173_upload_survey.py b/artsurvey/migrations/0173_upload_survey.py new file mode 100644 index 0000000..18425de --- /dev/null +++ b/artsurvey/migrations/0173_upload_survey.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-06-20 15:06 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0172_remove_artsurveysource_lastvisited'), + ] + + operations = [ + migrations.AddField( + model_name='upload', + name='survey', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='uploads', to='artsurvey.artsurvey'), + ), + ] diff --git a/artsurvey/migrations/0174_upload_primary.py b/artsurvey/migrations/0174_upload_primary.py new file mode 100644 index 0000000..074d966 --- /dev/null +++ b/artsurvey/migrations/0174_upload_primary.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-20 16:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0173_upload_survey'), + ] + + operations = [ + migrations.AddField( + model_name='upload', + name='primary', + field=models.BooleanField(default=False, help_text='Primary'), + ), + ] diff --git a/artsurvey/migrations/0175_artsurveysource_bibtex_paper.py b/artsurvey/migrations/0175_artsurveysource_bibtex_paper.py new file mode 100644 index 0000000..583851e --- /dev/null +++ b/artsurvey/migrations/0175_artsurveysource_bibtex_paper.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-23 10:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0174_upload_primary'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='bibtex_paper', + field=models.TextField(blank=True, max_length=2000, null=True, verbose_name='bibtex paper'), + ), + ] diff --git a/artsurvey/migrations/0176_auto_20220624_1826.py b/artsurvey/migrations/0176_auto_20220624_1826.py new file mode 100644 index 0000000..7f2b1f0 --- /dev/null +++ b/artsurvey/migrations/0176_auto_20220624_1826.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.13 on 2022-06-24 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0175_artsurveysource_bibtex_paper'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveysource', + name='bibtex_paper', + field=models.TextField(blank=True, max_length=5000, null=True, verbose_name='bibtex paper'), + ), + migrations.AlterField( + model_name='artsurveysource', + name='follow_up', + field=models.TextField(blank=True, max_length=5000, null=True, verbose_name='follow-up'), + ), + migrations.AlterField( + model_name='artsurveysource', + name='notes_paper', + field=models.TextField(blank=True, max_length=5000, null=True, verbose_name='notes paper'), + ), + ] diff --git a/artsurvey/migrations/0177_auto_20220707_1819.py b/artsurvey/migrations/0177_auto_20220707_1819.py new file mode 100644 index 0000000..c31d498 --- /dev/null +++ b/artsurvey/migrations/0177_auto_20220707_1819.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-07-07 15:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0176_auto_20220624_1826'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='detlike_max', + field=models.FloatField(blank=True, default=None, help_text='Detlike maximum value', null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='detlike_min', + field=models.FloatField(blank=True, default=None, help_text='Detlike minimum value', null=True), + ), + ] diff --git a/artsurvey/migrations/0178_auto_20220707_1825.py b/artsurvey/migrations/0178_auto_20220707_1825.py new file mode 100644 index 0000000..814a343 --- /dev/null +++ b/artsurvey/migrations/0178_auto_20220707_1825.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-07-07 15:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0177_auto_20220707_1819'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='detlike_max', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='detlike_min', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0179_artsurveyparams_exposure_min.py b/artsurvey/migrations/0179_artsurveyparams_exposure_min.py new file mode 100644 index 0000000..fe37637 --- /dev/null +++ b/artsurvey/migrations/0179_artsurveyparams_exposure_min.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-08-31 09:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0178_auto_20220707_1825'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='exposure_min', + field=models.FloatField(blank=True, default=None, help_text='Minimal exposure time', null=True), + ), + ] diff --git a/artsurvey/migrations/0180_auto_20221107_1529.py b/artsurvey/migrations/0180_auto_20221107_1529.py new file mode 100644 index 0000000..7e93233 --- /dev/null +++ b/artsurvey/migrations/0180_auto_20221107_1529.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-11-07 12:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0179_artsurveyparams_exposure_min'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='dec_max', + field=models.FloatField(blank=True, default=None, help_text='J2000 Declination max', null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='dec_min', + field=models.FloatField(blank=True, default=None, help_text='J2000 Declination min', null=True), + ), + ] diff --git a/artsurvey/migrations/0181_artsurvey_modified.py b/artsurvey/migrations/0181_artsurvey_modified.py new file mode 100644 index 0000000..894819d --- /dev/null +++ b/artsurvey/migrations/0181_artsurvey_modified.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-12-23 15:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0180_auto_20221107_1529'), + ] + + operations = [ + migrations.AddField( + model_name='artsurvey', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/artsurvey/migrations/0182_artsurveysource_ads_hidden.py b/artsurvey/migrations/0182_artsurveysource_ads_hidden.py new file mode 100644 index 0000000..ff0040e --- /dev/null +++ b/artsurvey/migrations/0182_artsurveysource_ads_hidden.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2023-01-13 15:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nasa', '0011_ads_hidden'), + ('artsurvey', '0181_artsurvey_modified'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='ads_hidden', + field=models.ManyToManyField(related_name='artsurveysource_hidden', to='nasa.Ads'), + ), + ] diff --git a/artsurvey/migrations/0183_artbasket.py b/artsurvey/migrations/0183_artbasket.py new file mode 100644 index 0000000..917bde5 --- /dev/null +++ b/artsurvey/migrations/0183_artbasket.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.13 on 2023-01-17 12:15 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('artsurvey', '0182_artsurveysource_ads_hidden'), + ] + + operations = [ + migrations.CreateModel( + name='ArtBasket', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=128)), + ('notes', models.TextField(blank=True, max_length=2000, null=True, validators=[django.core.validators.MaxLengthValidator(1600)], verbose_name='Notes')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('sources', models.ManyToManyField(blank=True, to='artsurvey.ArtSurveySource')), + ], + ), + ] diff --git a/artsurvey/migrations/0184_artsurveysource_modified.py b/artsurvey/migrations/0184_artsurveysource_modified.py new file mode 100644 index 0000000..83935fd --- /dev/null +++ b/artsurvey/migrations/0184_artsurveysource_modified.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-04-17 10:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0183_artbasket'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/artsurvey/migrations/0185_artsurveyparams_cname_startswith.py b/artsurvey/migrations/0185_artsurveyparams_cname_startswith.py new file mode 100644 index 0000000..d69a6af --- /dev/null +++ b/artsurvey/migrations/0185_artsurveyparams_cname_startswith.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-04-19 14:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0184_artsurveysource_modified'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='cname_startswith', + field=models.CharField(blank=True, default='', max_length=36, null=True), + ), + ] diff --git a/artsurvey/migrations/0186_rename_cname_startswith_artsurveyparams_cname_contains.py b/artsurvey/migrations/0186_rename_cname_startswith_artsurveyparams_cname_contains.py new file mode 100644 index 0000000..00d87f6 --- /dev/null +++ b/artsurvey/migrations/0186_rename_cname_startswith_artsurveyparams_cname_contains.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-04-19 14:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0185_artsurveyparams_cname_startswith'), + ] + + operations = [ + migrations.RenameField( + model_name='artsurveyparams', + old_name='cname_startswith', + new_name='cname_contains', + ), + ] diff --git a/artsurvey/migrations/0187_auto_20230516_1536.py b/artsurvey/migrations/0187_auto_20230516_1536.py new file mode 100644 index 0000000..bca4bb8 --- /dev/null +++ b/artsurvey/migrations/0187_auto_20230516_1536.py @@ -0,0 +1,63 @@ +# Generated by Django 3.2.13 on 2023-05-16 12:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0186_rename_cname_startswith_artsurveyparams_cname_contains'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='ml_dec', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_detlike', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_exp', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_flux_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_nfalse', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_ra', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_radec_err_90', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_radec_err_98', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='artsurveysource', + name='ml_sig', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0188_artsurveysource_log_ml_nfalse.py b/artsurvey/migrations/0188_artsurveysource_log_ml_nfalse.py new file mode 100644 index 0000000..712eea7 --- /dev/null +++ b/artsurvey/migrations/0188_artsurveysource_log_ml_nfalse.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-05-16 13:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0187_auto_20230516_1536'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='log_ml_nfalse', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0189_auto_20230516_1704.py b/artsurvey/migrations/0189_auto_20230516_1704.py new file mode 100644 index 0000000..a9c1266 --- /dev/null +++ b/artsurvey/migrations/0189_auto_20230516_1704.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2023-05-16 14:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0188_artsurveysource_log_ml_nfalse'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='log_ml_nfalse_max', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='log_ml_nfalse_min', + field=models.FloatField(blank=True, default=-1.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0190_auto_20231023_1559.py b/artsurvey/migrations/0190_auto_20231023_1559.py new file mode 100644 index 0000000..0af9ea4 --- /dev/null +++ b/artsurvey/migrations/0190_auto_20231023_1559.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.13 on 2023-10-23 12:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0189_auto_20230516_1704'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveymatchparams', + name='band1', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('E5', '3-15 keV')], default='E0', max_length=10), + ), + migrations.AlterField( + model_name='artsurveymatchparams', + name='band2', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('E5', '3-15 keV')], default='E0', max_length=10), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('E5', '3-12 keV'), ('ALL', 'All bands')], default='E0', help_text='E0: 4-12 keV, E1: 4-7 keV, E2: 7-12 keV, E3: 12-20 keV, E4: 20-30 keV', max_length=10), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='exclude_band', + field=models.CharField(choices=[('E0', '4-12 keV'), ('E1', '4-7 keV'), ('E2', '7-12 keV'), ('E3', '12-20 keV'), ('E4', '20-30 keV'), ('E5', '3-12 keV'), ('ALL', 'All bands')], default='E0', max_length=10), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='log_nfalse_min', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0191_artsurveysource_bj2021.py b/artsurvey/migrations/0191_artsurveysource_bj2021.py new file mode 100644 index 0000000..cdb9080 --- /dev/null +++ b/artsurvey/migrations/0191_artsurveysource_bj2021.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2023-10-23 13:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0081_alter_bjfromvizier_error_radius'), + ('artsurvey', '0190_auto_20231023_1559'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='bj2021', + field=models.ManyToManyField(to='astrobasis.BJfromVizieR'), + ), + ] diff --git a/artsurvey/migrations/0192_artsurveysource_gaiadr3.py b/artsurvey/migrations/0192_artsurveysource_gaiadr3.py new file mode 100644 index 0000000..f262c95 --- /dev/null +++ b/artsurvey/migrations/0192_artsurveysource_gaiadr3.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2023-10-25 09:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0086_gaiadr3fromvizier'), + ('artsurvey', '0191_artsurveysource_bj2021'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='gaiadr3', + field=models.ManyToManyField(to='astrobasis.GAIADR3fromVizieR'), + ), + ] diff --git a/artsurvey/migrations/0193_auto_20231025_1735.py b/artsurvey/migrations/0193_auto_20231025_1735.py new file mode 100644 index 0000000..5471ef6 --- /dev/null +++ b/artsurvey/migrations/0193_auto_20231025_1735.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.13 on 2023-10-25 14:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0086_gaiadr3fromvizier'), + ('artsurvey', '0192_artsurveysource_gaiadr3'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='bj2021_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.bjfromvizier'), + ), + migrations.AddField( + model_name='artsurveysource', + name='gaiadr3_primary', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='artsurveysource_primary', to='astrobasis.gaiadr3fromvizier'), + ), + ] diff --git a/artsurvey/migrations/0194_auto_20231026_1224.py b/artsurvey/migrations/0194_auto_20231026_1224.py new file mode 100644 index 0000000..ae8c0a8 --- /dev/null +++ b/artsurvey/migrations/0194_auto_20231026_1224.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.13 on 2023-10-26 09:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0193_auto_20231025_1735'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveymatchparams', + name='log_ml_nfalse_max1', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='log_ml_nfalse_max2', + field=models.FloatField(blank=True, default=None, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='log_ml_nfalse_min1', + field=models.FloatField(blank=True, default=-1.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + migrations.AddField( + model_name='artsurveymatchparams', + name='log_ml_nfalse_min2', + field=models.FloatField(blank=True, default=-1.0, help_text='not applied to extended sources (EXT>1)', null=True), + ), + ] diff --git a/artsurvey/migrations/0195_artsurveyparams_exclude_log_ml_nfalse.py b/artsurvey/migrations/0195_artsurveyparams_exclude_log_ml_nfalse.py new file mode 100644 index 0000000..3bcc0d4 --- /dev/null +++ b/artsurvey/migrations/0195_artsurveyparams_exclude_log_ml_nfalse.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-10-26 09:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0194_auto_20231026_1224'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='exclude_log_ml_nfalse', + field=models.FloatField(blank=True, default=-1.0, null=True), + ), + ] diff --git a/artsurvey/migrations/0196_alter_artsurveyparams_exclude_log_nfalse.py b/artsurvey/migrations/0196_alter_artsurveyparams_exclude_log_nfalse.py new file mode 100644 index 0000000..3d8370a --- /dev/null +++ b/artsurvey/migrations/0196_alter_artsurveyparams_exclude_log_nfalse.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-10-26 09:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0195_artsurveyparams_exclude_log_ml_nfalse'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='exclude_log_nfalse', + field=models.FloatField(blank=True, default=-1.0, help_text='not applied', null=True), + ), + ] diff --git a/artsurvey/migrations/0197_alter_artsurveyparams_exclude_log_nfalse.py b/artsurvey/migrations/0197_alter_artsurveyparams_exclude_log_nfalse.py new file mode 100644 index 0000000..c76a9dd --- /dev/null +++ b/artsurvey/migrations/0197_alter_artsurveyparams_exclude_log_nfalse.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-10-26 09:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0196_alter_artsurveyparams_exclude_log_nfalse'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='exclude_log_nfalse', + field=models.FloatField(blank=True, default=-1.0, help_text='not active, do not use it.', null=True), + ), + ] diff --git a/artsurvey/migrations/0198_auto_20231026_1613.py b/artsurvey/migrations/0198_auto_20231026_1613.py new file mode 100644 index 0000000..b261152 --- /dev/null +++ b/artsurvey/migrations/0198_auto_20231026_1613.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.13 on 2023-10-26 13:13 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0197_alter_artsurveyparams_exclude_log_nfalse'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='category_unclassified', + field=models.BooleanField(default=False, help_text='Show sources without any Category'), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='category', + field=models.ForeignKey(blank=True, help_text='Show sources with this Category only', null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.category'), + ), + ] diff --git a/artsurvey/migrations/0199_auto_20231028_1500.py b/artsurvey/migrations/0199_auto_20231028_1500.py new file mode 100644 index 0000000..3367b54 --- /dev/null +++ b/artsurvey/migrations/0199_auto_20231028_1500.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2023-10-28 12:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0198_auto_20231026_1613'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='glat_max', + field=models.FloatField(blank=True, default=None, help_text='Galactic latitude max', null=True), + ), + migrations.AddField( + model_name='artsurveyparams', + name='glat_min', + field=models.FloatField(blank=True, default=None, help_text='Galactic latitude min', null=True), + ), + ] diff --git a/artsurvey/migrations/0200_artsurveysource_turkish_date.py b/artsurvey/migrations/0200_artsurveysource_turkish_date.py new file mode 100644 index 0000000..a9919a9 --- /dev/null +++ b/artsurvey/migrations/0200_artsurveysource_turkish_date.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-11-28 08:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0199_auto_20231028_1500'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='turkish_date', + field=models.DateTimeField(null=True), + ), + ] diff --git a/artsurvey/migrations/0201_auto_20231130_1125.py b/artsurvey/migrations/0201_auto_20231130_1125.py new file mode 100644 index 0000000..acfbc89 --- /dev/null +++ b/artsurvey/migrations/0201_auto_20231130_1125.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2023-11-30 08:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0200_artsurveysource_turkish_date'), + ] + + operations = [ + migrations.RenameField( + model_name='artsurveyparams', + old_name='sign_max', + new_name='sign_ml_max', + ), + migrations.RenameField( + model_name='artsurveyparams', + old_name='sign_min', + new_name='sign_ml_min', + ), + ] diff --git a/artsurvey/migrations/0202_auto_20231130_1127.py b/artsurvey/migrations/0202_auto_20231130_1127.py new file mode 100644 index 0000000..2d6b380 --- /dev/null +++ b/artsurvey/migrations/0202_auto_20231130_1127.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2023-11-30 08:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0201_auto_20231130_1125'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='sign_ml_max', + field=models.FloatField(blank=True, default=None, help_text='ML significance', null=True), + ), + migrations.AlterField( + model_name='artsurveyparams', + name='sign_ml_min', + field=models.FloatField(blank=True, default=None, help_text='ML significance', null=True), + ), + ] diff --git a/artsurvey/migrations/0203_artsurveysource_marshall.py b/artsurvey/migrations/0203_artsurveysource_marshall.py new file mode 100644 index 0000000..b7c5b63 --- /dev/null +++ b/artsurvey/migrations/0203_artsurveysource_marshall.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-12-04 10:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0202_auto_20231130_1127'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveysource', + name='marshall', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/artsurvey/migrations/0204_artsurveyparams_marshall.py b/artsurvey/migrations/0204_artsurveyparams_marshall.py new file mode 100644 index 0000000..23546ef --- /dev/null +++ b/artsurvey/migrations/0204_artsurveyparams_marshall.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-12-04 10:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0203_artsurveysource_marshall'), + ] + + operations = [ + migrations.AddField( + model_name='artsurveyparams', + name='marshall', + field=models.CharField(choices=[('MSFC_FILT', 'Filter MSFC sources'), ('MSFC_EXCL', 'Exclude MSFC sources')], default=None, help_text='MSFC sources in NEP', max_length=10, null=True), + ), + ] diff --git a/artsurvey/migrations/0205_alter_artsurveyparams_marshall.py b/artsurvey/migrations/0205_alter_artsurveyparams_marshall.py new file mode 100644 index 0000000..82e93be --- /dev/null +++ b/artsurvey/migrations/0205_alter_artsurveyparams_marshall.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-12-04 10:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0204_artsurveyparams_marshall'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='marshall', + field=models.CharField(choices=[('', '----'), ('MSFC_FILT', 'NEP filter'), ('MSFC_EXCL', 'NEP exclude')], default=None, help_text='MSFC sources in NEP', max_length=10, null=True), + ), + ] diff --git a/artsurvey/migrations/0206_alter_artsurveyparams_marshall.py b/artsurvey/migrations/0206_alter_artsurveyparams_marshall.py new file mode 100644 index 0000000..d973125 --- /dev/null +++ b/artsurvey/migrations/0206_alter_artsurveyparams_marshall.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-12-04 10:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0205_alter_artsurveyparams_marshall'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='marshall', + field=models.CharField(choices=[('', 'None'), ('MSFC_FILT', 'NEP filter'), ('MSFC_EXCL', 'NEP exclude')], default=None, help_text='MSFC sources in NEP', max_length=10, null=True), + ), + ] diff --git a/artsurvey/migrations/0207_alter_artsurveyparams_marshall.py b/artsurvey/migrations/0207_alter_artsurveyparams_marshall.py new file mode 100644 index 0000000..e655e2a --- /dev/null +++ b/artsurvey/migrations/0207_alter_artsurveyparams_marshall.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-12-04 10:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0206_alter_artsurveyparams_marshall'), + ] + + operations = [ + migrations.AlterField( + model_name='artsurveyparams', + name='marshall', + field=models.CharField(blank=True, choices=[('', 'None'), ('MSFC_FILT', 'NEP filter'), ('MSFC_EXCL', 'NEP exclude')], default=None, help_text='MSFC sources in NEP', max_length=10, null=True), + ), + ] diff --git a/artsurvey/migrations/__init__.py b/artsurvey/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/artsurvey/models.py b/artsurvey/models.py new file mode 100644 index 0000000..229e759 --- /dev/null +++ b/artsurvey/models.py @@ -0,0 +1,1285 @@ +from django.db import models +from django.contrib.auth.models import User +from django.core.exceptions import ValidationError +from django.core.validators import MaxLengthValidator,MinValueValidator,MaxValueValidator +from math import isfinite, sqrt +import math +import time +from django.core.exceptions import PermissionDenied +import urllib.parse + +from django.core.files.storage import FileSystemStorage +import hashlib +import sys, os, os.path as osp +from django.dispatch import receiver +from django import forms + +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import BarycentricTrueEcliptic + +from heasarc.models import HeasarcBase +from heasarc.models import HeasarcSwiftBAT105m +from heasarc.models import Heasarc4XMMDR9 +from heasarc.models import HeasarcRASS2RXS +from heasarc.models import HeasarcIntRefCat +from heasarc.models import HeasarcSimpleClass +from heasarc.models import HeasarcUserCat + +from astrobasis.models import GAIADR2 +from astrobasis.models import GAIADR3 +from astrobasis.models import Simbad +from astrobasis.models import AllWise +from astrobasis.models import NVSS +from astrobasis.models import FIRST +from astrobasis.models import SUMSS +from astrobasis.models import VLASS +from astrobasis.models import VLASSfromVizieR +from astrobasis.models import BJfromVizieR +from astrobasis.models import GAIADR3fromVizieR +from astropy.table import Table + +from nasa.models import Ads +import numpy + +FS = FileSystemStorage(location='/export/django/Uploads/artsurvey') +VALID_EXTENSIONS = ['.pdf', '.ppt', '.pptx', '.png','.csv', '.txt', '.dat', '.xlsx','.docx','.gz','.jpg','.jpeg','.gz','.fits.gz','.fits','.reg','.FTZ','.evt','.fz','.qdp',] + +def validate_upload(value): + file = value.file + _, ext = osp.splitext(file.name) + if ext not in VALID_EXTENSIONS: + raise forms.ValidationError('Extension %s not supported.' % ext) + +def upload_filename(instance, filename): + _, ext = osp.splitext(filename) + + tohash = filename + str(time.clock()) + #new_fname = hashlib.md5(tohash).hexdigest() + new_fname = hashlib.md5(tohash.encode('utf-8')).hexdigest() + return new_fname + ext + +# Create your models here. + + + + +class Category(models.Model): + name = models.CharField(max_length=200) + slug = models.SlugField() + parent = models.ForeignKey('self',blank=True, null=True ,related_name='children', on_delete=models.CASCADE) + + class Meta: + unique_together = ('slug', 'parent',) #enforcing that there can not be two + verbose_name_plural = "categories" #categories under a parent with same + #slug + def get_absolute_url(self): + return "/artsurvey/category/%i/show" % (self.pk) + + def __str__(self): # __str__ method elaborated later in + full_path = [self.name] # post. use __unicode__ in place of + # __str__ if you are using python 2 + k = self.parent + + while k is not None: + full_path.append(k.name) + k = k.parent + + return ' -> '.join(full_path[::-1]) + +class Catalog(models.Model): + """ General catalog class """ + name = models.CharField(max_length=36) + #test = models.CharField(max_length=36,default='') + year = models.PositiveIntegerField(default=0) + + def __str__(self): + return "{} ({})".format(self.name, self.year) + +class MetaSource(models.Model): + """ + ArtSurvey MetaSource class + Dynamic information !!! + """ + catalog = models.ForeignKey(Catalog, null=True, blank=True, on_delete=models.SET_NULL) + name = models.CharField(max_length=36) + healpix = models.BigIntegerField(db_index=True, default=0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + def __str__(self): + return "{} ({})".format(self.name, self.year) + glon = models.FloatField(default=0.0) + glat = models.FloatField(default=0.0) + elon = models.FloatField(default=0.0) + elat = models.FloatField(default=0.0) + radec_error = models.FloatField(default=0.0) + + class Meta: + ordering = ["-ra"] + + def __str__(self): + return "{}".format(self.name) + + def get_absolute_url(self): + return "/artsurvey/%i/metasource" % (self.pk) + def get_delete_url(self): + return "/artsurvey/%i/metasource/delete" % (self.pk) + def get_delete_paper_url(self): + return "/artsurvey/%i/metasource/delete/paper" % (self.pk) + def get_delete_upload_url(self): + return "/artsurvey/{}/metasource/upload/delete".format(self.id) + def get_edit_upload_url(self): + return "/artsurvey/{}/metasource/upload/edit".format(self.id) + def get_upload_url(self): + return "/artsurvey/%i/metasource/upload" % (self.pk) + + def get_comments(self): + body = [] + comments = self.comments.all() + for comment in comments: + body.append("{}: {}".format(comment.owner, comment.body)) + return body + +class MetaComment(models.Model): + metasource = models.ForeignKey(MetaSource,on_delete=models.CASCADE,related_name='comments') + body = models.TextField() + created = models.DateTimeField(auto_now_add=True) + owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) + + class Meta: + ordering = ['created'] + def __str__(self): + return '[{}] {}: {}'.format(self.owner, self.metasource.name, self.body) + def get_delete_url(self): + return "/artsurvey/{}/comment/delete".format(self.id) + +class ArtSurvey(models.Model): + """ ArtSurvey makes high-level description of ART-XC catalogue. """ + version = models.FloatField(default=0.0, unique=False) + active = models.BooleanField(default=False) + hidden = models.BooleanField(default=False) + archived = models.BooleanField(default=False) + inwork = models.BooleanField(default=False) + label = models.CharField(max_length=12,blank=True, default='') + name = models.CharField(max_length=12,blank=True, default='') + parent = models.ForeignKey('self',blank=True, null=True ,related_name='children', on_delete=models.CASCADE) + notes = models.TextField("notes",max_length=3000, null=True, blank=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + ordering = ["version"] + + def get_notes_url(self): + return "/artsurvey/%i/notes/survey" % (self.pk) + + def get_absolute_url(self): + return "/artsurvey/catalog" + + def set_primary_upload_url(self): + return "/artsurvey/%i/survey/upload/primary" % (self.pk) + def clean_primary_upload_url(self): + return "/artsurvey/%i/survey/upload/primary/clean" % (self.pk) + + def get_upload_url(self): + return "/artsurvey/%i/survey/upload" % (self.pk) + def get_delete_upload_url(self): + return "/artsurvey/{}/survey/upload/delete".format(self.pk) + def get_edit_upload_url(self): + return "/artsurvey/{}/survey/upload/edit".format(self.pk) + + def __str__(self): + if self.name == '': + name = "{:.1f}".format(self.version) + else: + name = "{}v{:.2f}".format(self.name,self.version) + return "{}S{}{}{}{}{}{}".format(self.label,name, + 'R' if self.parent else '', + 'H' if self.hidden else '', + '*' if self.active else '', + 'A' if self.archived else '', + 'W' if self.inwork else '') + +class EnergyBand(models.Model): + """ ArtSurvey energy band """ + e1 = models.FloatField(default=0.0) + e2 = models.FloatField(default=0.0) + slug = models.SlugField() + def __str__(self): + return "{:.1f}-{:.1f} keV".format(self.e1,self.e2) + + +class ArtSurveySourceManager(models.Manager): + def create_source(self, src, survey, band): + + if (src.skymap): + field = src.skymap.SMAPNR + else: + field=0 + + log10 = -100.0 + if(src.nfalse): + if(src.nfalse > 0.0): + log10 = math.log10(src.nfalse) + + ml_log10 = -100.0 + if(src.ml_nfalse): + if(src.ml_nfalse > 0.0): + ml_log10 = math.log10(src.ml_nfalse) + + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + src.ecl_l=crd.transform_to(BarycentricTrueEcliptic).lon.value + src.ecl_b=crd.transform_to(BarycentricTrueEcliptic).lat.value + src.save() # may be not neccesary + + # set MSFC flag + marshall = True if(src.ecl_b > 82) else False + + + source = self.create(band=band, survey=survey, + ra=src.ra, dec=src.dec, + #ra=src.ml_ra, dec=src.ml_dec, + healpix=src.healpix, + healpix_plate=src.healpix_plate, + lii=src.lii, bii=src.bii, + ecl_l=src.ecl_l, ecl_b=src.ecl_b, + #radec_error = src.radec_error, + radec_error = src.ml_radec_err_98, + marshall = marshall, + name = src.name, + name_orig = src.name_orig, + field = field, + SMAPNR = field, + cnts = src.cnts, + cnts_err = src.cnts_err, + cnts_bg = src.cnts_bg, + exptime = src.exptime, + rate = src.rate, + rate_err = src.rate_err, + flux = src.flux, + #flux = src.ml_flux, + flux_err = src.flux_err, + #flux_err = src.ml_flux_err, + fluxerr_up = src.fluxerr_up, + fluxerr_lo = src.fluxerr_lo, + flux_uplim = src.flux_uplim, + cnts0 = src.cnts0, + cnts_er0 = src.cnts_er0, + cnts_bg0 = src.cnts_bg0, + cnts_t0 = src.cnts_t0, + exptime0 = src.exptime0, + flux0 = src.flux0, + flux_er0 = src.flux_er0, + + cnts1 = src.cnts1, + cnts_er1 = src.cnts_er1, + cnts_bg1 = src.cnts_bg1, + cnts_t1 = src.cnts_t1, + exptime1 = src.exptime1, + flux1 = src.flux1, + flux_er1 = src.flux_er1, + + cnts2 = src.cnts2, + cnts_er2 = src.cnts_er2, + cnts_bg2 = src.cnts_bg2, + cnts_t2 = src.cnts_t2, + exptime2 = src.exptime2, + flux2 = src.flux2, + flux_er2 = src.flux_er2, + + cnts3 = src.cnts3, + cnts_er3 = src.cnts_er3, + cnts_bg3 = src.cnts_bg3, + cnts_t3 = src.cnts_t3, + exptime3 = src.exptime3, + flux3 = src.flux3, + flux_er3 = src.flux_er3, + + cnts4 = src.cnts4, + cnts_er4 = src.cnts_er4, + cnts_bg4 = src.cnts_bg4, + cnts_t4 = src.cnts_t4, + exptime4 = src.exptime4, + flux4 = src.flux4, + flux_er4 = src.flux_er4, + ext = src.ext, + ext_id = src.ext_id, + + sig = src.sig, + #sig = src.ml_sig, + nfalse = src.nfalse, + #nfalse = src.ml_nfalse, + detlike = src.detlike, + #detlike = src.ml_detlike, + log_nfalse=log10, + #log_nfalse=ml_log10, + + log_ml_nfalse = ml_log10, + ml_sig = src.ml_sig, + ml_detlike = src.ml_detlike, + ml_nfalse = src.ml_nfalse, + ml_ra = src.ml_ra, + ml_dec = src.ml_dec, + ml_radec_err_90 = src.ml_radec_err_90, + ml_radec_err_98 = src.ml_radec_err_98, + ml_flux = src.ml_flux, + ml_flux_err = src.ml_flux_err, + ml_exp = src.ml_exp) + src.artsurvey_source=source + src.save() + return source + +class ArtSurveySource(models.Model): + #catalog_version = models.PositiveIntegerField(default=0) + catalog = models.ManyToManyField(Catalog) + SMAPNR = models.PositiveIntegerField(db_index=True, default=0) + parent = models.ForeignKey('self',blank=True, null=True ,related_name='children', on_delete=models.SET_NULL) + category = models.ForeignKey(Category, null=True, blank=True, on_delete=models.SET_NULL) + survey = models.ForeignKey(ArtSurvey, null=True, on_delete=models.CASCADE) + metasource = models.ForeignKey(MetaSource, null=True, blank=True, on_delete=models.SET_NULL) + metasource_sep = models.FloatField(default=0.0) + metasource_primary = models.ForeignKey(MetaSource, null=True, blank=True, on_delete=models.SET_NULL,related_name="artsurveysource_primary") + band = models.ForeignKey(EnergyBand, null=True, blank=True, on_delete=models.SET_NULL) + gaia3 = models.ManyToManyField(GAIADR3) + gaia3_primary = models.ForeignKey(GAIADR3, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) + notes = models.TextField("notes",max_length=2000, null=True, blank=True) + notes_paper = models.TextField("notes paper",max_length=5000, null=True, blank=True) + bibtex_paper = models.TextField("bibtex paper",max_length=5000, null=True, blank=True) + #notes_public = models.BooleanField(default=True) + follow_up = models.TextField("follow-up",max_length=5000, null=True, blank=True) + ads = models.ManyToManyField(Ads) + ads_hidden = models.ManyToManyField(Ads, related_name="artsurveysource_hidden") + refid = models.BigIntegerField(default=0) + heasarc = models.ManyToManyField(HeasarcBase) + heasarc_linked = models.ManyToManyField(HeasarcBase, related_name='artsurveysource_linked') + bj2021 = models.ManyToManyField(BJfromVizieR) + bj2021_primary = models.ForeignKey(BJfromVizieR, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + gaiadr3 = models.ManyToManyField(GAIADR3fromVizieR) + gaiadr3_primary = models.ForeignKey(GAIADR3fromVizieR, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + vlass = models.ManyToManyField(VLASSfromVizieR) + vlass_primary = models.ForeignKey(VLASSfromVizieR, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + nvss = models.ManyToManyField(NVSS) + nvss_primary = models.ForeignKey(NVSS, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + first = models.ManyToManyField(FIRST) + first_primary = models.ForeignKey(FIRST, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + sumss = models.ManyToManyField(SUMSS) + sumss_primary = models.ForeignKey(SUMSS, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + allwise = models.ManyToManyField(AllWise) + allwise_primary = models.ForeignKey(AllWise, null=True, blank=True, related_name='artsurveysource_primary', on_delete=models.SET_NULL) + simbad = models.ManyToManyField(Simbad) + simbad_failed = models.BooleanField(default=False, null=True) + xrays = models.BooleanField(default=False, null=True) + """ Simbad failed to respond """ + simbad_notfound = models.BooleanField(default=False, null=True) + """ No astronomical object found """ + erosita_data = models.BooleanField(default=False, null=True) + """ False if not found in ART-XC/eRosita match table """ + #wise_primary = models.ForeignKey(WiseCounterpart, null=True, blank=True, on_delete=models.SET_NULL) + wise_data = models.BooleanField(default=False, null=True) + marshall = models.BooleanField(default=False, null=True) + turkish = models.BooleanField(default=False, null=True) + turkish_date = models.DateTimeField(null=True) + turkpossible = models.BooleanField(default=False, null=True) + """ + sources suitable for TUBITAK (exclude Swift/BAT 105m (2018), + INTEGRAL Reference Catalog (last updated on 16 June 2019) and RXTE + """ + healpix = models.BigIntegerField(db_index=True, default=0) + healpix_plate = models.IntegerField(db_index=True, default=0) + field = models.IntegerField(default=0) + field_ind = models.IntegerField(default=0) + ra = models.FloatField(default=0.0) + """ Right Ascension of the Source """ + dec = models.FloatField(default=0.0) + """ Declination of the Source """ + lii = models.FloatField(default=0.0) + """ Galactic Longitude """ + bii = models.FloatField(default=0.0) + """ Galactic Latitude """ + ecl_l = models.FloatField(default=0.0) + ecl_b = models.FloatField(default=0.0) + radec_error = models.FloatField(default=0.0) + """ Positional Uncertainty (arcsec) + + Must be exactly radec_error to be consistent with EroTransSource and EroSurveySource + """ + name = models.CharField(max_length=36,blank=True, default='') + name_orig = models.CharField(max_length=22, null=True) + cname = models.CharField(max_length=36,blank=True, null=True,default='') + class_id = models.IntegerField(default=0) + """ Browse Object Classification """ + object_class = models.ForeignKey(HeasarcSimpleClass, on_delete=models.SET_NULL, null=True) + """ Related object in HeasarcObjectClass """ + class_tentative = models.BooleanField(default=False, null=True) + + cnts = models.FloatField(default=0.0,null=True) + cnts_err = models.FloatField(default=0.0,null=True) + cnts_bg = models.FloatField(default=0.0,null=True) + exptime = models.FloatField(default=0.0,null=True) + rate = models.FloatField(default=0.0,null=True) + rate_err = models.FloatField(default=0.0,null=True) + flux = models.FloatField(default=0.0,null=True) + flux_err = models.FloatField(default=0.0,null=True) + fluxerr_lo = models.FloatField(default=0.0,null=True) + fluxerr_up = models.FloatField(default=0.0,null=True) + flux_uplim = models.FloatField(default=0.0,null=True) + + cnts0 = models.FloatField(default=0.0,null=True) + cnts_er0 = models.FloatField(default=0.0,null=True) + cnts_bg0 = models.FloatField(default=0.0,null=True) + cnts_t0 = models.FloatField(default=0.0,null=True) + exptime0 = models.FloatField(default=0.0,null=True) + flux0 = models.FloatField(default=0.0,null=True) + flux_er0 = models.FloatField(default=0.0,null=True) + + cnts1 = models.FloatField(default=0.0,null=True) + cnts_er1 = models.FloatField(default=0.0,null=True) + cnts_bg1 = models.FloatField(default=0.0,null=True) + cnts_t1 = models.FloatField(default=0.0,null=True) + exptime1 = models.FloatField(default=0.0,null=True) + flux1 = models.FloatField(default=0.0,null=True) + flux_er1 = models.FloatField(default=0.0,null=True) + + cnts2 = models.FloatField(default=0.0,null=True) + cnts_er2 = models.FloatField(default=0.0,null=True) + cnts_bg2 = models.FloatField(default=0.0,null=True) + cnts_t2 = models.FloatField(default=0.0,null=True) + exptime2 = models.FloatField(default=0.0,null=True) + flux2 = models.FloatField(default=0.0,null=True) + flux_er2 = models.FloatField(default=0.0,null=True) + + cnts3 = models.FloatField(default=0.0,null=True) + cnts_er3 = models.FloatField(default=0.0,null=True) + cnts_bg3 = models.FloatField(default=0.0,null=True) + cnts_t3 = models.FloatField(default=0.0,null=True) + exptime3 = models.FloatField(default=0.0,null=True) + flux3 = models.FloatField(default=0.0,null=True) + flux_er3 = models.FloatField(default=0.0,null=True) + + cnts4 = models.FloatField(default=0.0,null=True) + cnts_er4 = models.FloatField(default=0.0,null=True) + cnts_bg4 = models.FloatField(default=0.0,null=True) + cnts_t4 = models.FloatField(default=0.0,null=True) + exptime4 = models.FloatField(default=0.0,null=True) + flux4 = models.FloatField(default=0.0,null=True) + flux_er4 = models.FloatField(default=0.0,null=True) + + #ext = models.FloatField(default=0.0) + ext = models.IntegerField(default=0,null=True) + ext_id = models.CharField(max_length=100,blank=True, default='',null=True) + + sig = models.FloatField(default=0.0,null=True) + detlike = models.FloatField(default=0.0,null=True) + nfalse = models.FloatField(default=0.0,null=True) + log_nfalse = models.FloatField(default=0.0,null=True) + redshift = models.FloatField(default=0.0, null=True, blank=True) + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + """ Date and time of creation. Fully automatic. """ + + ml_sig = models.FloatField(default=0.0,null=True) + ml_detlike = models.FloatField(default=0.0,null=True) + ml_nfalse = models.FloatField(default=0.0,null=True) + log_ml_nfalse = models.FloatField(default=0.0,null=True) + ml_ra = models.FloatField(default=0.0,null=True) + ml_dec = models.FloatField(default=0.0,null=True) + ml_radec_err_90 = models.FloatField(default=0.0,null=True) + ml_radec_err_98 = models.FloatField(default=0.0,null=True) + ml_flux = models.FloatField(default=0.0,null=True) + ml_flux_err = models.FloatField(default=0.0,null=True) + ml_exp = models.FloatField(default=0.0,null=True) + + objects = ArtSurveySourceManager() + class Meta: + get_latest_by = 'created' + + + def save(self, *args, **kwargs): + #print("Save model ArtSurveySource") + #sys.stdout.write(".") + if(self.survey.archived): + message="{} is archived".format(self.get_absolute_url(), + self) + raise PermissionDenied(message) + + super(ArtSurveySource, self).save(*args, **kwargs) + def __str__(self): + return "{} {} {}".format(self.name,self.survey,self.band.slug) + def load_skyview_image(self): + return "/artsurvey/{}/load/skyview".format(self.pk) + def load_ps1_color_image(self): + return "/artsurvey/{}/load/ps1/color".format(self.pk) + def load_ps1_gray_image(self): + return "/artsurvey/{}/load/ps1/gray".format(self.pk) + + def load_ps1_fits_image(self): + return "/artsurvey/{}/load/ps1/fits".format(self.pk) + + def get_r98(self): + return sqrt(self.ml_radec_err_98**2 + 10**2) + + def get_galplane_image_path(self): + """ + subpath = self.survey.name[-3:] + if not subpath: + return None + return "/srg/a1/work/rodion/SRG/art-xc/data/galmap/field_data/{}".format(subpath) + """ + + path = "/srg/a1/work/rodion/SRG/art-xc/data/galmap/field_data" + crd = SkyCoord(self.ra, self.dec, frame="fk5", unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + glon = int(5.0*round(lon/5.0)) + if(glon == 360): + glon = 0 + glat = int(5.0*round(lat/5.0)) + subpath="{:03d}+0".format(glon) + dirpath="{}/{}".format(path,subpath) + if(osp.exists("{}/{}".format(dirpath,"img_flux_0.fits.fz")) and abs(lat) < 2.5): + return dirpath + else: + return None + + def get_SMAPNR(self): + return "{:06d}".format(self.SMAPNR) + + def get_galplane_image_url(self): + """ + subpath = self.survey.name[-3:] + if not subpath: + return None + dirpath="{}/{}".format(path,subpath) + if(osp.exists(dirpath)): + return "/galplane/{}".format(subpath) + else: + pass + """ + + + path = "/srg/a1/work/rodion/SRG/art-xc/data/galmap/field_data" + crd = SkyCoord(self.ra, self.dec, frame="fk5", unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + + # map to 5degx5deg tiles + glon = int(5.0*round(lon/5.0)) + print(glon) + if(glon == 360): + glon = 0 + glat = int(5.0*round(lat/5.0)) + subpath="{:03d}".format(glon) + dirpath="{}/{}/12g".format(path,subpath) + if(osp.exists("{}/{}".format(dirpath,"img_flux_0.fits.fz")) and abs(lat) < 5.0): + return "/galplane/{}".format(subpath) + else: + return None + + + + def get_survey_primary_image_url(self): + upload = self.survey.uploads.all().filter(primary=True).first() + if upload: + return upload.get_download_url() + else: + return None + + def get_absolute_url(self): + return "/artsurvey/%i/artsource" % (self.pk) + def get_transfer_url(self): + return "/artsurvey/%i/transfer" % (self.pk) + def get_add_to_selection_url(self): + return "/artsurvey/%i/selection/add" % (self.pk) + def get_remove_from_selection_url(self): + return "/artsurvey/%i/selection/remove" % (self.pk) + def get_remove_from_selection_profile_url(self): + return "/artsurvey/%i/selection/remove/profile" % (self.pk) + def get_remove_from_basket_url(self): + return "/artsurvey/%i/selection/remove/basket" % (self.pk) + def new_heasarc_source(self): + return "/artsurvey/%i/usercat" % (self.pk) + def get_ds9_region(self): + return "/artsurvey/%i/artsource/ds9" % (self.pk) + def get_ds9_region_linked(self): + return "/artsurvey/%i/artsource/ds9/linked" % (self.pk) + def get_markbad_url(self): + return "/artsurvey/%i/markbad" % (self.pk) + def get_delete_url(self): + return "/artsurvey/%i/delete" % (self.pk) + def get_base_url(self): + return "/artsurvey/%i/" % (self.pk) + def get_clean_wise_url(self): + return "/artsurvey/%i/wise/clean/" % (self.pk) + def get_clean_allwise_url(self): + return "/artsurvey/%i/allwise/clean/" % (self.pk) + def get_clean_nvss_url(self): + return "/artsurvey/%i/nvss/clean/" % (self.pk) + def get_clean_vlass_url(self): + return "/artsurvey/%i/nvss/clean/" % (self.pk) + def get_clean_first_url(self): + return "/artsurvey/%i/first/clean/" % (self.pk) + def get_clean_sumss_url(self): + return "/artsurvey/%i/sumss/clean/" % (self.pk) + def get_clean_gaiadr3_url(self): + return "/artsurvey/%i/gaiadr3/clean/" % (self.pk) + def get_clean_bj2021_url(self): + return "/artsurvey/%i/bj2021/clean/" % (self.pk) + def get_vizier_url(self): + return "/artsurvey/%i/vizier" % (self.pk) + def get_type_url(self): + return "/artsurvey/%i/type" % (self.pk) + def get_notes_url(self): + return "/artsurvey/%i/notes" % (self.pk) + def get_cname_url(self): + return "/artsurvey/%i/cname" % (self.pk) + def get_update_category_url(self): + return "/artsurvey/category/{}/update".format(self.pk) + def get_skymap_url(self): + return "/srgcat/{}/field_radec/{}/{}".format(self.field,self.ra,self.dec) + def get_mark_tentative_url(self): + return "/artsurvey/tentative/{}/mark".format(self.pk) + def get_remove_tentative_url(self): + return "/artsurvey/tentative/{}/remove".format(self.pk) + def get_mark_newxray_url(self): + return "/artsurvey/newxray/{}/mark".format(self.pk) + def get_remove_newxray_url(self): + return "/artsurvey/newxray/{}/remove".format(self.pk) + def get_mark_turkish_url(self): + return "/artsurvey/turkish/{}/mark".format(self.pk) + def get_remove_turkish_url(self): + return "/artsurvey/turkish/{}/remove".format(self.pk) + def get_clean_class_url(self): + return "/artsurvey/type/{}/clean".format(self.pk) + #def get_make_notes_public_url(self): + # return "/artsurvey/notes_public/{}/make".format(self.pk) + #def get_remove_notes_public_url(self): + # return "/artsurvey/notes_public/{}/remove".format(self.pk) + + def get_add_bibcode_url(self): + return "/nasa/%i/bibcode" % (self.pk) + def get_add_counterpart_url(self): + return "/artsurvey/%i/counterpart" % (self.pk) + def get_delete_bibcode_url(self): + return "/nasa/%i/bibcode/delete" % (self.pk) + def get_update_bibcode_url(self): + return "/nasa/%i/bibcode/update" % (self.pk) + def get_hide_bibcode_url(self): + return "/nasa/%i/bibcode/hide" % (self.pk) + def get_unhide_bibcode_url(self): + return "/nasa/%i/bibcode/unhide" % (self.pk) + + def get_metasource_group_url(self): + return "/artsurvey/%i/metasource/group" % (self.pk) + def get_add_other_name_url(self): + return "/artsurvey/%i/oname" % (self.pk) + def get_update_other_name_url(self): + return "/artsurvey/%i/oname/update" % (self.pk) + + def get_update_source_url(self): + return "/artsurvey/%i/update/source" % (self.pk) + + def get_swiftbat_count(self): + swiftbat = HeasarcBase.objects.instance_of(HeasarcSwiftBAT105m).filter(healpix=self.healpix) + return swiftbat.count() + + def get_4xmm_count(self): + return HeasarcBase.objects.instance_of(Heasarc4XMMDR9).filter(healpix=self.healpix).count() + + def get_rosat_count(self): + return HeasarcBase.objects.instance_of(HeasarcRASS2RXS).filter(healpix=self.healpix).count() + + def get_integral_count(self): + return HeasarcBase.objects.instance_of(HeasarcIntRefCat).filter(healpix=self.healpix).count() + + def get_upload_url(self): + return "/artsurvey/%i/upload" % (self.pk) + def get_delete_upload_url(self): + return "/artsurvey/{}/upload/delete".format(self.id) + def get_edit_upload_url(self): + return "/artsurvey/{}/upload/edit".format(self.id) + + @property + def get_category(self): + return self.category + + def get_log10_nfalse(self): + """ + log10 = -100.0 + if(self.nfalse > 0.0): + log10 = math.log10(self.nfalse) + return log10 + """ + return self.log_nfalse + + def get_detlike100(self): + if(self.detlike < 100): + return "{:.2f}".format(self.detlike) + else: + return "> 100" + + + + def get_erosita_ratio(self): + ratio=-1.0 + if(hasattr(self, 'erositamatch')): + ratio = float(self.flux/self.erositamatch.flux) + return ratio + + def get_radio_total(self): + return self.nvss.all().count() + self.first.all().count() + self.sumss.all().count() + self.vlass.all().count() + + def get_link_heasarc_url(self): + return "/artsurvey/%i/link/heasarc/" % (self.pk) + def get_unlink_heasarc_url(self): + return "/artsurvey/%i/unlink/heasarc/" % (self.pk) + def get_unlink_heasarc_all_url(self): + return "/artsurvey/%i/unlink/heasarc/all" % (self.pk) + + def get_otype(self): + + if(self.object_class): + otype=self.object_class.class_name + else: + return '' + + s='CV' + if(otype.startswith(s)): + otype=s + + s='STAR' + if(otype.startswith(s)): + otype=s + + s='LMXB' + if(otype.startswith(s)): + otype=s + + s='HMXB' + if(otype.startswith(s)): + otype=s + + s='AGN' + if(otype.startswith(s)): + otype=s + + s='SEYFERT' + if(otype.startswith(s)): + otype='SEYFERT' + + if(otype.startswith('LINER')): + otype='SEYFERT' + + s='UNIDENTIFIED' + if(otype.startswith(s)): + otype='UNIDENT' + + + if(self.class_tentative): + otype+='?' + + + return otype + + def if_linked_to(self, heasarc): + return True if heasarc in self.heasarc_linked.all() else False + + def get_heasarc_linked_all(self): + return True if self.heasarc_linked.all().count() else False + + def get_heasarc_linked_optics(self): + lst = [] + linked = self.heasarc_linked.all() + for h in linked: + + if(h.gaia3_primary): + lst.append({'ra':h.gaia3_primary.ra,'dec':h.gaia3_primary.dec,'name':h.gaia3_primary.name,'color':'#00FF00'}) + else: + for s in h.gaia3.all(): + lst.append({'ra':s.ra,'dec':s.dec,'name':'Gaia3','color':''}) + + if(h.allwise_primary): + lst.append({'ra':h.allwise_primary.ra,'dec':h.allwise_primary.dec,'name':h.allwise_primary.name,'color':'#00FF00'}) + else: + for s in h.allwise.all(): + lst.append({'ra':s.ra,'dec':s.dec,'name':'AllWise','color':'green'}) + + if(h.glimpse_primary): + lst.append({'ra':h.glimpse_primary.ra,'dec':h.glimpse_primary.dec,'name':h.glimpse_primary.name,'color':'#00FF00'}) + else: + for s in h.glimpse.all(): + lst.append({'ra':s.ra,'dec':s.dec,'name':'GLIMPSE','color':'green'}) + + if(h.vlass_primary): + lst.append({'ra':h.vlass_primary.ra,'dec':h.vlass_primary.dec,'name':h.vlass_primary.name,'color':'#00FF00'}) + else: + for s in h.vlass.all(): + lst.append({'ra':s.ra,'dec':s.dec,'name':'VLASS','color':'green'}) + + if(h.twomass_primary): + lst.append({'ra':h.twomass_primary.ra,'dec':h.twomass_primary.dec,'name':h.twomass_primary.name,'color':'#00FF00'}) + else: + for s in h.twomass.all(): + lst.append({'ra':s.ra,'dec':s.dec,'name':'2MASS','color':'green'}) + + for s in h.usercat.all(): + lst.append({'ra':s.ra,'dec':s.dec,'name':'UserCat','color':'#00FF00'}) + + return lst + + def get_otype(self): + + if(self.object_class): + otype=self.object_class.class_name + else: + return '' + + s='CV' + if(otype.startswith(s)): + otype=s + + s='STAR' + if(otype.startswith(s)): + otype=s + + s='LMXB' + if(otype.startswith(s)): + otype=s + + s='HMXB' + if(otype.startswith(s)): + otype=s + + s='AGN' + if(otype.startswith(s)): + otype=s + + s='SEYFERT' + if(otype.startswith(s)): + otype='SEYFERT' + + if(otype.startswith('LINER')): + otype='SEYFERT' + + s='UNIDENT' + if(otype.startswith(s)): + otype='UNIDENT' + + + if(self.class_tentative): + otype+='?' + + return otype + + def get_flux_str_m11power(self): + koeff=1e-12 + #return "{:.2f} $\pm$ {}".format(self.flux/koeff,self.flux_err) + if(self.fluxerr_lo > 0.0 and self.fluxerr_up > 0.0): + pass + else: + return "{:.2f}".format(self.flux/koeff) + er1 = int((self.fluxerr_lo/koeff)*100) + er2 = int((self.fluxerr_up/koeff)*100) + if(er1 == er2): + return "{:.2f} $\pm$ {:.2f}".format(self.flux/koeff,self.fluxerr_lo/koeff) + else: + return "{:.2f} $_{{ -{:.2f} }}^{{ +{:.2f} }}$ ".format(self.flux/koeff,self.fluxerr_lo/koeff,self.fluxerr_up/koeff) + + def get_cname_ads_url(self): + return "https://ui.adsabs.harvard.edu/search/q=object%3A%22{}%22&sort=date%20desc%2C%20bibcode%20desc&p_=0".format(urllib.parse.quote_plus(self.cname)) + +class OpticalCounterpart(models.Model): + source = models.ForeignKey(ArtSurveySource, on_delete=models.CASCADE) + name = models.CharField(max_length=60,default='') + objid = models.BigIntegerField(default=0) + t_objid = models.CharField(max_length=15,default='') + redshift = models.FloatField(default=0,validators=[MinValueValidator(0.0)]) + mag = models.FloatField(default=0,validators=[MinValueValidator(0.0)], null=True, blank=True) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + def __str__(self): + return self.source.name + +class OtherName(models.Model): + source = models.ForeignKey(ArtSurveySource, on_delete=models.CASCADE) + name = models.CharField(max_length=60,default='') + owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) + def __str__(self): + return "{}".format(self.name) + def get_update_url(self): + return "/artsurvey/%i/oname/update" % (self.pk) + def get_delete_url(self): + return "/artsurvey/%i/oname/delete" % (self.pk) + +class WiseCounterpart(models.Model): + source = models.ForeignKey(ArtSurveySource, on_delete=models.CASCADE) + sep = models.FloatField(default=0,validators=[MinValueValidator(0.0)]) + objID = models.BigIntegerField(default=0) + raStack = models.FloatField(default=0.0) + decStack = models.FloatField(default=0.0) + raStackErr = models.FloatField(default=0.0) + decStackErr = models.FloatField(default=0.0) + raMean = models.FloatField(default=0.0) + decMean = models.FloatField(default=0.0) + raMeanErr = models.FloatField(default=0.0) + decMeanErr = models.FloatField(default=0.0) + objInfoFlag = models.BigIntegerField(default=0) + qualityFlag = models.IntegerField(default=0) + primaryDetection = models.BooleanField(default=False, null=True) + bestDetection = models.BooleanField(default=False, null=True) #models.IntegerField(default=0) + gPSFMag = models.FloatField(default=0.0, null=True) + gPSFMagErr = models.FloatField(default=0.0) + gApMag = models.FloatField(default=0.0) + gApMagErr= models.FloatField(default=0.0) + gKronMag= models.FloatField(default=0.0) + gKronMagErr= models.FloatField(default=0.0) + rPSFMag= models.FloatField(default=0.0, null=True) + rPSFMagErr= models.FloatField(default=0.0) + rApMag= models.FloatField(default=0.0) + rApMagErr= models.FloatField(default=0.0) + rKronMag= models.FloatField(default=0.0) + rKronMagErr= models.FloatField(default=0.0) + iPSFMag= models.FloatField(default=0.0, null=True) + iPSFMagErr= models.FloatField(default=0.0) + iApMag= models.FloatField(default=0.0) + iApMagErr= models.FloatField(default=0.0) + iKronMag= models.FloatField(default=0.0) + iKronMagErr= models.FloatField(default=0.0) + zPSFMag= models.FloatField(default=0.0, null=True) + zPSFMagErr= models.FloatField(default=0.0) + zApMag= models.FloatField(default=0.0) + zApMagErr= models.FloatField(default=0.0) + zKronMag= models.FloatField(default=0.0) + zKronMagErr= models.FloatField(default=0.0) + yPSFMag= models.FloatField(default=0.0, null=True) + yPSFMagErr= models.FloatField(default=0.0) + yApMag= models.FloatField(default=0.0) + yApMagErr= models.FloatField(default=0.0) + yKronMag= models.FloatField(default=0.0) + yKronMagErr= models.FloatField(default=0.0) + wisefield= models.CharField(max_length=8, default='') + fitext = models.BooleanField(default=False, null=True) + devaucou = models.BooleanField(default=False, null=True) + star = models.BooleanField(default=False, null=True) + w1flux= models.FloatField(default=0.0, null=True) + dw1flux= models.FloatField(default=0.0, null=True) + w1mag= models.FloatField(default=0.0, null=True) + dw1mag= models.FloatField(default=0.0, null=True) + w1reff= models.FloatField(default=0.0, null=True) + w2flux= models.FloatField(default=0.0, null=True) + dw2flux= models.FloatField(default=0.0, null=True) + w2mag= models.FloatField(default=0.0, null=True) + dw2mag= models.FloatField(default=0.0, null=True) + w2reff= models.FloatField(default=0.0, null=True) + primary = models.BooleanField(default=False, null=True) + def __str__(self): + return self.source.name + def w1w2(self): + if self.w1mag and self.w2mag: + return self.w1mag-self.w2mag + else: + return None + + def w1w2_error(self): + if self.dw1mag and self.dw2mag: + return sqrt(self.dw1mag*self.dw1mag + self.dw2mag*self.dw2mag) + else: + return None + def get_irsa_url(self): + return "http://irsa.ipac.caltech.edu/applications/wise/#id=Hydra_wise_wise_1&DoSearch=true&schema=allwise-multiband&intersect=CENTER&subsize=0.20&mcenter=mcen&band=1,2,3,4&dpLevel=3a&UserTargetWorldPt={};{};EQ_J2000&SimpleTargetPanel.field.resolvedBy=simbadthenned&coaddId=&projectId=wise&searchName=wise_1&startIdx=0&pageSize=0&shortDesc=Position&isBookmarkAble=true&isDrillDownRoot=true&isSearchResult=true".format(self.raStack,self.decStack) + + + +class eRositaMatch(models.Model): + source = models.OneToOneField(ArtSurveySource, on_delete=models.CASCADE) + ero_exp = models.FloatField(default=0.0) + lkh = models.FloatField(default=0.0) + sep_soft = models.FloatField(default=0.0, null=True) + match_soft = models.BooleanField(default=False, null=True) #models.CharField(max_length=5, blank=True, null=True, default='') + srcname_soft = models.CharField(max_length=21, blank=True, null=True, default='') + hpidx_soft = models.BigIntegerField(default=0, null=True) + RA_fin_soft = models.FloatField(default=0.0, null=True) + DEC_fin_soft = models.FloatField(default=0.0, null=True) + RA_soft = models.FloatField(default=0.0, null=True) + DEC_soft = models.FloatField(default=0.0, null=True) + RADEC_ERR_soft = models.FloatField(default=0.0, null=True) + DET_LIKE_0_soft = models.FloatField(default=0.0, null=True) + ML_FLUX_0_soft = models.FloatField(default=0.0, null=True) + ML_FLUX_ERR_0_soft = models.FloatField(default=0.0, null=True) + ML_CTS_0_soft = models.FloatField(default=0.0, null=True) + ML_CTS_ERR_0_soft = models.FloatField(default=0.0, null=True) + ML_BKG_0_soft = models.FloatField(default=0.0, null=True) + EXT_soft = models.FloatField(default=0.0, null=True) + EXT_LIKE_soft = models.FloatField(default=0.0, null=True) + EXT_ERR_soft = models.FloatField(default=0.0, null=True) + ML_EXP_1_soft = models.FloatField(default=0.0, null=True) + sep_med = models.FloatField(default=0.0, null=True) + match_med = models.BooleanField(default=False, null=True) #models.CharField(max_length=5, blank=True, null=True, default='') + srcname_med = models.CharField(max_length=21, blank=True, null=True, default='') + hpidx_med = models.BigIntegerField(default=0, null=True) + RA_fin_med = models.FloatField(default=0.0, null=True) + DEC_fin_med = models.FloatField(default=0.0, null=True) + RA_med = models.FloatField(default=0.0, null=True) + DEC_med = models.FloatField(default=0.0, null=True) + RADEC_ERR_med = models.FloatField(default=0.0, null=True) + DET_LIKE_0_med = models.FloatField(default=0.0, null=True) + ML_FLUX_0_med = models.FloatField(default=0.0, null=True) + ML_FLUX_ERR_0_med = models.FloatField(default=0.0, null=True) + ML_CTS_0_med = models.FloatField(default=0.0, null=True) + ML_CTS_ERR_0_med = models.FloatField(default=0.0, null=True) + ML_BKG_0_med = models.FloatField(default=0.0, null=True) + EXT_med = models.FloatField(default=0.0, null=True) + EXT_LIKE_med = models.FloatField(default=0.0, null=True) + EXT_ERR_med = models.FloatField(default=0.0, null=True) + ML_EXP_1_med = models.FloatField(default=0.0, null=True) + sep_hard = models.FloatField(default=0.0, null=True) + match_hard = models.BooleanField(default=False, null=True) #models.CharField(max_length=5, blank=True, null=True, default='') + srcname_hard = models.CharField(max_length=21, blank=True, null=True, default='') + hpidx_hard = models.BigIntegerField(default=0, null=True) + RA_fin_hard = models.FloatField(default=0.0, null=True) + DEC_fin_hard = models.FloatField(default=0.0, null=True) + RA_hard = models.FloatField(default=0.0, null=True) + DEC_hard = models.FloatField(default=0.0, null=True) + RADEC_ERR_hard = models.FloatField(default=0.0, null=True) + DET_LIKE_0_hard = models.FloatField(default=0.0, null=True) + ML_FLUX_0_hard = models.FloatField(default=0.0, null=True) + ML_FLUX_ERR_0_hard = models.FloatField(default=0.0, null=True) + ML_CTS_0_hard = models.FloatField(default=0.0, null=True) + ML_CTS_ERR_0_hard = models.FloatField(default=0.0, null=True) + ML_BKG_0_hard = models.FloatField(default=0.0, null=True) + EXT_hard = models.FloatField(default=0.0, null=True) + EXT_LIKE_hard = models.FloatField(default=0.0, null=True) + EXT_ERR_hard = models.FloatField(default=0.0, null=True) + ML_EXP_1_hard = models.FloatField(default=0.0, null=True) + sep_simb = models.FloatField(default=0.0, null=True) + match_simb = models.BooleanField(default=False, null=True) # models.CharField(max_length=5, blank=True, null=True, default='') + ra_match = models.FloatField(default=0.0, null=True) + dec_match = models.FloatField(default=0.0, null=True) + RAJ2000_simb = models.FloatField(default=0.0, null=True) + DEJ2000_simb = models.FloatField(default=0.0, null=True) + z_simb = models.FloatField(default=0.0, null=True) + otype_simb = models.CharField(max_length=17, blank=True, null=True, default='') + + def get_fields(self): + return [(field.name, field.value_to_string(self)) for field in eRositaMatch._meta.fields] + def __str__(self): + return self.source.name + + +class ArtSurveyParams(models.Model): + ALLSKY = 'allsky' + RU = 'ru' + DE = 'de' + #RUDE = 'ru,de' + OWNERSHIP = [ + (ALLSKY,'All-sky'), + (RU, 'RU-sky'), + (DE, 'DE-sky'), + #(RUDE, 'RU,DE'), + ] + + E0 = 'E0' + E1 = 'E1' + E2 = 'E2' + E3 = 'E3' + E4 = 'E4' + E5 = 'E5' + ALL = 'ALL' + BANDS = [ + (E0,'4-12 keV'), + (E1,'4-7 keV'), + (E2,'7-12 keV'), + (E3,'12-20 keV'), + (E4,'20-30 keV'), + (E5,'3-12 keV'), + (ALL,'All bands'), + ] + + MSFC_FILT = 'MSFC_FILT' + MSFC_EXCL = 'MSFC_EXCL' + MARSHALL_SELECT = [ + ("", "None"), + (MSFC_FILT,'NEP filter'), + (MSFC_EXCL,'NEP exclude'), + ] + + owner = models.OneToOneField(User, related_name="artsurveyparams", on_delete=models.CASCADE) + history = models.TextField(default='', editable=False) + sign_ml_min = models.FloatField(default=None, blank=True, null=True, help_text='ML significance') # help_text='not applied to extended sources (EXT>1)') + sign_ml_max = models.FloatField(default=None, blank=True, null=True, help_text='ML significance') #help_text='not applied to extended sources (EXT>1)') + log_nfalse_min = models.FloatField(default=None, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + log_nfalse_max = models.FloatField(default=None, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + log_ml_nfalse_min = models.FloatField(default=-1.0, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + log_ml_nfalse_max = models.FloatField(default=None, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + exposure_min = models.FloatField(default=None, blank=True, null=True, help_text='Minimal exposure time') + detlike_min = models.FloatField(default=None, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + detlike_max = models.FloatField(default=None, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + ext_min = models.IntegerField(default=None, blank=True, null=True, help_text='Minimal extention in arcseconds') + ext_max = models.IntegerField(default=None, blank=True, null=True, help_text='Maximum extention in arcseconds') + class_startswith = models.CharField(max_length=36, blank=True, null=True, default='') + cname_contains = models.CharField(max_length=36, blank=True, null=True, default='') + survey = models.ForeignKey(ArtSurvey, null=True, blank=True, on_delete=models.SET_NULL, help_text='Survey number N=[1..8] of version V=[0..9] is named as SN.V;
The combined survey is labeled as SN<...>N.V;
Tags: active (*), in work (W), archived (A)') + exclude_survey = models.ForeignKey(ArtSurvey, related_name="exclude_survey", null=True, blank=True, on_delete=models.SET_NULL) + exclude_log_nfalse = models.FloatField(default=-1.0, blank=True, null=True, help_text='not active, do not use it.') + exclude_log_ml_nfalse = models.FloatField(default=-1.0, blank=True, null=True) + category = models.ForeignKey(Category, null=True, blank=True, on_delete=models.SET_NULL, help_text='Show sources with this Category only') + exclude_category = models.ForeignKey(Category, null=True, blank=True, on_delete=models.SET_NULL, related_name="exclude_category", help_text='Hide sources with this Category') + category_unclassified = models.BooleanField(default=False, help_text='Show sources without any Category') + ecl_lat_min = models.FloatField(default=None, blank=True, null=True, help_text='Ecliptic latitude min') + ecl_lat_max = models.FloatField(default=None, blank=True, null=True, help_text='Ecliptic latitude max') + dec_min = models.FloatField(default=None, blank=True, null=True, help_text='J2000 Declination min') + dec_max = models.FloatField(default=None, blank=True, null=True, help_text='J2000 Declination max') + glat_min = models.FloatField(default=None, blank=True, null=True, help_text='Galactic latitude min') + glat_max = models.FloatField(default=None, blank=True, null=True, help_text='Galactic latitude max') + circle_ra = models.FloatField(default=0.0, blank=True, null=True, help_text='Selection circle RA (not implemented yet)') + circle_dec = models.FloatField(default=0.0, blank=True, null=True, help_text='Selection circle Dec (not implemented yet)') + circle_rmax_deg = models.FloatField(default=0.0, blank=True, null=True, help_text='Selection circle Rmax in degrees (not implemented yet)') + gaia_primary = models.BooleanField(default=False, help_text='Select sources with Gaia counterpart') + allwise_primary = models.BooleanField(default=False, help_text='Select sources with AllWISE counterpart') + turk_possible = models.BooleanField(default=False, help_text='Sources suitable for TUBITAK*') + #marshall_filter = models.BooleanField(default=False, help_text='Show MSFC sources in NEP') + #marshall_exclude = models.BooleanField(default=False, help_text='Exclude MSFC sources in NEP') + + sky = models.CharField( + max_length=6, + choices=OWNERSHIP, + default=ALLSKY, + ) + + band = models.CharField( + max_length=10, + choices=BANDS, + default=E0, + help_text='E0: 4-12 keV, E1: 4-7 keV, E2: 7-12 keV, E3: 12-20 keV, E4: 20-30 keV' + ) + + exclude_band = models.CharField( + max_length=10, + choices=BANDS, + default=E0, + ) + + marshall = models.CharField( + max_length=10, + choices=MARSHALL_SELECT, + default=None, null=True,blank=True, + help_text='MSFC sources in NEP' + ) + +class ArtSurveyMatchParams(models.Model): + E0 = 'E0' + E1 = 'E1' + E2 = 'E2' + E3 = 'E3' + E4 = 'E4' + E5 = 'E5' + BANDS = [ + (E0,'4-12 keV'), + (E1,'4-7 keV'), + (E2,'7-12 keV'), + (E3,'12-20 keV'), + (E4,'20-30 keV'), + (E5,'3-15 keV'), + ] + + owner = models.OneToOneField(User, related_name="artsurveymatchparams", on_delete=models.CASCADE) + survey1 = models.ForeignKey(ArtSurvey, related_name='matchsurvey1', null=True, blank=True, on_delete=models.SET_NULL) + survey2 = models.ForeignKey(ArtSurvey, related_name='matchsurvey2', null=True, blank=True, on_delete=models.SET_NULL) + band1 = models.CharField(max_length=10,choices=BANDS,default=E0,) + band2 = models.CharField(max_length=10,choices=BANDS,default=E0,) + + sign_min1 = models.FloatField(default=0.0, blank=True, null=True) + sign_max1 = models.FloatField(default=1000.0, blank=True, null=True) + #detlike_min1 = models.FloatField(default=None, blank=True, null=True) + + + sign_min2 = models.FloatField(default=0.0, blank=True, null=True) + sign_max2 = models.FloatField(default=1000.0, blank=True, null=True) + #detlike_min2 = models.FloatField(default=None, blank=True, null=True) + + log_ml_nfalse_min1 = models.FloatField(default=-1.0, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + log_ml_nfalse_max1 = models.FloatField(default=None, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + + log_ml_nfalse_min2 = models.FloatField(default=-1.0, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + log_ml_nfalse_max2 = models.FloatField(default=None, blank=True, null=True, help_text='not applied to extended sources (EXT>1)') + + Rmax = models.FloatField(default=40.0, blank=True, null=True) + maxdist = models.FloatField(default=120.0, blank=True, null=True) + f2f1_ratio_min = models.FloatField(default=0.001, blank=True, null=True) + f2f1_ratio_max = models.FloatField(default=1000.0, blank=True, null=True) + + +class Upload(models.Model): + owner = models.ForeignKey(User, on_delete=models.CASCADE, default=None, unique=False, related_name="uploads") + artsources = models.ManyToManyField(ArtSurveySource, blank=True, related_name="uploads") + metasource = models.ForeignKey(MetaSource, null=True, blank=True, on_delete=models.SET_NULL, related_name="uploads") + survey = models.ForeignKey(ArtSurvey, null=True, blank=True, on_delete=models.SET_NULL, related_name="uploads") + title = models.CharField(max_length=128) + notes = models.TextField("notes",max_length=2000, null=True, blank=True) + filename = models.CharField(max_length=256) + filefield = models.FileField(upload_to=upload_filename, storage=FS, validators=[validate_upload]) + uploaded = models.DateTimeField(auto_now_add=True) + primary = models.BooleanField(default=False, help_text='Primary') + def get_delete_file_url(self): + return "/artsurvey/upload/%d/delete/file" % (self.id) + def delete(self, remove_file=True): + if remove_file: + FS.delete(str(self.filefield)) + return models.Model.delete(self) + def __str__(self): + return self.title + def get_download_url(self): + return "/uploads/artsurvey/{}".format(self.filefield) + def get_absolute_url(self): + return "/artsurvey/upload/{}/show".format(self.id) + @property + def isfits(self): + return self.filename.endswith("fits") or self.filename.endswith("evt") or self.filename.endswith("evt.gz") or self.filename.endswith("fits.gz") or self.filename.endswith("FTZ") or self.filename.endswith("fz") + + @property + def isreg(self): + return self.filename.endswith("reg") + +@receiver(models.signals.post_delete, sender=Upload) +def auto_delete_file_on_delete(sender, instance, **kwargs): + """ + Deletes file from filesystem + when corresponding `Upload` object is deleted. + """ + print("Upload: trigger post_delete for {}".format(instance)) + if instance.filefield: + if os.path.isfile(instance.filefield.path): + os.remove(instance.filefield.path) + +class ArtSelection(models.Model): + owner = models.OneToOneField(User, null=True, on_delete=models.CASCADE) + sources = models.ManyToManyField(ArtSurveySource, blank=True) + def __str__(self): + return self.user + +class ArtBasket(models.Model): + #owner = models.ForeignKey(User, on_delete=models.CASCADE, default=None, unique=False, verbose_name="ArtBasket", related_name="baskets") + owner = models.ForeignKey(User, on_delete=models.CASCADE) + sources = models.ManyToManyField(ArtSurveySource, blank=True) + title = models.CharField(max_length=128) + notes = models.TextField("Notes",max_length=2000, null=True, blank=True, validators=[MaxLengthValidator(1600)]) + #begin = models.DateTimeField(blank=True, null=True, auto_now_add=True) + #end = models.DateTimeField(blank=True, null=True) + #active = models.BooleanField(default=True) + #created = models.DateTimeField(auto_now_add=True) + def __str__(self): + return "{}".format(self.title) + def get_absolute_url(self): + return "/artsurvey/basket/{}/show".format(self.id) + def get_update_url(self): + return "/artsurvey/basket/{}/update".format(self.id) + def get_delete_url(self): + return "/artsurvey/basket/{}/delete".format(self.id) + def get_sources_count(self): + return "{}".format(self.sources.all().count()) + +class ArtBasketForm(forms.ModelForm): + class Meta: + model=ArtBasket + fields=('title','notes',) + + diff --git a/artsurvey/templatetags/__init__.py b/artsurvey/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/artsurvey/templatetags/artsurvey_templates.py b/artsurvey/templatetags/artsurvey_templates.py new file mode 100644 index 0000000..2e1ac0a --- /dev/null +++ b/artsurvey/templatetags/artsurvey_templates.py @@ -0,0 +1,57 @@ +from django import template +from django.urls import reverse + +from artsurvey.models import Catalog +from artsurvey.models import ArtSurvey + +register = template.Library() + +@register.filter +def get_total(survey, band): + try: + return survey.artsurveysource_set.all().filter(band=band).count() + except: + return 'N/A' + +@register.filter +def get_total_5sigma(survey, band): + try: + return survey.artsurveysource_set.all().filter(band=band).filter(sig__gt=5).exclude(category__slug__exact='false').count() + except: + return 'N/A' + +@register.filter +def is_bibcode_hidden(src, bibcode): + return True if bibcode in src.ads_hidden.all() else False + + + +@register.filter +def get_metasource_for_band(metasource, band): + #catalog=Catalog.objects.get(year=2021) + try: + return metasource.artsurveysource_set.all().filter(band=band).filter(catalog__in=[metasource.catalog]) + except: + return None + + +@register.simple_tag +def heasarc_linked_to(src, heasarc): + return src.if_linked_to(heasarc) + +@register.filter +def set_latest_galsurvey(name): + try: + survey = ArtSurvey.objects.filter(name=name).order_by("-version").first() + return("/artsurvey/SGP/{}".format(survey.id)) + except Exception as e: + #print("{}".format(e)) + return reverse('artsurvey index') + +@register.filter +def remove_srga(value): + return value.replace("SRGA ","") + +@register.filter +def remove_class(value): + return value.replace("class=","") diff --git a/artsurvey/tests.py b/artsurvey/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/artsurvey/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/artsurvey/urls.py b/artsurvey/urls.py new file mode 100644 index 0000000..06d133c --- /dev/null +++ b/artsurvey/urls.py @@ -0,0 +1,157 @@ +from django.urls import path, re_path, register_converter, include +from django.conf.urls import handler404 + +from . import views + +urlpatterns = [ + path('', views.artsurvey_params, name='artsurvey index'), + path('resolve/name//', views.resolve_name, name='artsurvey resolve name'), + path('profile', views.profile, name='artsurvey profile'), + path('history', views.history, name='artsurvey history'), + path('search/coords', views.search_coords_form, name='artsurvey search by coords'), + path('search/name', views.search_name_form, name='artsurvey search by name'), + path('search/resolve', views.resolve_name, name='artsurvey resolve name'), + path('catalog', views.index, name='artsurvey catalog'), + path('tex', views.tex, name='artsurvey tex'), + + path('paper', views.index_catalog, name='artsurvey paper'), + path('paper/tex', views.get_tex_catalog, name='artsurvey tex catalog'), + path('paper/tex/notes', views.get_tex_notes, name='artsurvey tex notes'), + + path('paper2/cat', views.index_catalog_paper2, name='artsurvey cat paper2'), + path('paper2/csv', views.get_csv_catalog_paper2, name='artsurvey csv paper2'), + path('paper2/tex/catalog', views.get_tex_paper2_catalog, name='artsurvey tex paper2 catalog'), + path('paper2/tex/notes', views.get_tex_paper2_notes, name='artsurvey tex paper2 notes'), + + path('types', views.types, name='artsurvey types'), + path('match', views.artsurvey_crossmatch, name='artsurvey crossmatch'), + path('summary', views.summary, name='artsurvey summary'), + path('match/integral2020', views.integral2020, name='artsurvey integral2020'), + path('sort/ra/asc', views.index_sort_ra_asc, name='artsurvey sort ra asc'), + path('sort/ra/desc', views.index_sort_ra_desc, name='artsurvey sort ra desc'), + path('sort/nfalse/asc', views.index_sort_nfalse_asc, name='artsurvey sort nfalse asc'), + path('sort/nfalse/desc', views.index_sort_nfalse_desc, name='artsurvey sort nfalse desc'), + path('sort/dec/asc', views.index_sort_dec_asc, name='artsurvey sort dec asc'), + path('sort/dec/desc', views.index_sort_dec_desc, name='artsurvey sort dec desc'), + path('sort/sign/asc', views.index_sort_sign_asc, name='artsurvey sort sign asc'), + path('sort/sign/desc', views.index_sort_sign_desc, name='artsurvey sort sign desc'), + path('sort/flux/asc', views.index_sort_flux_asc, name='artsurvey sort flux asc'), + path('sort/flux/desc', views.index_sort_flux_desc, name='artsurvey sort flux desc'), + path('/new', views.create_artsurvey, name='Create ArtSurveySource'), + path('/delete', views.delete_artsurvey_source, name='Delete ArtSurveySource'), + path('/markbad', views.markbad_artsurvey_source, name='Mark bad ArtSurveySource'), + path('/usercat', views.new_heasarc_source, name='new heasarc source'), + path('/artsource', views.show_artsurvey_source, name='Show ArtSurveySource'), + path('/artsource/ds9', views.get_artsurvey_source_ds9, name='Show ArtSurveySource DS9'), + path('/artsource/ds9/linked', views.get_artsurvey_source_ds9_linked, name='Show ArtSurveySource DS9 linked'), + path('/metasource', views.show_metasource, name='Show MetaSource'), + path('/metasource/', views.show_metasource_band, name='Show MetaSource band'), + path('/metasource/delete', views.delete_metasource, name='Delete MetaSource'), + path('/metasource/delete/paper', views.delete_paper_metasource, name='Delete Paper MetaSource'), + path('/gaia//', views.set_artsurvey_gaia, name='set ArtSurveySource GAIA'), + path('/link/heasarc//', views.link_heasarc, name='Link HEASARC'), + path('/unlink/heasarc//', views.unlink_heasarc, name='unLink HEASARC'), + path('/unlink/heasarc/all', views.unlink_heasarc_all, name='unLink HEASARC all'), + path('/gaia//', views.set_artsurvey_gaia, name='set ArtSurveySource GAIA'), + path('/allwise//', views.set_artsurvey_allwise, name='set ArtSurveySource AllWISE'), + path('/nvss//', views.set_artsurvey_nvss, name='set ArtSurveySource NVSS'), + path('/vlass//', views.set_artsurvey_vlass, name='set ArtSurveySource VLASS'), + path('/gaiadr3//', views.set_artsurvey_gaiadr3, name='set ArtSurveySource GAIADR3'), + path('/bj2021//', views.set_artsurvey_bj2021, name='set ArtSurveySource BJ2021'), + path('/first//', views.set_artsurvey_first, name='set ArtSurveySource FIRST'), + path('/sumss//', views.set_artsurvey_sumss, name='set ArtSurveySource SUMSS'), + path('/wise//', views.set_artsurvey_wise, name='set ArtSurveySource WISE'), + path('/allwise/clean/', views.clean_artsurvey_allwise, name='clean ArtSurveySource AllWISE'), + path('/gaiadr3/clean/', views.clean_artsurvey_gaiadr3, name='clean ArtSurveySource GAIADR3'), + path('/bj2021/clean/', views.clean_artsurvey_bj2021, name='clean ArtSurveySource BJ2021'), + path('/nvss/clean/', views.clean_artsurvey_nvss, name='clean ArtSurveySource NVSS'), + path('/first/clean/', views.clean_artsurvey_first, name='clean ArtSurveySource FIRST'), + path('/sumss/clean/', views.clean_artsurvey_sumss, name='clean ArtSurveySource SUMSS'), + path('/wise/clean/', views.clean_artsurvey_wise, name='clean ArtSurveySource WISE'), + path('/gaia/clean/', views.clean_artsurvey_gaia, name='clean ArtSurveySource GAIA'), + path('/type', views.type_artsurvey_source, name='Type ArtSurveySource'), + path('/notes', views.notes, name='Notes ArtSurveySource'), + path('/notes/survey', views.notes_survey, name='Notes ArtSurvey'), + path('/cname', views.cname, name='CName ArtSurveySource'), + path('/update/source', views.update_artsurveysource, name='Update ArtSurveySource'), + path('/load/skyview//', views.load_skyview_image, name='load SkyView image'), + path('/load/ps1/color', views.load_ps1_color_image, name='load PS1 color image'), + path('/load/ps1/gray', views.load_ps1_gray_image, name='load PS1 gray image'), + path('/load/ps1/fits/', views.load_ps1_fits_image, name='load PS1 fits image'), + path('type//show', views.show_type, name='artsurvey show type'), + path('ajax/autocomplete/', views.ajax_autocomplete, name='artsurvey ajax autocomplete'), + path('/update/', views.update_type, name='artsurvey update type'), + path('params', views.artsurvey_params, name='artsurvey params'), + path('params/reset', views.artsurvey_params_reset, name='artsurvey params reset'), + path('csv/selection', views.get_csv_selection, name='artsurvey csv selection'), + path('csv/all', views.get_csv_all, name='artsurvey csv all'), + path('ds9/all', views.get_ds9_all, name='artsurvey ds9 all'), + path('ds9/all/flux', views.get_ds9_all_flux, name='artsurvey ds9 all flux'), + path('ds9/all/paper', views.get_ds9_all_paper, name='artsurvey ds9 all paper'), + path('ds9/all/paper2', views.get_ds9_all_paper2, name='artsurvey ds9 all paper2'), + path('ds9/all/paper/newonly', views.get_ds9_all_paper_newonly, name='artsurvey ds9 all paper newonly'), + path('ds9/all/paper/known', views.get_ds9_all_paper_known, name='artsurvey ds9 all paper known'), + path('csv/catalog', views.get_csv_catalog, name='artsurvey csv catalog'), + path('/counterpart', views.counterpart, name='counterpart new'), + path('/vizier', views.vizier, name='artsurvey vizier'), + path('category//update', views.update_category, name='artsurvey update category'), + path('category//show', views.show_category, name='artsurvey show category'), + path('/oname', views.other_name_add, name='artsurvey add other name'), + path('/oname/delete/', views.other_name_delete, name='artsurvey delete other name'), + path('/oname/update/', views.other_name_update, name='artsurvey update other name'), + path('tentative//mark', views.mark_tentative, name='mark tentative'), + path('tentative//remove', views.remove_tentative, name='remove tentative'), + path('turkish', views.show_turkish, name='artsurvey show turkish'), + path('turkish//mark', views.mark_turkish, name='mark turkish'), + path('turkish//remove', views.remove_turkish, name='remove turkish'), + path('newxray//mark', views.mark_newxray, name='mark new X-ray'), + path('newxray//remove', views.remove_newxray, name='remove new X-ray'), + path('type//clean', views.clean_class, name='clean class'), + path('swiftbat105m/filter', views.swiftbat105m_filter, name='artsurvey swiftbat105m filter'), + path('swiftbat105m/exclude', views.swiftbat105m_exclude, name='artsurvey swiftbat105m exclude'), + path('swiftbat105m/filter/csv', views.swiftbat105m_filter_csv, name='artsurvey swiftbat105m filter csv'), + path('swiftbat105m/exclude/csv', views.swiftbat105m_exclude_csv, name='artsurvey swiftbat105m exclude csv'), + path('xmmdr10', views.xmmdr10, name='artsurvey xmmdr10'), + path('xmmdr12', views.xmmdr12, name='artsurvey xmmdr12'), + path('xmmsl2', views.xmmsl2, name='artsurvey xmmsl2'), + path('rass2rxs/filter', views.rass2rxs_filter, name='artsurvey rass2rxs filter'), + path('rass2rxs/filter/csv', views.rass2rxs_filter_csv, name='artsurvey rass2rxs filter csv'), + path('rass2rxs/exclude', views.rass2rxs_exclude, name='artsurvey rass2rxs exclude'), + path('rass2rxs/exclude/csv', views.rass2rxs_exclude_csv, name='artsurvey rass2rxs exclude csv'), + path('rxte', views.rxte, name='artsurvey rxte'), + path('intfercat', views.intrefcat, name='artsurvey intrefcat'), + path('maxigsc', views.maxigsc, name='artsurvey maxigsc'), + path('3maxi', views.maxi3, name='artsurvey 3maxi'), + path('maxi7yr', views.maxi7yr, name='artsurvey maxi7yr'), + path('fermi', views.fermi, name='artsurvey fermi'), + path('exclude_xray', views.exclude_xray, name='artsurvey exclude xray'), + #path('tubitak', views.show_turkpossible, name='artsurvey turk possible'), + path('filter_xray', views.filter_xray, name='artsurvey filter xray'), + path('/upload', views.artsource_upload, name='artsource upload'), + path('/upload/delete/', views.delete_upload, name='artsource delete upload'), + path('/upload/edit/', views.edit_upload, name='artsource edit upload'), + + path('/survey/upload/primary/clean', views.clean_primary_upload_survey, name='artsource upload'), + path('/survey/upload/primary/', views.set_primary_upload_survey, name='artsource upload'), + path('/survey/upload', views.artsource_upload_survey, name='artsource upload'), + path('/survey/upload/delete/', views.delete_upload_survey, name='artsource delete upload'), + path('/survey/upload/edit/', views.edit_upload_survey, name='artsource edit upload'), + + path('/metasource/group/', views.metasource_group, name='metasource group'), + path('/metasource/upload', views.metasource_upload, name='metasource upload'), + path('/metasource/upload/delete/', views.delete_upload_metasource, name='artsource delete upload'), + path('/metasource/upload/edit/', views.edit_upload_metasource, name='artsource edit upload metasource'), + path('/selection/add', views.add_to_selection, name='artsource add to selection'), + path('/selection/remove', views.remove_from_selection, name='artsource remove from selection'), + path('/selection/remove/profile', views.remove_from_selection_profile, name='artsource remove from selection profile'), + path('/selection/remove/basket/', views.remove_from_basket, name='artsource remove from basket'), + path('bad_button', views.bad_button, name='ajax bad button'), + path('SGP/', views.set_galplane_survey, name='artsurvey set galplane survey'), + path('/comment/delete', views.delete_comment, name='artsource delete comment'), + path('/transfer/', views.transfer, name='artsource transfer from'), + path('basket/create', views.basket_create, name='artsurvey basket create'), + path('basket//update', views.basket_update, name='artsurvey basket update'), + path('basket//show', views.basket_show, name='artsurvey basket show'), + path('basket//delete', views.basket_delete, name='artsurvey basket delete'), +] + diff --git a/artsurvey/utils.py b/artsurvey/utils.py new file mode 100644 index 0000000..86fd2dc --- /dev/null +++ b/artsurvey/utils.py @@ -0,0 +1,683 @@ +import random +import math +from django.db.models import DecimalField, Value +from django.db.models import IntegerField, BooleanField +from django.db.models import Max, Subquery, OuterRef, Exists, Min, Case, When, Min, Func +from math import isfinite, sqrt + +from PIL import Image +from io import BytesIO +import pylab +import numpy as np + +from astroquery.skyview import SkyView + +import requests +import logging +import astropy +from astropy.io import fits +from astropy.io import ascii +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +from astropy_healpix import HEALPix, neighbours +from astropy import units as au +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch +from astropy.table import Table + + +from django.db.models import Q + +from django.db.models import FloatField, Value + +from artsurvey.models import MetaSource +from artsurvey.models import ArtSurvey, EnergyBand +from artsurvey.models import ArtSurveySource +from artsurvey.models import ArtSurveyParams + +from erosurvey.models import NSIDE_PLATES, ORDER_PLATES +from heasarc.models import NSIDE_SOURCES, ORDER +from heasarc.models import HeasarcBase +from heasarc.models import HeasarcSwiftBAT105m +from heasarc.models import HeasarcIntRefCat +from heasarc.models import HeasarcXTEASSCAT +from heasarc.models import HeasarcXMMSL2 +from heasarc.models import Heasarc4FGL +from heasarc.models import HeasarcIntegral2020 +from heasarc.models import HeasarcMAXI7YR +from heasarc.models import HeasarcRASS2RXS + +from srgcat.models import SkyMaps + + +from monthplan.models import SurveyHealpixPlate + +def get_sources(owner, altsurvey=None): + q = ArtSurveySource.objects.all() + q=q.annotate(exclude=Value(False, output_field=BooleanField())) + + try: + params = ArtSurveyParams.objects.get(owner=owner) + except: + return q + + if not altsurvey is None: + q=q.filter(survey=altsurvey) + else: + if not params.survey is None: + q=q.filter(survey=params.survey) + + if not params.band is None: + q=q.filter(band__slug__exact=params.band) + + if not params.glat_min is None: + q=q.filter(bii__gte=params.glat_min) + + if not params.glat_max is None: + q=q.filter(bii__lt=params.glat_max) + + if not params.ecl_lat_min is None: + q=q.filter(ecl_b__gte=params.ecl_lat_min) + + if not params.ecl_lat_max is None: + q=q.filter(ecl_b__lt=params.ecl_lat_max) + + if not params.dec_min is None: + q=q.filter(dec__gte=params.dec_min) + + if not params.dec_max is None: + q=q.filter(dec__lt=params.dec_max) + + if not params.sign_ml_min is None: + """ exclude extended sources from this filtering """ + #q=q.filter(Q(ml_sig__gte=params.sign_ml_min) | Q(ext__gte=1)) + q=q.filter(ml_sig__gte=params.sign_ml_min) + + if not params.sign_ml_max is None: + """ exclude extended sources from this filtering """ + #q=q.filter(Q(ml_sig__lt=params.sign_ml_max) | Q(ext__gte=1)) + q=q.filter(ml_sig__lt=params.sign_ml_max) + + if not params.exposure_min is None: + q=q.filter(exptime__gte=params.exposure_min) + + if not params.detlike_min is None: + """ exclude extended sources from this filtering """ + q=q.filter(Q(detlike__gte=params.detlike_min) | Q(ext__gte=1)) + + if not params.detlike_max is None: + """ exclude extended sources from this filtering """ + q=q.filter(Q(detlike__lt=params.detlike_max) | Q(ext__gte=1)) + + if not params.log_nfalse_min is None: + """ exclude extended sources from this filtering """ + q=q.filter(Q(log_nfalse__lte=params.log_nfalse_min) | Q(ext__gte=1)) + + if not params.log_nfalse_max is None: + """ exclude extended sources from this filtering """ + q=q.filter(Q(log_nfalse__gt=params.log_nfalse_max) | Q(ext__gte=1)) + + if not params.log_ml_nfalse_min is None: + """ exclude extended sources from this filtering """ + q=q.filter(Q(log_ml_nfalse__lte=params.log_ml_nfalse_min) | Q(ext__gte=1)) + + if not params.log_ml_nfalse_max is None: + """ exclude extended sources from this filtering """ + q=q.filter(Q(log_ml_nfalse__gt=params.log_ml_nfalse_max) | Q(ext__gte=1)) + + if not params.ext_min is None: + q=q.filter(ext__gte=params.ext_min) + + if not params.ext_max is None: + q=q.filter(ext__lt=params.ext_max) + + if not params.class_startswith is None: + q=q.filter(object_class__class_name__startswith=params.class_startswith) + + if not params.cname_contains is None: + #q=q.filter(cname__startswith=params.cname_startswith) + q=q.filter(cname__contains=params.cname_contains) + + if not params.category is None: + q=q.filter(category=params.category) + + if not params.exclude_category is None: + q=q.exclude(category=params.exclude_category) + + if params.category_unclassified: + q=q.filter(category=None) + + if params.gaia_primary: + q=q.filter(gaia3_primary__isnull=False) + + if params.turk_possible: + q=q.filter(turkpossible__exact=True) + + if params.allwise_primary: + q=q.filter(allwise_primary__isnull=False) + + if params.sky == 'allsky': + pass + elif params.sky == 'ru': + q=q.filter(lii__lt=180) + pass + elif params.sky == 'de': + q=q.filter(lii__gt=180) + pass + elif params.sky == 'ru,de': + pass + else: + pass + + + if params.marshall == None: + pass + elif params.marshall == 'MSFC_FILT': + q=q.filter(ecl_b__gt=82) + pass + elif params.marshall == 'MSFC_EXCL': + q=q.filter(ecl_b__lt=82) + pass + else: + pass + + + if not params.exclude_survey is None: + #print('params.exclude_survey') + subquery=MetaSource.objects.filter(artsurveysource=OuterRef("pk")) + survey_condition=Q(artsurveysource__survey=params.exclude_survey) + + if not params.exclude_band == 'ALL': + #print('params.exclude_band apply E band') + band_condition=Q(artsurveysource__band__slug__exact=params.exclude_band) + else: + """ Take all bands """ + #print('params.exclude_band apply ALL bands') + band_condition=Q(artsurveysource__band__slug__contains='E') + + + print("exclude band: {}".format(params.exclude_band)) + + """ switched to ML """ + if not params.exclude_log_ml_nfalse is None: + #print('params.exclude_log_nfalse apply') + log_nfalse_condition=Q(artsurveysource__log_ml_nfalse__lte=params.exclude_log_ml_nfalse) + else: + """ Take all sources """ + #print('params.exclude_log_nfalse default') + log_nfalse_condition=Q(artsurveysource__log_ml_nfalse__lte=-1) + + q=q.annotate(exclude=Exists(subquery.filter(survey_condition & band_condition & log_nfalse_condition))) + + q=q.exclude(exclude=True) + + return q + + +def get_sources_sorted(owner,refsrc,rmax_deg=10): + + ref_crd = SkyCoord(refsrc.ra, refsrc.dec, frame="fk5", unit="deg") + srcs0 = get_sources(owner,altsurvey=refsrc.survey).annotate(separation=Value(0.0, output_field=DecimalField())) + srcs=[] + for src in srcs0: + src_crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + sep=ref_crd.separation(src_crd).arcmin + if((sep/60) <= rmax_deg): + src.separation=sep + srcs.append(src) + return sorted(srcs, key=lambda x: x.separation, reverse=False) + +def get_heasarc_around(owner,refsrc,rmax_arcmin=60): + + ref_crd = SkyCoord(refsrc.ra, refsrc.dec, frame="fk5", unit="deg") + srcs0 = get_sources(owner,altsurvey=refsrc.survey).annotate(separation=Value(0.0, output_field=DecimalField())) + srcs=[] + for src in srcs0: + src_crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + sep=ref_crd.separation(src_crd).arcmin + if(sep <= rmax_arcmin): + heasarcs = src.heasarc.all() + for h in heasarcs: + srcs.append(h) + return srcs + + +def get_skymap_by_coords(ra, dec): + skymap=None + + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix_plate = hp_plate.skycoord_to_healpix(crd) + try: + my_hp_plate = SurveyHealpixPlate.objects.get(healpix=healpix_plate) + skymaps = my_hp_plate.skymaps_set.all() + sep_min=10000.0 + for smap in skymaps: + smap_crd = SkyCoord(smap.RA_CEN, smap.DE_CEN, frame=FK5(), unit="deg") + sep=crd.separation(smap_crd).arcmin + if(sep < sep_min): + sep_min = sep + skymap = smap + except Exception as e: + print("{}".format(e)) + + pass + + return skymap + + + +def search_by_name_artsurvey(name): + srcs = ArtSurveySource.objects.all().filter(Q(name__contains=name) | Q(cname__contains=name)) + return sorted(srcs, key=lambda x: x.name, reverse=False)[:100] + + +def search_by_coords_artsurvey(ra, dec, rmax=40,maxdist=120.0): + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + heal = hp.cone_search_skycoord(crd, radius=maxdist*u.arcsecond) + srcs_cone = ArtSurveySource.objects.all().filter(healpix__in=heal) + srcs=[] + for src in srcs_cone: + src_crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + sep=crd.separation(src_crd).arcsecond + if(sep <= rmax): + src.separation=sep + srcs.append(src) + return sorted(srcs, key=lambda x: x.separation, reverse=False) + #return srcs + +def find_heasarc(srcs,maxdist=120.0,minrad=40.0, syserr=0.0, offset=False): + """ version for paper2 """ + count=0 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + + excluded_heasarc=0 + + for s in srcs: + # don't clean at the beginning + #s.heasarc.clear() + + crd = SkyCoord(s.ra, s.dec, frame="fk5", unit="deg") + if(offset): + # random angle for estimating spurious matches + alpha = 2 * math.pi * random.random() + position_angle = alpha * 180 / math.pi * u.deg + separation = 45*60 * u.arcsecond + + crd=crd.directional_offset_by(position_angle, separation) + heal = hp.cone_search_skycoord(crd, radius=maxdist*u.arcsecond) + try: + heasarcs = HeasarcBase.objects.instance_of(HeasarcSwiftBAT105m,HeasarcXMMSL2,Heasarc4FGL,HeasarcIntegral2020,HeasarcMAXI7YR,HeasarcRASS2RXS) + heasarcs = heasarcs.filter(healpix__in=heal) + + #heasarcs = HeasarcBase.objects.all().filter(healpix__in=heal) + #print("{} --> {} HEASARCs".format(s,heasarcs.count())) + except Exception as e: + print('%s' % type(e)) + continue + + radec_error = sqrt(s.ml_radec_err_98**2 + syserr**2) + + heasarcs_clean=heasarcs + + if(offset): + for h in heasarcs: + if(h in s.heasarc.all()): + print(s,"*** Already exists in HEASARC list ***",h) + heasarcs_clean=heasarcs_clean.exclude(healpix=h.healpix) + excluded_heasarc+=1 + + s.heasarc.clear() + + if not (heasarcs_clean.count()>0): + continue + + #print("{} --> {} HEASARCs === {}".format(s,heasarcs.count(), heasarcs_clean.count())) + + for h in heasarcs_clean: + heasarc_crd = SkyCoord(h.ra, h.dec, frame="fk5", unit="deg") + sep=crd.separation(heasarc_crd).arcsecond + + if(sep <= radec_error or sep <= (radec_error + h.error_radius)): + s.heasarc.add(h) + s.save() + + print("### Total HEASARC excluded {} ###".format(excluded_heasarc)) + + +def find_heasarc_paper1(srcs,maxdist=120.0,minrad=40.0, offset=False): + """ saved version used for paper1 """ + count=0 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + + for s in srcs: + s.heasarc.clear() + + crd = SkyCoord(s.ra, s.dec, frame="fk5", unit="deg") + if(offset): + # random angle + alpha = 2 * math.pi * random.random() + position_angle = alpha * 180 / math.pi * u.deg + separation = 45*60 * u.arcsecond + + crd=crd.directional_offset_by(position_angle, separation) + heal = hp.cone_search_skycoord(crd, radius=maxdist*u.arcsecond) + try: + heasarcs = HeasarcBase.objects.all().filter(healpix__in=heal) + # print("--> HEASARC {}".format(heasarcs.count())) + except: + continue + + if(s.ext): + if(s.ext > minrad): + radec_error=float(s.ext) + else: + radec_error=minrad + else: + radec_error=minrad + + for h in heasarcs: + heasarc_crd = SkyCoord(h.ra, h.dec, frame="fk5", unit="deg") + sep=crd.separation(heasarc_crd).arcsecond + + if(sep <= radec_error or sep <= h.error_radius): + s.heasarc.add(h) + s.save() + +def update_marshall(srcs): + count=0 + + srcs0 = srcs.filter(ecl_b__gt=82.0) + for s in srcs0: + s.marshall=True + s.save() + + return srcs0.count() + +def update_metasource(srcs,maxdist=120.0,minrad=40.0): + count=0 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + + for s in srcs: + crd = SkyCoord(s.ra, s.dec, frame="fk5", unit="deg") + heal = hp.cone_search_skycoord(crd, radius=maxdist*u.arcsecond) + try: + meta_srcs = MetaSource.objects.all().filter(healpix__in=heal) + except: + continue + + sep_min=1000.0 + msource=None + for sm in meta_srcs: + meta_crd = SkyCoord(sm.ra, sm.dec, frame="fk5", unit="deg") + sep=crd.separation(meta_crd).arcsecond + if(sep <= sm.radec_error and sep < sep_min): + msource=sm + sep_min=sep + count=count+1 + + if not (msource): + sep_min=0.0 + if(s.ext): + if(s.ext > minrad): + radec_error=float(s.ext) + else: + radec_error=minrad + else: + radec_error=minrad + msource = MetaSource(name=s.name, healpix=s.healpix, ra=s.ra, dec=s.dec, glon=s.lii, glat=s.bii, elon=s.ecl_l, elat=s.ecl_b, radec_error=radec_error) + msource.save() + + s.metasource=msource + s.metasource_sep=sep_min + s.save() + + return count + + + +def setup_metasource(master_name,slave_name): + + try: + srcs1= ArtSurveySource.objects.filter(name__contains=master_name) + print("Master: Found {}".format(srcs1.count())) + for src in srcs1: + print("Master:--> {} --> {}".format(src,src.metasource)) + except: + print("Master: Not found: {}".format(master_name)) + return + + try: + srcs2= ArtSurveySource.objects.filter(name__contains=slave_name) + print("Slave: Found {}".format(srcs2.count())) + for src in srcs2: + print("Slave:--> {} --> {}".format(src,src.metasource)) + except: + print("Slave: Not found: {}".format(slave_name)) + return + + print("*** BEGIN ***") + for master in srcs1: + if not master.metasource: + continue + master_crd = SkyCoord(master.ra, master.dec, frame="fk5", unit="deg") + meta_crd = SkyCoord(master.metasource.ra, master.metasource.dec, frame="fk5", unit="deg") + print("Master {} (<--{})".format(master.metasource,master)) + for slave in srcs2: + if not slave.metasource: + continue + if slave.metasource == master.metasource: + print("Already satisfied") + continue + slave_crd = SkyCoord(slave.ra, slave.dec, frame="fk5", unit="deg") + metasource_sep=slave_crd.separation(meta_crd).arcsecond + slave.metasource=master.metasource + slave.metasource_sep=metasource_sep + slave.save() + print("--> Add slave {} at {:.2f} arcseconds".format(slave,metasource_sep)) + + print("*** END ***") + +def get_artsurvey(version, archived=False): + try: + survey = ArtSurvey.objects.get(Q(version=version) & Q(archived=archived)) + return survey + except ArtSurvey.DoesNotExist: + return None + +def get_energy_band(slug): + try: + band = EnergyBand.objects.get(slug=slug) + return band + except EnergyBand.DoesNotExist: + return None + +def find_skymaps(): + skymaps = SkyMaps.objects.all() + + print("Selected {} skymaps".format(skymaps.count())) + + srcs = ArtSurveySource.objects.all().filter(SMAPNR__exact=0) + print("Selected {} total".format(srcs.count())) + + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + min_sep=400 + for smap in skymaps: + crd_map = SkyCoord(smap.RA_CEN, smap.DE_CEN, frame="fk5", unit="deg") + sep=crd.separation(crd_map).deg + if(sep < min_sep): + min_sep=sep + src.SMAPNR=smap.SMAPNR + if(min_sep < 360): + print("Set SkyMap for {} at offset {} degrees".format(src,min_sep)) + src.save() + +def select_turkpossible(survey_version,slug): + minrad=40 + maxdist=600 + + try: + band = EnergyBand.objects.get(slug=slug) + except EnergyBand.DoesNotExist: + print("EnergyBand not found") + return + + """ + try: + survey = ArtSurvey.objects.get(version=survey_version) + print("use ArtSurvey {}".format(survey)) + except ArtSurvey.DoesNotExist: + print("ArtSurvey {} not found".format(survey_version)) + return + srcs = survey.artsurveysource_set.all().filter(band=band) + """ + + srcs = ArtSurveySource.objects.all().filter(Q(survey__version=survey_version)&Q(band=band)) + print("Selected {}".format(srcs.count())) + + + for src in srcs: + src.turkpossible=True + src.save() + + for src in srcs: + heasarcs = src.heasarc.all() + + if (heasarcs.filter(instance_of=HeasarcSwiftBAT105m).count()): + src.turkpossible=False + src.save() + + if (heasarcs.filter(instance_of=HeasarcIntRefCat).count()): + src.turkpossible=False + src.save() + + if (heasarcs.filter(instance_of=HeasarcXTEASSCAT).count()): + src.turkpossible=False + src.save() + + selection=srcs.filter(turkpossible__exact=True) + print("Total selected {}".format(selection.count())) + + +def ps1_getimages(ra,dec,size=240,filters="grizy"): + + """Query ps1filenames.py service to get a list of images + + ra, dec = position in degrees + size = image size in pixels (0.25 arcsec/pixel) + filters = string with filters to include + Returns a table with the results + """ + + service = "https://ps1images.stsci.edu/cgi-bin/ps1filenames.py" + url = ("{service}?ra={ra}&dec={dec}&size={size}&format=fits" + "&filters={filters}").format(**locals()) + table = Table.read(url, format='ascii') + return table + +def ps1_geturl(ra, dec, size=240, output_size=None, filters="grizy", format="jpg", color=False): + + """Get URL for images in the table + + ra, dec = position in degrees + size = extracted image size in pixels (0.25 arcsec/pixel) + output_size = output (display) image size in pixels (default = size). + output_size has no effect for fits format images. + filters = string with filters to include + format = data format (options are "jpg", "png" or "fits") + color = if True, creates a color image (only for jpg or png format). + Default is return a list of URLs for single-filter grayscale images. + Returns a string with the URL + """ + + if color and format == "fits": + raise ValueError("color images are available only for jpg or png formats") + if format not in ("jpg","png","fits"): + raise ValueError("format must be one of jpg, png, fits") + table = ps1_getimages(ra,dec,size=size,filters=filters) + url = ("https://ps1images.stsci.edu/cgi-bin/fitscut.cgi?" + "ra={ra}&dec={dec}&size={size}&format={format}").format(**locals()) + if output_size: + url = url + "&output_size={}".format(output_size) + # sort filters from red to blue + flist = ["yzirg".find(x) for x in table['filter']] + table = table[np.argsort(flist)] + if color: + if len(table) > 3: + # pick 3 filters + table = table[[0,len(table)//2,len(table)-1]] + for i, param in enumerate(["red","green","blue"]): + url = url + "&{}={}".format(param,table['filename'][i]) + else: + urlbase = url + "&red=" + url = [] + for filename in table['filename']: + url.append(urlbase+filename) + return url + +def skyview_query(ra, dec, size=240, survey=None): + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + urls = SkyView.get_image_list(position=crd, survey=[survey]) + r = requests.get(urls[0]) + return r.content + +def ps1_getcolorim(ra, dec, size=240, output_size=None, filters="grizy", format="jpg"): + + """Get color image at a sky position + + ra, dec = position in degrees + size = extracted image size in pixels (0.25 arcsec/pixel) + output_size = output (display) image size in pixels (default = size). + output_size has no effect for fits format images. + filters = string with filters to include + format = data format (options are "jpg", "png") + Returns the image + """ + + if format not in ("jpg","png"): + raise ValueError("format must be jpg or png") + url = ps1_geturl(ra,dec,size=size,filters=filters,output_size=output_size,format=format,color=True) + r = requests.get(url) + #im = Image.open(BytesIO(r.content)) + return r.content + + +def ps1_getgrayim(ra, dec, size=240, output_size=None, filter="g", format="jpg"): + + """Get grayscale image at a sky position + + ra, dec = position in degrees + size = extracted image size in pixels (0.25 arcsec/pixel) + output_size = output (display) image size in pixels (default = size). + output_size has no effect for fits format images. + filter = string with filter to extract (one of grizy) + format = data format (options are "jpg", "png") + Returns the image + """ + + if format not in ("jpg","png"): + raise ValueError("format must be jpg or png") + if filter not in list("grizy"): + raise ValueError("filter must be one of grizy") + url = ps1_geturl(ra,dec,size=size,filters=filter,output_size=output_size,format=format) + r = requests.get(url[0]) + #im = Image.open(BytesIO(r.content)) + return r.content + +def ps1_fitsim(ra, dec, size=240, output_size=None, filter="g", format="fits"): + + if format not in ("fits"): + raise ValueError("format must be fits") + if filter not in list("grizy"): + raise ValueError("filter must be one of grizy") + url = ps1_geturl(ra,dec,size=size,filters=filter,output_size=output_size,format=format) + r = requests.get(url[0]) + #im = Image.open(BytesIO(r.content)) + return r.content diff --git a/artsurvey/views.py b/artsurvey/views.py new file mode 100644 index 0000000..e427f86 --- /dev/null +++ b/artsurvey/views.py @@ -0,0 +1,3480 @@ +import json +import csv +import time +from math import pi +import math +from collections import OrderedDict +import numpy as np + +from django.shortcuts import render +from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden +from django.shortcuts import get_object_or_404 +from django.urls import reverse +from django.db.models import IntegerField, BooleanField +from django.db.models import F +from django.db.models import Q +from django.db.models import DecimalField, Value +from django.db.models import Max, Subquery, OuterRef, Min, Case, When, Min, Func +from django.db.models.functions.math import Sqrt, Cos, Power, Abs +from django.contrib.auth.models import User +from django.utils import timezone +from django.db.models import OuterRef, Exists +from django.db.models import Count +from django.core.files.base import ContentFile +from django.core.exceptions import PermissionDenied, FieldDoesNotExist +from django.db.models import BooleanField, DecimalField, CharField, Value + +from astroquery.vizier import Vizier +from astroquery.skyview import SkyView + +from django.utils.encoding import smart_str + +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import BarycentricTrueEcliptic +from astropy.coordinates import HeliocentricTrueEcliptic +from astropy.coordinates import GeocentricTrueEcliptic +from astropy_healpix import HEALPix +from astropy.coordinates.name_resolve import get_icrs_coordinates + +from srglib.utils import srg_auth, srg_group_auth, srg_group_auth_stats, status_code +from srglib.utils import crossmatch +from srglib.utils import make_source_name +from srglib.utils import transfer_data + +from artsurvey.models import MetaSource +from artsurvey.models import MetaComment +from artsurvey.models import ArtSurvey +from artsurvey.models import ArtSurveySource +from artsurvey.models import ArtSurveyParams +from artsurvey.models import ArtSurveyMatchParams +from artsurvey.models import OpticalCounterpart +from artsurvey.models import Category +from artsurvey.models import OtherName +from artsurvey.models import EnergyBand +from artsurvey.models import WiseCounterpart +from artsurvey.models import Catalog +from artsurvey.models import Upload +from artsurvey.models import ArtSelection +from artsurvey.models import ArtBasket, ArtBasketForm + +from artsurvey.utils import get_artsurvey +from artsurvey.utils import get_energy_band +from artsurvey.utils import search_by_coords_artsurvey +from artsurvey.utils import search_by_name_artsurvey +from artsurvey.utils import get_sources_sorted +from artsurvey.utils import get_sources +from artsurvey.utils import get_heasarc_around +from artsurvey.utils import get_skymap_by_coords +from artsurvey.utils import ps1_getcolorim +from artsurvey.utils import ps1_getgrayim +from artsurvey.utils import ps1_fitsim +from artsurvey.utils import skyview_query + +from artsurvey.forms import OpticalCounterpartForm +from artsurvey.forms import OtherNameForm +from artsurvey.forms import NotesForm +from artsurvey.forms import NotesSurveyForm +from artsurvey.forms import CnameForm +from artsurvey.forms import ArtSurveySourceForm +from artsurvey.forms import ArtSurveyParamsForm +from artsurvey.forms import ArtSurveyMatchParamsForm +from artsurvey.forms import CategoryForm +from artsurvey.forms import UploadForm +from artsurvey.forms import SearchCoordsForm +from artsurvey.forms import SearchNameForm +from artsurvey.forms import ResolveNameForm +from artsurvey.forms import MetaCommentForm +from artsurvey.forms import UserArtBasketListForm + + +from srgcat.models import SkyMapSource +from srgcat.models import SkyMaps + +from heasarc.models import HeasarcSimpleClass, Heasarc4XMMDR9 +from heasarc.models import GreenSNR +from heasarc.models import HeasarcUserCat, HeasarcUserCatForm + +from astrobasis.models import GAIADR2 +from astrobasis.models import GAIADR3 # this is in fact EDR3 +from astrobasis.models import AllWise +from astrobasis.models import NVSS +from astrobasis.models import FIRST +from astrobasis.models import SUMSS +from astrobasis.models import VLASSfromVizieR +from astrobasis.models import BJfromVizieR +from astrobasis.models import GAIADR3fromVizieR # this is real DR3 + +from heasarc.models import HeasarcALLWISEAGN +from heasarc.models import HeasarcBase +from heasarc.utils import healpix_usercat +from heasarc.utils import match_usercat_all + +from nasa.models import Category as AdsCategory +from nasa.models import Ads + + +def write_history(request, record): + user = get_object_or_404(User, pk=request.user.id) + params = get_object_or_404(ArtSurveyParams, owner=user) + now = timezone.localtime(timezone.now()) + params.history+="{} (MSK) [{}] {}
".format(now.strftime("%d.%m.%Y %H:%M"), + request.user, record) + + params.save() + +def history(request): + users = User.objects.filter(groups__name__exact='srg-artxc-catalog').distinct() + return render(request,'artsurvey/history.html', {'users':users,}) + +def update_selected(user): + repeat = 0 + srcs = user.artselection.sources.all() + for src in srcs: + print("Finding latest children for {}".format(src)) + try: + child=src.children.all().filter(survey__name__exact=src.survey.name).latest() + """ latest child in the same survey """ + except Exception as e: + print("--- {} {}".format(src,e)) + continue + if(child): + user.artselection.sources.remove(src) + user.artselection.sources.add(child) + repeat = repeat + child.children.all().count() + return repeat + +def profile(request): + while True: + repeat = update_selected(request.user) + if not repeat: + break + return render(request,'artsurvey/profile.html', {'user':request.user,}) + +def resolve_name(request,name): + srcs = ArtSurveySource.objects.filter(name__contains=name) + return HttpResponse("

{}

".format(name)) + #return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +# Create your views here. + + +def bad_button(request): + if request.method =="POST": + if request.POST.get("operation") == "bad_submit" and request.is_ajax(): + content_id=request.POST.get("content_id",None) + + content=get_object_or_404(ArtSurveySource,pk=content_id) + print("AJAX Request: bad_submit for {}".format(content)) + bad_category = get_object_or_404(Category,slug='false') + + if content.category: + badmarked=False + else: + content.category=bad_category + content.save() + badmarked=True + + ctx={"badmarked":badmarked,"content_id":content_id} + return HttpResponse(json.dumps(ctx), content_type='application/json') + else: + ctx={"badmarked":False,"content_id":content_id} + return HttpResponse(json.dumps(ctx), content_type='application/json') + + + +def custom_error_403(request, exception): + return render(request, '403.html', {'exception':exception}) + +def custom_error_404(request, exception): + return render(request, '404.html', {}) + +def HttpResponseCSV(srcs, key=''): + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.csv".format(key,time.strftime("%Y%m%d-%H%M%S")) + writer = csv.writer(response, csv.excel, delimiter=",") + #response.write(u'\ufeff'.encode('utf8')) + + #response.write(u''.encode('utf8')) + #response.write(smart_str(u"#updated,2015-10-17T00:00:00Z")) + writer.writerow([ + smart_str(u"NAME"), + smart_str(u"RA"), + smart_str(u"Dec"), + smart_str(u"GLON"), + smart_str(u"GLAT"), + smart_str(u"ELON"), + smart_str(u"ELAT"), + smart_str(u"EXT"), + smart_str(u"CNTS"), + smart_str(u"CNTS_ERR"), + smart_str(u"CNTS_BG"), + smart_str(u"EXPTIME"), + smart_str(u"FLUX"), + smart_str(u"FLUXERR_LO"), + smart_str(u"FLUXERR_UP"), + smart_str(u"SIG"), + smart_str(u"LOG_NFALSE"), + smart_str(u"DETLIKE"), + smart_str(u"ML_SIG"), + smart_str(u"ML_DETLIKE"), + smart_str(u"ML_NFALSE"), + smart_str(u"LOG_ML_NFALSE"), + smart_str(u"ML_RA"), + smart_str(u"ML_DEC"), + smart_str(u"ML_RADEC_ERR_90"), + smart_str(u"ML_RADEC_ERR_98"), + smart_str(u"ML_FLUX"), + smart_str(u"ML_FLUX_ERR"), + smart_str(u"ML_EXP"), + smart_str(u"class_id"), + smart_str(u"object_class"), + smart_str(u"redshift"), + smart_str(u"cname"), + smart_str(u"category"), +# smart_str(u"erosita data"), +# smart_str(u"erosita match"), +# smart_str(u"erosita match soft"), +# smart_str(u"erosita match med"), +# smart_str(u"erosita match hard"), +# smart_str(u"erosita soft"), +# smart_str(u"erosita med"), +# smart_str(u"erosita hard"), +# smart_str(u"erosita sep soft"), +# smart_str(u"erosita sep med"), +# smart_str(u"erosita sep hard"), + smart_str(u"notes"), + smart_str(u"paper_notes"), + smart_str(u"follow-up"), +# smart_str(u"bibcode"), + smart_str(u"turkish"), + smart_str(u"4XMM_count"), + smart_str(u"4XMM(offset, name, ra, dec, ep_5_flux)"), + smart_str(u"tentative"), + smart_str(u"gaia_primary"), + smart_str(u"gaia_ra"), + smart_str(u"gaia_dec"), + smart_str(u"gaia_sep"), + ]) + + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + bibcode=[] + bibcode_notes=[] + for b in src.ads.all(): + bibcode.append(b.bibcode) + bibcode_notes.append(b.notes) + bibcode_szip = zip(bibcode, bibcode_notes) + + xmm = src.heasarc.instance_of(Heasarc4XMMDR9) + xmm_name=[] + xmm_separation=[] + xmm_ra=[] + xmm_dec=[] + xmm_ep_5_flux=[] + xmm_ep_5_flux_err=[] + xmm_count=xmm.count() + for xmmsrc in xmm: + xmm_name.append(xmmsrc.name) + xmm_ra.append(xmmsrc.ra) + xmm_dec.append(xmmsrc.dec) + xmm_ep_5_flux.append(xmmsrc.EP_5_FLUX) + xmm_ep_5_flux_err.append(xmmsrc.EP_5_FLUX_ERR) + xmm_separation.append(crd.separation(SkyCoord(xmmsrc.ra, xmmsrc.dec, frame="fk5", unit="deg")).arcsecond) + + xmm_szip = sorted(zip(xmm_separation, xmm_name, xmm_ra, xmm_dec, xmm_ep_5_flux, xmm_ep_5_flux_err)) + + #crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + ecl_l=crd.transform_to(BarycentricTrueEcliptic).lon.value + ecl_b=crd.transform_to(BarycentricTrueEcliptic).lat.value + + erosita_sep_soft=None + erosita_sep_med=None + erosita_sep_hard=None + erosita_match_soft=None + erosita_match_med=None + erosita_match_hard=None + erosita_soft=None + erosita_med=None + erosita_hard=None + erositamatch=hasattr(src, 'erositamatch') + if(erositamatch): + erosita_match_soft=src.erositamatch.match_soft + erosita_match_med=src.erositamatch.match_med + erosita_match_hard=src.erositamatch.match_hard + + erosita_soft=src.erositamatch.ML_FLUX_0_soft + erosita_med=src.erositamatch.ML_FLUX_0_med + erosita_hard=src.erositamatch.ML_FLUX_0_hard + + if(src.erositamatch.RA_fin_soft): + erosita_sep_soft = crd.separation(SkyCoord(src.erositamatch.RA_fin_soft, src.erositamatch.DEC_fin_soft, frame="fk5", unit="deg")).arcsecond + if(src.erositamatch.RA_fin_med): + erosita_sep_med = crd.separation(SkyCoord(src.erositamatch.RA_fin_med, src.erositamatch.DEC_fin_med, frame="fk5", unit="deg")).arcsecond + if(src.erositamatch.RA_fin_hard): + erosita_sep_hard = crd.separation(SkyCoord(src.erositamatch.RA_fin_hard, src.erositamatch.DEC_fin_hard, frame="fk5", unit="deg")).arcsecond + + + gaia_primary=False + gaia_ra=False + gaia_dec=False + gaia_sep=False + if not src.gaia3_primary is None: + gaia_primary=True + gaia_ra=src.gaia3_primary.ra + gaia_dec=src.gaia3_primary.dec + gaia_crd = SkyCoord(gaia_ra, gaia_dec, frame="fk5", unit="deg") + gaia_sep=gaia_crd.separation(crd).arcsecond + + + writer.writerow([ + smart_str(src.name), + smart_str(src.ra), + smart_str(src.dec), + smart_str(src.lii), + smart_str(src.bii), + smart_str(ecl_l), + smart_str(ecl_b), + smart_str(src.ext), + smart_str(src.cnts), + smart_str(src.cnts_err), + smart_str(src.cnts_bg), + smart_str(src.exptime), + smart_str(src.flux), + smart_str(src.fluxerr_lo), + smart_str(src.fluxerr_up), + smart_str(src.sig), + smart_str(src.log_nfalse), + smart_str(src.detlike), + smart_str(src.ml_sig ), + smart_str(src.ml_detlike ), + smart_str(src.ml_nfalse), + smart_str(src.log_ml_nfalse), + smart_str(src.ml_ra), + smart_str(src.ml_dec), + smart_str(src.ml_radec_err_90), + smart_str(src.ml_radec_err_98), + smart_str(src.ml_flux), + smart_str(src.ml_flux_err), + smart_str(src.ml_exp), + smart_str(src.class_id), + smart_str(src.object_class), + smart_str(src.redshift), + smart_str(src.cname), + smart_str(src.category), +# smart_str(src.erosita_data), +# smart_str(erositamatch), +# smart_str(erosita_match_soft), +# smart_str(erosita_match_med), +# smart_str(erosita_match_hard), +# smart_str(erosita_soft), +# smart_str(erosita_med), +# smart_str(erosita_hard), +# smart_str(erosita_sep_soft), +# smart_str(erosita_sep_med), +# smart_str(erosita_sep_hard), + smart_str(src.notes), + smart_str(src.notes_paper), + smart_str(src.follow_up), +# smart_str([bibcode for bibcode in bibcode_szip]), + smart_str(src.turkish), + smart_str(xmm_count), + smart_str([xmm_name for xmm_name in xmm_szip]), + smart_str(src.class_tentative), + smart_str(gaia_primary), + smart_str(gaia_ra), + smart_str(gaia_dec), + smart_str(gaia_sep), + ]) + return response + +def print_head(reg): + reg.append("#\n# SRG/ART-XC X-ray catalog system; Author: Roman Krivonos, email: krivonos@cosmos.ru\n") + reg.append("# (c) Space Research Institute, Moscow, Russia\n#\n") + reg.append("# Generated on {}\n#\n".format(time.strftime("%Y-%m-%d %H:%M:%S"))) + reg.append("""# Description of region colours: +# Green colour represents SRG/ART-XC source with 15 arcsec radius (solid, for convenience) and 40 arcsec radius (dash, selection radius) +# Yellow color shows HEASARC counterparts with _their_ counterparts in magenta colour.\n#\n""") + + +def append_ds9_region_primary(reg,cntp): + if(cntp): + if (cntp.error_radius>0.0 and np.isnan(cntp.error_radius) == False): + reg.append("fk5;circle({}, {}, {}) # color=magenta text={{{}}}\n".format(cntp.ra,cntp.dec,cntp.error_radius/60/60,cntp.name)) + else: + reg.append("fk5;point({}, {}) # point=cross color=magenta text={{{}}}\n".format(cntp.ra,cntp.dec,cntp.name)) + +def HttpResponseDS9_catalog(srcs, key='', print_flux=False): + reg=[] + print_head(reg) + reg.append("# This catalog contains SRGA sources and \"linked\" HEASARC sources with their associated (\"primary\") optical/radio counterparts\n#\n") + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + name = src.cname if(src.cname) else src.name + + if(name == 'NEW X-RAY SOURCE'): + name = src.name + + if(print_flux): + otype=src.get_otype() + otype = "?" if not (otype) else otype + + reg.append("fk5;circle({}, {}, {}) # width=2 color=green text={{{} ({:.1E} {})}}\n".format(src.ra,src.dec,15/60/60,name,src.flux,otype)) + else: + reg.append("fk5;circle({}, {}, {}) # width=2 color=green text={{{}}}\n".format(src.ra,src.dec,15/60/60,name)) + + for heasarc in src.heasarc_linked.all(): + if (heasarc.error_radius>0.0): + reg.append("fk5;circle({}, {}, {}) # color=yellow text={{{}}}\n".format(heasarc.ra, + heasarc.dec, + heasarc.error_radius/60/60, + heasarc.name)) + else: + reg.append("fk5;point({}, {}) # color=yellow text={{{}}}\n".format(heasarc.ra, + heasarc.dec, + heasarc.name)) + + append_ds9_region_primary(reg,heasarc.glimpse_primary) + append_ds9_region_primary(reg,heasarc.vlass_primary) + append_ds9_region_primary(reg,heasarc.gaia3_primary) + append_ds9_region_primary(reg,heasarc.twomass_primary) + append_ds9_region_primary(reg,heasarc.allwise_primary) + + response = HttpResponse(reg,content_type='text/plain') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.reg".format(key,time.strftime("%Y%m%d-%H%M%S")) + return response + +def HttpResponseDS9_catalog_paper(srcs, key=''): + reg=[] + print_head(reg) + reg.append("# This catalog contains SRGA sources\n#\n") + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + name = src.cname if(src.cname) else src.name + + if(name == 'NEW X-RAY SOURCE'): + name = src.name + + reg.append("fk5;circle({}, {}, {}) # width=2 color=green text={{{}}}\n".format(src.ra,src.dec,3/60,name)) + + response = HttpResponse(reg,content_type='text/plain') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.reg".format(key,time.strftime("%Y%m%d-%H%M%S")) + return response + +def HttpResponseDS9_catalog_paper2(srcs, key=''): + reg=[] + print_head(reg) + reg.append("# This catalog contains SRGA sources\n#\n") + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + name = src.cname if(src.cname) else src.name + + if(name == 'NEW X-RAY SOURCE'): + name = src.name + + reg.append("fk5;circle({}, {}, {}) # width=2 color=green\n".format(src.ra,src.dec,3/60)) + reg.append("fk5;text({}, {}) # color=green textrotate=0 text={{{}}}\n".format(src.ra,src.dec,name)) + + response = HttpResponse(reg,content_type='text/plain') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.reg".format(key,time.strftime("%Y%m%d-%H%M%S")) + return response + + +def HttpResponseDS9_catalog_paper_newonly(srcs, key=''): + reg=[] + print_head(reg) + reg.append("# This catalog contains SRGA NEW X-RAY sources\n#\n") + for src in srcs: + name = src.cname if(src.cname) else src.name + if(name == 'NEW X-RAY SOURCE'): + name = src.name + reg.append("fk5;circle({}, {}, {}) # width=2 color=green text={{{}}}\n".format(src.ra,src.dec,3/60,name)) + + response = HttpResponse(reg,content_type='text/plain') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.reg".format(key,time.strftime("%Y%m%d-%H%M%S")) + return response + +def HttpResponseDS9_catalog_paper_known(srcs, key=''): + reg=[] + print_head(reg) + reg.append("# This catalog contains SRGA known sources\n#\n") + for src in srcs: + if not (src.cname == 'NEW X-RAY SOURCE'): + reg.append("fk5;circle({}, {}, {}) # width=2 color=magenta text={{{}}}\n".format(src.ra,src.dec,3/60,src.cname)) + + response = HttpResponse(reg,content_type='text/plain') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.reg".format(key,time.strftime("%Y%m%d-%H%M%S")) + return response + + +def append_ds9_region_m2m(reg,cntps): + for cntp in cntps: + if (cntp.error_radius>0.0): + reg.append("fk5;circle({}, {}, {}) # color=magenta text={{{}}}\n".format(cntp.ra,cntp.dec,cntp.error_radius/60/60,cntp.name)) + else: + reg.append("fk5;point({}, {}) # point=cross color=magenta text={{{}}}\n".format(cntp.ra,cntp.dec,cntp.name)) + +def HttpResponseDS9_source(src, key=''): + reg=[] + print_head(reg) + reg.append("# {} from {} version {}\n#\n".format(src.name,src.survey.name,src.survey.version)) + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + name = src.cname if(src.cname) else src.name + reg.append("fk5;circle({}, {}, {}) # color=green text={{{}}}\n".format(src.ra,src.dec,15/60/60,name)) + reg.append("fk5;circle({}, {}, {}) # width=2 color=green dash=1 text={{{}}}\n".format(src.ra,src.dec,40/60/60,name)) + for heasarc in src.heasarc.all(): + if (heasarc.error_radius>0.0): + reg.append("fk5;circle({}, {}, {}) # color=yellow text={{{}}}\n".format(heasarc.ra, + heasarc.dec, + heasarc.error_radius/60/60, + heasarc.name)) + else: + reg.append("fk5;point({}, {}) # color=yellow text={{{}}}\n".format(heasarc.ra, + heasarc.dec, + heasarc.name)) + append_ds9_region_m2m(reg,heasarc.glimpse.all()) + append_ds9_region_m2m(reg,heasarc.vlass.all()) + append_ds9_region_m2m(reg,heasarc.gaia3.all()) + append_ds9_region_m2m(reg,heasarc.twomass.all()) + append_ds9_region_m2m(reg,heasarc.allwise.all()) + + response = HttpResponse(reg,content_type='text/plain') + response['Content-Disposition'] = 'attachment; filename='+"{}_all_{}.reg".format(src.name.replace(" ", "_"),time.strftime("%Y%m%d-%H%M%S")) + return response + +def HttpResponseDS9_source_linked(src, key=''): + reg=[] + print_head(reg) + reg.append("# {} from {} version {}\n#\n".format(src.name,src.survey.name,src.survey.version)) + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + name = src.cname if(src.cname) else src.name + reg.append("fk5;circle({}, {}, {}) # color=green text={{{}}}\n".format(src.ra,src.dec,15/60/60,name)) + reg.append("fk5;circle({}, {}, {}) # width=2 color=green dash=1 text={{{}}}\n".format(src.ra,src.dec,40/60/60,name)) + for heasarc in src.heasarc_linked.all(): + if (heasarc.error_radius>0.0): + reg.append("fk5;circle({}, {}, {}) # color=yellow text={{{}}}\n".format(heasarc.ra, + heasarc.dec, + heasarc.error_radius/60/60, + heasarc.name)) + else: + reg.append("fk5;point({}, {}) # color=yellow text={{{}}}\n".format(heasarc.ra, + heasarc.dec, + heasarc.name)) + append_ds9_region_primary(reg,heasarc.glimpse_primary) + append_ds9_region_primary(reg,heasarc.vlass_primary) + append_ds9_region_primary(reg,heasarc.gaia3_primary) + append_ds9_region_primary(reg,heasarc.twomass_primary) + append_ds9_region_primary(reg,heasarc.allwise_primary) + + + response = HttpResponse(reg,content_type='text/plain') + response['Content-Disposition'] = 'attachment; filename='+"{}_linked_{}.reg".format(src.name.replace(" ", "_"),time.strftime("%Y%m%d-%H%M%S")) + return response + + + +""" +S1E0_log_nfalse +S1E0_flux +S1E0_err + +S2E0_log_nfalse +S2E0_flux +S2E0_err + +S1S2E0_log_nfalse +S1S2E0_flux +S1S2E0_err + +S1S2E1_log_nfalse +S1S2E1_flux +S1S2E1_err + +S1S2E2_log_nfalse +S1S2E2_flux +S1S2E2_err + +S1S2E3_log_nfalse +S1S2E3_flux +S1S2E3_err + +EXTENDED YES/NO +TYPE +""" + +def HttpResponseCSV_catalog(srcs, key=''): + + s12=get_artsurvey(12.5, archived=True) + s1=get_artsurvey(1.7) + s2=get_artsurvey(2.7) + + e0=get_energy_band('E0') + e1=get_energy_band('E1') + e2=get_energy_band('E2') + e3=get_energy_band('E3') + + + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.csv".format(key,time.strftime("%Y%m%d-%H%M%S")) + writer = csv.writer(response, csv.excel) + writer.writerow([ + smart_str(u"SRCID"), + smart_str(u"NAME"), + smart_str(u"RA"), + smart_str(u"DEC"), + #smart_str(u"GLON"), + #smart_str(u"GLAT"),bg + #smart_str(u"ELON"), + #smart_str(u"ELAT"), + #smart_str(u"S1E0_flux"), + #smart_str(u"S1E0_error"), + #smart_str(u"S1E0_ml"), + #smart_str(u"S2E0_flux"), + #smart_str(u"S2E0_error"), + #smart_str(u"S2E0_ml"), + smart_str(u"SNR"), + smart_str(u"Flux"), + smart_str(u"Flux_err_lo"), + smart_str(u"Flux_err_up"), + #smart_str(u"S1S2E0_ml"), + #smart_str(u"S1S2E1_flux"), + #smart_str(u"S1S2E1_error"), + #smart_str(u"S1S2E1_ml"), + #smart_str(u"S1S2E2_flux"), + #smart_str(u"S1S2E2_error"), + #smart_str(u"S1S2E2_ml"), + #smart_str(u"S1S2E3_flux"), + #smart_str(u"S1S2E3_error"), + #smart_str(u"S1S2E3_ml"), + smart_str(u"EXTENDED"), + smart_str(u"CNAME"), + smart_str(u"NEWXRAY"), + smart_str(u"OTYPE"), + smart_str(u"NOTES"), + ]) + + counter=0 + for src in srcs: + counter=counter+1 + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + ecl_l=crd.transform_to(BarycentricTrueEcliptic).lon.value + ecl_b=crd.transform_to(BarycentricTrueEcliptic).lat.value + lii=crd.transform_to(Galactic).l.value + bii=crd.transform_to(Galactic).b.value + + notes='' + cname='' + otype='' + extended=False + + try: + s1e0=src.artsurveysource_set.all().get(survey=s1,band=e0,catalog__in=[src.catalog]) + S1E0_ml=s1e0.log_nfalse + S1E0_flux=s1e0.flux + S1E0_error=s1e0.flux_err + cname=s.cname + otype=s.get_otype() + except: + S1E0_ml=None + S1E0_flux=None + S1E0_error=None + + try: + s2e0=src.artsurveysource_set.all().get(survey=s2,band=e0,catalog__in=[src.catalog]) + S2E0_ml=s2e0.log_nfalse + S2E0_flux=s2e0.flux + S2E0_error=s2e0.flux_err + cname=s.cname + otype=s.get_otype() + except: + S2E0_ml=None + S2E0_flux=None + S2E0_error=None + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e0,catalog__in=[src.catalog]) + S12E0_ml=s.log_nfalse + S12E0_flux=s.flux + S12E0_err_lo=None + S12E0_err_up=None + cname=s.cname + otype=s.get_otype() + extended = True if (s.ext>1) else False + notes=s.notes_paper + if(notes==None): + notes='' + + if(cname=='NEW X-RAY SOURCE'): + newxray=True + cname='' + else: + newxray=False + + k=1e-12 + err_up = s.fluxerr_up/k + err_lo = s.fluxerr_lo/k + + if (s.ext>1): + extended = True + sign=None + flux0=s.flux/k + + if(flux0>=10 and flux0<100): + flux=round(flux0/10)*10 + if(flux0>=100 and flux0<1000): + flux=round(flux0/100)*100 + if(flux0>=1000): + flux=flux0 + + + S12E0_flux="{:.1f}".format(flux) + else: + extended = False + sign = "{:.1f}".format(s.sig) + S12E0_flux="{:.1f}".format(s.flux/k) + S12E0_err_lo="{:.1f}".format(err_lo) + S12E0_err_up="{:.1f}".format(err_up) + + + except Exception as e: + print("{} {}".format(src,e)) + S12E0_ml=None + S12E0_flux=None + S12E0_err_lo=None + S12E0_err_up=None + sign=None + newxray=False + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e1,catalog__in=[src.catalog]) + S12E1_ml=s.log_nfalse + S12E1_flux=s.flux + S12E1_error=s.flux_err + cname=s.cname + otype=s.get_otype() + except: + S12E1_ml=None + S12E1_flux=None + S12E1_error=None + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e2,catalog__in=[src.catalog]) + S12E2_ml=s.log_nfalse + S12E2_flux=s.flux + S12E2_error=s.flux_err + cname=s.cname + otype=s.get_otype() + except: + S12E2_ml=None + S12E2_flux=None + S12E2_error=None + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e3,catalog__in=[src.catalog]) + S12E3_ml=s.log_nfalse + S12E3_flux=s.flux + S12E3_error=s.flux_err + cname=s.cname + otype=s.get_otype() + except: + S12E3_ml=None + S12E3_flux=None + S12E3_error=None + + + if not cname.isprintable(): + cname="*** NON-PRINTABLE *** {}".format(cname) + + if not notes.isprintable(): + notes="*** NON-PRINTABLE *** {}".format(notes) + + writer.writerow([ + smart_str(counter), + smart_str(src.name), + smart_str(src.ra), + smart_str(src.dec), + #smart_str(lii), + #smart_str(bii), + #smart_str(ecl_l), + #smart_str(ecl_b), + #smart_str(S1E0_flux), + #smart_str(S1E0_error), + #smart_str(S1E0_ml), + #smart_str(S2E0_flux), + #smart_str(S2E0_error), + #smart_str(S2E0_ml), + smart_str(sign), + smart_str(S12E0_flux), + smart_str(S12E0_err_lo), + smart_str(S12E0_err_up), + #smart_str(S12E0_ml), + #smart_str(S12E1_flux), + #smart_str(S12E1_error), + #smart_str(S12E1_ml), + #smart_str(S12E2_flux), + #smart_str(S12E2_error), + #smart_str(S12E2_ml), + #smart_str(S12E3_flux), + #smart_str(S12E3_error), + #smart_str(S12E3_ml), + smart_str(extended), + smart_str(cname), + smart_str(newxray), + smart_str(otype), + smart_str(notes), + ]) + return response + +def HttpResponseCSV_catalog_paper2(srcs, key=''): + + s15=get_artsurvey(12345.14, archived=True) + e5=get_energy_band('E5') + + + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.csv".format(key,time.strftime("%Y%m%d-%H%M%S")) + writer = csv.writer(response, csv.excel) + writer.writerow([ + smart_str(u"SRCID"), + smart_str(u"NAME"), + smart_str(u"ML RA"), + smart_str(u"ML DEC"), + smart_str(u"ML R98"), + smart_str(u"ML SNR"), + smart_str(u"ML LOG_NFALSE"), + smart_str(u"ML Flux"), + smart_str(u"ML Flux_err"), + smart_str(u"CNAME"), + smart_str(u"NEWXRAY"), + smart_str(u"OTYPE"), + ]) + + counter=0 + for src in srcs: + counter=counter+1 + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + ecl_l=crd.transform_to(BarycentricTrueEcliptic).lon.value + ecl_b=crd.transform_to(BarycentricTrueEcliptic).lat.value + lii=crd.transform_to(Galactic).l.value + bii=crd.transform_to(Galactic).b.value + + notes='' + cname='' + otype='' + otype=src.get_otype() + cname=src.cname + + if(src.cname=='NEW X-RAY SOURCE'): + newxray=True + cname='' + else: + newxray=False + + k=1e-12 + + if not cname.isprintable(): + cname="*** NON-PRINTABLE *** {}".format(cname) + + if not notes.isprintable(): + notes="*** NON-PRINTABLE *** {}".format(notes) + + writer.writerow([ + smart_str(counter), + smart_str(src.name), + smart_str(src.ml_ra), + smart_str(src.ml_dec), + smart_str(src.ml_radec_err_98), + smart_str(src.ml_sig), + smart_str(src.log_ml_nfalse), + smart_str(src.ml_flux/k), + smart_str(src.ml_flux_err/k), + smart_str(cname), + smart_str(newxray), + smart_str(otype), + ]) + return response + +def HttpResponseTEX_catalog(srcs, key=''): + + s12=get_artsurvey(12.5, archived=True) + s1=get_artsurvey(1.7) + s2=get_artsurvey(2.7) + + e0=get_energy_band('E0') + e1=get_energy_band('E1') + e2=get_energy_band('E2') + e3=get_energy_band('E3') + + + response = HttpResponse(content_type='application/x-latex') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.tex".format(key,time.strftime("%Y%m%d-%H%M%S")) + writer = csv.writer(response, csv.excel, delimiter='&') + + negative_flux_counter=0 + counter=0 + for src in srcs: + counter+=1 + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + cname='' + otype='' + bands='' + halfs='' + notes_paper='' + extended=False + redshift=0.0 + + k=1e-12 + + # survey 1 + try: + s=src.artsurveysource_set.all().get(survey=s1,band=e0,catalog=src.catalog) + err_up = s.fluxerr_up/k + err_lo = s.fluxerr_lo/k if(s.flux > s.fluxerr_lo) else s.flux + S1E0="${:.1f} -{:.1f} +{:.1f}$".format(s.flux/k,err_lo,err_up) + if(s.redshift): + redshift=s.redshift + cname=s.cname + otype=s.get_otype() + halfs+='1,' + except Exception as e: + print("{} {}".format(src,e)) + S1E0='' + + # survey 2 + try: + s=src.artsurveysource_set.all().get(survey=s2,band=e0,catalog=src.catalog) + S2E0="${:.1f} -{:.1f} +{:.1f}$".format(s.flux/k,s.fluxerr_lo/k,s.fluxerr_up/k) + if(s.redshift): + redshift=s.redshift + cname=s.cname + otype=s.get_otype() + halfs+='2,' + except: + S2E0='' + + # survey 1+2 + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e0,catalog=src.catalog) + err_up = s.fluxerr_up/k + err_lo = s.fluxerr_lo/k + if(s.flux < s.fluxerr_lo): + err_lo = s.flux/k + negative_flux_counter=negative_flux_counter+1 + print("{}: {}, sign: {:.1f}, flux: {:.1f} -{:.1f} +{:.1f}".format(negative_flux_counter,s,s.sig,s.flux/k,s.fluxerr_lo/k,s.fluxerr_up/k)) + + if(s.redshift): + redshift=s.redshift + if(s.notes_paper): + notes_paper='$^{c)}$' + cname=s.cname + otype="{}{}".format(s.get_otype(),notes_paper) + # extended information is only available in S12! + + if (s.ext>1): + extended = True + sign="" + flux0=s.flux/k + if(flux0>=10 and flux0<100): + flux=round(flux0/10)*10 + if(flux0>=100 and flux0<1000): + flux=round(flux0/100)*100 + + S12E0="${{\\sim}}{}$".format(int(flux)) + else: + extended = False + sign = "{:.1f}".format(s.sig) + S12E0="${:.1f} (-{:.1f} +{:.1f})$".format(s.flux/k,err_lo,err_up) + except: + S12E0='' + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e1,catalog=src.catalog) + S12E1="${:.1f} -{:.1f} +{:.1f}$".format(s.flux/k,s.fluxerr_lo/k,s.fluxerr_up/k) + if not redshift>0.0: + redshift=s.redshift + if not cname: + cname=s.cname + if not otype: + otype=s.get_otype() + bands+='1,' + except: + S12E1='' + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e2,catalog=src.catalog) + S12E2="${:.1f} -{:.1f} +{:.1f}$".format(s.flux/k,s.fluxerr_lo/k,s.fluxerr_up/k) + if not redshift>0.0: + redshift=s.redshift + if not cname: + cname=s.cname + if not otype: + otype=s.get_otype() + bands+='2,' + except: + S12E2='' + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e3,catalog__in=[src.catalog]) + S12E3="${:.1f} -{:.1f} +{:.1f}$".format(s.flux/k,s.fluxerr_lo/k,s.fluxerr_up/k) + if not redshift>0.0: + redshift=s.redshift + if not cname: + cname=s.cname + if not otype: + otype=s.get_otype() + bands+='3,' + except: + S12E3='' + + if(bands): + bands="E{}".format(bands)[:-1] + + if(halfs): + halfs="S{}".format(halfs)[:-1] + + name=src.name.replace('SRGA','').replace('-','--') + if(cname=='NEW X-RAY SOURCE'): + name="{{\\bf {} }}".format(name) + cname='' + + if(extended): + name="{}$^{{b)}}$".format(name) + + if(redshift>0.0): + print_redshift="{:.4f}".format(redshift) + else: + print_redshift='' + + writer.writerow([ + smart_str("{:03d}".format(counter)), + smart_str(name), + smart_str("{:.4f}".format(src.ra)), + smart_str("{:.4f}".format(src.dec)), + ##smart_str(S1E0), + ##smart_str(S2E0), + smart_str(sign), + smart_str(S12E0), + ##smart_str(S12E1), + ##smart_str(S12E2), + ##smart_str(S12E3), + #smart_str(bands), + #smart_str(halfs), + smart_str(cname), + smart_str(print_redshift), + smart_str(otype+'\\\\'), + ]) + return response + +def HttpResponseTEX_notes(srcs, key=''): + + s12=get_artsurvey(12.5, archived=True) + s1=get_artsurvey(1.7) + s2=get_artsurvey(2.7) + + e0=get_energy_band('E0') + e1=get_energy_band('E1') + e2=get_energy_band('E2') + e3=get_energy_band('E3') + + + file_data='' + counter=0 + for src in srcs: + counter+=1 + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + notes='' + + # survey 1 + try: + s=src.artsurveysource_set.all().get(survey=s1,band=e0,catalog=src.catalog) + cname=s.notes_paper + except Exception as e: + print("{} {}".format(src,e)) + pass + + # survey 2 + try: + s=src.artsurveysource_set.all().get(survey=s2,band=e0,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + # survey 1+2 + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e0,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e1,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e2,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e3,catalog__in=[src.catalog]) + notes=s.notes_paper + except: + pass + + if(notes): + file_data+="\\subsection*{{ {} }}\n".format(src.name) + file_data+="{}\n".format(notes) + + response = HttpResponse(file_data, content_type='application/text charset=utf-8') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.tex".format(key,time.strftime("%Y%m%d-%H%M%S")) + + return response + +def HttpResponseTEX_notes_paper2(srcs, key=''): + + file_data='' + for src in srcs: + notes='' + notes=src.notes_paper + if(notes): + file_data+="\\subsection*{{ {} }}\n".format(src.name) + file_data+="{}\n".format(notes) + + response = HttpResponse(file_data, content_type='application/text charset=utf-8') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.tex".format(key,time.strftime("%Y%m%d-%H%M%S")) + + return response + +def HttpResponseTEX_catalog_paper2(srcs, key=''): + + response = HttpResponse(content_type='application/x-latex') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.tex".format(key,time.strftime("%Y%m%d-%H%M%S")) + writer = csv.writer(response, csv.excel, delimiter='&') + + negative_flux_counter=0 + counter=0 + for src in srcs: + counter+=1 + crd = SkyCoord(src.ml_ra, src.ml_dec, frame="fk5", unit="deg") + + cname='' + otype='' + bands='' + halfs='' + notes_paper='' + redshift=0.0 + + k=1e-12 + cname=src.cname + otype=src.get_otype() + if(src.redshift): + redshift=src.redshift + + if(src.notes_paper): + notes_paper='$^{c)}$' + cname=src.cname + otype="{}{}".format(src.get_otype(),notes_paper) + + #name=src.name.replace('SRGA','').replace('-','--') + name=src.name.replace('SRGA','').replace('-','$-$') + + if(cname=='NEW X-RAY SOURCE'): + #name="{{\\bf {} }}".format(name) + name="{}$^{{b)}}$".format(name) + cname='' + + if(redshift>0.0): + print_redshift="{:.4f}".format(redshift) + else: + print_redshift='' + + if not cname.isprintable(): + cname="*** NON-PRINTABLE *** {}".format(cname) + + sign_str = '$-$' if(src.ml_dec<0.0) else '' + + writer.writerow([ + smart_str("N{:03d}".format(counter)), + smart_str(name), + smart_str("{:.4f}".format(src.ml_ra)), + smart_str("{}{:.4f}".format(sign_str,abs(src.ml_dec))),# make $-$ !!! + smart_str("{:.1f}".format(src.ml_radec_err_98)), + smart_str("{:.2f}".format(src.ml_sig)), + smart_str("{:.2f} $\\pm$ {:.2f}".format(src.ml_flux/k,src.ml_flux_err/k)), + smart_str(cname), + smart_str(print_redshift), + smart_str(otype+'\\\\'), + ]) + return response + +def HttpResponseTEX_notes(srcs, key=''): + + s12=get_artsurvey(12.5, archived=True) + s1=get_artsurvey(1.7) + s2=get_artsurvey(2.7) + + e0=get_energy_band('E0') + e1=get_energy_band('E1') + e2=get_energy_band('E2') + e3=get_energy_band('E3') + + + file_data='' + counter=0 + for src in srcs: + counter+=1 + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + notes='' + + # survey 1 + try: + s=src.artsurveysource_set.all().get(survey=s1,band=e0,catalog=src.catalog) + cname=s.notes_paper + except Exception as e: + print("{} {}".format(src,e)) + pass + + # survey 2 + try: + s=src.artsurveysource_set.all().get(survey=s2,band=e0,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + # survey 1+2 + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e0,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e1,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e2,catalog=src.catalog) + notes=s.notes_paper + except: + pass + + try: + s=src.artsurveysource_set.all().get(survey=s12,band=e3,catalog__in=[src.catalog]) + notes=s.notes_paper + except: + pass + + if(notes): + file_data+="\\subsection*{{ {} }}\n".format(src.name) + file_data+="{}\n".format(notes) + + response = HttpResponse(file_data, content_type='application/text charset=utf-8') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.tex".format(key,time.strftime("%Y%m%d-%H%M%S")) + + return response + + + +def annotate_separation(srcs, erosita, erosita_crd, source_crd): + srcs = srcs.annotate(separation=Value(0.0, output_field=DecimalField())) + srcs = srcs.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + src.separation=source_crd.separation(crd).arcsecond + if not erosita is None: + src.erosita_separation=erosita_crd.separation(crd).arcsecond + return srcs + + +def get_csv_selection(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseCSV(srcs, key='artsurvey_selection') + +def get_csv_catalog(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + catalog=Catalog.objects.get(year=2021) + bands = EnergyBand.objects.all() + srcs = MetaSource.objects.filter(catalog=catalog).order_by("ra") + return HttpResponseCSV_catalog(srcs, key='artxc_catalog') + +def get_csv_catalog_paper2(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + catalog=Catalog.objects.get(year=2023) + srcs = ArtSurveySource.objects.all().filter(catalog=catalog).order_by("ra") + return HttpResponseCSV_catalog_paper2(srcs, key='artxc_catalog_paper2') + +def get_tex_paper2_catalog(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + catalog=Catalog.objects.get(year=2023) + srcs = ArtSurveySource.objects.all().filter(catalog=catalog).order_by("ra") + return HttpResponseTEX_catalog_paper2(srcs, key='artxc_paper2_catalog') + +def get_tex_paper2_notes(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + catalog=Catalog.objects.get(year=2023) + srcs = ArtSurveySource.objects.all().filter(catalog=catalog).order_by("ra") + return HttpResponseTEX_notes_paper2(srcs, key='artxc_paper2_notes') + +def get_tex_catalog(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + catalog=Catalog.objects.get(year=2021) + bands = EnergyBand.objects.all() + srcs = MetaSource.objects.filter(catalog=catalog).order_by("ra") + + return HttpResponseTEX_catalog(srcs, key='artsurvey_catalog') + +def get_tex_notes(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + catalog=Catalog.objects.get(year=2021) + bands = EnergyBand.objects.all() + srcs = MetaSource.objects.filter(catalog=catalog).order_by("ra") + return HttpResponseTEX_notes(srcs, key='artsurvey_notes') + +def get_csv_all(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseCSV(srcs, key='artsurvey_catalog') + +def get_ds9_all(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseDS9_catalog(srcs, key='artxc_catalog') + +def get_ds9_all_flux(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseDS9_catalog(srcs, key='artxc_catalog_extended',print_flux=True) + +def get_ds9_all_paper(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseDS9_catalog_paper(srcs, key='artxc_catalog_paper') + +def get_ds9_all_paper2(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseDS9_catalog_paper2(srcs, key='artxc_catalog_paper2') + +def get_ds9_all_paper_newonly(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseDS9_catalog_paper_newonly(srcs, key='artxc_catalog_paper_newonly') + +def get_ds9_all_paper_known(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return HttpResponseDS9_catalog_paper_known(srcs, key='artxc_catalog_paper_known') + + +def get_artsurvey_source_ds9(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + return HttpResponseDS9_source(src, key='artxc_source') + +def get_artsurvey_source_ds9_linked(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + return HttpResponseDS9_source_linked(src, key='artxc_source') + + + +""" +def get_sources2(owner): + try: + params = ArtSurveyParams.objects.get(owner=owner) + if not params.sign_threshold is None: + return ArtSurveySource.objects.all().filter(sig__gt=params.sign_threshold) + except ArtSurveyParams.DoesNotExist: + pass + return ArtSurveySource.objects.all() +""" + +def types(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + params = get_object_or_404(ArtSurveyParams, owner=request.user) + survey = params.survey + band = get_object_or_404(EnergyBand, slug=params.band) + + types = HeasarcSimpleClass.objects.all() + return render(request,'artsurvey/types.html', {'types':types,'survey':survey,}) + +def summary(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + surveys = ArtSurvey.objects.all()#.filter(name__startswith="GP").exclude(hidden=True) + bands = EnergyBand.objects.all() + return render(request,'artsurvey/summary.html', {'surveys':surveys,'bands':bands,}) + +""" +def show_turkish(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = ArtSurveySource.objects.all().filter(Q(turkish=True)&Q(survey__hidden=False)) + return render(request,'artsurvey/turkish.html', {'srcs':srcs,'ntotal':srcs.count(),'categories':Category.objects.all(),}) +""" + +def show_turkish(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs=MetaSource.objects.filter(artsurveysource__turkish=True).distinct().order_by("ra") + + #srcs = MetaSource.objects.filter(catalog=catalog).order_by("ra") + return render(request,'artsurvey/turkish.html', {'srcs':srcs,'ntotal':srcs.count(),}) + + +def index(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra").annotate(allwiseagn=Value(0, output_field=IntegerField())) + for src in srcs: + src.allwiseagn=src.heasarc.all().instance_of(HeasarcALLWISEAGN).count() + + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'categories':Category.objects.all(),}) + +def tex(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + srcs = srcs.annotate(refs=Value('', output_field=CharField())) + + refdict = {} + counter=1 + for src in srcs: + if(src.object_class): + #src.otype=fixtype(src.catalog.object_class) + src.otype=src.get_otype() + for ref in src.ads.all(): + if(ref in src.ads_hidden.all()): + continue + """ + if(ref.hidden): + continue + """ + k="\cite{%s}" % ref.bibcode + if k in refdict: + pass + else: + refdict[k]=counter + counter=counter+1 + src.refs=src.refs+"{},".format(refdict[k]) + if(src.refs): + src.refs = src.refs[:-1] + src.refs=src.refs+';' + + bibtex = [] + bibtex_paper = [] + for src in srcs: + for ref in src.ads.all(): + if ref.bibtex: + bibtex.append(ref.bibtex) + if src.bibtex_paper: + bibtex_paper.append(src.bibtex_paper) + + + return render(request,'artsurvey/tex.html', {'srcs':srcs,'ntotal':srcs.count(),'refdict':refdict,'bibtex':bibtex,'bibtex_paper':bibtex_paper}) + +def show_type(request, type_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(object_class__class_id=type_id).order_by("ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'categories':Category.objects.all(),}) + + +def artsurvey_crossmatch(request): + """ Edit user session parameters """ + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + if request.method == 'POST': + form = ArtSurveyMatchParamsForm(request.POST) + if form.is_valid(): + band1 = form.cleaned_data['band1'] + survey1 = form.cleaned_data['survey1'] + band2 = form.cleaned_data['band2'] + survey2 = form.cleaned_data['survey2'] + #log_nfalse_min1 = form.cleaned_data['log_nfalse_min1'] + sign_min1 = form.cleaned_data['sign_min1'] + sign_max1 = form.cleaned_data['sign_max1'] + log_ml_nfalse_min1 = form.cleaned_data['log_ml_nfalse_min1'] + log_ml_nfalse_max1 = form.cleaned_data['log_ml_nfalse_max1'] + #log_nfalse_min2 = form.cleaned_data['log_nfalse_min2'] + sign_min2 = form.cleaned_data['sign_min2'] + sign_max2 = form.cleaned_data['sign_max2'] + log_ml_nfalse_min2 = form.cleaned_data['log_ml_nfalse_min2'] + log_ml_nfalse_max2 = form.cleaned_data['log_ml_nfalse_max2'] + Rmax = form.cleaned_data['Rmax'] + maxdist = form.cleaned_data['maxdist'] + f2f1_ratio_min = form.cleaned_data['f2f1_ratio_min'] + f2f1_ratio_max = form.cleaned_data['f2f1_ratio_max'] + try: + params = ArtSurveyMatchParams.objects.get(owner=request.user) + except ArtSurveyMatchParams.DoesNotExist: + params=form.save(commit=False) + params.owner=request.user + params.save() + + params.survey1=survey1 + params.band1=band1 + params.survey2=survey2 + params.band2=band2 + #params.log_nfalse_min1=log_nfalse_min1 + params.sign_max1=sign_max1 + params.sign_min1=sign_min1 + params.log_ml_nfalse_min1=log_ml_nfalse_min1 + params.log_ml_nfalse_max1=log_ml_nfalse_max1 + #params.log_nfalse_min2=log_nfalse_min2 + params.sign_max2=sign_max2 + params.sign_min2=sign_min2 + params.log_ml_nfalse_min2=log_ml_nfalse_min2 + params.log_ml_nfalse_max2=log_ml_nfalse_max2 + if(Rmax): + params.Rmax=Rmax + else: + params.Rmax=40 + if(maxdist): + params.maxdist=maxdist + else: + params.maxdist=120 + params.f1f2_ratio_min=f2f1_ratio_min + params.f1f2_ratio_max=f2f1_ratio_max + params.save() + + b1 = get_object_or_404(EnergyBand, slug=band1) + b2 = get_object_or_404(EnergyBand, slug=band2) + + srcs1=survey1.artsurveysource_set.all().filter(band=b1) + + """ + if not params.log_ml_nfalse_min is None: + q=q.filter(Q(log_ml_nfalse__lte=params.log_ml_nfalse_min) | Q(ext__gte=1)) + + if not params.log_ml_nfalse_max is None: + q=q.filter(Q(log_ml_nfalse__gt=params.log_ml_nfalse_max) | Q(ext__gte=1)) + """ + + try: + if(sign_min1): + srcs1=srcs1.filter(sig__gt=sign_min1) + if(log_ml_nfalse_min1): + srcs1=srcs1.filter(log_ml_nfalse__lte=log_ml_nfalse_min1) + if(log_ml_nfalse_max1): + srcs1=srcs1.filter(log_ml_nfalse__gt=log_ml_nfalse_max1) + if(sign_max1): + srcs1=srcs1.filter(sig__lt=sign_max1) + except: + pass + + srcs2=survey2.artsurveysource_set.all().filter(band=b2) + + try: + if(sign_min2): + srcs2=srcs2.filter(sig__gt=sign_min2) + if(sign_max1): + srcs2=srcs2.filter(sig__lt=sign_max2) + if(log_ml_nfalse_min2): + srcs2=srcs2.filter(log_ml_nfalse__lte=log_ml_nfalse_min2) + if(log_ml_nfalse_max2): + srcs2=srcs2.filter(log_ml_nfalse__gt=log_ml_nfalse_max2) + except: + pass + + + match_count, match1_count,match2_count,match3_count,match4_count, match_list, missed_count1, missed_list1, missed_count2, missed_list2, match_count2,match1_count2, match2_count2,match3_count2,match4_count2 = crossmatch(srcs1, srcs2, minrad=Rmax, maxdist=maxdist, ratio_min=f2f1_ratio_min, ratio_max=f2f1_ratio_max) + + return render(request, 'artsurvey/crossmatch.html', {'form':form,'params':params, + 'nrows1':srcs1.count(), + 'nrows2':srcs2.count(), + 'match_count':match_count, + 'match_count2':match_count2, + 'match1_count':match1_count, + 'match2_count':match2_count, + 'match3_count':match3_count, + 'match4_count':match4_count, + 'match1_count2':match1_count2, + 'match2_count2':match2_count2, + 'match3_count2':match3_count2, + 'match4_count2':match4_count2, + 'match_list':match_list, + 'missed_count1':missed_count1, + 'missed_list1':missed_list1, + 'missed_count2':missed_count2, + 'missed_list2':missed_list2,}) + else: + pass + + try: + params = ArtSurveyMatchParams.objects.get(owner=request.user) + except ArtSurveyMatchParams.DoesNotExist: + params = ArtSurveyMatchParams() + params.owner=request.user + params.save() + + form = ArtSurveyMatchParamsForm(instance=params) + + return render(request, 'artsurvey/crossmatch.html', {'form': form,'params':params,}) + +def get_heasarc_statistics(srcs): + histogram = {} + for s in srcs: + cnt = s.count_heasarc + if not cnt in histogram.keys(): + histogram[cnt] = {'total': 0,} + histogram[cnt]['total'] += 1 + return OrderedDict(sorted(histogram.items())) + + +def get_reverse_matching_statistics(srcs,class_name): + my_filter = {} + my_filter["{}__isnull".format(class_name)] = False + reverse_match={} + for s in srcs: + already_taken=[] + heasarcs = s.heasarc.filter(**my_filter) + for h in heasarcs: + if not h.name in reverse_match.keys(): + reverse_match[h.name] = {'total': 1,'list': "{}".format(s.get_absolute_url(),s.name)} + already_taken.append(s.pk) + else: + if not s.pk in already_taken: + reverse_match[h.name]['total'] += 1 + reverse_match[h.name]['list'] += ", {}".format(s.get_absolute_url(),s.name) + already_taken.append(s.pk) + return reverse_match + + +def swiftbat105m_filter(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcswiftbat105m__isnull=False).annotate(count_heasarc=Count('heasarc')).order_by("-ra") + #.distinct('name')#.order_by("-ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarcswiftbat105m"),}) +def swiftbat105m_exclude(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).exclude(heasarc__heasarcswiftbat105m__isnull=False).annotate(count_heasarc=Count('heasarc')).order_by("-ra") + #.distinct('name')#.order_by("-ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarcswiftbat105m"),}) +def swiftbat105m_filter_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcswiftbat105m__isnull=False).annotate(count_heasarc=Count('heasarc')).order_by("-ra") + #.distinct('name')#.order_by("-ra") + return HttpResponseCSV(srcs, key='artsurvey_filter_bat105m') + +def swiftbat105m_exclude_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).exclude(heasarc__heasarcswiftbat105m__isnull=False).annotate(count_heasarc=Count('heasarc')).order_by("-ra") + #.distinct('name')#.order_by("-ra") + return HttpResponseCSV(srcs, key='artsurvey_exclude_bat105m') + + +def rass2rxs_filter(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcrass2rxs__isnull=False).annotate(count_heasarc=Count('heasarc')) + #.distinct('name')#.order_by("-ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarcrass2rxs"),}) +def rass2rxs_exclude(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).exclude(heasarc__heasarcrass2rxs__isnull=False).annotate(count_heasarc=Count('heasarc')) + #.distinct('name')#.order_by("-ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarcrass2rxs"),}) +def rass2rxs_filter_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcrass2rxs__isnull=False).annotate(count_heasarc=Count('heasarc')) + return HttpResponseCSV(srcs, key='artsurvey_filter_rass') + +def rass2rxs_exclude_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).exclude(heasarc__heasarcrass2rxs__isnull=False).annotate(count_heasarc=Count('heasarc')) + return HttpResponseCSV(srcs, key='artsurvey_exclude_rass') + +def xmmdr10(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarc4xmmdr10__isnull=False).distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def xmmdr12(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarc4xmmdr12__isnull=False).distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + + +def xmmsl2(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcxmmsl2__isnull=False).annotate(count_heasarc=Count('heasarc')) + srcs = srcs.annotate(count_heasarc=Count('heasarc')) + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarcxmmsl2"),}) + +def intrefcat(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcintrefcat__isnull=False).distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def maxigsc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcmaxigschgl__isnull=False).distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def maxi3(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarc3maxi__isnull=False).distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def maxi7yr(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcmaxi7yr__isnull=False).annotate(count_heasarc=Count('heasarc')) + #.distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarcmaxi7yr"),}) + +def fermi(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarc4fgl__isnull=False).annotate(count_heasarc=Count('heasarc')).order_by("-ra") + #.distinct('name') + + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarc4fgl"),}) + +def exclude_xray(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).exclude(xrays__exact=True) + """ see 00_artsurvey_mark_xrays script """ + #srcs=srcs.exclude(heasarc__heasarc4fgl__isnull=False) + #srcs = srcs.exclude(heasarc__heasarcrass2rxs__isnull=False) + #srcs = srcs.exclude(heasarc__heasarcswiftbat105m__isnull=False) + #srcs = srcs.exclude(heasarc__heasarcxmmsl2__isnull=False) + #srcs = srcs.exclude(heasarc__heasarcmaxi7yr__isnull=False) + #srcs = srcs.exclude(heasarc__heasarcintegral2020__isnull=False).distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + + + +def filter_xray(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(xrays__exact=True).filter(cname__contains='NEW X-RAY SOURCE') + + #.distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def rxte(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcxteasscat__isnull=False).distinct('name') + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def integral2020(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).filter(heasarc__heasarcintegral2020__isnull=False).annotate(count_heasarc=Count('heasarc')).order_by("-ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'histogram':get_heasarc_statistics(srcs), + 'reverse':get_reverse_matching_statistics(srcs,"heasarcintegral2020"),}) + +def index_sort_ra_desc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("-ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_sort_ra_asc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count()}) + +def index_sort_nfalse_desc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("-log_ml_nfalse") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_sort_nfalse_asc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("log_ml_nfalse") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count()}) + + +def index_sort_dec_desc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("-dec") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_sort_dec_asc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).all().order_by("dec") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_sort_sign_desc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("-ml_sig") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_sort_sign_asc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ml_sig") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_sort_flux_desc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("-ml_flux") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_sort_flux_asc(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + srcs = get_sources(request.user).order_by("ml_flux") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def index_catalog(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + catalog=Catalog.objects.get(year=2021) + bands = EnergyBand.objects.all() + srcs = MetaSource.objects.filter(catalog=catalog).order_by("ra") + return render(request,'artsurvey/catalog.html', {'srcs':srcs,'bands':bands,'ntotal':srcs.count(),}) + +def index_catalog_paper2(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + catalog=Catalog.objects.get(year=2023) + #bands = EnergyBand.objects.all() + survey = get_object_or_404(ArtSurvey, version=12345.14) + band = EnergyBand.objects.get(slug='E5') + srcs = ArtSurveySource.objects.all().filter(survey=survey).filter(band=band).filter(catalog=catalog).order_by("ra") + return render(request,'artsurvey/catalog_paper2.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def create_artsurvey(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + try: + src = SkyMapSource.objects.get(pk=source_id) + except: + return HttpResponse("This SkyMapSource was not found") + + if not (src.artsurvey_source): + artsource = ArtSurveySource.objects.create_source(src) + artsource.save() + + srcs = get_sources(request.user) + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),}) + +def show_metasource(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(MetaSource, pk=source_id) + srcs = src.artsurveysource_set.all().order_by("-modified")#.order_by("survey__version") + bands = EnergyBand.objects.all() + comment_form = MetaCommentForm() + comments = src.comments.all() + if request.method == 'POST': + comment_form = MetaCommentForm(data=request.POST) + if comment_form.is_valid(): + new_comment = comment_form.save(commit=False) + new_comment.metasource = src + new_comment.owner = request.user + new_comment.save() + else: + pass + comment_form = MetaCommentForm() + return render(request,'artsurvey/metasource.html', {'comments':comments,'comment_form':comment_form,'src':src,'srcs':srcs,'bands':bands,}) + +def show_metasource_band(request, source_id, band_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(MetaSource, pk=source_id) + band = get_object_or_404(EnergyBand, pk=band_id) + srcs = src.artsurveysource_set.all().filter(band=band).order_by("survey__version") + bands = EnergyBand.objects.all() + comment_form = MetaCommentForm() + comments = src.comments.all() + if request.method == 'POST': + comment_form = MetaCommentForm(data=request.POST) + if comment_form.is_valid(): + new_comment = comment_form.save(commit=False) + new_comment.metasource = src + new_comment.owner = request.user + new_comment.save() + else: + comment_form = MetaCommentForm() + return render(request,'artsurvey/metasource.html', {'comments':comments,'comment_form':comment_form,'src':src,'srcs':srcs,'bands':bands,}) + + +def markbad_artsurvey_source(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + write_history(request, 'Source {} MARK BAD'.format(src)) + bad_category = get_object_or_404(Category,slug='false') + src.category=bad_category + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + #return HttpResponseRedirect(reverse('artsurvey index')) + +def delete_artsurvey_source(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + write_history(request, 'Source {} DELETE'.format(src)) + src.delete() + #bad_category = get_object_or_404(Category,slug='false') + #src.category=bad_category + #src.save() + #return HttpResponseRedirect(src.get_absolute_url()) + return HttpResponseRedirect(reverse('artsurvey index')) + +def delete_metasource(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(MetaSource, pk=source_id) + write_history(request, 'MetaSource {} DELETE'.format(src)) + src.delete() + return HttpResponseRedirect(reverse('artsurvey index')) + +def delete_paper_metasource(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(MetaSource, pk=source_id) + write_history(request, 'MetaSource {} DELETE Paper'.format(src)) + srcs = src.artsurveysource_set.all().order_by("survey__version") + bands = EnergyBand.objects.all() + src.catalog = None + src.save() + return render(request,'artsurvey/metasource.html', {'src':src,'srcs':srcs,'bands':bands,}) + + + +def show_artsurvey_source(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + c = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + try: + erosita = src.erositamatch + erosita_crd = SkyCoord(erosita.RA_fin_soft, erosita.DEC_fin_soft, frame="fk5", unit="deg") + except Exception as e: + print("{} {}".format(src,e)) + erosita = None + erosita_crd = None + + heasarcs = src.heasarc.all().annotate(separation=Value(0.0, output_field=DecimalField())) + heasarcs = heasarcs.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for heasarc in heasarcs: + c0 = SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg") + heasarc.separation=c.separation(c0).arcsecond + if not erosita is None: + heasarc.erosita_separation=erosita_crd.separation(c0).arcsecond + + + simbad = src.simbad.all().annotate(separation=Value(0.0, output_field=DecimalField())) + simbad = simbad.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for simsrc in simbad: + simbad_crd=SkyCoord(simsrc.ra, simsrc.dec, frame="fk5", unit="deg") + simsrc.separation=c.separation(simbad_crd).arcsecond + if not erosita is None: + simsrc.erosita_separation=erosita_crd.separation(simbad_crd).arcsecond + + allwise = src.allwise.all().annotate(separation=Value(0.0, output_field=DecimalField())) + allwise = allwise.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for asrc in allwise: + allwise_crd = SkyCoord(asrc.ra, asrc.dec, frame="fk5", unit="deg") + asrc.separation=c.separation(allwise_crd).arcsecond + if not erosita is None: + asrc.erosita_separation=erosita_crd.separation(allwise_crd).arcsecond + + first = annotate_separation(src.first.all(), erosita, erosita_crd, c) + nvss = annotate_separation(src.nvss.all(), erosita, erosita_crd, c) + sumss = annotate_separation(src.sumss.all(), erosita, erosita_crd, c) + vlass = annotate_separation(src.vlass.all(), erosita, erosita_crd, c) + bj2021 = annotate_separation(src.bj2021.all(), erosita, erosita_crd, c) + gaiadr3 = annotate_separation(src.gaiadr3.all(), erosita, erosita_crd, c) + + wise = src.wisecounterpart_set.all().order_by('sep') + wise = wise.annotate(separation=Value(0.0, output_field=DecimalField())) + wise = wise.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for w in wise: + if not (w.raStack < -90.0 or w.decStack < -90.0): + wise_crd = SkyCoord(w.raStack, w.decStack, frame="fk5", unit="deg") + w.separation=c.separation(wise_crd).arcsecond + if not erosita is None: + w.erosita_separation=erosita_crd.separation(wise_crd).arcsecond + + gaia = src.gaia3.all() + + gaia_ex = gaia.exclude(Q(pmra__abs__gt=F("pmra_error")*Value(4.)) | Q(pmdec__abs__gt=F("pmdec_error")*Value(4.)) | Q(parallax__abs__gt=F("parallax_error")*Value(4.))) + gaia_ex = gaia_ex.annotate(separation=Value(0.0, output_field=DecimalField())) + gaia_ex = gaia_ex.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for gaiasrc in gaia_ex: + gaia_crd=SkyCoord(gaiasrc.ra, gaiasrc.dec, frame="fk5", unit="deg") + gaiasrc.separation=c.separation(SkyCoord(gaiasrc.ra, gaiasrc.dec, frame="fk5", unit="deg")).arcsecond + if not erosita is None: + gaiasrc.erosita_separation=erosita_crd.separation(gaia_crd).arcsecond + + gaia_pm = gaia.filter(Q(pmra__abs__gt=F("pmra_error")*Value(4.)) | Q(pmdec__abs__gt=F("pmdec_error")*Value(4.))) + gaia_pm = gaia_pm.annotate(separation=Value(0.0, output_field=DecimalField())) + gaia_pm = gaia_pm.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for gaiasrc in gaia_pm: + gaia_crd=SkyCoord(gaiasrc.ra, gaiasrc.dec, frame="fk5", unit="deg") + gaiasrc.separation=c.separation(gaia_crd).arcsecond + if not erosita is None: + gaiasrc.erosita_separation=erosita_crd.separation(gaia_crd).arcsecond + + gaia_plx = gaia.filter(Q(parallax__abs__gt=F("parallax_error")*Value(4.))) + gaia_plx = gaia_plx.annotate(separation=Value(0.0, output_field=DecimalField())) + gaia_plx = gaia_plx.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for gaiasrc in gaia_plx: + gaia_crd=SkyCoord(gaiasrc.ra, gaiasrc.dec, frame="fk5", unit="deg") + gaiasrc.separation=c.separation(SkyCoord(gaiasrc.ra, gaiasrc.dec, frame="fk5", unit="deg")).arcsecond + if not erosita is None: + gaiasrc.erosita_separation=erosita_crd.separation(gaia_crd).arcsecond + + """ find original source """ + try: + orig = SkyMapSource.objects.get(artsurvey_source=src) + #plate = get_object_or_404(SkyMaps, SMAPNR=orig.skymap.SMAPNR) + #neib=[orig.skymap.SMAPNR] + #for p in plate.neighbours.all(): + # neib.append(p.SMAPNR) + srcs = get_sources_sorted(request.user, src) + heasarc_around = get_heasarc_around(request.user, src) + ##s1 = ArtSurvey.objects.get(version=1.7) + ##s2 = ArtSurvey.objects.get(version=2.7) + ##s12 = ArtSurvey.objects.get(Q(version=12.5) & Q(archived=True)) + # all_sources = ArtSurveySource.objects.filter(survey__in=[s1,s2,s12]).filter(SMAPNR__in=neib).order_by("ra") + all_sources=[] + + #all_sources0 = ArtSurveySource.objects.filter(SMAPNR__in=neib).order_by("ra") + #for s in all_sources0: + # s_crd=SkyCoord(s.ra, s.dec, frame="fk5", unit="deg") + # if(s_crd.separation(c).arcmin < 60): + # all_sources.append(s) + #print("Heasarc around {}".format(heasarc_around.count())) + except Exception as e: + print("{} --> {}".format(src,e)) + orig = None + srcs = None + all_sources = None + heasarc_around = None + + if(orig==None and src.parent): + """ if orig is not found, take it from parent """ + try: + orig = SkyMapSource.objects.get(artsurvey_source=src.parent) + plate = get_object_or_404(SkyMaps, SMAPNR=orig.skymap.SMAPNR) + except Exception as e: + print("{} --> {}".format(src,e)) + pass + + if(src.metasource): + metasource_list = src.metasource.artsurveysource_set.all()#.sort_by("version") + else: + metasource_list=None + + + skyview = OrderedDict(sorted(SkyView.survey_dict.items())) + + + basketform=UserArtBasketListForm() + basketform.fields["folder"].queryset=request.user.artbasket_set.all() + baskets = src.artbasket_set.all().filter(owner=request.user) + + if request.method == 'POST': # If the form has been submitted... + form = UserArtBasketListForm(request.POST) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + selection = form.cleaned_data['folder'] + selection.sources.add(src) + pass + #return HttpResponseForbidden("{}".format(baskets)) + else: + pass + + + return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,'wise':wise,'srcs': srcs, + 'basketform': basketform,'baskets': baskets, + 'all_sources': all_sources, + 'erosita':erosita, + 'heasarcs':heasarcs,'heasarc_around':heasarc_around, + 'simbad':simbad,'bj2021':bj2021,'gaiadr3':gaiadr3, + 'allwise':allwise, + 'nvss':nvss,'first':first,'sumss':sumss,'vlass':vlass, + 'gaia_ex':gaia_ex, + 'gaia_pm':gaia_pm, + 'gaia_plx':gaia_plx, + 'orig':orig,'skyview':skyview, + 'metasource_list':metasource_list,}) + +def type_artsurvey_source(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + try: + src = ArtSurveySource.objects.get(pk=source_id) + except: + return HttpResponse("This ArtSurveySource was not found") + + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + +def set_galplane_survey(request, survey_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + survey = get_object_or_404(ArtSurvey, pk=survey_id) + params = get_object_or_404(ArtSurveyParams, owner=request.user) + params.survey=survey + params.save() + return HttpResponseRedirect(reverse('artsurvey index')) + + + + +def clean_class(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.class_id = 0 + src.object_class = None + src.save() + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + +def mark_tentative(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.class_tentative=True + src.save() + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + +def remove_tentative(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.class_tentative=False + src.save() + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + +def mark_newxray(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.cname='NEW X-RAY SOURCE' + src.save() + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + +def remove_newxray(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.cname=None + src.save() + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + + +def mark_turkish(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.turkish=True + src.turkish_date=timezone.localtime(timezone.now()) + src.save() + + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + +def remove_turkish(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.turkish=False + src.turkish_date=None + src.save() + #return render(request,'artsurvey/show_artsurvey_source.html', {'src':src,}) + return HttpResponseRedirect(src.get_absolute_url()) + +def ajax_autocomplete(request): + if request.is_ajax(): + q = request.GET.get('term', '').upper() + search_qs = HeasarcSimpleClass.objects.filter(class_name__startswith=q) + results = [] + print(q) + for r in search_qs: + results.append(r.class_name) + data = json.dumps(results) + else: + data = 'fail' + mimetype = 'application/json' + return HttpResponse(data, mimetype) + +def update_type(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': + text=request.POST['txtSearch'] + try: + object_class = HeasarcSimpleClass.objects.get(class_name=text) + except: + return HttpResponse("Error when trying to find HeasarcSimpleClass {}".format(text)) + src.object_class=object_class + src.class_id=object_class.class_id + src.owner=request.user + src.save() + print("CHANGE TYPE: >{}<".format(text)) + return HttpResponseRedirect(src.get_absolute_url()) + else: + return HttpResponseRedirect(reverse('art survey index')) + +def notes(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': + form = NotesForm(request.POST) + if form.is_valid(): + src.notes=form.cleaned_data['notes'] + src.notes_paper=form.cleaned_data['notes_paper'] + src.bibtex_paper=form.cleaned_data['bibtex_paper'] + src.follow_up=form.cleaned_data['follow_up'] + src.redshift=form.cleaned_data['redshift'] + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + pass + initial = {'redshift': src.redshift, 'notes': src.notes,'notes_paper': src.notes_paper, 'bibtex_paper':src.bibtex_paper,'follow_up': src.follow_up} + form = NotesForm(initial=initial) + return render(request, 'artsurvey/notes.html', {'form': form,'src':src,}) + +def notes_survey(request, survey_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + survey = get_object_or_404(ArtSurvey, pk=survey_id) + + if request.method == 'POST': + form = NotesSurveyForm(request.POST) + if form.is_valid(): + survey.notes=form.cleaned_data['notes'] + survey.save() + return HttpResponseRedirect(reverse('artsurvey index')) + else: + pass + initial = {'notes': survey.notes,} + form = NotesSurveyForm(initial=initial) + return render(request, 'artsurvey/notes.html', {'form': form,}) + + +def cname(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': + form = CnameForm(request.POST) + if form.is_valid(): + src.cname=form.cleaned_data['cname'] + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + pass + initial = {'cname': src.cname} + form = CnameForm(initial=initial) + return render(request, 'artsurvey/cname.html', {'form': form,'src':src,}) + +def update_artsurveysource(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': + form = ArtSurveySourceForm(request.POST) + if form.is_valid(): + src.name=form.cleaned_data['name'] + src.ra=form.cleaned_data['ra'] + src.dec=form.cleaned_data['dec'] + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + src.ecl_l=crd.transform_to(BarycentricTrueEcliptic).lon.value + src.ecl_b=crd.transform_to(BarycentricTrueEcliptic).lat.value + src.lii = crd.galactic.l.value + src.bii = crd.galactic.b.value + src.flux=form.cleaned_data['flux'] + src.fluxerr_lo=form.cleaned_data['fluxerr_lo'] + src.fluxerr_up=form.cleaned_data['fluxerr_up'] + src.ext=form.cleaned_data['ext'] + src.nfalse=form.cleaned_data['nfalse'] + src.sig=form.cleaned_data['sig'] + log10 = -100.0 + if(src.nfalse > 0.0): + src.log_nfalse = math.log10(src.nfalse) + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + pass + initial = {'name': src.name,'ra':src.ra,'dec':src.dec, + 'flux':src.flux,'fluxerr_lo':src.fluxerr_lo,'fluxerr_up':src.fluxerr_up, + 'nfalse':src.nfalse,'sig':src.sig} + form = ArtSurveySourceForm(initial=initial) + return render(request, 'artsurvey/update_artsurvey_source.html', {'form': form,'src':src,}) + + +def artsurvey_params(request): + """ Edit user session parameters """ + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + if request.method == 'POST': + form = ArtSurveyParamsForm(request.POST) + if form.is_valid(): + gaia_primary = form.cleaned_data['gaia_primary'] + allwise_primary = form.cleaned_data['allwise_primary'] + log_nfalse_min = form.cleaned_data['log_nfalse_min'] + log_nfalse_max = form.cleaned_data['log_nfalse_max'] + log_ml_nfalse_min = form.cleaned_data['log_ml_nfalse_min'] + log_ml_nfalse_max = form.cleaned_data['log_ml_nfalse_max'] + detlike_min = form.cleaned_data['detlike_min'] + detlike_max = form.cleaned_data['detlike_max'] + dec_min = form.cleaned_data['dec_min'] + dec_max = form.cleaned_data['dec_max'] + sign_ml_min = form.cleaned_data['sign_ml_min'] + sign_ml_max = form.cleaned_data['sign_ml_max'] + exposure_min = form.cleaned_data['exposure_min'] + ext_min = form.cleaned_data['ext_min'] + ext_max = form.cleaned_data['ext_max'] + class_startswith = form.cleaned_data['class_startswith'] + cname_contains = form.cleaned_data['cname_contains'] + sky = form.cleaned_data['sky'] + marshall = form.cleaned_data['marshall'] + band = form.cleaned_data['band'] + exclude_band = form.cleaned_data['exclude_band'] + exclude_log_nfalse = form.cleaned_data['exclude_log_nfalse'] + exclude_log_ml_nfalse = form.cleaned_data['exclude_log_ml_nfalse'] + survey = form.cleaned_data['survey'] + exclude_survey = form.cleaned_data['exclude_survey'] + category = form.cleaned_data['category'] + exclude_category = form.cleaned_data['exclude_category'] + category_unclassified = form.cleaned_data['category_unclassified'] + ecl_lat_min = form.cleaned_data['ecl_lat_min'] + ecl_lat_max = form.cleaned_data['ecl_lat_max'] + glat_min = form.cleaned_data['glat_min'] + glat_max = form.cleaned_data['glat_max'] + turk_possible = form.cleaned_data['turk_possible'] + try: + params = ArtSurveyParams.objects.get(owner=request.user) + except ArtSurveyParams.DoesNotExist: + params=form.save(commit=False) + params.owner=request.user + params.save() + + params.dec_min = dec_min + params.dec_max = dec_max + params.ext_min = ext_min + params.ext_max = ext_max + params.detlike_min = detlike_min + params.detlike_max = detlike_max + params.log_nfalse_min = log_nfalse_min + params.log_nfalse_max = log_nfalse_max + params.log_ml_nfalse_min = log_ml_nfalse_min + params.log_ml_nfalse_max = log_ml_nfalse_max + params.exposure_min = exposure_min + params.sign_ml_min = sign_ml_min + params.sign_ml_max = sign_ml_max + params.class_startswith = class_startswith + params.cname_contains = cname_contains + params.survey=survey + params.exclude_survey=exclude_survey + params.category=category + params.exclude_category=exclude_category + params.category_unclassified=category_unclassified + params.sky=sky + params.marshall=marshall + params.band=band + params.gaia_primary=gaia_primary + params.allwise_primary=allwise_primary + params.exclude_band=exclude_band + params.exclude_log_nfalse=exclude_log_nfalse + params.exclude_log_ml_nfalse=exclude_log_ml_nfalse + params.ecl_lat_min = ecl_lat_min + params.ecl_lat_max = ecl_lat_max + params.glat_min = glat_min + params.glat_max = glat_max + params.turk_possible = turk_possible + params.save() + + ntotal = get_sources(request.user).count() + return render(request, 'artsurvey/params.html', {'form':form,'params':params,'ntotal':ntotal,}) + else: + #form = UserSessionParametersForm() + pass + + try: + params = ArtSurveyParams.objects.get(owner=request.user) + except ArtSurveyParams.DoesNotExist: + params = ArtSurveyParams() + params.owner=request.user + params.save() + + form = ArtSurveyParamsForm(instance=params) + + return render(request, 'artsurvey/params.html', {'form': form,'params':params,}) + +def artsurvey_params_reset(request): + """ Edit user session parameters """ + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + params = get_object_or_404(ArtSurveyParams, owner=request.user) + params.ext_min = ArtSurveyParams._meta.get_field('ext_min').get_default() + params.ext_max = ArtSurveyParams._meta.get_field('ext_max').get_default() + params.dec_min = ArtSurveyParams._meta.get_field('dec_min').get_default() + params.dec_max = ArtSurveyParams._meta.get_field('dec_max').get_default() + params.log_nfalse_min = ArtSurveyParams._meta.get_field('log_nfalse_min').get_default() + params.log_nfalse_max = ArtSurveyParams._meta.get_field('log_nfalse_max').get_default() + params.log_ml_nfalse_min = ArtSurveyParams._meta.get_field('log_ml_nfalse_min').get_default() + params.log_ml_nfalse_max = ArtSurveyParams._meta.get_field('log_ml_nfalse_max').get_default() + params.detlike_min = ArtSurveyParams._meta.get_field('detlike_min').get_default() + params.detlike_max = ArtSurveyParams._meta.get_field('detlike_max').get_default() + params.sign_ml_min = ArtSurveyParams._meta.get_field('sign_ml_min').get_default() + params.sign_ml_max = ArtSurveyParams._meta.get_field('sign_ml_max').get_default() + params.exposure_min = ArtSurveyParams._meta.get_field('exposure_min').get_default() + params.class_startswith = ArtSurveyParams._meta.get_field('class_startswith').get_default() + params.cname_contains = ArtSurveyParams._meta.get_field('cname_contains').get_default() + params.exclude_survey = ArtSurveyParams._meta.get_field('exclude_survey').get_default() + params.category = ArtSurveyParams._meta.get_field('category').get_default() + params.exclude_category = ArtSurveyParams._meta.get_field('exclude_category').get_default() + params.category_unclassified = ArtSurveyParams._meta.get_field('category_unclassified').get_default() + params.sky = ArtSurveyParams._meta.get_field('sky').get_default() + params.marshall = ArtSurveyParams._meta.get_field('marshall').get_default() + params.band = ArtSurveyParams._meta.get_field('band').get_default() + params.gaia_primary = ArtSurveyParams._meta.get_field('gaia_primary').get_default() + params.allwise_primary = ArtSurveyParams._meta.get_field('allwise_primary').get_default() + params.exclude_band = ArtSurveyParams._meta.get_field('exclude_band').get_default() + params.exclude_log_nfalse = ArtSurveyParams._meta.get_field('exclude_log_nfalse').get_default() + params.ecl_lat_min = ArtSurveyParams._meta.get_field('ecl_lat_min').get_default() + params.ecl_lat_max = ArtSurveyParams._meta.get_field('ecl_lat_max').get_default() + params.glat_min = ArtSurveyParams._meta.get_field('glat_min').get_default() + params.glat_max = ArtSurveyParams._meta.get_field('glat_max').get_default() + params.turk_possible = ArtSurveyParams._meta.get_field('turk_possible').get_default() + params.save() + return HttpResponseRedirect(reverse('artsurvey params')) + + #form = ArtSurveyParamsForm(instance=params) + #ntotal = get_sources(request.user).count() + #return render(request, 'artsurvey/params.html', {'form':form,'params':params,'ntotal':ntotal,}) + + +def counterpart(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': + form = OpticalCounterpartForm(request.POST) + if form.is_valid(): + c=form.save(commit=False) + c.source=src + c.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + pass + + form = OpticalCounterpartForm() + return render(request, 'artsurvey/counterpart.html', {'form': form,'src':src,}) + +def show_category(request,category_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + category = get_object_or_404(Category, pk=category_id) + srcs = get_sources(request.user).filter(category=category).order_by("ra") + return render(request,'artsurvey/index.html', {'srcs':srcs,'ntotal':srcs.count(),'categories':Category.objects.all(),}) + +def update_category(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': # If the form has been submitted... + form = CategoryForm(request.POST,instance=src) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save(commit=False) + category = form.cleaned_data['category'] + new_form.category=category + new_form.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + return render(request,'artsurvey/category.html', {'form':form,'src':src,}) + + else: + form = CategoryForm(instance=src) + return render(request,'artsurvey/category.html', {'form':form,'src':src,}) + +def vizier(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + return render(request, 'artsurvey/vizier.html', {'src':src,}) + +def set_artsurvey_gaia(request,source_id,gaia_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.gaia3_primary = get_object_or_404(GAIADR3, pk=gaia_id) + src.save() + + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_allwise(request,source_id,allwise_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + allwise = get_object_or_404(AllWise, pk=allwise_id) + src.allwise_primary=allwise + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_nvss(request,source_id,nvss_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + nvss=get_object_or_404(NVSS, pk=nvss_id) + src.nvss_primary=nvss + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_vlass(request,source_id,vlass_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + vlass=get_object_or_404(VLASSfromVizieR, pk=vlass_id) + src.vlass_primary=vlass + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_gaiadr3(request,source_id,gaiadr3_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + gaiadr3=get_object_or_404(GAIADR3fromVizieR, pk=gaiadr3_id) + src.gaiadr3_primary=gaiadr3 + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_bj2021(request,source_id,bj2021_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + bj2021=get_object_or_404(BJfromVizieR, pk=bj2021_id) + src.bj2021_primary=bj2021 + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_first(request,source_id,first_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.first_primary=get_object_or_404(FIRST, pk=first_id) + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_sumss(request,source_id,sumss_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.sumss_primary=get_object_or_404(SUMSS, pk=sumss_id) + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def set_artsurvey_wise(request,source_id,wise_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + wise = get_object_or_404(WiseCounterpart, pk=wise_id) + wise_all = src.wisecounterpart_set.all() + for w in wise_all: + w.primary=False + w.save() + wise.primary=True + wise.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def clean_artsurvey_wise(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + wise_all = src.wisecounterpart_set.all() + for w in wise_all: + w.primary=False + w.save() + return HttpResponseRedirect(src.get_absolute_url()) + + +def clean_artsurvey_gaia(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.gaia3_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + + +def clean_artsurvey_allwise(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.allwise_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def clean_artsurvey_nvss(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.nvss_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def clean_artsurvey_gaiadr3(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.gaiadr3_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def clean_artsurvey_bj2021(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.bj2021_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def clean_artsurvey_vlass(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.vlass_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def clean_artsurvey_first(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.first_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + + +def clean_artsurvey_sumss(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.sumss_primary=None + src.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def other_name_add(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': + form = OtherNameForm(request.POST) + if form.is_valid(): + oname=form.save(commit=False) + oname.source=src + oname.owner=request.user + oname.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + pass + + form = OtherNameForm() + return render(request, 'artsurvey/other_name.html', {'form': form,'src':src,}) + + +def other_name_delete(request, othername_id): + oname = get_object_or_404(OtherName, pk=othername_id) + src=oname.source + oname.delete() + return HttpResponseRedirect(src.get_absolute_url()) + +def other_name_update(request, othername_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + oname = get_object_or_404(OtherName, pk=othername_id) + src=oname.source + + if request.method == 'POST': + form = OtherNameForm(request.POST) + if form.is_valid(): + oname.name=form.cleaned_data['name'] + oname.owner=request.user + oname.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + pass + initial = {'name':oname.name,} + form = OtherNameForm(initial=initial) + return render(request, 'artsurvey/other_name.html', {'form': form,'src':src,}) + +def link_heasarc(request,source_id,heasarc_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + heasarc = get_object_or_404(HeasarcBase, pk=heasarc_id) + src.heasarc_linked.add(heasarc) + src.save() + if(heasarc.ads): + src.ads.add(heasarc.ads) + + return HttpResponseRedirect(src.get_absolute_url()) + +def unlink_heasarc(request,source_id,heasarc_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + heasarc = get_object_or_404(HeasarcBase, pk=heasarc_id) + src.heasarc_linked.remove(heasarc) + src.save() + if(heasarc.ads): + src.ads.remove(heasarc.ads) + + return HttpResponseRedirect(src.get_absolute_url()) + +def unlink_heasarc_all(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + src.heasarc_linked.clear() + src.save() + + return HttpResponseRedirect(src.get_absolute_url()) + +def load_skyview_image(request, source_id, survey): + src = get_object_or_404(ArtSurveySource, pk=source_id) + print("Loading SkyView survey {}".format(survey)) + + try: + im = skyview_query(src.ra, src.dec, size=1440, survey=survey) + except Exception as e: + print("{} {}".format(src,e)) + return HttpResponseRedirect(src.get_absolute_url()) + + upload = Upload(owner=request.user,title="{}".format(survey),filename="skyview_image.fits", + notes="Loaded from the SkyView server via astroquery") + new_file = ContentFile(im) + new_file.name = upload.filename + upload.filefield = new_file + upload.save() + upload.artsources.add(src) + + return HttpResponseRedirect(src.get_absolute_url()) + +def load_ps1_color_image(request, source_id): + src = get_object_or_404(ArtSurveySource, pk=source_id) + try: + im = ps1_getcolorim(src.ra, src.dec, size=1440) + except Exception as e: + print("{} {}".format(src,e)) + return HttpResponseRedirect(src.get_absolute_url()) + upload = Upload(owner=request.user,title='PS1 color image',filename="ps1_color_image.jpg", + notes="Loaded from the PS1 image server") + new_file = ContentFile(im) + new_file.name = upload.filename + upload.filefield = new_file + upload.save() + upload.artsources.add(src) + return HttpResponseRedirect(src.get_absolute_url()) + +def load_ps1_gray_image(request, source_id): + src = get_object_or_404(ArtSurveySource, pk=source_id) + try: + im = ps1_getgrayim(src.ra, src.dec, size=1440) + except Exception as e: + print("{} {}".format(src,e)) + return HttpResponseRedirect(src.get_absolute_url()) + upload = Upload(owner=request.user,title='PS1 gray image',filename="ps1_gray_image.jpg", + notes="Loaded from the PS1 image server") + new_file = ContentFile(im) + new_file.name = upload.filename + upload.filefield = new_file + upload.save() + upload.artsources.add(src) + return HttpResponseRedirect(src.get_absolute_url()) + +def load_ps1_fits_image(request, source_id, filtername): + src = get_object_or_404(ArtSurveySource, pk=source_id) + try: + im = ps1_fitsim(src.ra, src.dec, size=1440, format="fits", filter=filtername) + except Exception as e: + print("{} {}".format(src,e)) + return HttpResponseRedirect(src.get_absolute_url()) + upload = Upload(owner=request.user, + title="PS1 filter {}".format(filtername), + filename="ps1_image_{}.fits".format(filtername), + notes="Loaded from the PS1 image server") + new_file = ContentFile(im) + new_file.name = upload.filename + upload.filefield = new_file + upload.save() + upload.artsources.add(src) + return HttpResponseRedirect(src.get_absolute_url()) + + + +def artsource_upload(request, source_id): + src = get_object_or_404(ArtSurveySource, pk=source_id) + + if request.method == 'POST': + form = UploadForm(request.POST, request.FILES) + if form.is_valid(): + upload = form.save(commit=False) + upload.owner=request.user + upload.save() + upload.artsources.add(src) + """ + if(src.metasource): + upload.metasource=src.metasource + """ + upload.title=form.cleaned_data['title'] + upload.filename = request.FILES['filefield'].name#.encode('utf-8') + upload.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + form = UploadForm() + + return render(request,'artsurvey/upload.html', {'source':src, 'form': form,}) + +def artsource_upload_survey(request, survey_id): + survey = get_object_or_404(ArtSurvey, pk=survey_id) + + if request.method == 'POST': + form = UploadForm(request.POST, request.FILES) + if form.is_valid(): + upload = form.save(commit=False) + upload.owner=request.user + upload.save() + upload.survey = survey + upload.title=form.cleaned_data['title'] + upload.filename = request.FILES['filefield'].name#.encode('utf-8') + upload.save() + return HttpResponseRedirect(reverse('artsurvey catalog')) + else: + form = UploadForm() + + return render(request,'artsurvey/upload.html', {'src':survey, 'form': form,}) + +def metasource_upload(request, source_id): + src = get_object_or_404(MetaSource, pk=source_id) + + if request.method == 'POST': + form = UploadForm(request.POST, request.FILES) + if form.is_valid(): + upload = form.save(commit=False) + upload.owner=request.user + upload.save() + upload.metasource=src + upload.title=form.cleaned_data['title'] + upload.filename = request.FILES['filefield'].name#.encode('utf-8') + upload.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + form = UploadForm() + + return render(request,'artsurvey/upload.html', {'src':src, 'form': form,}) + +def set_primary_upload_survey(request, survey_id, upload_id): + survey = get_object_or_404(ArtSurvey, pk=survey_id) + for upload in survey.uploads.all(): + upload.primary = False + upload.save() + upload = get_object_or_404(Upload, pk=upload_id) + upload.primary=True + upload.save() + return HttpResponseRedirect(reverse('artsurvey catalog')) + +def clean_primary_upload_survey(request, survey_id): + survey = get_object_or_404(ArtSurvey, pk=survey_id) + for upload in survey.uploads.all(): + upload.primary = False + upload.save() + return HttpResponseRedirect(reverse('artsurvey catalog')) + +def delete_upload(request, source_id, upload_id): + src = get_object_or_404(ArtSurveySource, pk=source_id) + upload = get_object_or_404(Upload, pk=upload_id) + upload.delete() + return HttpResponseRedirect(src.get_absolute_url()) + +def delete_upload_survey(request, survey_id, upload_id): + survey = get_object_or_404(ArtSurvey, pk=survey_id) + upload = get_object_or_404(Upload, pk=upload_id) + upload.delete() + return HttpResponseRedirect(reverse('artsurvey catalog')) + +def delete_upload_metasource(request, source_id, upload_id): + src = get_object_or_404(MetaSource, pk=source_id) + upload = get_object_or_404(Upload, pk=upload_id) + upload.delete() + return HttpResponseRedirect(src.get_absolute_url()) + +def new_heasarc_source(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + c = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + try: + erosita = src.erositamatch + erosita_crd = SkyCoord(erosita.RA_fin_soft, erosita.DEC_fin_soft, frame="fk5", unit="deg") + except Exception as e: + print("{} {}".format(src,e)) + erosita = None + erosita_crd = None + + heasarcs = src.heasarc.all().annotate(separation=Value(0.0, output_field=DecimalField())) + heasarcs = heasarcs.annotate(erosita_separation=Value(0.0, output_field=DecimalField())) + for heasarc in heasarcs: + c0 = SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg") + heasarc.separation=c.separation(c0).arcsecond + if not erosita is None: + heasarc.erosita_separation=erosita_crd.separation(c0).arcsecond + + if request.method == 'POST': + form = HeasarcUserCatForm(request.POST) + if form.is_valid(): + new_form=form.save(commit=False) + + + #bibcode=form.cleaned_data['bibcode'] + #notes=form.cleaned_data['notes'] + ra=float(form.cleaned_data['ra']) + dec=float(form.cleaned_data['dec']) + name=form.cleaned_data['name'] + autoname=form.cleaned_data['autoname'] + if(autoname): + new_form.name=make_source_name(name,ra,dec) + new_form.autoname=False + new_form.save() + + if(form.cleaned_data['bibcode']): + category_open = AdsCategory.objects.get(slug="open") + ads = Ads(bibcode=form.cleaned_data['bibcode']) + ads.save() + ads.category.add(category_open) + src.ads.add(ads) + new_form.ads=ads + new_form.save() + + healpix_usercat(new_form) + match_usercat_all(new_form) + src.heasarc.add(new_form) + src.heasarc_linked.add(new_form) + return HttpResponseRedirect(src.get_absolute_url()) + else: + pass + + form = HeasarcUserCatForm() + return render(request, 'artsurvey/usercat.html', {'form': form,'src':src,}) + + +def resolve_name(request): + if request.method == 'POST': + form = ResolveNameForm(request.POST) + if form.is_valid(): + name = form.cleaned_data['name'] + rmax = float(form.cleaned_data['rmax']) + maxdist = float(form.cleaned_data['maxdist']) + try: + cord=get_icrs_coordinates(name) + cord=cord.fk5 + except: + return render(request, 'artsurvey/search_coords.html', {'form':form, 'notresolved':True,}) + skymap = get_skymap_by_coords(cord.ra.value, cord.dec.value) + srcs = search_by_coords_artsurvey(cord.ra.value, cord.dec.value, rmax=rmax, maxdist=maxdist) + return render(request, 'artsurvey/search_coords.html', {'form':form,'srcs':srcs,'skymap':skymap,'ra':cord.ra.value,'dec':cord.dec.value,}) + else: + form = ResolveNameForm() + return render(request, 'artsurvey/search_coords.html', {'form':form,}) + +def search_coords_form(request): + + # if this is a POST request we need to process the form data + if request.method == 'POST': + # create a form instance and populate it with data from the request: + form = SearchCoordsForm(request.POST) + # check whether it's valid: + if form.is_valid(): + ra = float(form.cleaned_data['ra']) + dec = float(form.cleaned_data['dec']) + rmax = float(form.cleaned_data['rmax']) + maxdist = float(form.cleaned_data['maxdist']) + skymap = get_skymap_by_coords(ra, dec) + srcs = search_by_coords_artsurvey(ra, dec, rmax=rmax, maxdist=maxdist) + return render(request, 'artsurvey/search_coords.html', {'form':form,'srcs':srcs,'skymap':skymap,'ra':ra,'dec':dec,}) + else: + form = SearchCoordsForm() + return render(request, 'artsurvey/search_coords.html', {'form':form,}) + +def search_name_form(request): + + # if this is a POST request we need to process the form data + if request.method == 'POST': + # create a form instance and populate it with data from the request: + form = SearchNameForm(request.POST) + # check whether it's valid: + if form.is_valid(): + name = form.cleaned_data['name'] + srcs = search_by_name_artsurvey(name) + return render(request, 'artsurvey/search_name.html', {'form':form,'srcs':srcs,}) + else: + form = SearchNameForm() + return render(request, 'artsurvey/search_name.html', {'form':form,}) + +def edit_upload(request,source_id,upload_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(ArtSurveySource, pk=source_id) + upload = get_object_or_404(Upload, pk=upload_id) + + if request.method == 'POST': # If the form has been submitted... + form = UploadForm(request.POST,instance=upload) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save(commit=False) + #category = form.cleaned_data['category'] + #new_form.category=category + new_form.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + return render(request,'artsurvey/upload.html', {'form':form,'src':src,}) + + else: + form = UploadForm(instance=upload) + return render(request,'artsurvey/upload.html', {'form':form,'src':src,}) + +def edit_upload_survey(request,survey_id,upload_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + survey = get_object_or_404(ArtSurvey, pk=survey_id) + upload = get_object_or_404(Upload, pk=upload_id) + + if request.method == 'POST': # If the form has been submitted... + form = UploadForm(request.POST,instance=upload) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save(commit=False) + #category = form.cleaned_data['category'] + #new_form.category=category + new_form.save() + return HttpResponseRedirect(survey.get_absolute_url()) + else: + return render(request,'artsurvey/upload.html', {'form':form,'src':survey,}) + + else: + form = UploadForm(instance=upload) + return render(request,'artsurvey/upload.html', {'form':form,'src':survey,}) + +def edit_upload_metasource(request,source_id,upload_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + src = get_object_or_404(MetaSource, pk=source_id) + upload = get_object_or_404(Upload, pk=upload_id) + + if request.method == 'POST': # If the form has been submitted... + form = UploadForm(request.POST,instance=upload) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save(commit=False) + #category = form.cleaned_data['category'] + #new_form.category=category + new_form.save() + return HttpResponseRedirect(src.get_absolute_url()) + else: + return render(request,'artsurvey/upload.html', {'form':form,'src':src,}) + + else: + form = UploadForm(instance=upload) + return render(request,'artsurvey/upload.html', {'form':form,'src':src,}) + + +def add_to_selection(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + + try: + selection = request.user.artselection + except: + selection = ArtSelection(owner=request.user) + selection.save() + + selection.sources.add(src) + selection.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def remove_from_selection(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + + try: + selection = request.user.artselection + except: + return HttpResponse("Source Selection was not found") + selection.sources.remove(src) + selection.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def remove_from_selection_profile(request,source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + + try: + selection = request.user.artselection + except: + return HttpResponse("Source Selection was not found") + selection.sources.remove(src) + selection.save() + return HttpResponseRedirect(reverse('artsurvey profile')) + +def remove_from_basket(request,source_id,basket_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(ArtSurveySource, pk=source_id) + basket = get_object_or_404(ArtBasket, pk=basket_id) + basket.sources.remove(src) + return HttpResponseRedirect(basket.get_absolute_url()) + +def delete_comment(request, comment_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + comment = get_object_or_404(MetaComment, pk=comment_id) + metasource = comment.metasource + if(request.user == comment.owner): + comment.delete() + else: + raise PermissionDenied("You don't have rights to delete this comment. Go back to {}.".format(metasource.get_absolute_url(),metasource)) + return HttpResponseRedirect(metasource.get_absolute_url()) + +def metasource_group(request, source_id, friend_id): + src = get_object_or_404(ArtSurveySource, pk=source_id) + friend = get_object_or_404(ArtSurveySource, pk=friend_id) + if not src.metasource: + raise PermissionDenied("(main) {} has no MetaSource".format(src.get_absolute_url(),src)) + if not friend.metasource: + raise PermissionDenied("(friend) {} has no MetaSource".format(friend.get_absolute_url(),friend)) + src_crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + meta_crd = SkyCoord(src.metasource.ra, src.metasource.dec, frame="fk5", unit="deg") + friend_crd = SkyCoord(friend.ra, friend.dec, frame="fk5", unit="deg") + + if friend.metasource == src.metasource: + raise PermissionDenied("{} MetaSource has been already added ({} == {})".format(src.get_absolute_url(),src,friend.metasource,src.metasource)) + + devmax=3600 + metasource_sep=friend_crd.separation(meta_crd).arcsecond + if metasource_sep > devmax: + raise PermissionDenied("{} too far away ({:.2f} > {} arcseconds)".format(src.get_absolute_url(),src,metasource_sep, devmax)) + + srcs = src.metasource.artsurveysource_set.all() + for s in srcs: + s.metasource_primary = None + s.save() + + src.metasource_primary=src.metasource + src.save() + + friend.metasource_primary=None + friend.metasource=src.metasource + friend.metasource_sep=metasource_sep + friend.save() + return HttpResponseRedirect(src.get_absolute_url()) + +def transfer(request,source_id,transfer_from_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + destination = get_object_or_404(ArtSurveySource, pk=source_id) + source = get_object_or_404(ArtSurveySource, pk=transfer_from_id) + destination.parent=source + destination.save() + transfer_data(destination,clean_ads=False) + + return HttpResponseRedirect(destination.get_absolute_url()) + +def basket_create(request): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + if request.method == 'POST': # If the form has been submitted... + form = ArtBasketForm(request.POST) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save(commit=False) + new_form.owner=request.user + new_form.save() + + return HttpResponseRedirect(reverse('artsurvey index')) + else: + return render(request,'artsurvey/basket.html', {'form':form,}) + else: + form = ArtBasketForm() + return render(request,'artsurvey/basket.html', {'form':form,}) + +def basket_update(request, basket_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + + basket = get_object_or_404(ArtBasket, pk=basket_id) + + if request.method == 'POST': # If the form has been submitted... + form = ArtBasketForm(request.POST,instance=basket) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save(commit=False) + new_form.owner=request.user + new_form.save() + return HttpResponseRedirect(new_form.get_absolute_url()) + else: + return render(request,'artsurvey/basket.html', {'form':form,'basket':basket,}) + else: + form = ArtBasketForm(instance=basket) + return render(request,'artsurvey/basket.html', {'form':form,'basket':basket,}) + +def basket_show(request, basket_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + basket = get_object_or_404(ArtBasket, pk=basket_id) + """ + baskets = ArtBasket.objects.all() + if not basket in user_profile.baskets.all(): + write_history(request, "Trying to access not allowed basket {}.".format(basket)) + return HttpResponseForbidden("You are really bad guy!") + """ + return render(request,'artsurvey/basket.html', {'basket':basket,}) + +def basket_delete(request, basket_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + basket = get_object_or_404(ArtBasket, pk=basket_id) + basket.delete() + return HttpResponseRedirect(reverse('artsurvey index')) + diff --git a/astrobasis/__init__.py b/astrobasis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/astrobasis/admin.py b/astrobasis/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/astrobasis/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/astrobasis/apps.py b/astrobasis/apps.py new file mode 100644 index 0000000..357422f --- /dev/null +++ b/astrobasis/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class AstrobasisConfig(AppConfig): + name = 'astrobasis' diff --git a/astrobasis/management/commands/00_astrobasis_2mass_healpix.py b/astrobasis/management/commands/00_astrobasis_2mass_healpix.py new file mode 100644 index 0000000..7538bde --- /dev/null +++ b/astrobasis/management/commands/00_astrobasis_2mass_healpix.py @@ -0,0 +1,11 @@ +from django.core.management.base import BaseCommand, CommandError + +from astrobasis.utils import astrobasis_assign_healpix +from astrobasis.models import TwoMASS + +class Command(BaseCommand): + help = 'Initiates healpix for 2MASS' + + def handle(self, *args, **options): + astrobasis_assign_healpix(TwoMASS) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/astrobasis/management/commands/00_astrobasis_2mass_init.py b/astrobasis/management/commands/00_astrobasis_2mass_init.py new file mode 100644 index 0000000..7f87947 --- /dev/null +++ b/astrobasis/management/commands/00_astrobasis_2mass_init.py @@ -0,0 +1,88 @@ +from django.core.management.base import BaseCommand, CommandError +import time +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass +from heasarc.models import NSIDE_SOURCES, ORDER +from astrobasis.models import TwoMASS + +from srglib.utils import notnan +from srglib.utils import boolean_notnan + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + + +def bulk_load_heasarc_table(filename,reset=False): + batch_size = 500 + + print("Delete all 2MASS") + if(reset): + srcs = TwoMASS.objects.all() + srcs.delete() + + print("Loading {}".format(filename)) + data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1') + print(data.info) + + print("Bulk load to DB") + + """ + cols = data.columns + for key, value in cols.items(): + print(key, '->', value) + """ + + + objs = (TwoMASS(name=data[i]['2MASS'], + ra=float(data[i]['RAJ2000']), + dec=float(data[i]['DEJ2000']), + jmag = notnan(data[i]['Jmag']), + e_jmag = notnan(data[i]['e_Jmag']), + hmag = notnan(data[i]['Hmag']), + e_hmag = notnan(data[i]['e_Hmag']), + kmag = notnan(data[i]['Kmag']), + e_kmag = notnan(data[i]['e_Kmag']), + qkfg = data[i]['Qflg'], + rkfg = data[i]['Rflg'], + bkfg = data[i]['Bflg'], + ckfg = data[i]['Cflg'], + prox = notnan(data[i]['prox']), + xflg = boolean_notnan(data[i]['Xflg']), + aflg = boolean_notnan(data[i]['Aflg']),) + for i in range(len(data))) + TwoMASS.objects.bulk_create(objs,batch_size) + + print('--> Successfully loaded "%s"' % filename) + pass + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + start_time = time.time() + bulk_load_heasarc_table('/data/2MASS/l_0_180.tsv',reset=True) + print("--- %s seconds ---" % (time.time() - start_time)) + + start_time = time.time() + bulk_load_heasarc_table('/data/2MASS/l_180_360.tsv') + print("--- %s seconds ---" % (time.time() - start_time)) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/astrobasis/management/commands/00_astrobasis_glimpse_healpix.py b/astrobasis/management/commands/00_astrobasis_glimpse_healpix.py new file mode 100644 index 0000000..98e5833 --- /dev/null +++ b/astrobasis/management/commands/00_astrobasis_glimpse_healpix.py @@ -0,0 +1,11 @@ +from django.core.management.base import BaseCommand, CommandError + +from astrobasis.utils import astrobasis_assign_healpix +from astrobasis.models import GLIMPSE + +class Command(BaseCommand): + help = 'Initiates healpix for GLIMPSE' + + def handle(self, *args, **options): + astrobasis_assign_healpix(GLIMPSE) + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/astrobasis/management/commands/00_astrobasis_glimpse_init.py b/astrobasis/management/commands/00_astrobasis_glimpse_init.py new file mode 100644 index 0000000..05f154b --- /dev/null +++ b/astrobasis/management/commands/00_astrobasis_glimpse_init.py @@ -0,0 +1,159 @@ +from django.core.management.base import BaseCommand, CommandError +import time +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass +from heasarc.models import NSIDE_SOURCES, ORDER +from astrobasis.models import GLIMPSE + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +def bulk_load_heasarc_table(filename,reset=False): + batch_size = 300 + + print("Delete all GLIMPSE") + if(reset): + tables = GLIMPSE.objects.all() + tables.delete() + + print("Loading {}".format(filename)) + data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1') + print(data.info) + print("Bulk load to DB") + + """ + cols = data.columns + for key, value in cols.items(): + print(key, '->', value) + """ + + start_time = time.time() + + objs = (GLIMPSE(name=data[i]['GLIMPSE'], + ra=float(data[i]['RAJ2000']), + dec=float(data[i]['DEJ2000']), + twomass = data[i]['2MASS'], + jmag = data[i]['Jmag'], + hmag = data[i]['Hmag'], + kmag = data[i]['Kmag'], + b1mag = data[i]['3.6mag'], + e_b1mag = data[i]['e_3.6mag'], + b2mag = data[i]['4.5mag'], + e_b2mag = data[i]['e_4.5mag'], + b3mag = data[i]['5.8mag'], + e_b3mag = data[i]['e_5.8mag'], + b4mag = data[i]['8.0mag'], + e_b4mag = data[i]['e_8.0mag'],) + for i in range(len(data))) + GLIMPSE.objects.bulk_create(objs,batch_size) + print("--- %s seconds ---" % (time.time() - start_time)) + + print('--> Successfully loaded "%s"' % filename) + pass + +def bulk_healpix_update_tooslow(): + glimpse_all = GLIMPSE.objects.all() + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + objs = [] + start_time = time.time() + for obj in glimpse_all: + crd = SkyCoord(obj.ra, obj.dec, frame=FK5(), unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + healpix = hp.skycoord_to_healpix(crd) + obj.healpix = healpix + obj.glon = lon + obj.glat = lat + objs.append(obj) + GLIMPSE.objects.bulk_update(objs, ['healpix','glon','glat'], batch_size=1000) + print("--- %s seconds ---" % (time.time() - start_time)) + +def load_heasarc_table(filename,reset=False): + + if(reset): + tables = GLIMPSE.objects.all() + tables.delete() + + return + + data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1') + print(data.info) + + """ + cols = data.columns + for key, value in cols.items(): + print(key, '->', value) + """ + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + for item in data: + + healpix = 0 + ra = 0.0 + dec = 0.0 + lon = 0.0 + lat = 0.0 + if not (ma.is_masked(item['RAJ2000']) or ma.is_masked(item['DEJ2000'])): + ra = float(item['DEJ2000']) + dec = float(item['DEJ2000']) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + healpix = hp.skycoord_to_healpix(crd) + else: + continue + + + obj = GLIMPSE.objects.create(healpix=healpix, + ra=ra, + dec=dec, + glon = lon, + glat = lat, + name=item['GLIMPSE']) + twomass = item['2MASS'] + jmag = item['Jmag'] + hmag = item['Hmag'] + kmag = item['Kmag'] + b1mag = item['3.6mag'] + e_b1mag = item['e_3.6mag'] + b2mag = item['4.5mag'] + e_b2mag = item['e_4.5mag'] + b3mag = item['5.8mag'] + e_b3mag = item['e_5.8mag'] + b4mag = item['8.0mag'] + e_b4mag = item['e_8.0mag'] + obj.save() +#C;GLIMPSE;2MASS;RAJ2000;DEJ2000;Jmag;Hmag;Kmag;3.6mag;e_3.6mag;4.5mag;e_4.5mag;5.8mag;e_5.8mag;8.0mag;e_8.0mag + + print('--> Successfully loaded "%s"' % filename) + pass + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + start_time = time.time() + bulk_load_heasarc_table('/data/GLIMPSE/glimpse_spitzer.tsv',reset=True) + print("--- %s seconds ---" % (time.time() - start_time)) + # Model.objects.filter(amount__isnull=True).update(amount=F('pre_tax') * 1.2) + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/astrobasis/management/commands/00_astrobasis_vlass_healpix.py b/astrobasis/management/commands/00_astrobasis_vlass_healpix.py new file mode 100644 index 0000000..5af8a45 --- /dev/null +++ b/astrobasis/management/commands/00_astrobasis_vlass_healpix.py @@ -0,0 +1,76 @@ +from django.core.management.base import BaseCommand, CommandError +from django.core.paginator import Paginator + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import VLASS + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from heasarc.models import NSIDE_SOURCES, ORDER + +import os +import glob +import logging +import time + +class Command(BaseCommand): + help = 'Initiates healpix for Gaia' + + def handle(self, *args, **options): + start_time = time.time() + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5) + + + #srcs = VLASS.objects.all().filter(healpix__gt=0)[:50] + #for src in srcs: + # crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + # heal = hp.skycoord_to_healpix(crd) + # print(src.ra,src.dec,src.healpix,heal) + #return + + + srcs = VLASS.objects.all()#.filter(healpix__exact=0) + paginator = Paginator(srcs, 50000) + for page_idx in range(1, paginator.num_pages): + start_time0 = time.time() + ra=paginator.page(page_idx).object_list.values_list('ra', flat=True) + dec=paginator.page(page_idx).object_list.values_list('dec', flat=True) + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + heal = hp.skycoord_to_healpix(crd) + index=0 + updates=[] + for src in paginator.page(page_idx).object_list: + # here you can do what you want with the row + #crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + #lon = crd.galactic.l.value + #lat = crd.galactic.b.value + #heal = hp.skycoord_to_healpix(crd) + src.healpix=heal[index] + src.glon=lon[index] + src.glat=lat[index] + #src.save() + updates.append(src) + #print(src.ra,src.dec,heal) + index=index+1 + VLASS.objects.bulk_update(updates, ["healpix","glon","glat"], batch_size=200) + elapsed = time.time() - start_time0 + estimated = (elapsed*(paginator.num_pages-page_idx))/60/60 + print("Page {}/{} for {:.2f} sec, est. {:.2f} hours".format(page_idx,paginator.num_pages,elapsed,estimated)) + + print("--- {:.2f} hours ---".format((time.time() - start_time)/60/60)) diff --git a/astrobasis/management/commands/00_astrobasis_vlass_init.py b/astrobasis/management/commands/00_astrobasis_vlass_init.py new file mode 100644 index 0000000..e811089 --- /dev/null +++ b/astrobasis/management/commands/00_astrobasis_vlass_init.py @@ -0,0 +1,161 @@ +from django.core.management.base import BaseCommand, CommandError +import time +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass +from heasarc.models import NSIDE_SOURCES, ORDER +from astrobasis.models import VLASS + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +def bulk_load_heasarc_table(filename,reset=False): + batch_size = 500 + + print("Delete all VLASS") + + if(reset): + tables = VLASS.objects.all() + tables.delete() + + print("Loading {}".format(filename)) + data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1') + #print(data.info) + + + + for col in data.columns: + print("--> ",col) + + + start_time = time.time() + print("Bulk load to DB") + + objs = (VLASS(name=data[i]['CompName'], + glon=float(data[i]['_Glon']), + glat=float(data[i]['_Glat']), + ra_m=float(data[i]['RAMdeg']), + dec_m=float(data[i]['DEMdeg']), + ra=float(data[i]['RAJ2000']), + e_ra=float(data[i]['e_RAJ2000']), + dec=float(data[i]['DEJ2000']), + e_dec=float(data[i]['e_DEJ2000']), + ftot = data[i]['Ftot'], + e_ftot = data[i]['e_Ftot'], + fpeak = data[i]['Fpeak'], + e_fpeak = data[i]['e_Fpeak'], + dupflag = int(data[i]['DupFlag']), + qualflag = int(data[i]['QualFlag'])) + for i in range(len(data))) + VLASS.objects.bulk_create(objs,batch_size) + print("--- %s seconds ---" % (time.time() - start_time)) + + print('--> Successfully loaded "%s"' % filename) + pass + +def bulk_healpix_update_tooslow(): + glimpse_all = GLIMPSE.objects.all() + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + objs = [] + start_time = time.time() + for obj in glimpse_all: + crd = SkyCoord(obj.ra, obj.dec, frame=FK5(), unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + healpix = hp.skycoord_to_healpix(crd) + obj.healpix = healpix + obj.glon = lon + obj.glat = lat + objs.append(obj) + GLIMPSE.objects.bulk_update(objs, ['healpix','glon','glat'], batch_size=1000) + print("--- %s seconds ---" % (time.time() - start_time)) + +def load_heasarc_table(filename,reset=False): + + if(reset): + tables = GLIMPSE.objects.all() + tables.delete() + + return + + data = astropy.table.Table.read(filename, format='ascii.csv',delimiter=';',comment='#',encoding='latin1') + print(data.info) + + """ + cols = data.columns + for key, value in cols.items(): + print(key, '->', value) + """ + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + for item in data: + + healpix = 0 + ra = 0.0 + dec = 0.0 + lon = 0.0 + lat = 0.0 + if not (ma.is_masked(item['RAJ2000']) or ma.is_masked(item['DEJ2000'])): + ra = float(item['DEJ2000']) + dec = float(item['DEJ2000']) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + healpix = hp.skycoord_to_healpix(crd) + else: + continue + + + obj = GLIMPSE.objects.create(healpix=healpix, + ra=ra, + dec=dec, + glon = lon, + glat = lat, + name=item['GLIMPSE']) + twomass = item['2MASS'] + jmag = item['Jmag'] + hmag = item['Hmag'] + kmag = item['Kmag'] + b1mag = item['3.6mag'] + e_b1mag = item['e_3.6mag'] + b2mag = item['4.5mag'] + e_b2mag = item['e_4.5mag'] + b3mag = item['5.8mag'] + e_b3mag = item['e_5.8mag'] + b4mag = item['8.0mag'] + e_b4mag = item['e_8.0mag'] + obj.save() +#C;GLIMPSE;2MASS;RAJ2000;DEJ2000;Jmag;Hmag;Kmag;3.6mag;e_3.6mag;4.5mag;e_4.5mag;5.8mag;e_5.8mag;8.0mag;e_8.0mag + + print('--> Successfully loaded "%s"' % filename) + pass + + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + start_time = time.time() + bulk_load_heasarc_table('/data/VizieR/VLASS_radio.tsv',reset=True) + #bulk_healpix_update_tooslow() + print("--- %s seconds ---" % (time.time() - start_time)) + # Model.objects.filter(amount__isnull=True).update(amount=F('pre_tax') * 1.2) + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/astrobasis/management/commands/00_init_gaia.py b/astrobasis/management/commands/00_init_gaia.py new file mode 100644 index 0000000..b8df155 --- /dev/null +++ b/astrobasis/management/commands/00_init_gaia.py @@ -0,0 +1,54 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import GaiaSourceFileDR3, GAIADR3 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + + +import os +import glob +import logging + +def init_gaia(filepath): + + filename_w_ext = os.path.basename(filepath) + filename, file_extension = os.path.splitext(filename_w_ext) + + gaia = GaiaSourceFileDR3(filename=filename) + gaia.save() + + # print('--> Successfully initiated "%s"' % filename) + pass + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + gaia_all = GAIADR3.objects.all() + gaia_all.delete() + + files = [] + for file in glob.glob("/data/tmp/george/Gaia/gedr3/gaia_source/fits/fits/*.fits"): + files.append(file) + + for file in files: + print(file) + init_gaia(file) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/astrobasis/management/commands/00_load_ztf_alerce.py b/astrobasis/management/commands/00_load_ztf_alerce.py new file mode 100644 index 0000000..ef7f2fe --- /dev/null +++ b/astrobasis/management/commands/00_load_ztf_alerce.py @@ -0,0 +1,179 @@ +from django.core.management.base import BaseCommand, CommandError + +import os +import io +import gzip +import tarfile +import warnings + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +from avro.datafile import DataFileReader, DataFileWriter +from avro.io import DatumReader, DatumWriter +import fastavro + +from astropy.time import Time +from astropy.time import Time, TimezoneInfo +from astropy.io import fits +import astropy.units as u +import aplpy + +from astropy.time import Time +from django.utils.timezone import localdate, localtime, now + +from collections import defaultdict + +import logging +import requests +import json +import pprint + +from astrobasis.models import ALeRCE +from srglib.ztf import find_ztf_in_survey, load_ztf_alerce + +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import re +import json +import pandas as pd +from astropy_healpix import HEALPix +from erotrans.models import EroTransSource, EroTrans +from erosurvey.models import NSIDE_PLATES, ORDER_PLATES +from heasarc.models import NSIDE_SOURCES, ORDER +from monthplan.models import Survey + +def load_ztf_alerce_to_django_local(item): + utc_moscow = TimezoneInfo(utc_offset=3*u.hour) + hp = HEALPix(nside=NSIDE_SOURCES, + order=ORDER, frame=FK5()) + + hp_plates = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, frame=FK5()) + + try: + ztf = ALeRCE.objects.get(oid=item['oid']) + print("ALeRCE ZTF alert %s is already loaded, skip" % item['oid']) + return + except ALeRCE.DoesNotExist: + print("ALeRCE ZTF alert %s is not loaded" % item['oid']) + pass + + ra = float(item['meanra']) + dec = float(item['meandec']) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + healpix_plate = hp_plates.skycoord_to_healpix(crd) + + first_tm = Time(item['firstmjd'], format='mjd', scale='utc') + last_tm = Time(item['lastmjd'], format='mjd', scale='utc') + first_dt = first_tm.to_datetime(timezone=utc_moscow) + last_dt = last_tm.to_datetime(timezone=utc_moscow) + + ztf = ALeRCE( + healpix = healpix, + healpix_plate = healpix_plate, + classearly = item['classearly'], + classrf = item['classrf'], + oid = item['oid'], + firstdate = first_dt, + lastdate = last_dt, + firstmjd = item['firstmjd'], + lastmjd = item['lastmjd'], + mean_magap_g = item['mean_magap_g'], + mean_magap_r = item['mean_magap_r'], + mean_magpsf_g = item['mean_magpsf_g'], + mean_magpsf_r = item['mean_magpsf_r'], + dec = dec, + ra = ra, + nobs = item['nobs'], + pclassearly = item['pclassearly'], + pclassrf = item['pclassrf'], + sigma_magap_g = item['sigma_magap_g'], + sigma_magap_r = item['sigma_magap_r'], + sigma_magpsf_g = item['sigma_magpsf_g'], + sigma_magpsf_r = item['sigma_magpsf_r'], + sigmadec = item['sigmadec'], + sigmara = item['sigmara']) + + ztf.save() + find_ztf_in_survey(ztf) + +def load_ztf_alerce_local(days, extra=None): + classrf = "sn ia" + pclassrf = 0.1 + pclassearly = 0.1 + sortBy = "firstmjd" + nobsmin = 2 + nobsmax = 40 + url = "https://ztf.alerce.online/query" + + current_utc = now() + current_mjd = Time(current_utc).mjd + firstmjd_min = current_mjd - days + + post={ + "sortBy": sortBy, + "records_per_pages": 100, + "query_parameters":{ + "filters": { + "nobs": { + "min": nobsmin, + "max": nobsmax + }, + #"classrf": classrf, + #"pclassrf": pclassrf, + 'pclassearly':pclassearly, + }, + "dates":{ + "firstmjd":{ + "min": firstmjd_min + } + } + } + } + + if(extra): + post['query_parameters']['filters'].update(extra) + + #pprint.pprint(post) + + r = requests.post(url = url, json = post) + data = r.json() + for item in data['result']: + """ loads first page """ + packet=data['result'][item] + #pprint.pprint(packet['oid']) + load_ztf_alerce_to_django_local(packet) + + total=int(data['total']) + num_pages=int(data['num_pages']) + page=int(data['page']) + + print('total ',total,'num_pages ',num_pages,'page ', page) + #return + pages = list(range(2,num_pages+1)) + for page in pages: + """ loads all other pages, staring from 2 """ + post.update( {'page':page,} ) + pprint.pprint(post) + r = requests.post(url = url, json = post) + data = r.json() + for item in data['result']: + packet=data['result'][item] + #pprint.pprint(packet['oid']) + load_ztf_alerce_to_django_local(packet) + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + #all = ALeRCE.objects.all() + #all.delete() + + load_ztf_alerce_local(2) + + print('Done') diff --git a/astrobasis/management/commands/00_load_ztf_archive.py b/astrobasis/management/commands/00_load_ztf_archive.py new file mode 100644 index 0000000..41b1222 --- /dev/null +++ b/astrobasis/management/commands/00_load_ztf_archive.py @@ -0,0 +1,256 @@ +from django.core.management.base import BaseCommand, CommandError + +import os +import io +import gzip +import tarfile +import warnings + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +from avro.datafile import DataFileReader, DataFileWriter +from avro.io import DatumReader, DatumWriter +import fastavro + +from astropy.time import Time +from astropy.io import fits +import astropy.units as u +import aplpy + +from collections import defaultdict + +import logging +import requests +import json +import pprint + +from astrobasis.models import ZTFAlert +from srglib.utils import find_ztf_in_survey + +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import re +import json +import pandas as pd +from astropy_healpix import HEALPix +from erotrans.models import EroTransSource, EroTrans +from erosurvey.models import NSIDE_PLATES, ORDER_PLATES +from heasarc.models import NSIDE_SOURCES, ORDER +from monthplan.models import Survey + + +""" +https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html + +Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample: + +rb >= 0.65 and +nbad = 0 and +fwhm <= 5 and +elong <= 1.2 and +abs(magdiff) <= 0.1 +""" + +def plot_lightcurve(dflc, ax=None, days_ago=True): + + filter_color = {1:'green', 2:'red', 3:'pink'} + if days_ago: + now = Time.now().jd + t = dflc.jd - now + xlabel = 'Days Ago' + else: + t = dflc.jd + xlabel = 'Time (JD)' + + if ax is None: + plt.figure() + for fid, color in filter_color.items(): + # plot detections in this filter: + w = (dflc.fid == fid) & ~dflc.magpsf.isnull() + if np.sum(w): + plt.errorbar(t[w],dflc.loc[w,'magpsf'], dflc.loc[w,'sigmapsf'],fmt='.',color=color) + wnodet = (dflc.fid == fid) & dflc.magpsf.isnull() + if np.sum(wnodet): + plt.scatter(t[wnodet],dflc.loc[wnodet,'diffmaglim'], marker='v',color=color,alpha=0.25) + + plt.gca().invert_yaxis() + plt.xlabel(xlabel) + plt.ylabel('Magnitude') + plt.title(dflc.objectId) + +def plot_cutout(stamp, fig=None, subplot=None, **kwargs): + with gzip.open(io.BytesIO(stamp), 'rb') as f: + with fits.open(io.BytesIO(f.read())) as hdul: + if fig is None: + fig = plt.figure(figsize=(4,4)) + if subplot is None: + subplot = (1,1,1) + ffig = aplpy.FITSFigure(hdul[0],figure=fig, subplot=subplot, **kwargs) + ffig.show_grayscale(stretch='arcsinh') + return ffig + +def show_stamps(packet): + #fig, axes = plt.subplots(1,3, figsize=(12,4)) + fig = plt.figure(figsize=(12,4)) + for i, cutout in enumerate(['Science','Template','Difference']): + stamp = packet['cutout{}'.format(cutout)]['stampData'] + ffig = plot_cutout(stamp, fig=fig, subplot = (1,3,i+1)) + ffig.set_title(cutout) + +def show_all(packet): + fig = plt.figure(figsize=(16,4)) + dflc = make_dataframe(packet) + plot_lightcurve(dflc,ax = plt.subplot(1,4,1)) + for i, cutout in enumerate(['Science','Template','Difference']): + stamp = packet['cutout{}'.format(cutout)]['stampData'] + ffig = plot_cutout(stamp, fig=fig, subplot = (1,4,i+2)) + ffig.set_title(cutout) + +def make_dataframe(packet): + dfc = pd.DataFrame(packet['candidate'], index=[0]) + df_prv = pd.DataFrame(packet['prv_candidates']) + dflc = pd.concat([dfc,df_prv], ignore_index=True, sort=False) + # we'll attach some metadata--not this may not be preserved after all operations + # https://stackoverflow.com/questions/14688306/adding-meta-information-metadata-to-pandas-dataframe + dflc.objectId = packet['objectId'] + dflc.candid = packet['candid'] + return dflc + +def is_alert_pure(packet): + pure = True + pure &= packet['candidate']['rb'] >= 0.65 + pure &= packet['candidate']['nbad'] == 0 + pure &= packet['candidate']['fwhm'] <= 5 + pure &= packet['candidate']['elong'] <= 1.2 + pure &= np.abs(packet['candidate']['magdiff']) <= 0.1 + return pure + +def generate_dictionaries(root_dir): + for fname in find_files(root_dir): + for packet in open_avro(fname): + yield packet + +def open_avro(fname): + with open(fname,'rb') as f: + freader = fastavro.reader(f) + # in principle there can be multiple packets per file + for packet in freader: + yield packet + +def find_files(root_dir): + for dir_name, subdir_list, file_list in os.walk(root_dir, followlinks=True): + for fname in file_list: + if fname.endswith('.avro'): + yield dir_name+'/'+fname + +def is_transient(dflc): + + candidate = dflc.loc[0] + + is_positive_sub = candidate['isdiffpos'] == 't' + + if (candidate['distpsnr1'] is None) or (candidate['distpsnr1'] > 1.5): + no_pointsource_counterpart = True + else: + if candidate['sgscore1'] < 0.5: + no_pointsource_counterpart = True + else: + no_pointsource_counterpart = False + + where_detected = (dflc['isdiffpos'] == 't') # nondetections will be None + if np.sum(where_detected) >= 2: + detection_times = dflc.loc[where_detected,'jd'].values + dt = np.diff(detection_times) + not_moving = np.max(dt) >= (30*u.minute).to(u.day).value + else: + not_moving = False + + no_ssobject = (candidate['ssdistnr'] is None) or (candidate['ssdistnr'] < 0) or (candidate['ssdistnr'] > 5) + + return is_positive_sub and no_pointsource_counterpart and not_moving and no_ssobject + +def load_ztf_packet(item): + hp = HEALPix(nside=NSIDE_SOURCES, + order=ORDER, frame=FK5()) + + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + + name = item['objectId'] + + + try: + ztf = ZTFAlert.objects.get(objectId=name) + print("ZTF alert ID %s is already loaded, skip" % name) + return + except ZTFAlert.DoesNotExist: + print("ZTF alert ID %s is not loaded" % name) + pass + #date_string = item['candidate']['wall_time'] + #dt = datetime.strptime(date_string, format_string) + ra = item['candidate']['ra'] + dec = item['candidate']['dec'] + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + healpix_plate = hp_plate.skycoord_to_healpix(crd) + ztf = ZTFAlert( + objectId = item['objectId'], + fid = item['candidate']['fid'], + lco_id = item['candid'], + healpix = healpix, healpix_plate = healpix_plate, + ra = ra, dec = dec, + programid = item['candidate']['programid'], + magpsf = item['candidate']['magpsf'], + sigmapsf = item['candidate']['sigmapsf'], + magap = item['candidate']['magap'], + magdiff = item['candidate']['magdiff'], + nbad = item['candidate']['nbad'], + fwhm = item['candidate']['fwhm'], + elong = item['candidate']['elong'], + sigmagap = item['candidate']['sigmagap'], + #wall_time = datetime.strptime(date_string, format_string), + diffmaglim = item['candidate']['diffmaglim'], + #deltamagref = item['candidate']['deltamagref'], + #deltamaglatest = item['candidate']['deltamaglatest'], + rb = item['candidate']['rb']) + ztf.save() + + find_ztf_in_survey(ztf) + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + tar_archive = '/data/ZTF/ztf_public_20200126.tar.gz' + + output_dir = tar_archive.split('/')[-1].split('.')[-3] + + + #archive = tarfile.open(tar_archive,'r:gz') + #archive.extractall(path=output_dir) + #archive.close() + + print (output_dir) + print('{} has {} avro files'.format(output_dir, len(list(find_files(output_dir))))) + + transient_alerts = [] + for packet in filter(is_alert_pure,generate_dictionaries(output_dir)): + dflc = make_dataframe(packet) + if is_transient(dflc): + transient_alerts.append(packet) + load_ztf_packet(packet) + + + """ + for packet in transient_alerts: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + show_all(packet) + """ diff --git a/astrobasis/management/commands/00_load_ztf_mars.py b/astrobasis/management/commands/00_load_ztf_mars.py new file mode 100644 index 0000000..172a7f8 --- /dev/null +++ b/astrobasis/management/commands/00_load_ztf_mars.py @@ -0,0 +1,65 @@ +from django.core.management.base import BaseCommand, CommandError + +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import re +import json +import pandas as pd +from astropy_healpix import HEALPix + +import os +import io +import gzip +import tarfile +import warnings + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +from avro.datafile import DataFileReader, DataFileWriter +from avro.io import DatumReader, DatumWriter +import fastavro + +from astropy.time import Time +from astropy.io import fits +import astropy.units as u +import aplpy + +from astrobasis.models import ztf_filter + +from collections import defaultdict + +import logging +import requests +import json +import pprint + +from astrobasis.models import ZTFAlert +from heasarc.models import NSIDE_SOURCES, ORDER, HeasarcBase +from srglib.utils import load_ztf_alerts + +""" +https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html + +Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample: + +rb >= 0.65 and +nbad = 0 and +fwhm <= 5 and +elong <= 1.2 and +abs(magdiff) <= 0.1 +""" + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + + all = ZTFAlert.objects.all() + all.delete() + + #load_ztf_alerts() diff --git a/astrobasis/management/commands/00_status_gaia.py b/astrobasis/management/commands/00_status_gaia.py new file mode 100644 index 0000000..94ce3fa --- /dev/null +++ b/astrobasis/management/commands/00_status_gaia.py @@ -0,0 +1,52 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import GaiaSourceFileDR3, GAIADR3 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + + +import os +import glob +import logging + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + total_empty=GaiaSourceFileDR3.objects.filter(status__exact='empty').count() + print("Empty %d" % total_empty) + + total_ready=GaiaSourceFileDR3.objects.filter(status__exact='ready').count() + print("Ready %d" % total_ready) + + total_loaded=GaiaSourceFileDR3.objects.filter(status__exact='loaded').count() + print("Loaded %d" % total_loaded) + + total_healpix=GaiaSourceFileDR3.objects.filter(status__exact='healpix').count() + print("Healpix %d" % total_healpix) + + total=GaiaSourceFileDR3.objects.count() + print("Total %d = %d" % (total, total_empty+total_loaded+total_healpix)) + + #total=GAIADR3.objects.count() + #print("Total Gaia %d" % total) + + return + + diff --git a/astrobasis/management/commands/00_test_gaia.py b/astrobasis/management/commands/00_test_gaia.py new file mode 100644 index 0000000..26da2ea --- /dev/null +++ b/astrobasis/management/commands/00_test_gaia.py @@ -0,0 +1,39 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import GaiaSourceFile, GAIADR3 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import time + + +import os +import glob +import logging + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + start_time = time.time() + srcs=GAIADR3.objects.filter(healpix=38837101).count() + #for src in srcs: + # print(src.name) + print("Total Gaia found %d" % srcs) + print("--- %s seconds ---" % (time.time() - start_time)) + diff --git a/astrobasis/management/commands/00_test_ztf.py b/astrobasis/management/commands/00_test_ztf.py new file mode 100644 index 0000000..ce22eb2 --- /dev/null +++ b/astrobasis/management/commands/00_test_ztf.py @@ -0,0 +1,221 @@ +from django.core.management.base import BaseCommand, CommandError + +import os +import io +import gzip +import tarfile +import warnings + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +from avro.datafile import DataFileReader, DataFileWriter +from avro.io import DatumReader, DatumWriter +import fastavro + +from astropy.time import Time +from astropy.io import fits +import astropy.units as u +import aplpy + +from collections import defaultdict + +import requests +import json +import pprint + +""" +https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html + +Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample: + +rb >= 0.65 and +nbad = 0 and +fwhm <= 5 and +elong <= 1.2 and +abs(magdiff) <= 0.1 +""" + +def plot_lightcurve(dflc, ax=None, days_ago=True): + + filter_color = {1:'green', 2:'red', 3:'pink'} + if days_ago: + now = Time.now().jd + t = dflc.jd - now + xlabel = 'Days Ago' + else: + t = dflc.jd + xlabel = 'Time (JD)' + + if ax is None: + plt.figure() + for fid, color in filter_color.items(): + # plot detections in this filter: + w = (dflc.fid == fid) & ~dflc.magpsf.isnull() + if np.sum(w): + plt.errorbar(t[w],dflc.loc[w,'magpsf'], dflc.loc[w,'sigmapsf'],fmt='.',color=color) + wnodet = (dflc.fid == fid) & dflc.magpsf.isnull() + if np.sum(wnodet): + plt.scatter(t[wnodet],dflc.loc[wnodet,'diffmaglim'], marker='v',color=color,alpha=0.25) + + plt.gca().invert_yaxis() + plt.xlabel(xlabel) + plt.ylabel('Magnitude') + plt.title(dflc.objectId) + +def plot_cutout(stamp, fig=None, subplot=None, **kwargs): + with gzip.open(io.BytesIO(stamp), 'rb') as f: + with fits.open(io.BytesIO(f.read())) as hdul: + if fig is None: + fig = plt.figure(figsize=(4,4)) + if subplot is None: + subplot = (1,1,1) + ffig = aplpy.FITSFigure(hdul[0],figure=fig, subplot=subplot, **kwargs) + ffig.show_grayscale(stretch='arcsinh') + return ffig + +def show_stamps(packet): + #fig, axes = plt.subplots(1,3, figsize=(12,4)) + fig = plt.figure(figsize=(12,4)) + for i, cutout in enumerate(['Science','Template','Difference']): + stamp = packet['cutout{}'.format(cutout)]['stampData'] + ffig = plot_cutout(stamp, fig=fig, subplot = (1,3,i+1)) + ffig.set_title(cutout) + +def show_all(packet): + fig = plt.figure(figsize=(16,4)) + dflc = make_dataframe(packet) + plot_lightcurve(dflc,ax = plt.subplot(1,4,1)) + for i, cutout in enumerate(['Science','Template','Difference']): + stamp = packet['cutout{}'.format(cutout)]['stampData'] + ffig = plot_cutout(stamp, fig=fig, subplot = (1,4,i+2)) + ffig.set_title(cutout) + +def make_dataframe(packet): + dfc = pd.DataFrame(packet['candidate'], index=[0]) + df_prv = pd.DataFrame(packet['prv_candidates']) + dflc = pd.concat([dfc,df_prv], ignore_index=True, sort=False) + # we'll attach some metadata--not this may not be preserved after all operations + # https://stackoverflow.com/questions/14688306/adding-meta-information-metadata-to-pandas-dataframe + dflc.objectId = packet['objectId'] + dflc.candid = packet['candid'] + return dflc + +def is_alert_pure(packet): + pure = True + pure &= packet['candidate']['rb'] >= 0.65 + pure &= packet['candidate']['nbad'] == 0 + pure &= packet['candidate']['fwhm'] <= 5 + pure &= packet['candidate']['elong'] <= 1.2 + pure &= np.abs(packet['candidate']['magdiff']) <= 0.1 + return pure + +def generate_dictionaries(root_dir): + for fname in find_files(root_dir): + for packet in open_avro(fname): + yield packet + +def open_avro(fname): + with open(fname,'rb') as f: + freader = fastavro.reader(f) + # in principle there can be multiple packets per file + for packet in freader: + yield packet + +def find_files(root_dir): + for dir_name, subdir_list, file_list in os.walk(root_dir, followlinks=True): + for fname in file_list: + if fname.endswith('.avro'): + yield dir_name+'/'+fname + +def is_transient(dflc): + + candidate = dflc.loc[0] + + is_positive_sub = candidate['isdiffpos'] == 't' + + if (candidate['distpsnr1'] is None) or (candidate['distpsnr1'] > 1.5): + no_pointsource_counterpart = True + else: + if candidate['sgscore1'] < 0.5: + no_pointsource_counterpart = True + else: + no_pointsource_counterpart = False + + where_detected = (dflc['isdiffpos'] == 't') # nondetections will be None + if np.sum(where_detected) >= 2: + detection_times = dflc.loc[where_detected,'jd'].values + dt = np.diff(detection_times) + not_moving = np.max(dt) >= (30*u.minute).to(u.day).value + else: + not_moving = False + + no_ssobject = (candidate['ssdistnr'] is None) or (candidate['ssdistnr'] < 0) or (candidate['ssdistnr'] > 5) + + return is_positive_sub and no_pointsource_counterpart and not_moving and no_ssobject + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + query_86400='https://mars.lco.global/?format=json&sort_value=jd&sort_order=desc&objectId=&candid=&time__gt=&time__lt=&time__since=86400&jd__gt=&jd__lt=&filter=&cone=&objectcone=&objectidps=&ra__gt=&ra__lt=&dec__gt=&dec__lt=&l__gt=&l__lt=&b__gt=&b__lt=&magpsf__gte=&magpsf__lte=&sigmapsf__lte=&magap__gte=&magap__lte=&distnr__gte=&distnr__lte=&deltamaglatest__gte=&deltamaglatest__lte=&deltamagref__gte=&deltamagref__lte=&elong__lte=&nbad__lte=&rb__gte=&drb__gte=&classtar__gte=&classtar__lte=&fwhm__lte=' + + URL = "https://mars.lco.global/" + + # defining a params dict for the parameters to be sent to the API + #data = {"queries": [{"time_since": 100,},]} + + PARAMS={'format':'json', 'time__since':86400} + + # sending get request and saving the response as response object + r = requests.get(url = URL, params = PARAMS) + + + # extracting data in json format + data = r.json() + for d in data: + print(d) + + + #pprint.pprint(data['results']) + + for item in data['results']: + print(item['lco_id'],item['objectId'],item['candidate']['wall_time']) + + return + + tar_archive = '/export/django/srg/data/ZTF/ztf_public_20200125.tar.gz' + output_dir = tar_archive.split('/')[-1].split('.')[-3] + print (output_dir) + + print('{} has {} avro files'.format(output_dir, len(list(find_files(output_dir))))) + + + #archive = tarfile.open(tar_archive,'r:gz') + #archive.extractall(path=output_dir) + #archive.close() + + programs = defaultdict(int) + #for packet in generate_dictionaries(output_dir): + # programs[packet['candidate']['programid']] += 1 + #print(programs) + + for packet in filter(is_alert_pure,generate_dictionaries(output_dir)): + programs[packet['candidate']['programid']] += 1 + print(programs) + + transient_alerts = [] + for packet in filter(is_alert_pure,generate_dictionaries(output_dir)): + dflc = make_dataframe(packet) + if is_transient(dflc): + print(packet['objectId'],packet['candidate']['ra'],packet['candidate']['dec']) + transient_alerts.append(packet) + + """ + for packet in transient_alerts: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + show_all(packet) + """ diff --git a/astrobasis/management/commands/01_load_gaia.py b/astrobasis/management/commands/01_load_gaia.py new file mode 100644 index 0000000..0c76c1a --- /dev/null +++ b/astrobasis/management/commands/01_load_gaia.py @@ -0,0 +1,71 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import GaiaSourceFile, GAIADR2 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +import os +import glob +import logging + +def load_gaia_bulk(): + + path='/data/Gaia/gaia_source/csv/' + batch_size = 500 + gaia_empty=GaiaSourceFile.objects.filter(status__exact='empty') + for gaia in gaia_empty: + srcs = gaia.gaiadr2_set.all() + if(srcs): + srcs.delete() + print("Clear ",gaia.filename," back ref: ",len(srcs)) + + filepath=path+gaia.filename+'.gz' + data = astropy.table.Table.read(filepath, format='ascii.csv',encoding='latin1') + gaia.nrows = len(data) + gaia.save() + print("Bulk create for %s" % gaia.filename) + objs = (GAIADR2(filename=gaia, + ra=data[i]['ra'], + dec=data[i]['dec'], + name=data[i]['designation'], + solution_id = data[i]['solution_id'], + source_id = data[i]['source_id'], + ref_epoch = data[i]['ref_epoch'], + ra_error = data[i]['ra_error'], + dec_error = data[i]['dec_error'], + parallax = data[i]['parallax'], + parallax_error = data[i]['parallax_error'], + pmra = data[i]['pmra'], + pmra_error = data[i]['pmra_error'], + pmdec = data[i]['pmdec'], + pmdec_error = data[i]['pmdec_error'], + phot_g_mean_mag = data[i]['phot_g_mean_mag'], + phot_bp_mean_mag = data[i]['phot_bp_mean_mag'], + phot_rp_mean_mag = data[i]['phot_rp_mean_mag'],) + for i in range(len(data))) + + GAIADR2.objects.bulk_create(objs,batch_size) + + pass + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + load_gaia_bulk() + diff --git a/astrobasis/management/commands/01_load_gaia_dr3.py b/astrobasis/management/commands/01_load_gaia_dr3.py new file mode 100644 index 0000000..4739121 --- /dev/null +++ b/astrobasis/management/commands/01_load_gaia_dr3.py @@ -0,0 +1,79 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma +import time + +from astrobasis.models import GaiaSourceFileDR3, GAIADR3 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +import os +import glob +import logging + +def load_gaia_bulk(): + start_time = time.time() + path='/data/tmp/george/Gaia/gedr3/gaia_source/fits/fits/' + batch_size = 500 + gaia_empty=GaiaSourceFileDR3.objects.filter(status__exact='empty') + + for gaia in gaia_empty: + srcs = gaia.gaiadr3_set.all() + if(srcs): + srcs.delete() + print("Clear ",gaia.filename," back ref: ",len(srcs)) + + filepath=path+gaia.filename+'.fits' + data = astropy.table.Table.read(filepath, format='fits') + + + gaia.nrows = len(data) + gaia.save() + print("Bulk create for %s" % gaia.filename) + objs = (GAIADR3(filename=gaia, + ra=data[i]['ra'], + dec=data[i]['dec'], + name=data[i]['designation'], + #solution_id = data[i]['solution_id'], + source_id = data[i]['source_id'], + #ref_epoch = data[i]['ref_epoch'], + ra_error = data[i]['ra_error'], + dec_error = data[i]['dec_error'], + parallax = data[i]['parallax'], + parallax_error = data[i]['parallax_error'], + pmra = data[i]['pmra'], + pmra_error = data[i]['pmra_error'], + pmdec = data[i]['pmdec'], + pmdec_error = data[i]['pmdec_error'], + phot_g_mean_mag = data[i]['phot_g_mean_mag'], + phot_bp_mean_mag = data[i]['phot_bp_mean_mag'], + phot_rp_mean_mag = data[i]['phot_rp_mean_mag'],) + for i in range(len(data))) + + GAIADR3.objects.bulk_create(objs,batch_size) + gaia.status="ready" + gaia.save() + + + print("--- %s seconds ---" % (time.time() - start_time)) + pass + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + load_gaia_bulk() + diff --git a/astrobasis/management/commands/02_delete_gaia.py b/astrobasis/management/commands/02_delete_gaia.py new file mode 100644 index 0000000..5771352 --- /dev/null +++ b/astrobasis/management/commands/02_delete_gaia.py @@ -0,0 +1,52 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import GaiaSourceFile, GAIADR2 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + + +import os +import glob +import logging + +def delete_gaia(): + + gaia_loaded = GaiaSourceFile.objects.all() #filter(status__exact='loaded') + total_loaded = gaia_loaded.count() + total = 0 + for gaia in gaia_loaded: + total_linked = gaia.gaiadr2_set.all().count() + print("%s %s linked: %d nrows: %d" % (gaia.filename, gaia.status,total_linked,gaia.nrows)) + + linked = gaia.gaiadr2_set.all() + linked.delete() + gaia.status='empty' + gaia.save() + total+=1 + + print("Total %d removed from %d" % (total, total_loaded,)) + pass + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + delete_gaia() + + + diff --git a/astrobasis/management/commands/02_mark_gaia.py b/astrobasis/management/commands/02_mark_gaia.py new file mode 100644 index 0000000..8b5adb0 --- /dev/null +++ b/astrobasis/management/commands/02_mark_gaia.py @@ -0,0 +1,65 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import GaiaSourceFile, GAIADR2 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + + +import os +import glob +import logging + +def mark_gaia(): + + gaia_empty = GaiaSourceFile.objects.filter(status__exact='empty') + total_empty = gaia_empty.count() + total = 0 + for gaia in gaia_empty: + total_linked = gaia.gaiadr2_set.all().count() + #print("%s %s linked: %d nrows: %d" % (gaia.filename, gaia.status,total_linked,gaia.nrows)) + + if (gaia.nrows == 0): + """ empty and not still started to load """ + continue + + if (total_linked == 0): + """ looks like currently loading """ + continue + + if (total_linked == gaia.nrows): + print(gaia.filename+" -- loaded") + gaia.status='loaded' + gaia.save() + total+=1 + else: + """ delete leftovers if any """ + #linked = gaia.gaiadr2_set.all() + #linked.delete() + print(gaia.filename+" "+str(total_linked)+" != "+str(gaia.nrows)) + + print("Total %d marked as loaded from %d empty selected" % (total, total_empty,)) + pass + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + mark_gaia() + + + diff --git a/astrobasis/management/commands/03_healpix_gaia.py b/astrobasis/management/commands/03_healpix_gaia.py new file mode 100644 index 0000000..237eec4 --- /dev/null +++ b/astrobasis/management/commands/03_healpix_gaia.py @@ -0,0 +1,108 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from heasarc.models import INPUT_DATA_DIR +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine +import numpy.ma as ma + +from astrobasis.models import GaiaSourceFileDR3, GAIADR3 + +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from heasarc.models import NSIDE_SOURCES, ORDER + +import os +import glob +import logging +import time + +from multiprocessing import Pool, cpu_count + + +USE_NCORES=1 + +def myfunc(n): + print(n) + + batch_size=500 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5) + + plates=GaiaSourceFileDR3.objects.filter(status__exact="ready") + + total_ready = len(plates) + print('Total Gaia files: '+str(total_ready)) + + step=int(total_ready/USE_NCORES)+1 + idx1 = step*n + idx2 = step*(n+1) + if idx2 > total_ready: + idx2=total_ready + for i in range(idx1,idx2): + print(str(n)+": "+plates[i].filename) + phots=plates[i].gaiadr3_set.exclude(healpix__gt=0.0) + ra=phots.values_list('ra', flat=True) + dec=phots.values_list('dec', flat=True) + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + heal = hp.skycoord_to_healpix(crd) + for j, phot in enumerate(phots): + phot.healpix=heal[j] + GAIADR3.objects.bulk_update(phots, ['healpix'], batch_size=batch_size) + + plates[i].status='healpix' + plates[i].save() + + pass + +def run_healpix(): + + start_time = time.time() + + batch_size=500 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5) + + plates=GaiaSourceFileDR3.objects.filter(status__exact="ready") + + total_ready = len(plates) + + print('Total Gaia files: '+str(total_ready)) + + for plate in plates: + print(plate.filename) + phots=plate.gaiadr3_set.exclude(healpix__gt=0.0) + ra=phots.values_list('ra', flat=True) + dec=phots.values_list('dec', flat=True) + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + heal = hp.skycoord_to_healpix(crd) + for j, phot in enumerate(phots): + phot.healpix=heal[j] + GAIADR3.objects.bulk_update(phots, ['healpix'], batch_size=batch_size) + + plate.status='healpix' + plate.save() + + print("--- %s seconds ---" % (time.time() - start_time)) + pass + +class Command(BaseCommand): + help = 'Initiates healpix for Gaia' + + def handle(self, *args, **options): + + run_healpix() + + """ + does not work for multitheading for some reason + + with Pool(10) as p: + p.map(myfunc, range(USE_NCORES)) + """ diff --git a/astrobasis/management/commands/show_celery.py b/astrobasis/management/commands/show_celery.py new file mode 100644 index 0000000..7446598 --- /dev/null +++ b/astrobasis/management/commands/show_celery.py @@ -0,0 +1,15 @@ +from django.core.management.base import BaseCommand, CommandError +from celery.task.control import inspect + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + i = inspect() + + print('Show the items that have an ETA or are scheduled for later processing') + print(i.scheduled()) + print('Show tasks that are currently active.') + print(i.active()) + print('Show tasks that have been claimed by workers') + print(i.reserved()) diff --git a/astrobasis/migrations/0001_initial.py b/astrobasis/migrations/0001_initial.py new file mode 100644 index 0000000..92808fd --- /dev/null +++ b/astrobasis/migrations/0001_initial.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.6 on 2020-01-21 15:05 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='GaiaSourceFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('filename', models.CharField(blank=True, default='', max_length=80)), + ('status', models.CharField(default='empty', max_length=7)), + ('nrows', models.IntegerField(default=0)), + ], + ), + migrations.CreateModel( + name='GAIADR2', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('solution_id', models.CharField(blank=True, default='', max_length=19)), + ('name', models.CharField(blank=True, default='', max_length=30)), + ('source_id', models.CharField(blank=True, default='', max_length=19)), + ('ref_epoch', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('ra_error', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('dec_error', models.FloatField(default=0.0)), + ('parallax', models.FloatField(default=0.0)), + ('parallax_error', models.FloatField(default=0.0)), + ('pmra', models.FloatField(default=0.0)), + ('pmra_error', models.FloatField(default=0.0)), + ('pmdec', models.FloatField(default=0.0)), + ('pmdec_error', models.FloatField(default=0.0)), + ('phot_g_mean_mag', models.FloatField(default=0.0)), + ('phot_bp_mean_mag', models.FloatField(default=0.0)), + ('phot_rp_mean_mag', models.FloatField(default=0.0)), + ('filename', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='astrobasis.GaiaSourceFile')), + ], + ), + ] diff --git a/astrobasis/migrations/0002_ztfalert.py b/astrobasis/migrations/0002_ztfalert.py new file mode 100644 index 0000000..5d18cca --- /dev/null +++ b/astrobasis/migrations/0002_ztfalert.py @@ -0,0 +1,35 @@ +# Generated by Django 2.2.6 on 2020-01-28 18:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ZTFAlert', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('objectId', models.CharField(blank=True, default='', max_length=12)), + ('fid', models.IntegerField(db_index=True, default=0)), + ('lco_id', models.BigIntegerField(db_index=True, default=0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('programid', models.IntegerField(db_index=True, default=0)), + ('magpsf', models.FloatField(default=0.0)), + ('sigmapsf', models.FloatField(default=0.0)), + ('magap', models.FloatField(default=0.0)), + ('sigmagap', models.FloatField(default=0.0)), + ('wall_time', models.DateTimeField(blank=True)), + ('diffmaglim', models.FloatField(default=0.0)), + ('deltamagref', models.FloatField(default=0.0)), + ('deltamaglatest', models.FloatField(default=0.0)), + ('rb', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0003_remove_ztfalert_wall_time.py b/astrobasis/migrations/0003_remove_ztfalert_wall_time.py new file mode 100644 index 0000000..3eac659 --- /dev/null +++ b/astrobasis/migrations/0003_remove_ztfalert_wall_time.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.6 on 2020-01-28 18:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0002_ztfalert'), + ] + + operations = [ + migrations.RemoveField( + model_name='ztfalert', + name='wall_time', + ), + ] diff --git a/astrobasis/migrations/0004_ztfalert_wall_time.py b/astrobasis/migrations/0004_ztfalert_wall_time.py new file mode 100644 index 0000000..26fd202 --- /dev/null +++ b/astrobasis/migrations/0004_ztfalert_wall_time.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-01-28 18:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0003_remove_ztfalert_wall_time'), + ] + + operations = [ + migrations.AddField( + model_name='ztfalert', + name='wall_time', + field=models.CharField(blank=True, default='', max_length=29), + ), + ] diff --git a/astrobasis/migrations/0005_auto_20200128_2138.py b/astrobasis/migrations/0005_auto_20200128_2138.py new file mode 100644 index 0000000..21d15dc --- /dev/null +++ b/astrobasis/migrations/0005_auto_20200128_2138.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-01-28 18:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0004_ztfalert_wall_time'), + ] + + operations = [ + migrations.AlterField( + model_name='ztfalert', + name='deltamaglatest', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='ztfalert', + name='deltamagref', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/astrobasis/migrations/0006_delete_ztfalert.py b/astrobasis/migrations/0006_delete_ztfalert.py new file mode 100644 index 0000000..6bbb3f4 --- /dev/null +++ b/astrobasis/migrations/0006_delete_ztfalert.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.6 on 2020-01-28 18:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0005_auto_20200128_2138'), + ] + + operations = [ + migrations.DeleteModel( + name='ZTFAlert', + ), + ] diff --git a/astrobasis/migrations/0007_ztfalert.py b/astrobasis/migrations/0007_ztfalert.py new file mode 100644 index 0000000..93a6d69 --- /dev/null +++ b/astrobasis/migrations/0007_ztfalert.py @@ -0,0 +1,36 @@ +# Generated by Django 2.2.6 on 2020-01-28 18:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0006_delete_ztfalert'), + ] + + operations = [ + migrations.CreateModel( + name='ZTFAlert', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('objectId', models.CharField(blank=True, default='', max_length=12)), + ('fid', models.IntegerField(db_index=True, default=0)), + ('lco_id', models.BigIntegerField(db_index=True, default=0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('programid', models.IntegerField(db_index=True, default=0)), + ('magpsf', models.FloatField(default=0.0)), + ('sigmapsf', models.FloatField(default=0.0)), + ('magap', models.FloatField(default=0.0)), + ('sigmagap', models.FloatField(default=0.0)), + ('wall_time', models.CharField(blank=True, default='', max_length=29)), + ('diffmaglim', models.FloatField(default=0.0)), + ('deltamagref', models.FloatField(default=0.0, null=True)), + ('deltamaglatest', models.FloatField(default=0.0, null=True)), + ('rb', models.FloatField(default=0.0)), + ('created', models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/astrobasis/migrations/0008_ztfalert_healpix_plate.py b/astrobasis/migrations/0008_ztfalert_healpix_plate.py new file mode 100644 index 0000000..a896c6f --- /dev/null +++ b/astrobasis/migrations/0008_ztfalert_healpix_plate.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-01-29 07:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0007_ztfalert'), + ] + + operations = [ + migrations.AddField( + model_name='ztfalert', + name='healpix_plate', + field=models.IntegerField(default=0), + ), + ] diff --git a/astrobasis/migrations/0009_ztfalert_survey.py b/astrobasis/migrations/0009_ztfalert_survey.py new file mode 100644 index 0000000..9c6b7e1 --- /dev/null +++ b/astrobasis/migrations/0009_ztfalert_survey.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-01-29 07:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0002_auto_20191209_2233'), + ('astrobasis', '0008_ztfalert_healpix_plate'), + ] + + operations = [ + migrations.AddField( + model_name='ztfalert', + name='survey', + field=models.ManyToManyField(to='monthplan.Survey'), + ), + ] diff --git a/astrobasis/migrations/0010_auto_20200129_1339.py b/astrobasis/migrations/0010_auto_20200129_1339.py new file mode 100644 index 0000000..1714be1 --- /dev/null +++ b/astrobasis/migrations/0010_auto_20200129_1339.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.6 on 2020-01-29 10:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0009_ztfalert_survey'), + ] + + operations = [ + migrations.AddField( + model_name='ztfalert', + name='elong', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='ztfalert', + name='fwhm', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='ztfalert', + name='magdiff', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='ztfalert', + name='nbad', + field=models.IntegerField(default=0), + ), + ] diff --git a/astrobasis/migrations/0011_remove_ztfalert_wall_time.py b/astrobasis/migrations/0011_remove_ztfalert_wall_time.py new file mode 100644 index 0000000..ee5cf6f --- /dev/null +++ b/astrobasis/migrations/0011_remove_ztfalert_wall_time.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.6 on 2020-01-29 12:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0010_auto_20200129_1339'), + ] + + operations = [ + migrations.RemoveField( + model_name='ztfalert', + name='wall_time', + ), + ] diff --git a/astrobasis/migrations/0012_ztfalert_wall_time.py b/astrobasis/migrations/0012_ztfalert_wall_time.py new file mode 100644 index 0000000..6c4cafb --- /dev/null +++ b/astrobasis/migrations/0012_ztfalert_wall_time.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-01-29 12:52 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0011_remove_ztfalert_wall_time'), + ] + + operations = [ + migrations.AddField( + model_name='ztfalert', + name='wall_time', + field=models.DateTimeField(blank=True, default=datetime.datetime.now), + ), + ] diff --git a/astrobasis/migrations/0013_alerce.py b/astrobasis/migrations/0013_alerce.py new file mode 100644 index 0000000..5cfe3e5 --- /dev/null +++ b/astrobasis/migrations/0013_alerce.py @@ -0,0 +1,44 @@ +# Generated by Django 2.2.6 on 2020-01-29 15:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0002_auto_20191209_2233'), + ('astrobasis', '0012_ztfalert_wall_time'), + ] + + operations = [ + migrations.CreateModel( + name='ALeRCE', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('healpix_plate', models.IntegerField(default=0)), + ('created', models.DateTimeField(auto_now_add=True)), + ('classearly', models.IntegerField(default=0)), + ('classrf', models.IntegerField(default=0)), + ('oid', models.CharField(default='', max_length=12)), + ('firstmjd', models.FloatField(default=0.0)), + ('lastmjd', models.FloatField(default=0.0)), + ('mean_magap_g', models.FloatField(default=0.0)), + ('mean_magap_r', models.FloatField(default=0.0)), + ('mean_magpsf_g', models.FloatField(default=0.0)), + ('mean_magpsf_r', models.FloatField(default=0.0)), + ('meandec', models.FloatField(default=0.0)), + ('meanra', models.FloatField(default=0.0)), + ('nobs', models.IntegerField(default=0)), + ('pclassearly', models.FloatField(default=0.0)), + ('pclassrf', models.FloatField(default=0.0)), + ('sigma_magap_g', models.FloatField(default=0.0)), + ('sigma_magap_r', models.FloatField(default=0.0)), + ('sigma_magpsf_g', models.FloatField(default=0.0)), + ('sigma_magpsf_r', models.FloatField(default=0.0)), + ('sigmadec', models.FloatField(default=0.0)), + ('sigmara', models.FloatField(default=0.0)), + ('survey', models.ManyToManyField(to='monthplan.Survey')), + ], + ), + ] diff --git a/astrobasis/migrations/0014_auto_20200129_1948.py b/astrobasis/migrations/0014_auto_20200129_1948.py new file mode 100644 index 0000000..572a125 --- /dev/null +++ b/astrobasis/migrations/0014_auto_20200129_1948.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-01-29 16:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0013_alerce'), + ] + + operations = [ + migrations.AlterField( + model_name='alerce', + name='classrf', + field=models.IntegerField(default=0, null=True), + ), + ] diff --git a/astrobasis/migrations/0015_auto_20200129_1949.py b/astrobasis/migrations/0015_auto_20200129_1949.py new file mode 100644 index 0000000..20e24c1 --- /dev/null +++ b/astrobasis/migrations/0015_auto_20200129_1949.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.6 on 2020-01-29 16:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0014_auto_20200129_1948'), + ] + + operations = [ + migrations.AlterField( + model_name='alerce', + name='mean_magap_g', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='mean_magap_r', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='mean_magpsf_g', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='mean_magpsf_r', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/astrobasis/migrations/0016_auto_20200129_1949.py b/astrobasis/migrations/0016_auto_20200129_1949.py new file mode 100644 index 0000000..85e4ca1 --- /dev/null +++ b/astrobasis/migrations/0016_auto_20200129_1949.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-01-29 16:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0015_auto_20200129_1949'), + ] + + operations = [ + migrations.AlterField( + model_name='alerce', + name='pclassearly', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='pclassrf', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/astrobasis/migrations/0017_auto_20200129_1950.py b/astrobasis/migrations/0017_auto_20200129_1950.py new file mode 100644 index 0000000..dae33ef --- /dev/null +++ b/astrobasis/migrations/0017_auto_20200129_1950.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.6 on 2020-01-29 16:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0016_auto_20200129_1949'), + ] + + operations = [ + migrations.AlterField( + model_name='alerce', + name='sigma_magap_g', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='sigma_magap_r', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='sigma_magpsf_g', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='sigma_magpsf_r', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/astrobasis/migrations/0018_auto_20200130_1300.py b/astrobasis/migrations/0018_auto_20200130_1300.py new file mode 100644 index 0000000..afba9b8 --- /dev/null +++ b/astrobasis/migrations/0018_auto_20200130_1300.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-01-30 10:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0017_auto_20200129_1950'), + ] + + operations = [ + migrations.AlterField( + model_name='alerce', + name='classearly', + field=models.IntegerField(default=0, null=True), + ), + ] diff --git a/astrobasis/migrations/0019_auto_20200130_1455.py b/astrobasis/migrations/0019_auto_20200130_1455.py new file mode 100644 index 0000000..477e9c9 --- /dev/null +++ b/astrobasis/migrations/0019_auto_20200130_1455.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-01-30 11:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0018_auto_20200130_1300'), + ] + + operations = [ + migrations.AlterField( + model_name='alerce', + name='sigmadec', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='alerce', + name='sigmara', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/astrobasis/migrations/0020_auto_20200130_1726.py b/astrobasis/migrations/0020_auto_20200130_1726.py new file mode 100644 index 0000000..a79a3a7 --- /dev/null +++ b/astrobasis/migrations/0020_auto_20200130_1726.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.6 on 2020-01-30 14:26 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0019_auto_20200130_1455'), + ] + + operations = [ + migrations.AddField( + model_name='alerce', + name='firstdate', + field=models.DateTimeField(default=datetime.datetime.now), + ), + migrations.AddField( + model_name='alerce', + name='lastdate', + field=models.DateTimeField(default=datetime.datetime.now), + ), + ] diff --git a/astrobasis/migrations/0021_gaiadr2_error_radius.py b/astrobasis/migrations/0021_gaiadr2_error_radius.py new file mode 100644 index 0000000..6aff79b --- /dev/null +++ b/astrobasis/migrations/0021_gaiadr2_error_radius.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-01-31 10:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0020_auto_20200130_1726'), + ] + + operations = [ + migrations.AddField( + model_name='gaiadr2', + name='error_radius', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/0022_alerce_error_radius.py b/astrobasis/migrations/0022_alerce_error_radius.py new file mode 100644 index 0000000..becdd86 --- /dev/null +++ b/astrobasis/migrations/0022_alerce_error_radius.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-01-31 12:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0021_gaiadr2_error_radius'), + ] + + operations = [ + migrations.AddField( + model_name='alerce', + name='error_radius', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/0023_auto_20200131_1555.py b/astrobasis/migrations/0023_auto_20200131_1555.py new file mode 100644 index 0000000..260eb62 --- /dev/null +++ b/astrobasis/migrations/0023_auto_20200131_1555.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.6 on 2020-01-31 12:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0022_alerce_error_radius'), + ] + + operations = [ + migrations.RemoveField( + model_name='alerce', + name='meandec', + ), + migrations.RemoveField( + model_name='alerce', + name='meanra', + ), + ] diff --git a/astrobasis/migrations/0024_auto_20200131_1556.py b/astrobasis/migrations/0024_auto_20200131_1556.py new file mode 100644 index 0000000..83709ad --- /dev/null +++ b/astrobasis/migrations/0024_auto_20200131_1556.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-01-31 12:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0023_auto_20200131_1555'), + ] + + operations = [ + migrations.AddField( + model_name='alerce', + name='dec', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='alerce', + name='ra', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/0025_sdssdr12spec.py b/astrobasis/migrations/0025_sdssdr12spec.py new file mode 100644 index 0000000..c67dd0f --- /dev/null +++ b/astrobasis/migrations/0025_sdssdr12spec.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.6 on 2020-02-03 13:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0024_auto_20200131_1556'), + ] + + operations = [ + migrations.CreateModel( + name='SDSSDR12Spec', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('MJD', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('obj_class', models.CharField(default='', max_length=6)), + ('obj_subclass', models.CharField(default='', max_length=21)), + ('z', models.FloatField(default=0.0)), + ('z_err', models.FloatField(default=0.0)), + ('specobjid', models.CharField(default='', max_length=22)), + ('targetobjid', models.CharField(default='', max_length=22)), + ], + ), + ] diff --git a/astrobasis/migrations/0026_flesch_z.py b/astrobasis/migrations/0026_flesch_z.py new file mode 100644 index 0000000..6f42b40 --- /dev/null +++ b/astrobasis/migrations/0026_flesch_z.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.6 on 2020-02-10 15:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0025_sdssdr12spec'), + ] + + operations = [ + migrations.CreateModel( + name='Flesch_z', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('name', models.CharField(default='', max_length=25)), + ('obj_class', models.CharField(default='', max_length=4)), + ('Rmag', models.FloatField(default=0.0)), + ('Bmag', models.FloatField(default=0.0)), + ('z', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0027_delete_flesch_z.py b/astrobasis/migrations/0027_delete_flesch_z.py new file mode 100644 index 0000000..b91cdce --- /dev/null +++ b/astrobasis/migrations/0027_delete_flesch_z.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.6 on 2020-02-10 15:43 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0026_flesch_z'), + ] + + operations = [ + migrations.DeleteModel( + name='Flesch_z', + ), + ] diff --git a/astrobasis/migrations/0028_flesch.py b/astrobasis/migrations/0028_flesch.py new file mode 100644 index 0000000..39bfecd --- /dev/null +++ b/astrobasis/migrations/0028_flesch.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.6 on 2020-02-10 15:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0027_delete_flesch_z'), + ] + + operations = [ + migrations.CreateModel( + name='Flesch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('name', models.CharField(default='', max_length=25)), + ('obj_class', models.CharField(default='', max_length=4)), + ('Rmag', models.FloatField(default=0.0)), + ('Bmag', models.FloatField(default=0.0)), + ('z', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0029_simbad.py b/astrobasis/migrations/0029_simbad.py new file mode 100644 index 0000000..af0052d --- /dev/null +++ b/astrobasis/migrations/0029_simbad.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.6 on 2020-02-22 19:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0028_flesch'), + ] + + operations = [ + migrations.CreateModel( + name='Simbad', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('error_radius', models.FloatField(default=0.0)), + ('main_id', models.CharField(default='', max_length=25)), + ('obj_class', models.CharField(default='', max_length=4)), + ('coo_bibcode', models.CharField(default='', max_length=25)), + ('coo_wavelength', models.CharField(default='', max_length=3)), + ('z', models.FloatField(default=0.0)), + ('otype', models.CharField(default='', max_length=21)), + ], + ), + ] diff --git a/astrobasis/migrations/0030_auto_20200223_0814.py b/astrobasis/migrations/0030_auto_20200223_0814.py new file mode 100644 index 0000000..32d8818 --- /dev/null +++ b/astrobasis/migrations/0030_auto_20200223_0814.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-23 05:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0029_simbad'), + ] + + operations = [ + migrations.AlterField( + model_name='simbad', + name='main_id', + field=models.CharField(max_length=25, unique=True), + ), + ] diff --git a/astrobasis/migrations/0031_simbad_healpix.py b/astrobasis/migrations/0031_simbad_healpix.py new file mode 100644 index 0000000..583f0a4 --- /dev/null +++ b/astrobasis/migrations/0031_simbad_healpix.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-23 11:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0030_auto_20200223_0814'), + ] + + operations = [ + migrations.AddField( + model_name='simbad', + name='healpix', + field=models.BigIntegerField(db_index=True, default=0), + ), + ] diff --git a/astrobasis/migrations/0032_auto_20200223_1650.py b/astrobasis/migrations/0032_auto_20200223_1650.py new file mode 100644 index 0000000..4046e46 --- /dev/null +++ b/astrobasis/migrations/0032_auto_20200223_1650.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-23 13:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0031_simbad_healpix'), + ] + + operations = [ + migrations.AlterField( + model_name='simbad', + name='main_id', + field=models.CharField(max_length=51, unique=True), + ), + ] diff --git a/astrobasis/migrations/0033_auto_20200225_1604.py b/astrobasis/migrations/0033_auto_20200225_1604.py new file mode 100644 index 0000000..af3ee54 --- /dev/null +++ b/astrobasis/migrations/0033_auto_20200225_1604.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-25 13:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0032_auto_20200223_1650'), + ] + + operations = [ + migrations.AlterField( + model_name='simbad', + name='coo_wavelength', + field=models.CharField(default='', max_length=5), + ), + ] diff --git a/astrobasis/migrations/0034_auto_20200225_1608.py b/astrobasis/migrations/0034_auto_20200225_1608.py new file mode 100644 index 0000000..27e1e0d --- /dev/null +++ b/astrobasis/migrations/0034_auto_20200225_1608.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-25 13:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0033_auto_20200225_1604'), + ] + + operations = [ + migrations.AlterField( + model_name='simbad', + name='obj_class', + field=models.CharField(default='', max_length=10), + ), + ] diff --git a/astrobasis/migrations/0035_auto_20200225_1625.py b/astrobasis/migrations/0035_auto_20200225_1625.py new file mode 100644 index 0000000..724ac2e --- /dev/null +++ b/astrobasis/migrations/0035_auto_20200225_1625.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-25 13:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0034_auto_20200225_1608'), + ] + + operations = [ + migrations.AlterField( + model_name='simbad', + name='obj_class', + field=models.CharField(default='', max_length=21), + ), + ] diff --git a/astrobasis/migrations/0036_wise.py b/astrobasis/migrations/0036_wise.py new file mode 100644 index 0000000..2292df8 --- /dev/null +++ b/astrobasis/migrations/0036_wise.py @@ -0,0 +1,78 @@ +# Generated by Django 3.0.8 on 2020-07-22 14:28 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0035_auto_20200225_1625'), + ] + + operations = [ + migrations.CreateModel( + name='Wise', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sep', models.FloatField(default=0, validators=[django.core.validators.MinValueValidator(0.0)])), + ('objID', models.BigIntegerField(default=0)), + ('raStack', models.FloatField(default=0.0)), + ('decStack', models.FloatField(default=0.0)), + ('raStackErr', models.FloatField(default=0.0)), + ('decStackErr', models.FloatField(default=0.0)), + ('raMean', models.FloatField(default=0.0)), + ('decMean', models.FloatField(default=0.0)), + ('raMeanErr', models.FloatField(default=0.0)), + ('decMeanErr', models.FloatField(default=0.0)), + ('objInfoFlag', models.BigIntegerField(default=0)), + ('qualityFlag', models.IntegerField(default=0)), + ('primaryDetection', models.IntegerField(default=0)), + ('bestDetection', models.IntegerField(default=0)), + ('gPSFMag', models.FloatField(default=0.0)), + ('gPSFMagErr', models.FloatField(default=0.0)), + ('gApMag', models.FloatField(default=0.0)), + ('gApMagErr', models.FloatField(default=0.0)), + ('gKronMag', models.FloatField(default=0.0)), + ('gKronMagErr', models.FloatField(default=0.0)), + ('rPSFMag', models.FloatField(default=0.0)), + ('rPSFMagErr', models.FloatField(default=0.0)), + ('rApMag', models.FloatField(default=0.0)), + ('rApMagErr', models.FloatField(default=0.0)), + ('rKronMag', models.FloatField(default=0.0)), + ('rKronMagErr', models.FloatField(default=0.0)), + ('iPSFMag', models.FloatField(default=0.0)), + ('iPSFMagErr', models.FloatField(default=0.0)), + ('iApMag', models.FloatField(default=0.0)), + ('iApMagErr', models.FloatField(default=0.0)), + ('iKronMag', models.FloatField(default=0.0)), + ('iKronMagErr', models.FloatField(default=0.0)), + ('zPSFMag', models.FloatField(default=0.0)), + ('zPSFMagErr', models.FloatField(default=0.0)), + ('zApMag', models.FloatField(default=0.0)), + ('zApMagErr', models.FloatField(default=0.0)), + ('zKronMag', models.FloatField(default=0.0)), + ('zKronMagErr', models.FloatField(default=0.0)), + ('yPSFMag', models.FloatField(default=0.0)), + ('yPSFMagErr', models.FloatField(default=0.0)), + ('yApMag', models.FloatField(default=0.0)), + ('yApMagErr', models.FloatField(default=0.0)), + ('yKronMag', models.FloatField(default=0.0)), + ('yKronMagErr', models.FloatField(default=0.0)), + ('wisefield', models.CharField(default='', max_length=8)), + ('fitext', models.BooleanField(default=False, null=True)), + ('devaucou', models.BooleanField(default=False, null=True)), + ('star', models.BooleanField(default=False, null=True)), + ('w1flux', models.FloatField(default=0.0)), + ('dw1flux', models.FloatField(default=0.0)), + ('w1mag', models.FloatField(default=0.0)), + ('dw1mag', models.FloatField(default=0.0)), + ('w1reff', models.FloatField(default=0.0)), + ('w2flux', models.FloatField(default=0.0)), + ('dw2flux', models.FloatField(default=0.0)), + ('w2mag', models.FloatField(default=0.0)), + ('dw2mag', models.FloatField(default=0.0)), + ('w2reff', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0037_delete_wise.py b/astrobasis/migrations/0037_delete_wise.py new file mode 100644 index 0000000..b38c5e5 --- /dev/null +++ b/astrobasis/migrations/0037_delete_wise.py @@ -0,0 +1,16 @@ +# Generated by Django 3.0.8 on 2020-07-22 14:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0036_wise'), + ] + + operations = [ + migrations.DeleteModel( + name='Wise', + ), + ] diff --git a/astrobasis/migrations/0038_allwise.py b/astrobasis/migrations/0038_allwise.py new file mode 100644 index 0000000..1e77ae6 --- /dev/null +++ b/astrobasis/migrations/0038_allwise.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.8 on 2020-08-04 06:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0037_delete_wise'), + ] + + operations = [ + migrations.CreateModel( + name='AllWise', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=51, unique=True)), + ('image_url', models.CharField(default='', max_length=150)), + ('W1mag', models.FloatField(default=0.0)), + ('e_W1mag', models.FloatField(default=0.0)), + ('W2mag', models.FloatField(default=0.0)), + ('e_W2mag', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0039_remove_allwise_image_url.py b/astrobasis/migrations/0039_remove_allwise_image_url.py new file mode 100644 index 0000000..fa50330 --- /dev/null +++ b/astrobasis/migrations/0039_remove_allwise_image_url.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2020-08-04 06:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0038_allwise'), + ] + + operations = [ + migrations.RemoveField( + model_name='allwise', + name='image_url', + ), + ] diff --git a/astrobasis/migrations/0040_nvss.py b/astrobasis/migrations/0040_nvss.py new file mode 100644 index 0000000..4031307 --- /dev/null +++ b/astrobasis/migrations/0040_nvss.py @@ -0,0 +1,26 @@ +# Generated by Django 3.0.8 on 2020-08-04 10:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0039_remove_allwise_image_url'), + ] + + operations = [ + migrations.CreateModel( + name='NVSS', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=51, unique=True)), + ('S14', models.FloatField(default=0.0)), + ('e_S14', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0041_auto_20200804_1457.py b/astrobasis/migrations/0041_auto_20200804_1457.py new file mode 100644 index 0000000..5dd772c --- /dev/null +++ b/astrobasis/migrations/0041_auto_20200804_1457.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2020-08-04 11:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0040_nvss'), + ] + + operations = [ + migrations.AddField( + model_name='nvss', + name='e_dec', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='nvss', + name='e_ra', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/0042_first.py b/astrobasis/migrations/0042_first.py new file mode 100644 index 0000000..15b9b31 --- /dev/null +++ b/astrobasis/migrations/0042_first.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.8 on 2020-08-04 12:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0041_auto_20200804_1457'), + ] + + operations = [ + migrations.CreateModel( + name='FIRST', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=51, unique=True)), + ('Fpeak', models.FloatField(default=0.0)), + ('Fint', models.FloatField(default=0.0)), + ('rms', models.FloatField(default=0.0)), + ('major_axis', models.FloatField(default=0.0)), + ('minor_axis', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0043_sumss.py b/astrobasis/migrations/0043_sumss.py new file mode 100644 index 0000000..b868da7 --- /dev/null +++ b/astrobasis/migrations/0043_sumss.py @@ -0,0 +1,32 @@ +# Generated by Django 3.0.8 on 2020-08-04 13:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0042_first'), + ] + + operations = [ + migrations.CreateModel( + name='SUMSS', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('e_ra', models.FloatField(default=0.0)), + ('e_dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=51, unique=True)), + ('Sp', models.FloatField(default=0.0)), + ('e_Sp', models.FloatField(default=0.0)), + ('St', models.FloatField(default=0.0)), + ('e_St', models.FloatField(default=0.0)), + ('major_axis', models.FloatField(default=0.0)), + ('minor_axis', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0044_auto_20200804_1703.py b/astrobasis/migrations/0044_auto_20200804_1703.py new file mode 100644 index 0000000..48a92eb --- /dev/null +++ b/astrobasis/migrations/0044_auto_20200804_1703.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2020-08-04 14:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0043_sumss'), + ] + + operations = [ + migrations.AddField( + model_name='sumss', + name='dec_orig', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='sumss', + name='ra_orig', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/0045_auto_20200804_1707.py b/astrobasis/migrations/0045_auto_20200804_1707.py new file mode 100644 index 0000000..5304b72 --- /dev/null +++ b/astrobasis/migrations/0045_auto_20200804_1707.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2020-08-04 14:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0044_auto_20200804_1703'), + ] + + operations = [ + migrations.AlterField( + model_name='sumss', + name='dec_orig', + field=models.CharField(max_length=51, unique=True), + ), + migrations.AlterField( + model_name='sumss', + name='ra_orig', + field=models.CharField(max_length=51, unique=True), + ), + ] diff --git a/astrobasis/migrations/0046_gaiadr3.py b/astrobasis/migrations/0046_gaiadr3.py new file mode 100644 index 0000000..c5a031a --- /dev/null +++ b/astrobasis/migrations/0046_gaiadr3.py @@ -0,0 +1,40 @@ +# Generated by Django 3.0.8 on 2021-02-12 10:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0045_auto_20200804_1707'), + ] + + operations = [ + migrations.CreateModel( + name='GAIADR3', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('solution_id', models.CharField(blank=True, default='', max_length=19)), + ('name', models.CharField(blank=True, default='', max_length=30)), + ('source_id', models.CharField(blank=True, default='', max_length=19)), + ('error_radius', models.FloatField(default=0.0)), + ('ref_epoch', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('ra_error', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('dec_error', models.FloatField(default=0.0)), + ('parallax', models.FloatField(default=0.0)), + ('parallax_error', models.FloatField(default=0.0)), + ('pmra', models.FloatField(default=0.0)), + ('pmra_error', models.FloatField(default=0.0)), + ('pmdec', models.FloatField(default=0.0)), + ('pmdec_error', models.FloatField(default=0.0)), + ('phot_g_mean_mag', models.FloatField(default=0.0)), + ('phot_bp_mean_mag', models.FloatField(default=0.0)), + ('phot_rp_mean_mag', models.FloatField(default=0.0)), + ('filename', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='astrobasis.GaiaSourceFile')), + ], + ), + ] diff --git a/astrobasis/migrations/0047_remove_gaiadr3_filename.py b/astrobasis/migrations/0047_remove_gaiadr3_filename.py new file mode 100644 index 0000000..32d08ae --- /dev/null +++ b/astrobasis/migrations/0047_remove_gaiadr3_filename.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2021-02-12 10:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0046_gaiadr3'), + ] + + operations = [ + migrations.RemoveField( + model_name='gaiadr3', + name='filename', + ), + ] diff --git a/astrobasis/migrations/0048_delete_gaiadr3.py b/astrobasis/migrations/0048_delete_gaiadr3.py new file mode 100644 index 0000000..ba758cf --- /dev/null +++ b/astrobasis/migrations/0048_delete_gaiadr3.py @@ -0,0 +1,16 @@ +# Generated by Django 3.0.8 on 2021-02-12 10:33 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0047_remove_gaiadr3_filename'), + ] + + operations = [ + migrations.DeleteModel( + name='GAIADR3', + ), + ] diff --git a/astrobasis/migrations/0049_gaiadr3.py b/astrobasis/migrations/0049_gaiadr3.py new file mode 100644 index 0000000..ffc46a3 --- /dev/null +++ b/astrobasis/migrations/0049_gaiadr3.py @@ -0,0 +1,40 @@ +# Generated by Django 3.0.8 on 2021-02-12 10:33 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0048_delete_gaiadr3'), + ] + + operations = [ + migrations.CreateModel( + name='GAIADR3', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('solution_id', models.CharField(blank=True, default='', max_length=19)), + ('name', models.CharField(blank=True, default='', max_length=30)), + ('source_id', models.CharField(blank=True, default='', max_length=19)), + ('error_radius', models.FloatField(default=0.0)), + ('ref_epoch', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('ra_error', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('dec_error', models.FloatField(default=0.0)), + ('parallax', models.FloatField(default=0.0)), + ('parallax_error', models.FloatField(default=0.0)), + ('pmra', models.FloatField(default=0.0)), + ('pmra_error', models.FloatField(default=0.0)), + ('pmdec', models.FloatField(default=0.0)), + ('pmdec_error', models.FloatField(default=0.0)), + ('phot_g_mean_mag', models.FloatField(default=0.0)), + ('phot_bp_mean_mag', models.FloatField(default=0.0)), + ('phot_rp_mean_mag', models.FloatField(default=0.0)), + ('filename', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='astrobasis.GaiaSourceFile')), + ], + ), + ] diff --git a/astrobasis/migrations/0050_delete_gaiadr3.py b/astrobasis/migrations/0050_delete_gaiadr3.py new file mode 100644 index 0000000..9c03637 --- /dev/null +++ b/astrobasis/migrations/0050_delete_gaiadr3.py @@ -0,0 +1,16 @@ +# Generated by Django 3.0.8 on 2021-02-12 12:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0049_gaiadr3'), + ] + + operations = [ + migrations.DeleteModel( + name='GAIADR3', + ), + ] diff --git a/astrobasis/migrations/0051_gaiadr3_gaiasourcefiledr3.py b/astrobasis/migrations/0051_gaiadr3_gaiasourcefiledr3.py new file mode 100644 index 0000000..4ee9ef7 --- /dev/null +++ b/astrobasis/migrations/0051_gaiadr3_gaiasourcefiledr3.py @@ -0,0 +1,49 @@ +# Generated by Django 3.0.8 on 2021-02-12 12:03 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0050_delete_gaiadr3'), + ] + + operations = [ + migrations.CreateModel( + name='GaiaSourceFileDR3', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('filename', models.CharField(blank=True, default='', max_length=80)), + ('status', models.CharField(default='empty', max_length=7)), + ('nrows', models.IntegerField(default=0)), + ], + ), + migrations.CreateModel( + name='GAIADR3', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('solution_id', models.CharField(blank=True, default='', max_length=19)), + ('name', models.CharField(blank=True, default='', max_length=30)), + ('source_id', models.CharField(blank=True, default='', max_length=19)), + ('error_radius', models.FloatField(default=0.0)), + ('ref_epoch', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('ra_error', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('dec_error', models.FloatField(default=0.0)), + ('parallax', models.FloatField(default=0.0)), + ('parallax_error', models.FloatField(default=0.0)), + ('pmra', models.FloatField(default=0.0)), + ('pmra_error', models.FloatField(default=0.0)), + ('pmdec', models.FloatField(default=0.0)), + ('pmdec_error', models.FloatField(default=0.0)), + ('phot_g_mean_mag', models.FloatField(default=0.0)), + ('phot_bp_mean_mag', models.FloatField(default=0.0)), + ('phot_rp_mean_mag', models.FloatField(default=0.0)), + ('filename', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='astrobasis.GaiaSourceFileDR3')), + ], + ), + ] diff --git a/astrobasis/migrations/0052_remove_gaiadr3_solution_id.py b/astrobasis/migrations/0052_remove_gaiadr3_solution_id.py new file mode 100644 index 0000000..0e047e7 --- /dev/null +++ b/astrobasis/migrations/0052_remove_gaiadr3_solution_id.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2021-02-12 12:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0051_gaiadr3_gaiasourcefiledr3'), + ] + + operations = [ + migrations.RemoveField( + model_name='gaiadr3', + name='solution_id', + ), + ] diff --git a/astrobasis/migrations/0053_remove_gaiadr3_ref_epoch.py b/astrobasis/migrations/0053_remove_gaiadr3_ref_epoch.py new file mode 100644 index 0000000..454fff8 --- /dev/null +++ b/astrobasis/migrations/0053_remove_gaiadr3_ref_epoch.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2021-02-12 12:15 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0052_remove_gaiadr3_solution_id'), + ] + + operations = [ + migrations.RemoveField( + model_name='gaiadr3', + name='ref_epoch', + ), + ] diff --git a/astrobasis/migrations/0054_auto_20210812_1858.py b/astrobasis/migrations/0054_auto_20210812_1858.py new file mode 100644 index 0000000..f94336f --- /dev/null +++ b/astrobasis/migrations/0054_auto_20210812_1858.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2021-08-12 15:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0053_remove_gaiadr3_ref_epoch'), + ] + + operations = [ + migrations.AlterField( + model_name='sumss', + name='dec_orig', + field=models.CharField(max_length=51), + ), + migrations.AlterField( + model_name='sumss', + name='ra_orig', + field=models.CharField(max_length=51), + ), + ] diff --git a/astrobasis/migrations/0055_auto_20211229_1259.py b/astrobasis/migrations/0055_auto_20211229_1259.py new file mode 100644 index 0000000..413bb3a --- /dev/null +++ b/astrobasis/migrations/0055_auto_20211229_1259.py @@ -0,0 +1,78 @@ +# Generated by Django 3.2.10 on 2021-12-29 09:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0054_auto_20210812_1858'), + ] + + operations = [ + migrations.AlterField( + model_name='alerce', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='allwise', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='first', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='flesch', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='gaiadr2', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='gaiadr3', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='gaiasourcefile', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='gaiasourcefiledr3', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='nvss', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='sdssdr12spec', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='simbad', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='sumss', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='ztfalert', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/astrobasis/migrations/0056_twomass.py b/astrobasis/migrations/0056_twomass.py new file mode 100644 index 0000000..b81cc66 --- /dev/null +++ b/astrobasis/migrations/0056_twomass.py @@ -0,0 +1,39 @@ +# Generated by Django 3.2.12 on 2022-03-05 16:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0055_auto_20211229_1259'), + ] + + operations = [ + migrations.CreateModel( + name='TwoMASS', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('error_radius', models.FloatField(default=0.5)), + ('name', models.CharField(blank=True, default='', max_length=16)), + ('jmag', models.FloatField(default=0.0, null=True)), + ('e_jmag', models.FloatField(default=0.0, null=True)), + ('hmag', models.FloatField(default=0.0, null=True)), + ('e_hmag', models.FloatField(default=0.0, null=True)), + ('kmag', models.FloatField(default=0.0, null=True)), + ('e_kmag', models.FloatField(default=0.0, null=True)), + ('qkfg', models.CharField(blank=True, default='', max_length=3, null=True)), + ('rkfg', models.CharField(blank=True, default='', max_length=3, null=True)), + ('bkfg', models.CharField(blank=True, default='', max_length=3, null=True)), + ('ckfg', models.CharField(blank=True, default='', max_length=3, null=True)), + ('prox', models.FloatField(default=0.0, null=True)), + ('xflg', models.BooleanField(default=False, null=True)), + ('aflg', models.BooleanField(default=False, null=True)), + ('glon', models.FloatField(default=0.0)), + ('glat', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0057_glimpse.py b/astrobasis/migrations/0057_glimpse.py new file mode 100644 index 0000000..d814889 --- /dev/null +++ b/astrobasis/migrations/0057_glimpse.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.12 on 2022-03-05 18:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0056_twomass'), + ] + + operations = [ + migrations.CreateModel( + name='GLIMPSE', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('error_radius', models.FloatField(default=0.5)), + ('name', models.CharField(blank=True, default='', max_length=17)), + ('twomass', models.CharField(blank=True, default='', max_length=16)), + ('jmag', models.FloatField(default=0.0, null=True)), + ('hmag', models.FloatField(default=0.0, null=True)), + ('kmag', models.FloatField(default=0.0, null=True)), + ('b1mag', models.FloatField(default=0.0, null=True)), + ('e_b1mag', models.FloatField(default=0.0, null=True)), + ('b2mag', models.FloatField(default=0.0, null=True)), + ('e_b2mag', models.FloatField(default=0.0, null=True)), + ('b3mag', models.FloatField(default=0.0, null=True)), + ('e_b3mag', models.FloatField(default=0.0, null=True)), + ('b4mag', models.FloatField(default=0.0, null=True)), + ('e_b4mag', models.FloatField(default=0.0, null=True)), + ('glon', models.FloatField(default=0.0)), + ('glat', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0058_alter_glimpse_healpix.py b/astrobasis/migrations/0058_alter_glimpse_healpix.py new file mode 100644 index 0000000..eaa7e38 --- /dev/null +++ b/astrobasis/migrations/0058_alter_glimpse_healpix.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-15 08:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0057_glimpse'), + ] + + operations = [ + migrations.AlterField( + model_name='glimpse', + name='healpix', + field=models.BigIntegerField(db_index=True, default=0, null=True), + ), + ] diff --git a/astrobasis/migrations/0059_auto_20220315_1125.py b/astrobasis/migrations/0059_auto_20220315_1125.py new file mode 100644 index 0000000..aac2e37 --- /dev/null +++ b/astrobasis/migrations/0059_auto_20220315_1125.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.12 on 2022-03-15 08:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0058_alter_glimpse_healpix'), + ] + + operations = [ + migrations.AlterField( + model_name='glimpse', + name='glat', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='glimpse', + name='glon', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/astrobasis/migrations/0060_alter_glimpse_options.py b/astrobasis/migrations/0060_alter_glimpse_options.py new file mode 100644 index 0000000..f24b0e6 --- /dev/null +++ b/astrobasis/migrations/0060_alter_glimpse_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.12 on 2022-03-23 08:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0059_auto_20220315_1125'), + ] + + operations = [ + migrations.AlterModelOptions( + name='glimpse', + options={'ordering': ['-id']}, + ), + ] diff --git a/astrobasis/migrations/0061_vlass.py b/astrobasis/migrations/0061_vlass.py new file mode 100644 index 0000000..5a51e8b --- /dev/null +++ b/astrobasis/migrations/0061_vlass.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.12 on 2022-03-25 14:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0060_alter_glimpse_options'), + ] + + operations = [ + migrations.CreateModel( + name='VLASS', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=51, unique=True)), + ('ftot', models.FloatField(default=0.0)), + ('e_ftot', models.FloatField(default=0.0)), + ('fpeak', models.FloatField(default=0.0)), + ('e_fpeak', models.FloatField(default=0.0)), + ], + ), + ] diff --git a/astrobasis/migrations/0062_auto_20220325_1731.py b/astrobasis/migrations/0062_auto_20220325_1731.py new file mode 100644 index 0000000..8e7da52 --- /dev/null +++ b/astrobasis/migrations/0062_auto_20220325_1731.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.12 on 2022-03-25 14:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0061_vlass'), + ] + + operations = [ + migrations.AddField( + model_name='vlass', + name='dec_m', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='vlass', + name='e_dec', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='vlass', + name='e_ra', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='vlass', + name='ra_m', + field=models.FloatField(default=0.0), + ), + migrations.AlterField( + model_name='vlass', + name='name', + field=models.CharField(max_length=31, unique=True), + ), + ] diff --git a/astrobasis/migrations/0063_auto_20220325_1742.py b/astrobasis/migrations/0063_auto_20220325_1742.py new file mode 100644 index 0000000..c9c74b4 --- /dev/null +++ b/astrobasis/migrations/0063_auto_20220325_1742.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.12 on 2022-03-25 14:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0062_auto_20220325_1731'), + ] + + operations = [ + migrations.AddField( + model_name='vlass', + name='dupflag', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='vlass', + name='qualflag', + field=models.IntegerField(default=0), + ), + migrations.AlterField( + model_name='vlass', + name='name', + field=models.CharField(max_length=31), + ), + ] diff --git a/astrobasis/migrations/0064_auto_20220325_1752.py b/astrobasis/migrations/0064_auto_20220325_1752.py new file mode 100644 index 0000000..f06300c --- /dev/null +++ b/astrobasis/migrations/0064_auto_20220325_1752.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.12 on 2022-03-25 14:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0063_auto_20220325_1742'), + ] + + operations = [ + migrations.AddField( + model_name='vlass', + name='glat', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='vlass', + name='glon', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/0065_auto_20220330_1313.py b/astrobasis/migrations/0065_auto_20220330_1313.py new file mode 100644 index 0000000..2745d11 --- /dev/null +++ b/astrobasis/migrations/0065_auto_20220330_1313.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.12 on 2022-03-30 10:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0064_auto_20220325_1752'), + ] + + operations = [ + migrations.AlterModelOptions( + name='first', + options={'ordering': ['-id']}, + ), + migrations.AlterModelOptions( + name='sumss', + options={'ordering': ['-id']}, + ), + migrations.AlterModelOptions( + name='vlass', + options={'ordering': ['-id']}, + ), + ] diff --git a/astrobasis/migrations/0066_alter_twomass_name.py b/astrobasis/migrations/0066_alter_twomass_name.py new file mode 100644 index 0000000..114520f --- /dev/null +++ b/astrobasis/migrations/0066_alter_twomass_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-31 10:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0065_auto_20220330_1313'), + ] + + operations = [ + migrations.AlterField( + model_name='twomass', + name='name', + field=models.CharField(blank=True, default='', max_length=17), + ), + ] diff --git a/astrobasis/migrations/0067_vlassfromvizier.py b/astrobasis/migrations/0067_vlassfromvizier.py new file mode 100644 index 0000000..89fc3b2 --- /dev/null +++ b/astrobasis/migrations/0067_vlassfromvizier.py @@ -0,0 +1,39 @@ +# Generated by Django 3.2.12 on 2022-04-04 13:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0066_alter_twomass_name'), + ] + + operations = [ + migrations.CreateModel( + name='VLASSfromVizieR', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('glon', models.FloatField(default=0.0)), + ('glat', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('e_ra', models.FloatField(default=0.0)), + ('e_dec', models.FloatField(default=0.0)), + ('ra_m', models.FloatField(default=0.0)), + ('dec_m', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=31)), + ('ftot', models.FloatField(default=0.0)), + ('e_ftot', models.FloatField(default=0.0)), + ('fpeak', models.FloatField(default=0.0)), + ('e_fpeak', models.FloatField(default=0.0)), + ('dupflag', models.IntegerField(default=0)), + ('qualflag', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['-id'], + }, + ), + ] diff --git a/astrobasis/migrations/0068_auto_20220404_1639.py b/astrobasis/migrations/0068_auto_20220404_1639.py new file mode 100644 index 0000000..dfc9c18 --- /dev/null +++ b/astrobasis/migrations/0068_auto_20220404_1639.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.12 on 2022-04-04 13:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0067_vlassfromvizier'), + ] + + operations = [ + migrations.RemoveField( + model_name='vlassfromvizier', + name='dec_m', + ), + migrations.RemoveField( + model_name='vlassfromvizier', + name='ra_m', + ), + ] diff --git a/astrobasis/migrations/0069_vlassfromvizier_recno.py b/astrobasis/migrations/0069_vlassfromvizier_recno.py new file mode 100644 index 0000000..593c0c8 --- /dev/null +++ b/astrobasis/migrations/0069_vlassfromvizier_recno.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-04-04 16:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0068_auto_20220404_1639'), + ] + + operations = [ + migrations.AddField( + model_name='vlassfromvizier', + name='recno', + field=models.IntegerField(default=0), + ), + ] diff --git a/astrobasis/migrations/0070_alter_twomass_options.py b/astrobasis/migrations/0070_alter_twomass_options.py new file mode 100644 index 0000000..a14c329 --- /dev/null +++ b/astrobasis/migrations/0070_alter_twomass_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.12 on 2022-04-07 14:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0069_vlassfromvizier_recno'), + ] + + operations = [ + migrations.AlterModelOptions( + name='twomass', + options={'ordering': ['-id']}, + ), + ] diff --git a/astrobasis/migrations/0071_astrobasisusercat.py b/astrobasis/migrations/0071_astrobasisusercat.py new file mode 100644 index 0000000..b643b2b --- /dev/null +++ b/astrobasis/migrations/0071_astrobasisusercat.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.13 on 2022-06-09 14:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0070_alter_twomass_options'), + ] + + operations = [ + migrations.CreateModel( + name='AstroBasisUserCat', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=51, unique=True)), + ('notes', models.TextField(blank=True, max_length=2000, null=True, verbose_name='notes')), + ('bibcode', models.CharField(blank=True, default='', max_length=36)), + ('autoname', models.BooleanField(default=False, help_text='Generate IAU name of the component in the form NAME JHHMMSS.s+DDMMSS,
where NAME is taken from the corresponding field above.')), + ], + options={ + 'ordering': ['-id'], + }, + ), + ] diff --git a/astrobasis/migrations/0072_delete_astrobasisusercat.py b/astrobasis/migrations/0072_delete_astrobasisusercat.py new file mode 100644 index 0000000..58aba5b --- /dev/null +++ b/astrobasis/migrations/0072_delete_astrobasisusercat.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.13 on 2022-06-09 14:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0071_astrobasisusercat'), + ] + + operations = [ + migrations.DeleteModel( + name='AstroBasisUserCat', + ), + ] diff --git a/astrobasis/migrations/0073_astrobasisusercat.py b/astrobasis/migrations/0073_astrobasisusercat.py new file mode 100644 index 0000000..f650221 --- /dev/null +++ b/astrobasis/migrations/0073_astrobasisusercat.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.13 on 2022-06-10 15:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0072_delete_astrobasisusercat'), + ] + + operations = [ + migrations.CreateModel( + name='AstroBasisUserCat', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.0)), + ('name', models.CharField(max_length=51, unique=True)), + ('notes', models.TextField(blank=True, max_length=2000, null=True, verbose_name='notes')), + ('bibcode', models.CharField(blank=True, default='', max_length=36)), + ('autoname', models.BooleanField(default=False, help_text='Generate IAU name of the component in the form NAME JHHMMSS.s+DDMMSS,
where NAME is taken from the corresponding field above.')), + ], + options={ + 'ordering': ['-id'], + }, + ), + ] diff --git a/astrobasis/migrations/0074_magnitude.py b/astrobasis/migrations/0074_magnitude.py new file mode 100644 index 0000000..766469c --- /dev/null +++ b/astrobasis/migrations/0074_magnitude.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.13 on 2022-06-13 13:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0073_astrobasisusercat'), + ] + + operations = [ + migrations.CreateModel( + name='Magnitude', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('magn', models.FloatField(default=0.0)), + ('band', models.CharField(choices=[('U', 'U'), ('B', 'B'), ('V', 'V'), ('R', 'R'), ('I', 'I'), ('Z', 'Z'), ('Y', 'Y'), ('', 'J'), ('H', 'H'), ('K', 'K'), ('Ks', 'Ks'), ('Br_gamma', 'Br_gamma'), ('u', 'u'), ('g', 'g'), ('r', 'r'), ('i', 'i'), ('z', 'z'), ('y', 'y'), ('G', 'G'), ('Gbr', 'Gbr'), ('Grp', 'Grp'), ('I1', 'I1'), ('I2', 'I2'), ('I3', 'I3'), ('I4', 'I4'), ('MIPS.24mu', 'MIPS.24mu'), ('MIPS.70mu', 'MIPS.70mu'), ('MIPS.160mu', 'MIPS.160mu'), ('W1', 'W1'), ('W2', 'W2'), ('W3', 'W3'), ('W4', 'W4')], default='W1', max_length=10)), + ], + options={ + 'verbose_name_plural': 'Magnitudes', + }, + ), + ] diff --git a/astrobasis/migrations/0075_astrobasisusercat_magnitudes.py b/astrobasis/migrations/0075_astrobasisusercat_magnitudes.py new file mode 100644 index 0000000..4b34df3 --- /dev/null +++ b/astrobasis/migrations/0075_astrobasisusercat_magnitudes.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-13 13:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0074_magnitude'), + ] + + operations = [ + migrations.AddField( + model_name='astrobasisusercat', + name='magnitudes', + field=models.ManyToManyField(to='astrobasis.Magnitude'), + ), + ] diff --git a/astrobasis/migrations/0076_magnitude_error.py b/astrobasis/migrations/0076_magnitude_error.py new file mode 100644 index 0000000..5129d8b --- /dev/null +++ b/astrobasis/migrations/0076_magnitude_error.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-14 14:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0075_astrobasisusercat_magnitudes'), + ] + + operations = [ + migrations.AddField( + model_name='magnitude', + name='error', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/0077_alter_magnitude_band.py b/astrobasis/migrations/0077_alter_magnitude_band.py new file mode 100644 index 0000000..a1d3b1a --- /dev/null +++ b/astrobasis/migrations/0077_alter_magnitude_band.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-06-24 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0076_magnitude_error'), + ] + + operations = [ + migrations.AlterField( + model_name='magnitude', + name='band', + field=models.CharField(choices=[('U', 'U'), ('B', 'B'), ('V', 'V'), ('R', 'R'), ('I', 'I'), ('Z', 'Z'), ('Y', 'Y'), ('J', 'J'), ('H', 'H'), ('K', 'K'), ('Ks', 'Ks'), ('Br_gamma', 'Br_gamma'), ('u', 'u'), ('g', 'g'), ('r', 'r'), ('Halpha', 'Halpha'), ('i', 'i'), ('z', 'z'), ('y', 'y'), ('G', 'G'), ('Gbr', 'Gbr'), ('Grp', 'Grp'), ('I1', 'I1'), ('I2', 'I2'), ('I3', 'I3'), ('I4', 'I4'), ('MIPS.24mu', 'MIPS.24mu'), ('MIPS.70mu', 'MIPS.70mu'), ('MIPS.160mu', 'MIPS.160mu'), ('W1', 'W1'), ('W2', 'W2'), ('W3', 'W3'), ('W4', 'W4')], default='W1', max_length=10), + ), + ] diff --git a/astrobasis/migrations/0078_alter_simbad_obj_class.py b/astrobasis/migrations/0078_alter_simbad_obj_class.py new file mode 100644 index 0000000..fffef6d --- /dev/null +++ b/astrobasis/migrations/0078_alter_simbad_obj_class.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-07-01 20:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0077_alter_magnitude_band'), + ] + + operations = [ + migrations.AlterField( + model_name='simbad', + name='obj_class', + field=models.CharField(default='', max_length=30), + ), + ] diff --git a/astrobasis/migrations/0079_bjfromvizier.py b/astrobasis/migrations/0079_bjfromvizier.py new file mode 100644 index 0000000..c27ea9c --- /dev/null +++ b/astrobasis/migrations/0079_bjfromvizier.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.13 on 2023-10-23 12:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0078_alter_simbad_obj_class'), + ] + + operations = [ + migrations.CreateModel( + name='BJfromVizieR', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('recno', models.IntegerField(default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('r_med_geo', models.FloatField(default=0.0)), + ('r_lo_geo', models.FloatField(default=0.0)), + ('r_hi_geo', models.FloatField(default=0.0)), + ('r_med_pgeo', models.FloatField(default=0.0)), + ('r_lo_pgeo', models.FloatField(default=0.0)), + ('r_hi_pgeo', models.FloatField(default=0.0)), + ('flag', models.CharField(max_length=5)), + ('source_id', models.BigIntegerField(db_index=True, default=0)), + ], + options={ + 'ordering': ['-id'], + }, + ), + ] diff --git a/astrobasis/migrations/0080_auto_20231023_1633.py b/astrobasis/migrations/0080_auto_20231023_1633.py new file mode 100644 index 0000000..fe9240e --- /dev/null +++ b/astrobasis/migrations/0080_auto_20231023_1633.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2023-10-23 13:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0079_bjfromvizier'), + ] + + operations = [ + migrations.AddField( + model_name='bjfromvizier', + name='error_radius', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='bjfromvizier', + name='healpix', + field=models.BigIntegerField(db_index=True, default=0), + ), + ] diff --git a/astrobasis/migrations/0081_alter_bjfromvizier_error_radius.py b/astrobasis/migrations/0081_alter_bjfromvizier_error_radius.py new file mode 100644 index 0000000..c193631 --- /dev/null +++ b/astrobasis/migrations/0081_alter_bjfromvizier_error_radius.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-10-23 13:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0080_auto_20231023_1633'), + ] + + operations = [ + migrations.AlterField( + model_name='bjfromvizier', + name='error_radius', + field=models.FloatField(default=0.1), + ), + ] diff --git a/astrobasis/migrations/0082_alter_bjfromvizier_error_radius.py b/astrobasis/migrations/0082_alter_bjfromvizier_error_radius.py new file mode 100644 index 0000000..b44cc5c --- /dev/null +++ b/astrobasis/migrations/0082_alter_bjfromvizier_error_radius.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-10-23 13:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0081_alter_bjfromvizier_error_radius'), + ] + + operations = [ + migrations.AlterField( + model_name='bjfromvizier', + name='error_radius', + field=models.FloatField(default=0.00035), + ), + ] diff --git a/astrobasis/migrations/0083_remove_bjfromvizier_recno.py b/astrobasis/migrations/0083_remove_bjfromvizier_recno.py new file mode 100644 index 0000000..bde0b17 --- /dev/null +++ b/astrobasis/migrations/0083_remove_bjfromvizier_recno.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2023-10-23 13:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0082_alter_bjfromvizier_error_radius'), + ] + + operations = [ + migrations.RemoveField( + model_name='bjfromvizier', + name='recno', + ), + ] diff --git a/astrobasis/migrations/0084_alter_bjfromvizier_flag.py b/astrobasis/migrations/0084_alter_bjfromvizier_flag.py new file mode 100644 index 0000000..bde6cae --- /dev/null +++ b/astrobasis/migrations/0084_alter_bjfromvizier_flag.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-10-23 14:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0083_remove_bjfromvizier_recno'), + ] + + operations = [ + migrations.AlterField( + model_name='bjfromvizier', + name='flag', + field=models.CharField(max_length=6), + ), + ] diff --git a/astrobasis/migrations/0085_remove_bjfromvizier_flag.py b/astrobasis/migrations/0085_remove_bjfromvizier_flag.py new file mode 100644 index 0000000..2fdb360 --- /dev/null +++ b/astrobasis/migrations/0085_remove_bjfromvizier_flag.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2023-10-23 14:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0084_alter_bjfromvizier_flag'), + ] + + operations = [ + migrations.RemoveField( + model_name='bjfromvizier', + name='flag', + ), + ] diff --git a/astrobasis/migrations/0086_gaiadr3fromvizier.py b/astrobasis/migrations/0086_gaiadr3fromvizier.py new file mode 100644 index 0000000..cc280a7 --- /dev/null +++ b/astrobasis/migrations/0086_gaiadr3fromvizier.py @@ -0,0 +1,37 @@ +# Generated by Django 3.2.13 on 2023-10-25 09:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0085_remove_bjfromvizier_flag'), + ] + + operations = [ + migrations.CreateModel( + name='GAIADR3fromVizieR', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('error_radius', models.FloatField(default=0.00035)), + ('source_id', models.BigIntegerField(db_index=True, default=0)), + ('e_ra', models.FloatField(default=0.0)), + ('e_dec', models.FloatField(default=0.0)), + ('plx', models.FloatField(default=0.0)), + ('e_plx', models.FloatField(default=0.0)), + ('rplx', models.FloatField(default=0.0)), + ('pm', models.FloatField(default=0.0)), + ('pmra', models.FloatField(default=0.0)), + ('e_pmra', models.FloatField(default=0.0)), + ('pmde', models.FloatField(default=0.0)), + ('e_pmde', models.FloatField(default=0.0)), + ], + options={ + 'ordering': ['-id'], + }, + ), + ] diff --git a/astrobasis/migrations/0087_auto_20231025_1924.py b/astrobasis/migrations/0087_auto_20231025_1924.py new file mode 100644 index 0000000..9643709 --- /dev/null +++ b/astrobasis/migrations/0087_auto_20231025_1924.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.13 on 2023-10-25 16:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0086_gaiadr3fromvizier'), + ] + + operations = [ + migrations.AddField( + model_name='gaiadr3fromvizier', + name='bpmag', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gaiadr3fromvizier', + name='e_bpmag', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gaiadr3fromvizier', + name='e_gmag', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gaiadr3fromvizier', + name='e_rpmag', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gaiadr3fromvizier', + name='gmag', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gaiadr3fromvizier', + name='rpmag', + field=models.FloatField(default=0.0), + ), + ] diff --git a/astrobasis/migrations/__init__.py b/astrobasis/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/astrobasis/models.py b/astrobasis/models.py new file mode 100644 index 0000000..4bc272b --- /dev/null +++ b/astrobasis/models.py @@ -0,0 +1,882 @@ +from django.db import models +from datetime import datetime +#from django.utils.timezone import localdate, localtime +#from monthplan.models import Survey +from math import isfinite, sqrt +from django import forms + +import urllib.parse +# Create your models here. + +# https://docs.djangoproject.com/en/3.0/howto/custom-lookups/ + +from django.db.models import Transform + +class AbsoluteValue(Transform): + lookup_name = 'abs' + function = 'ABS' + +models.FloatField.register_lookup(AbsoluteValue) + +class GLIMPSE(models.Model): + """ + GLIMPSE Source Catalog (I + II + 3D) (IPAC 2008) + Only high-reliability sources are loaded (marked as C) + """ + healpix = models.BigIntegerField(db_index=True, default=0, null=True) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + error_radius = models.FloatField(default=0.5) + """ additional field for cross-match """ + name = models.CharField(max_length=17,blank=True, default='') + twomass = models.CharField(max_length=16,blank=True, default='') + jmag = models.FloatField(default=0.0,null=True) + hmag = models.FloatField(default=0.0,null=True) + kmag = models.FloatField(default=0.0,null=True) + b1mag = models.FloatField(default=0.0,null=True) + """ 3.6um IRAC (Band 1) magnitude """ + e_b1mag = models.FloatField(default=0.0,null=True) + """ 1-{sigma} error on 3.6mag """ + b2mag = models.FloatField(default=0.0,null=True) + """ 4.5um IRAC (Band 2) magnitude """ + e_b2mag = models.FloatField(default=0.0,null=True) + """ 1-{sigma} error on 4.5mag """ + b3mag = models.FloatField(default=0.0,null=True) + """ 5.8um IRAC (Band 1) magnitude """ + e_b3mag = models.FloatField(default=0.0,null=True) + """ 1-{sigma} error on 5.8mag """ + b4mag = models.FloatField(default=0.0,null=True) + """ 8.0um IRAC (Band 1) magnitude """ + e_b4mag = models.FloatField(default=0.0,null=True) + """ 1-{sigma} error on 8.0mag """ + glon = models.FloatField(default=0.0,null=True) + glat = models.FloatField(default=0.0,null=True) + class Meta: + ordering = ['-id'] + def __str__(self): + return "{}".format(self.name) + +class TwoMASS(models.Model): + """ 2MASS All-Sky Catalog of Point Sources (Cutri+ 2003) """ + healpix = models.BigIntegerField(db_index=True, default=0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + error_radius = models.FloatField(default=0.5) + """ additional field for cross-match """ + name = models.CharField(max_length=17,blank=True, default='') + jmag = models.FloatField(default=0.0,null=True) + e_jmag = models.FloatField(default=0.0,null=True) + hmag = models.FloatField(default=0.0,null=True) + e_hmag = models.FloatField(default=0.0,null=True) + kmag = models.FloatField(default=0.0,null=True) + e_kmag = models.FloatField(default=0.0,null=True) + qkfg = models.CharField(max_length=3,blank=True,null=True,default='') + rkfg = models.CharField(max_length=3,blank=True,null=True,default='') + bkfg = models.CharField(max_length=3,blank=True,null=True,default='') + ckfg = models.CharField(max_length=3,blank=True,null=True,default='') + prox = models.FloatField(default=0.0,null=True) + """ Distance between source and nearest neighbour """ + xflg = models.BooleanField(default=False,null=True) + """ (gal_contam) Extended source contamination """ + aflg = models.BooleanField(default=False,null=True) + """ (mp_flg) Association with asteroid or comet """ + glon = models.FloatField(default=0.0) + glat = models.FloatField(default=0.0) + class Meta: + ordering = ['-id'] + + def __str__(self): + return "2MASS {}".format(self.name) + + #RAJ2000','DEJ2000','2MASS','Jmag','e_Jmag','Hmag','e_Hmag','Kmag','e_Kmag','Qflg','Rflg','Bflg','Cflg','prox','Xflg','Aflg','GLON','GLAT' + +class GaiaSourceFile(models.Model): + """ Used for Gaia loading and indexing """ + filename = models.CharField(max_length=80,blank=True, default='') + status = models.CharField(max_length=7, default='empty') # 00 empty, 01 loaded, 03 healpix + nrows = models.IntegerField(default=0) + """ Number of sources in file """ + +class GAIADR2(models.Model): + """ Gaia DR2 Catalog + + See detailed description of Gaia DR2 data model_. + + .. _model: https://gea.esac.esa.int/archive/documentation/GDR2/Gaia_archive/chap_datamodel/sec_dm_main_tables/ssec_dm_gaia_source.html + """ + + filename = models.ForeignKey(GaiaSourceFile, on_delete=models.CASCADE) + + healpix = models.BigIntegerField(db_index=True, default=0) + """ HEALPIX source index """ + + solution_id = models.CharField(max_length=19, blank=True, default='') + """ Solution Identifier (long) + + All Gaia data processed by the Data Processing and Analysis Consortium + comes tagged with a solution identifier. This is a numeric field attached + to each table row that can be used to unequivocally identify the version + of all the subsystems that where used in the generation of the data + as well as the input data used. It is mainly for internal DPAC use but + is included in the published data releases to enable end users to examine + the provenance of processed data products. To decode a given solution ID + visit https://gaia.esac.esa.int/decoder/solnDecoder.jsp + """ + + name = models.CharField(max_length=30,blank=True, default='') + """ Unique source designation (unique across all Data Releases) (string) """ + + source_id = models.CharField(max_length=19,blank=True, default='') + """ Unique source identifier (unique within a particular Data Release) (long) + + A unique numerical identifier of the source, encoding the approximate + position of the source (roughly to the nearest arcmin), the provenance + (data processing centre where it was created), a running number, + and a component number. + """ + + error_radius = models.FloatField(default=0.0) + """ additional field for cross-match """ + + ref_epoch = models.FloatField(default=0.0) + """ Reference epoch (double, Time[Julian Years]) """ + + ra = models.FloatField(default=0.0) + """ Right ascension (double, Angle[deg]) + + Barycentric right ascension of the source in ICRS at the reference epoch ref_epoch + """ + + ra_error = models.FloatField(default=0.0) + """ Standard error of right ascension (double, Angle[mas]) """ + + dec = models.FloatField(default=0.0) + """ Declination (double, Angle[deg]) + + Barycentric declination of the source in ICRS at the reference epoch ref_epoch + """ + + dec_error = models.FloatField(default=0.0) + """ Standard error of declination (double, Angle[mas]) """ + + parallax = models.FloatField(default=0.0) + """ Parallax (double, Angle[mas]) + + Absolute stellar parallax Ï– of the source at the reference epoch ref_epoch + """ + + parallax_error = models.FloatField(default=0.0) + """ Standard error of parallax (double, Angle[mas]) + + Standard error of the stellar parallax at the reference epoch ref_epoch + """ + + pmra = models.FloatField(default=0.0) + """ Proper motion in right ascension direction (double, Angular Velocity[mas/year]) """ + + pmra_error = models.FloatField(default=0.0) + """ Standard error of proper motion in right ascension direction (double, Angular Velocity[mas/year]) """ + + pmdec = models.FloatField(default=0.0) + """ Proper motion in declination direction (double, Angular Velocity[mas/year]) """ + + pmdec_error = models.FloatField(default=0.0) + """ Standard error of proper motion in declination direction (double, Angular Velocity[mas/year]) """ + + phot_g_mean_mag = models.FloatField(default=0.0) + """ G-band mean magnitude (float, Magnitude[mag]) + + Mean magnitude in the G band. This is computed from the G-band mean + flux applying the magnitude zero-point in the Vega scale. + """ + + phot_bp_mean_mag = models.FloatField(default=0.0) + """ Integrated BP mean magnitude (float, Magnitude[mag]) + + Mean magnitude in the integrated BP band. This is computed + from the BP-band mean flux applying the magnitude zero-point in the Vega scale. + """ + + phot_rp_mean_mag = models.FloatField(default=0.0) + """ Integrated RP mean magnitude (float, Magnitude[mag]) + + Mean magnitude in the integrated RP band. This is computed from the + RP-band mean flux applying the magnitude zero-point in the Vega scale. + """ + + # http://vizier.u-strasbg.fr/viz-bin/VizieR-S?Gaia%20DR2%20431160672954647552 + def get_absolute_url_vizier(self): + return "http://vizier.u-strasbg.fr/viz-bin/VizieR-S?" + urllib.parse.quote_plus(self.name) + + def get_absolute_url(self): + return "/astrobasis/%i/gaia" % (self.pk) + +class GaiaSourceFileDR3(models.Model): + """ Used for Gaia loading and indexing """ + filename = models.CharField(max_length=80,blank=True, default='') + status = models.CharField(max_length=7, default='empty') # 00 empty, 01 loaded, 03 healpix + nrows = models.IntegerField(default=0) + """ Number of sources in file """ + +class GAIADR3(models.Model): + """ Gaia DR3 Catalog """ + + filename = models.ForeignKey(GaiaSourceFileDR3, on_delete=models.CASCADE) + + healpix = models.BigIntegerField(db_index=True, default=0) + """ HEALPIX source index """ + + #solution_id = models.CharField(max_length=19, blank=True, default='') + """ Solution Identifier (long) + + All Gaia data processed by the Data Processing and Analysis Consortium + comes tagged with a solution identifier. This is a numeric field attached + to each table row that can be used to unequivocally identify the version + of all the subsystems that where used in the generation of the data + as well as the input data used. It is mainly for internal DPAC use but + is included in the published data releases to enable end users to examine + the provenance of processed data products. To decode a given solution ID + visit https://gaia.esac.esa.int/decoder/solnDecoder.jsp + """ + + name = models.CharField(max_length=30,blank=True, default='') + """ Unique source designation (unique across all Data Releases) (string) """ + + source_id = models.CharField(max_length=19,blank=True, default='') + """ Unique source identifier (unique within a particular Data Release) (long) + + A unique numerical identifier of the source, encoding the approximate + position of the source (roughly to the nearest arcmin), the provenance + (data processing centre where it was created), a running number, + and a component number. + """ + + error_radius = models.FloatField(default=0.0) + """ additional field for cross-match """ + + #ref_epoch = models.FloatField(default=0.0) + """ Reference epoch (double, Time[Julian Years]) """ + + ra = models.FloatField(default=0.0) + """ Right ascension (double, Angle[deg]) + + Barycentric right ascension of the source in ICRS at the reference epoch ref_epoch + """ + + ra_error = models.FloatField(default=0.0) + """ Standard error of right ascension (double, Angle[mas]) """ + + dec = models.FloatField(default=0.0) + """ Declination (double, Angle[deg]) + + Barycentric declination of the source in ICRS at the reference epoch ref_epoch + """ + + dec_error = models.FloatField(default=0.0) + """ Standard error of declination (double, Angle[mas]) """ + + parallax = models.FloatField(default=0.0) + """ Parallax (double, Angle[mas]) + + Absolute stellar parallax Ï– of the source at the reference epoch ref_epoch + """ + + parallax_error = models.FloatField(default=0.0) + """ Standard error of parallax (double, Angle[mas]) + + Standard error of the stellar parallax at the reference epoch ref_epoch + """ + + pmra = models.FloatField(default=0.0) + """ Proper motion in right ascension direction (double, Angular Velocity[mas/year]) """ + + pmra_error = models.FloatField(default=0.0) + """ Standard error of proper motion in right ascension direction (double, Angular Velocity[mas/year]) """ + + pmdec = models.FloatField(default=0.0) + """ Proper motion in declination direction (double, Angular Velocity[mas/year]) """ + + pmdec_error = models.FloatField(default=0.0) + """ Standard error of proper motion in declination direction (double, Angular Velocity[mas/year]) """ + + phot_g_mean_mag = models.FloatField(default=0.0) + """ G-band mean magnitude (float, Magnitude[mag]) + + Mean magnitude in the G band. This is computed from the G-band mean + flux applying the magnitude zero-point in the Vega scale. + """ + + phot_bp_mean_mag = models.FloatField(default=0.0) + """ Integrated BP mean magnitude (float, Magnitude[mag]) + + Mean magnitude in the integrated BP band. This is computed + from the BP-band mean flux applying the magnitude zero-point in the Vega scale. + """ + + phot_rp_mean_mag = models.FloatField(default=0.0) + """ Integrated RP mean magnitude (float, Magnitude[mag]) + + Mean magnitude in the integrated RP band. This is computed from the + RP-band mean flux applying the magnitude zero-point in the Vega scale. + """ + + # http://vizier.u-strasbg.fr/viz-bin/VizieR-S?Gaia%20DR2%20431160672954647552 + def get_absolute_url_vizier(self): + return "http://vizier.u-strasbg.fr/viz-bin/VizieR-S?" + urllib.parse.quote_plus(self.name) + + def get_absolute_url(self): + return "/astrobasis/%i/gaiadr3" % (self.pk) + def __str__(self): + return "{}".format(self.name) + + +def ztf_filter(x): + return { + 1: 'g', + 2: 'r', + 3: 'i', + }[x] + +class ZTFAlert(models.Model): + """ ZTF Alert + + See detailed description of ZTF data ztf-model_. + + .. _ztf-model: https://zwickytransientfacility.github.io/ztf-avro-alert/schema.html + """ + survey = models.ManyToManyField('monthplan.Survey') + """ Reference to Survey """ + + objectId = models.CharField(max_length=12, blank=True, default='') + + fid = models.IntegerField(db_index=True, default=0) + """ Filter ID (1=g; 2=r; 3=i) """ + + lco_id = models.BigIntegerField(db_index=True, default=0) + """ unique identifier for this object """ + + healpix = models.BigIntegerField(db_index=True, default=0) + """ HEALPIX source index """ + + healpix_plate = models.IntegerField(default=0) + """ HEALPIX plate index """ + + ra = models.FloatField(default=0.0) + """ Right ascension (double, Angle[deg]) """ + + dec = models.FloatField(default=0.0) + """ Declination (double, Angle[deg]) """ + + programid = models.IntegerField(db_index=True, default=0) + """ Program ID: encodes either public, collab, or caltech mode """ + + magpsf = models.FloatField(default=0.0) + """ magnitude from PSF-fit photometry [mag] """ + + sigmapsf = models.FloatField(default=0.0) + """ 1-sigma uncertainty in magpsf [mag] """ + + fwhm = models.FloatField(default=0.0) + """ Full Width Half Max assuming a Gaussian core, from SExtractor [pixels] """ + + elong = models.FloatField(default=0.0) + """ Ratio: aimage / bimage """ + + magap = models.FloatField(default=0.0) + """ Aperture mag using 8 pixel diameter aperture [mag] """ + + magdiff = models.FloatField(default=0.0) + """ Difference: magap - magpsf [mag] """ + + nbad = models.IntegerField(default=0) + """ number of prior-tagged bad pixels in a 5 x 5 pixel stamp """ + + + sigmagap = models.FloatField(default=0.0) + """ 1-sigma uncertainty in magap [mag] """ + + #wall_time = models.CharField(max_length=29, blank=True, default='') + wall_time = models.DateTimeField(default=datetime.now, blank=True) + diffmaglim = models.FloatField(default=0.0) + """ 5-sigma mag limit in difference image based on PSF-fit photometry [mag] """ + + deltamagref = models.FloatField(default=0.0,null=True) + + deltamaglatest = models.FloatField(default=0.0,null=True) + + rb = models.FloatField(default=0.0) + """ RealBogus quality score; range is 0 to 1 where closer to 1 is more reliable """ + + created = models.DateTimeField(auto_now_add=True) + """ Date and time of creation. Fully automatic. """ + + def get_alerce_url(self): + return "https://alerce.online/object/%s" % (self.objectId) + + def get_mars_url(self): + return "https://mars.lco.global/%i/" % (self.lco_id) + + def get_filter(self): + return "%s" % (ztf_filter(self.fid)) + + def __str__(self): + return "%s" % (self.objectId) + +def alerce_early_class(x): + return { + 18: 'AGN', + 19: 'SN', + 20: 'VS', + 21: 'Astrd', + 22: 'Bogus', + }[x] + +def alerce_late_class(x): + return 'N/A' + +class ALeRCE(models.Model): + """ ALeRCE ZTF Alerts + + See detailed description of ZTF data alerce-model_. + + .. _alerce-model: https://zwickytransientfacility.github.io/ztf-avro-alert/schema.html + """ + survey = models.ManyToManyField('monthplan.Survey') + """ Reference to Survey """ + + healpix = models.BigIntegerField(db_index=True, default=0) + """ HEALPIX source index """ + + healpix_plate = models.IntegerField(default=0) + """ HEALPIX plate index """ + + created = models.DateTimeField(auto_now_add=True) + """ Date and time of creation. Fully automatic. """ + + classearly = models.IntegerField(default=0, null=True) + """ Early Classifier (Stamp Classifier) """ + + error_radius = models.FloatField(default=0.0) + """ additional field for cross-match """ + + classrf = models.IntegerField(default=0, null=True) + """ Late Classifier (Random Forest) """ + firstdate = models.DateTimeField(default=datetime.now) + lastdate = models.DateTimeField(default=datetime.now) + oid = models.CharField(max_length=12, default='') + firstmjd = models.FloatField(default=0.0) + lastmjd = models.FloatField(default=0.0) + mean_magap_g = models.FloatField(default=0.0, null=True) + mean_magap_r = models.FloatField(default=0.0, null=True) + mean_magpsf_g = models.FloatField(default=0.0, null=True) + mean_magpsf_r = models.FloatField(default=0.0, null=True) + dec = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + nobs = models.IntegerField(default=0) + pclassearly = models.FloatField(default=0.0, null=True) + pclassrf = models.FloatField(default=0.0, null=True) + sigma_magap_g = models.FloatField(default=0.0, null=True) + sigma_magap_r = models.FloatField(default=0.0, null=True) + sigma_magpsf_g = models.FloatField(default=0.0, null=True) + sigma_magpsf_r = models.FloatField(default=0.0, null=True) + sigmadec = models.FloatField(default=0.0, null=True) + sigmara = models.FloatField(default=0.0, null=True) + + + def get_absolute_url(self): + return "/astrobasis/%i/alerce" % (self.pk) + + def get_alerce_url(self): + return "https://alerce.online/object/%s" % (self.oid) + + def get_tns_url(self): + return "https://wis-tns.weizmann.ac.il/object/%s" % (self.oid) + + def __str__(self): + return "ALeRCE %s" % (self.oid) + + def get_early_class(self): + if(self.classearly): + return "%s" % (alerce_early_class(self.classearly)) + else: + return None + +class SDSSDR12Spec(models.Model): + """ SDSS DR 12 Spectra """ + + healpix = models.BigIntegerField(db_index=True, default=0) + """ HEALPIX source index """ + + MJD = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + obj_class = models.CharField(max_length=6, default='') + obj_subclass = models.CharField(max_length=21, default='') + z = models.FloatField(default=0.0) + z_err = models.FloatField(default=0.0) + specobjid = models.CharField(max_length=22, default='') + targetobjid = models.CharField(max_length=22, default='') + + +class Flesch(models.Model): + """ Flesch 2019 catalog VII/283 """ + + healpix = models.BigIntegerField(db_index=True, default=0) + """ HEALPIX source index """ + + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + name = models.CharField(max_length=25, default="") + obj_class = models.CharField(max_length=4, default='') + Rmag = models.FloatField(default=0.0) + Bmag = models.FloatField(default=0.0) + z = models.FloatField(default=0.0) + +class Simbad(models.Model): + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + main_id = models.CharField(max_length=51, unique=True) + obj_class = models.CharField(max_length=30, default='') + coo_bibcode = models.CharField(max_length=25, default="") + coo_wavelength = models.CharField(max_length=5, default="") + z = models.FloatField(default=0.0) + otype = models.CharField(max_length=21, default="") + + def is_redshift_real(self): + return isfinite(self.z) + def is_error_radius_real(self): + return isfinite(self.error_radius) + + def __str__(self): + return "{}:{}/{}".format(self.main_id,self.obj_class,self.otype) + + +class AllWise(models.Model): + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + name = models.CharField(max_length=51, unique=True) + W1mag = models.FloatField(default=0.0) + e_W1mag = models.FloatField(default=0.0) + W2mag = models.FloatField(default=0.0) + e_W2mag = models.FloatField(default=0.0) + def __str__(self): + return "{}".format(self.name) + def w1w2(self): + if self.W1mag and self.W2mag: + return self.W1mag-self.W2mag + else: + return None + + def w1w2_error(self): + if self.e_W1mag and self.e_W2mag: + return sqrt(self.e_W1mag*self.e_W1mag + self.e_W2mag*self.e_W2mag) + else: + return None + + def get_vizier_url(self): + return "https://vizier.u-strasbg.fr/viz-bin/VizieR-5?-source=II/328/allwise&AllWISE==="+urllib.parse.quote(self.name) + def get_irsa_url(self): + return "http://irsa.ipac.caltech.edu/applications/wise/#id=Hydra_wise_wise_1&DoSearch=true&schema=allwise-multiband&intersect=CENTER&subsize=0.20&mcenter=mcen&band=1,2,3,4&dpLevel=3a&UserTargetWorldPt={};{};EQ_J2000&SimpleTargetPanel.field.resolvedBy=simbadthenned&coaddId=&projectId=wise&searchName=wise_1&startIdx=0&pageSize=0&shortDesc=Position&isBookmarkAble=true&isDrillDownRoot=true&isSearchResult=true".format(self.ra,self.dec) + +class NVSS(models.Model): + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + e_ra = models.FloatField(default=0.0) + e_dec = models.FloatField(default=0.0) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + name = models.CharField(max_length=51, unique=True) + S14 = models.FloatField(default=0.0) + e_S14 = models.FloatField(default=0.0) + def __str__(self): + return "{}".format(self.name) + def get_vizier_url(self): + return "https://vizier.u-strasbg.fr/viz-bin/VizieR-5?-source=VIII/65/nvss&NVSS=="+urllib.parse.quote(self.name) + +class FIRST(models.Model): + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + name = models.CharField(max_length=51, unique=True) + Fpeak = models.FloatField(default=0.0) + Fint = models.FloatField(default=0.0) + rms = models.FloatField(default=0.0) + major_axis = models.FloatField(default=0.0) + minor_axis = models.FloatField(default=0.0) + def __str__(self): + return "{}".format(self.name) + def get_vizier_url(self): + return "https://vizier.u-strasbg.fr/viz-bin/VizieR-5?-source=VIII/92/first14&FIRST=="+urllib.parse.quote(self.name) + class Meta: + ordering = ['-id'] + +class SUMSS(models.Model): + ra_orig = models.CharField(max_length=51, unique=False)# models.FloatField(default=0.0) + dec_orig = models.CharField(max_length=51, unique=False) # models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + e_ra = models.FloatField(default=0.0) + e_dec = models.FloatField(default=0.0) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + name = models.CharField(max_length=51, unique=True) + Sp = models.FloatField(default=0.0) + e_Sp = models.FloatField(default=0.0) + St = models.FloatField(default=0.0) + e_St = models.FloatField(default=0.0) + major_axis = models.FloatField(default=0.0) + minor_axis = models.FloatField(default=0.0) + def __str__(self): + return "{}".format(self.name) + def get_vizier_url(self): + return "https://vizier.u-strasbg.fr/viz-bin/VizieR-5?-source=VIII/81B/sumss212&RAJ2000=="+urllib.parse.quote(self.ra_orig) + class Meta: + ordering = ['-id'] + +class VLASS(models.Model): + glon = models.FloatField(default=0.0) + glat = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + e_ra = models.FloatField(default=0.0) + """ 1-sigma uncertainty in RA """ + e_dec = models.FloatField(default=0.0) + """ 1-sigma uncertainty in DEC """ + ra_m = models.FloatField(default=0.0) + """ Right Ascension (J2000) of the component peak brightness (RA_max) """ + dec_m = models.FloatField(default=0.0) + """ Declination (J2000) of the component peak brightness (DEC_max) """ + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + name = models.CharField(max_length=31, unique=False) + """ CompName (A31) IAU name of the component, VLASS1QLCIR JHHMMSS.ss+DDMMSS.s """ + ftot = models.FloatField(default=0.0) + """ The integrated flux density of the component in the S-band (2-4GHz) """ + e_ftot = models.FloatField(default=0.0) + """ 1-sigma uncertainty in total flux """ + fpeak = models.FloatField(default=0.0) + """ Peak brightness of the component in the S-band (2-4GHz) """ + e_fpeak = models.FloatField(default=0.0) + """ 1-sigma uncertainty in peak flux """ + dupflag = models.IntegerField(default=0) + """ DupFlag (I1) [0/2] Flag to identify duplicate detections (Duplicate) """ + qualflag = models.IntegerField(default=0) + """ QualFlag (I1) [0/7] Flag to denote the Quality of the detection """ + def __str__(self): + return "{}".format(self.name) + def get_vizier_url(self): + return "https://vizier.u-strasbg.fr/viz-bin/VizieR-5?-source=VIII/81B/sumss212&RAJ2000=="+urllib.parse.quote(self.ra_orig) + class Meta: + ordering = ['-id'] + +class VLASSfromVizieR(models.Model): + recno = models.IntegerField(default=0) + glon = models.FloatField(default=0.0) + glat = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + e_ra = models.FloatField(default=0.0) + """ 1-sigma uncertainty in RA """ + e_dec = models.FloatField(default=0.0) + """ 1-sigma uncertainty in DEC """ + #ra_m = models.FloatField(default=0.0) + """ Right Ascension (J2000) of the component peak brightness (RA_max) """ + #dec_m = models.FloatField(default=0.0) + """ Declination (J2000) of the component peak brightness (DEC_max) """ + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + name = models.CharField(max_length=31, unique=False) + """ CompName (A31) IAU name of the component, VLASS1QLCIR JHHMMSS.ss+DDMMSS.s """ + ftot = models.FloatField(default=0.0) + """ The integrated flux density of the component in the S-band (2-4GHz) """ + e_ftot = models.FloatField(default=0.0) + """ 1-sigma uncertainty in total flux """ + fpeak = models.FloatField(default=0.0) + """ Peak brightness of the component in the S-band (2-4GHz) """ + e_fpeak = models.FloatField(default=0.0) + """ 1-sigma uncertainty in peak flux """ + dupflag = models.IntegerField(default=0) + """ DupFlag (I1) [0/2] Flag to identify duplicate detections (Duplicate) """ + qualflag = models.IntegerField(default=0) + """ QualFlag (I1) [0/7] Flag to denote the Quality of the detection """ + def __str__(self): + return "{}".format(self.name) + def get_vizier_url(self): + return "https://vizier.cds.unistra.fr/viz-bin/VizieR-5?-source=J/ApJS/255/30/comp&recno={}".format(self.recno) + class Meta: + ordering = ['-id'] + +class BJfromVizieR(models.Model): + #recno = models.IntegerField(default=0) + #glon = models.FloatField(default=0.0) + #glat = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + r_med_geo = models.FloatField(default=0.0) + r_lo_geo = models.FloatField(default=0.0) + r_hi_geo = models.FloatField(default=0.0) + r_med_pgeo = models.FloatField(default=0.0) + r_lo_pgeo = models.FloatField(default=0.0) + r_hi_pgeo = models.FloatField(default=0.0) + #flag = models.CharField(max_length=6, unique=False) + source_id = models.BigIntegerField(db_index=True, default=0) + #source_id = models.CharField(max_length=19,blank=True, default='') + #Unique source identifier (unique within a particular Data Release) (long) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.00035) # arXiv:2202.08369v1 + + + def __str__(self): + return "{}".format(self.source_id) + def get_vizier_url(self): + return "http://vizier.cds.unistra.fr/viz-bin/VizieR-6?-out.form=%2bH%2bm&-source=I/350/gaia%2a&Source=%20{}".format(self.source_id) + class Meta: + ordering = ['-id'] + +class GAIADR3fromVizieR(models.Model): + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.00035) # arXiv:2202.08369v1 + source_id = models.BigIntegerField(db_index=True, default=0) + e_ra = models.FloatField(default=0.0) + e_dec = models.FloatField(default=0.0) + plx = models.FloatField(default=0.0) + e_plx = models.FloatField(default=0.0) + rplx = models.FloatField(default=0.0) + pm = models.FloatField(default=0.0) + pmra = models.FloatField(default=0.0) + e_pmra = models.FloatField(default=0.0) + pmde = models.FloatField(default=0.0) + e_pmde = models.FloatField(default=0.0) + gmag = models.FloatField(default=0.0) + e_gmag = models.FloatField(default=0.0) + bpmag = models.FloatField(default=0.0) + e_bpmag = models.FloatField(default=0.0) + rpmag = models.FloatField(default=0.0) + e_rpmag = models.FloatField(default=0.0) + + def __str__(self): + return "{}".format(self.source_id) + def get_vizier_url(self): + return "https://vizier.cds.unistra.fr" + class Meta: + ordering = ['-id'] + +class Magnitude(models.Model): + U='U' + B='B' + V='V' + R='R' + I='I' + Z='Z' + Y='Y' + J='J' + H='H' + K='K' + Ks='Ks' + Br_gamma='Br_gamma' + u='u' + g='g' + r='r' + i='i' + z='z' + y='y' + G='G' + Gbr='Gbr' + Grp='Grp' + I1='I1' + I2='I2' + I3='I3' + I4='I4' + MIPS24mu='MIPS.24mu' + MIPS70mu='MIPS.70mu' + MIPS160mu='MIPS.160mu' + W1='W1' + W2='W2' + W3='W3' + W4='W4' + Halpha='Halpha' + SELECTION = [ + (U,'U'), + (B,'B'), + (V,'V'), + (R,'R'), + (I,'I'), + (Z,'Z'), + (Y,'Y'), + (J,'J'), + (H,'H'), + (K,'K'), + (Ks,'Ks'), + (Br_gamma,'Br_gamma'), + (u,'u'), + (g,'g'), + (r,'r'), + (Halpha,'Halpha'), + (i,'i'), + (z,'z'), + (y,'y'), + (G,'G'), + (Gbr,'Gbr'), + (Grp,'Grp'), + (I1,'I1'), + (I2,'I2'), + (I3,'I3'), + (I4,'I4'), + (MIPS24mu,'MIPS.24mu'), + (MIPS70mu,'MIPS.70mu'), + (MIPS160mu,'MIPS.160mu'), + (W1,'W1'), + (W2,'W2'), + (W3,'W3'), + (W4,'W4'), + ] + magn = models.FloatField(default=0.0) + error = models.FloatField(default=0.0) + band = models.CharField(max_length=10,choices=SELECTION,default=W1,) + class Meta: + verbose_name_plural = "Magnitudes" + def __str__(self): + return "{} {}".format(self.magn, self.band) + def get_delete_url(self): + return "/astrobasis/{}/delmagn".format(self.pk) + +class MagnitudeForm(forms.ModelForm): + class Meta: + model = Magnitude + fields = ('band','magn','error') + + +class AstroBasisUserCat(models.Model): + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + healpix = models.BigIntegerField(db_index=True, default=0) + error_radius = models.FloatField(default=0.0) + name = models.CharField(max_length=51, unique=True) + notes = models.TextField("notes",max_length=2000, null=True, blank=True) + bibcode = models.CharField(max_length=36,blank=True, default='') + autoname = models.BooleanField(default=False, null=False, help_text="Generate IAU name of the component in the form NAME JHHMMSS.s+DDMMSS,
where NAME is taken from the corresponding field above.") + magnitudes = models.ManyToManyField('Magnitude') + + def __str__(self): + return "{}".format(self.name) + class Meta: + ordering = ['-id'] + def get_update_url(self): + return "/astrobasis/%i/usercat/update" % (self.pk) + def get_delete_url(self): + return "/astrobasis/%i/usercat/delete" % (self.pk) + def get_add_magn_url(self): + return "/astrobasis/%i/usercat/addmagn" % (self.pk) + +class AstroBasisUserCatForm(forms.ModelForm): + class Meta: + model = AstroBasisUserCat + fields = ('name','autoname','ra', 'dec', 'error_radius','bibcode','notes') + diff --git a/astrobasis/templatetags/__init__.py b/astrobasis/templatetags/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/astrobasis/templatetags/__init__.py @@ -0,0 +1 @@ + diff --git a/astrobasis/templatetags/simbadescape.py b/astrobasis/templatetags/simbadescape.py new file mode 100644 index 0000000..34d8e06 --- /dev/null +++ b/astrobasis/templatetags/simbadescape.py @@ -0,0 +1,7 @@ +from django import template + +register = template.Library() + +@register.filter +def simbad_escape(value): + return value.replace("+", "%2B").replace(" ", "+") diff --git a/astrobasis/tests.py b/astrobasis/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/astrobasis/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/astrobasis/urls.py b/astrobasis/urls.py new file mode 100644 index 0000000..d588095 --- /dev/null +++ b/astrobasis/urls.py @@ -0,0 +1,24 @@ +from django.urls import path + +from . import views + +"""check""" + +urlpatterns = [ + path('', views.index, name='ztf index'), + path('ztf_alerce_agn', views.ztf_alerce_agn, name='ztf alerce agn early'), + path('ztf_alerce_sn', views.ztf_alerce_sn, name='ztf alerce sn early'), + path('ztf_alerce_vs', views.ztf_alerce_vs, name='ztf alerce vs early'), + path('ztf_alerce_bogus', views.ztf_alerce_bogus, name='ztf alerce bogus early'), + path('ztf_alerce_asteroid', views.ztf_alerce_asteroid, name='ztf alerce asteroid early'), + path('/alerce', views.show_ztf_alerce, name='show ztf alerce'), + #path('/match', views.show_survey_match, name='show match'), + #path('/nomatch', views.show_survey_nomatch, name='show nomatch'), + #path('/transients', views.show_transients, name='show nomatch'), + #path('/gaia', views.show_survey_gaia_match, name='show gaia'), + path('/source', views.new_astrobasis_source, name='new astrobasis source'), + path('/usercat/delete/', views.delete_astrobasis_source, name='delete astrobasis source'), + path('/usercat/update/', views.update_astrobasis_source, name='update astrobasis source'), + path('/usercat/addmagn/', views.addmagn_astrobasis_source, name='addmagn astrobasis source'), + path('/delmagn/', views.delmagn, name='delmagn'), +] diff --git a/astrobasis/utils.py b/astrobasis/utils.py new file mode 100644 index 0000000..ea36400 --- /dev/null +++ b/astrobasis/utils.py @@ -0,0 +1,48 @@ +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import time + +from django.core.paginator import Paginator + +from heasarc.models import NSIDE_SOURCES, ORDER + +def astrobasis_assign_healpix(catalogname,perpage=50000,batch_size=200): + start_time = time.time() + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5) + + srcs = catalogname.objects.all()#.filter(healpix__exact=0) + + paginator = Paginator(srcs, perpage) + for page_idx in range(1, paginator.num_pages): + start_time0 = time.time() + ra=paginator.page(page_idx).object_list.values_list('ra', flat=True) + dec=paginator.page(page_idx).object_list.values_list('dec', flat=True) + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + lon = crd.galactic.l.value + lat = crd.galactic.b.value + heal = hp.skycoord_to_healpix(crd) + index=0 + updates=[] + for src in paginator.page(page_idx).object_list: + # here you can do what you want with the row + #crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + #lon = crd.galactic.l.value + #lat = crd.galactic.b.value + #heal = hp.skycoord_to_healpix(crd) + src.healpix=heal[index] + src.glon=lon[index] + src.glat=lat[index] + #src.save() + updates.append(src) + #print(src.ra,src.dec,heal) + index=index+1 + catalogname.objects.bulk_update(updates, ["healpix","glon","glat"], batch_size=batch_size) + elapsed = time.time() - start_time0 + estimated = (elapsed*(paginator.num_pages-page_idx))/60/60 + print("Page {}/{} for {:.2f} sec, est. {:.2f} hours".format(page_idx,paginator.num_pages,elapsed,estimated)) + + print("--- {:.2f} hours ---".format((time.time() - start_time)/60/60)) diff --git a/astrobasis/views.py b/astrobasis/views.py new file mode 100644 index 0000000..8ea6e37 --- /dev/null +++ b/astrobasis/views.py @@ -0,0 +1,267 @@ +from django.shortcuts import render +from astrobasis.models import ZTFAlert, ALeRCE +from django.db.models import Q +from srglib.utils import make_source_name +from srglib.utils import srg_auth, srg_group_auth, srg_group_auth_stats, status_code +from django.shortcuts import get_object_or_404 +from heasarc.models import HeasarcBase +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import BarycentricTrueEcliptic +from astropy.coordinates import HeliocentricTrueEcliptic +from astropy.coordinates import GeocentricTrueEcliptic +from astropy_healpix import HEALPix +from astropy.coordinates.name_resolve import get_icrs_coordinates +from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden + +from astrobasis.models import Magnitude, MagnitudeForm +from astrobasis.models import AstroBasisUserCat +from astrobasis.models import AstroBasisUserCatForm + +# Create your views here. + +def index(request): + """ + status=srg_group_auth(request.user, "srg-erosita-transients") + if status != 0: + return HttpResponse(status_code[status]) + """ + #ztf_alerts = ZTFAlert.objects.all().order_by('-created') + ztf_alerts = ALeRCE.objects.all() #.order_by('-created') + ztf_ntotal=ztf_alerts.count() + + agn_count = ALeRCE.objects.filter(classearly=18).count() + sn_count = ALeRCE.objects.filter(classearly=19).count() + vs_count = ALeRCE.objects.filter(classearly=20).count() + asteroid_count = ALeRCE.objects.filter(classearly=21).count() + bogus_count = ALeRCE.objects.filter(classearly=22).count() + + return render(request,'astrobasis/index.html', {#'ztf_alerts':ztf_alerts, + 'ztf_ntotal':ztf_ntotal, + 'agn_count':agn_count, + 'sn_count':sn_count, + 'vs_count':vs_count, + 'asteroid_count':asteroid_count, + 'bogus_count':bogus_count,}) + +def ztf_alerce_agn(request): + ztf_alerts = ALeRCE.objects.filter(classearly=18).order_by('-created') + ztf_ntotal=ztf_alerts.count() + + agn_count = ALeRCE.objects.filter(classearly=18).count() + sn_count = ALeRCE.objects.filter(classearly=19).count() + vs_count = ALeRCE.objects.filter(classearly=20).count() + asteroid_count = ALeRCE.objects.filter(classearly=21).count() + bogus_count = ALeRCE.objects.filter(classearly=22).count() + + return render(request,'astrobasis/index.html', {'ztf_alerts':ztf_alerts, + 'ztf_ntotal':ztf_ntotal, + 'agn_count':agn_count, + 'sn_count':sn_count, + 'vs_count':vs_count, + 'asteroid_count':asteroid_count, + 'bogus_count':bogus_count,}) + +def ztf_alerce_sn(request): + ztf_alerts = ALeRCE.objects.filter(classearly=19).order_by('-created') + ztf_ntotal=ztf_alerts.count() + + agn_count = ALeRCE.objects.filter(classearly=18).count() + sn_count = ALeRCE.objects.filter(classearly=19).count() + vs_count = ALeRCE.objects.filter(classearly=20).count() + asteroid_count = ALeRCE.objects.filter(classearly=21).count() + bogus_count = ALeRCE.objects.filter(classearly=22).count() + + return render(request,'astrobasis/index.html', {'ztf_alerts':ztf_alerts, + 'ztf_ntotal':ztf_ntotal, + 'agn_count':agn_count, + 'sn_count':sn_count, + 'vs_count':vs_count, + 'asteroid_count':asteroid_count, + 'bogus_count':bogus_count,}) + +def ztf_alerce_vs(request): + ztf_alerts = ALeRCE.objects.filter(classearly=20).order_by('-created') + ztf_ntotal=ztf_alerts.count() + + agn_count = ALeRCE.objects.filter(classearly=18).count() + sn_count = ALeRCE.objects.filter(classearly=19).count() + vs_count = ALeRCE.objects.filter(classearly=20).count() + asteroid_count = ALeRCE.objects.filter(classearly=21).count() + bogus_count = ALeRCE.objects.filter(classearly=22).count() + + return render(request,'astrobasis/index.html', {'ztf_alerts':ztf_alerts, + 'ztf_ntotal':ztf_ntotal, + 'agn_count':agn_count, + 'sn_count':sn_count, + 'vs_count':vs_count, + 'asteroid_count':asteroid_count, + 'bogus_count':bogus_count,}) + +def ztf_alerce_asteroid(request): + ztf_alerts = ALeRCE.objects.filter(classearly=21).order_by('-created') + ztf_ntotal=ztf_alerts.count() + + agn_count = ALeRCE.objects.filter(classearly=18).count() + sn_count = ALeRCE.objects.filter(classearly=19).count() + vs_count = ALeRCE.objects.filter(classearly=20).count() + asteroid_count = ALeRCE.objects.filter(classearly=21).count() + bogus_count = ALeRCE.objects.filter(classearly=22).count() + + return render(request,'astrobasis/index.html', {'ztf_alerts':ztf_alerts, + 'ztf_ntotal':ztf_ntotal, + 'agn_count':agn_count, + 'sn_count':sn_count, + 'vs_count':vs_count, + 'asteroid_count':asteroid_count, + 'bogus_count':bogus_count,}) + +def ztf_alerce_bogus(request): + ztf_alerts = ALeRCE.objects.filter(classearly=22).order_by('-created') + ztf_ntotal=ztf_alerts.count() + + agn_count = ALeRCE.objects.filter(classearly=18).count() + sn_count = ALeRCE.objects.filter(classearly=19).count() + vs_count = ALeRCE.objects.filter(classearly=20).count() + asteroid_count = ALeRCE.objects.filter(classearly=21).count() + bogus_count = ALeRCE.objects.filter(classearly=22).count() + + return render(request,'astrobasis/index.html', {'ztf_alerts':ztf_alerts,'ztf_ntotal':ztf_ntotal,'agn_count':agn_count, + 'sn_count':sn_count, + 'vs_count':vs_count, + 'asteroid_count':asteroid_count, + 'bogus_count':bogus_count,}) + +def show_ztf_alerce(request, alerce_pk): + try: + alerce=ALeRCE.objects.get(pk=alerce_pk) + except: + return HttpResponse("This ALeRCE ID was not found") + erotrans_set = alerce.erotranssource_set.all() + #erosurvey_set = alerce.erosurveysource_set.all() + #artcat_set = alerce.artsource_set.all() + return render(request,'astrobasis/show_ztf_alerce.html', + {'alerce':alerce, + #'artcat_set':artcat_set, + #'erosurvey_set':erosurvey_set, + 'erotrans_set':erotrans_set,}) + +def ztf_filter(request): + """ + status=srg_group_auth(request.user, "srg-erosita-transients") + if status != 0: + return HttpResponse(status_code[status]) + """ + ztf_alerts = ZTFAlert.objects.filter(Q(rb__gt=0.65) & Q(nbad=0) & Q(fwhm__lt=5) & Q(elong__lt=1.2) & Q(magdiff__gt=-0.1) & Q(magdiff__lt=0.1)).order_by('-created') + ntotal=ztf_alerts.count() + return render(request,'astrobasis/index.html', {'ztf_alerts':ztf_alerts,'ntotal':ntotal,}) + +def delete_astrobasis_source(request, source_id, heasarcbase_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(AstroBasisUserCat, pk=source_id) + base = get_object_or_404(HeasarcBase, pk=heasarcbase_id) + magns = src.magnitudes.all() + magns.delete() + src.delete() + return HttpResponseRedirect(base.get_base_url()) + +def new_astrobasis_source(request, source_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(HeasarcBase, pk=source_id) + c = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + if request.method == 'POST': + form = AstroBasisUserCatForm(request.POST) + if form.is_valid(): + new_form=form.save(commit=False) + #bibcode=form.cleaned_data['bibcode'] + #notes=form.cleaned_data['notes'] + ra=float(form.cleaned_data['ra']) + dec=float(form.cleaned_data['dec']) + name=form.cleaned_data['name'] + autoname=form.cleaned_data['autoname'] + if(autoname): + new_form.name=make_source_name(name,ra,dec) + new_form.autoname=False + new_form.save() + #healpix_usercat(new_form) + #match_usercat_all(new_form) + src.usercat.add(new_form) + return HttpResponseRedirect(src.get_base_url()) + else: + pass + + form = AstroBasisUserCatForm() + return render(request, 'astrobasis/usercat.html', {'form': form,'src':src,}) + + +def delmagn(request, magnitude_id,heasarcbase_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + magn = get_object_or_404(Magnitude, pk=magnitude_id) + base = get_object_or_404(HeasarcBase, pk=heasarcbase_id) + magn.delete() + return HttpResponseRedirect(base.get_base_url()) + +def addmagn_astrobasis_source(request, source_id,heasarcbase_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(AstroBasisUserCat, pk=source_id) + base = get_object_or_404(HeasarcBase, pk=heasarcbase_id) + if request.method == 'POST': + form = MagnitudeForm(request.POST) + if form.is_valid(): + new_form=form.save(commit=False) + magn=form.cleaned_data['magn'] + band=form.cleaned_data['band'] + new_form.save() + src.magnitudes.add(new_form) + return HttpResponseRedirect(base.get_base_url()) + else: + pass + + form = MagnitudeForm() + return render(request, 'astrobasis/magnitude.html', {'form': form,'src':src,'base':base,}) + + +def update_astrobasis_source(request, source_id, heasarcbase_id): + status=srg_group_auth_stats(request, "srg-artxc-catalog") + if status != 0: + return HttpResponse(status_code[status]) + src = get_object_or_404(AstroBasisUserCat, pk=source_id) + base = get_object_or_404(HeasarcBase, pk=heasarcbase_id) + + + if request.method == 'POST': + form = AstroBasisUserCatForm(request.POST, instance=src) + if form.is_valid(): + new_form=form.save(commit=False) + #bibcode=form.cleaned_data['bibcode'] + #notes=form.cleaned_data['notes'] + ra = float(form.cleaned_data['ra']) + dec = float(form.cleaned_data['dec']) + name = form.cleaned_data['name'] + autoname = form.cleaned_data['autoname'] + if(autoname): + new_form.name=make_source_name(name,ra,dec) + new_form.autoname=False + new_form.save() + #healpix_usercat(new_form) + #match_usercat_all(new_form) + #src.ra=ra + #src.dec=dec + #src.name=name + #src.save() + return HttpResponseRedirect(base.get_base_url()) + else: + pass + + form = AstroBasisUserCatForm(instance=src) + return render(request, 'astrobasis/update_usercat.html', {'form': form,'src':src,'base':base,}) diff --git a/genericsource/__init__.py b/genericsource/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/genericsource/admin.py b/genericsource/admin.py new file mode 100755 index 0000000..8c38f3f --- /dev/null +++ b/genericsource/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/genericsource/apps.py b/genericsource/apps.py new file mode 100755 index 0000000..f6e7a91 --- /dev/null +++ b/genericsource/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class GenericsourceConfig(AppConfig): + name = 'genericsource' diff --git a/genericsource/migrations/0001_initial.py b/genericsource/migrations/0001_initial.py new file mode 100644 index 0000000..a578594 --- /dev/null +++ b/genericsource/migrations/0001_initial.py @@ -0,0 +1,105 @@ +# Generated by Django 2.2.6 on 2020-05-01 14:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='GenericCatalog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=21)), + ('nsource', models.IntegerField(default=0)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('mjd_start', models.FloatField(default=0.0)), + ('mjd_stop', models.FloatField(default=0.0)), + ('tfield', models.FloatField(default=0.0)), + ('polymorphic_ctype', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polymorphic_genericsource.genericcatalog_set+', to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'base_manager_name': 'objects', + }, + ), + migrations.CreateModel( + name='GenericConnection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('separation', models.FloatField(default=0.0)), + ('probability', models.FloatField(default=1.0)), + ], + ), + migrations.CreateModel( + name='GenericSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('healpix_plate', models.IntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('lii', models.FloatField(default=0.0)), + ('bii', models.FloatField(default=0.0)), + ('error_radius', models.FloatField(default=0.0)), + ('sxflux', models.FloatField(default=0.0)), + ('mjdobs', models.FloatField(default=0.0)), + ('name', models.CharField(blank=True, default='', max_length=50)), + ('catalog', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genericsource.GenericCatalog')), + ('connections', models.ManyToManyField(through='genericsource.GenericConnection', to='genericsource.GenericSource')), + ('polymorphic_ctype', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polymorphic_genericsource.genericsource_set+', to='contenttypes.ContentType')), + ], + options={ + 'abstract': False, + 'base_manager_name': 'objects', + }, + ), + migrations.CreateModel( + name='SourceComment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField(max_length=1000)), + ('src', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genericsource.GenericSource')), + ], + ), + migrations.AddField( + model_name='genericconnection', + name='connected', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='connected', to='genericsource.GenericSource'), + ), + migrations.AddField( + model_name='genericconnection', + name='connectto', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='connectto', to='genericsource.GenericSource'), + ), + migrations.CreateModel( + name='ConnectionComment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField(max_length=1000)), + ('connection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genericsource.GenericConnection')), + ], + ), + migrations.CreateModel( + name='CatalogComment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField(max_length=10000)), + ('catalog', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='genericsource.GenericCatalog')), + ], + ), + migrations.AlterUniqueTogether( + name='genericconnection', + unique_together={('connectto', 'connected')}, + ), + ] diff --git a/genericsource/migrations/0002_auto_20200601_2248.py b/genericsource/migrations/0002_auto_20200601_2248.py new file mode 100644 index 0000000..9e742a8 --- /dev/null +++ b/genericsource/migrations/0002_auto_20200601_2248.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.6 on 2020-06-01 19:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='genericsource', + name='catalog', + ), + migrations.AddField( + model_name='genericsource', + name='catalog', + field=models.ManyToManyField(to='genericsource.GenericCatalog'), + ), + ] diff --git a/genericsource/migrations/0003_auto_20200626_1000.py b/genericsource/migrations/0003_auto_20200626_1000.py new file mode 100644 index 0000000..ea4b0b7 --- /dev/null +++ b/genericsource/migrations/0003_auto_20200626_1000.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.6 on 2020-06-26 07:00 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('genericsource', '0002_auto_20200601_2248'), + ] + + operations = [ + migrations.AddField( + model_name='sourcecomment', + name='author', + field=models.ForeignKey(default=3, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + preserve_default=False, + ), + migrations.AddField( + model_name='sourcecomment', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + ] diff --git a/genericsource/migrations/0004_auto_20200626_1523.py b/genericsource/migrations/0004_auto_20200626_1523.py new file mode 100644 index 0000000..78632e9 --- /dev/null +++ b/genericsource/migrations/0004_auto_20200626_1523.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-26 12:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0003_auto_20200626_1000'), + ] + + operations = [ + migrations.AlterField( + model_name='genericcatalog', + name='name', + field=models.CharField(blank=True, max_length=51), + ), + ] diff --git a/genericsource/migrations/0005_genericsource_src_class.py b/genericsource/migrations/0005_genericsource_src_class.py new file mode 100644 index 0000000..0e33224 --- /dev/null +++ b/genericsource/migrations/0005_genericsource_src_class.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-08-27 13:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0004_auto_20200626_1523'), + ] + + operations = [ + migrations.AddField( + model_name='genericsource', + name='src_class', + field=models.CharField(default='', max_length=21), + ), + ] diff --git a/genericsource/migrations/0006_srcauxdata.py b/genericsource/migrations/0006_srcauxdata.py new file mode 100644 index 0000000..5e498c5 --- /dev/null +++ b/genericsource/migrations/0006_srcauxdata.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.8 on 2020-09-11 09:41 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0005_genericsource_src_class'), + ] + + operations = [ + migrations.CreateModel( + name='SrcAuxDATA', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('spec', models.CharField(default='', max_length=101)), + ('arf', models.CharField(default='', max_length=101)), + ('rmf', models.CharField(default='', max_length=101)), + ('bkg', models.CharField(default='', max_length=101)), + ('img', models.CharField(default='', max_length=101)), + ('reg', models.CharField(default='', max_length=101)), + ('log', models.CharField(default='', max_length=101)), + ('src', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='genericsource.GenericSource')), + ], + ), + ] diff --git a/genericsource/migrations/0007_srcauxdata_specimg.py b/genericsource/migrations/0007_srcauxdata_specimg.py new file mode 100644 index 0000000..c2e0eb0 --- /dev/null +++ b/genericsource/migrations/0007_srcauxdata_specimg.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-09-11 17:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0006_srcauxdata'), + ] + + operations = [ + migrations.AddField( + model_name='srcauxdata', + name='specimg', + field=models.CharField(default='', max_length=101), + ), + ] diff --git a/genericsource/migrations/0008_genericcatalogsgroup.py b/genericsource/migrations/0008_genericcatalogsgroup.py new file mode 100644 index 0000000..c92aa84 --- /dev/null +++ b/genericsource/migrations/0008_genericcatalogsgroup.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.8 on 2020-09-16 07:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0007_srcauxdata_specimg'), + ] + + operations = [ + migrations.CreateModel( + name='GenericCatalogsGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=51)), + ('catalog', models.ManyToManyField(to='genericsource.GenericCatalog')), + ], + ), + ] diff --git a/genericsource/migrations/0009_srcauxdata_xcm.py b/genericsource/migrations/0009_srcauxdata_xcm.py new file mode 100644 index 0000000..a4c4d94 --- /dev/null +++ b/genericsource/migrations/0009_srcauxdata_xcm.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-09-17 09:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0008_genericcatalogsgroup'), + ] + + operations = [ + migrations.AddField( + model_name='srcauxdata', + name='xcm', + field=models.CharField(default='', max_length=101), + ), + ] diff --git a/genericsource/migrations/0010_auto_20201023_1729.py b/genericsource/migrations/0010_auto_20201023_1729.py new file mode 100644 index 0000000..1a7fe31 --- /dev/null +++ b/genericsource/migrations/0010_auto_20201023_1729.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.8 on 2020-10-23 14:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0009_srcauxdata_xcm'), + ] + + operations = [ + migrations.AlterModelOptions( + name='sourcecomment', + options={'ordering': ['created']}, + ), + ] diff --git a/genericsource/migrations/0011_auto_20201103_1325.py b/genericsource/migrations/0011_auto_20201103_1325.py new file mode 100644 index 0000000..5f8589b --- /dev/null +++ b/genericsource/migrations/0011_auto_20201103_1325.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.8 on 2020-11-03 10:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0010_auto_20201023_1729'), + ] + + operations = [ + migrations.AddField( + model_name='srcauxdata', + name='gamma', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='srcauxdata', + name='gammael', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='srcauxdata', + name='gammaeu', + field=models.FloatField(default=0.0), + ), + ] diff --git a/genericsource/migrations/0012_auto_20201104_1915.py b/genericsource/migrations/0012_auto_20201104_1915.py new file mode 100644 index 0000000..0e5b7fe --- /dev/null +++ b/genericsource/migrations/0012_auto_20201104_1915.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2020-11-04 16:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0011_auto_20201103_1325'), + ] + + operations = [ + migrations.AddField( + model_name='srcauxdata', + name='nH', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='srcauxdata', + name='norm', + field=models.FloatField(default=0.0), + ), + ] diff --git a/genericsource/migrations/0013_auto_20201106_1817.py b/genericsource/migrations/0013_auto_20201106_1817.py new file mode 100644 index 0000000..97309a1 --- /dev/null +++ b/genericsource/migrations/0013_auto_20201106_1817.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2020-11-06 15:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0012_auto_20201104_1915'), + ] + + operations = [ + migrations.RenameField( + model_name='srcauxdata', + old_name='nH', + new_name='nh', + ), + ] diff --git a/genericsource/migrations/0014_auto_20201112_1303.py b/genericsource/migrations/0014_auto_20201112_1303.py new file mode 100644 index 0000000..1e8282b --- /dev/null +++ b/genericsource/migrations/0014_auto_20201112_1303.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.8 on 2020-11-12 10:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0013_auto_20201106_1817'), + ] + + operations = [ + migrations.AddField( + model_name='srcauxdata', + name='lcfits', + field=models.CharField(default='', max_length=101), + ), + migrations.AddField( + model_name='srcauxdata', + name='lcpdf', + field=models.CharField(default='', max_length=101), + ), + ] diff --git a/genericsource/migrations/0015_gsbibliogrpahy.py b/genericsource/migrations/0015_gsbibliogrpahy.py new file mode 100644 index 0000000..f4bccd9 --- /dev/null +++ b/genericsource/migrations/0015_gsbibliogrpahy.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.8 on 2020-11-12 10:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('genericsource', '0014_auto_20201112_1303'), + ] + + operations = [ + migrations.CreateModel( + name='GSBibliogrpahy', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(default='', max_length=1000)), + ('code', models.CharField(default='', max_length=100)), + ('addedat', models.DateTimeField(auto_now_add=True)), + ('addedby', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('srcs', models.ManyToManyField(to='genericsource.GenericSource')), + ], + ), + ] diff --git a/genericsource/migrations/0016_auto_20211229_1259.py b/genericsource/migrations/0016_auto_20211229_1259.py new file mode 100644 index 0000000..2d3c749 --- /dev/null +++ b/genericsource/migrations/0016_auto_20211229_1259.py @@ -0,0 +1,58 @@ +# Generated by Django 3.2.10 on 2021-12-29 09:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('genericsource', '0015_gsbibliogrpahy'), + ] + + operations = [ + migrations.AlterField( + model_name='catalogcomment', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='connectioncomment', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='genericcatalog', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='genericcatalogsgroup', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='genericconnection', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='genericsource', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='gsbibliogrpahy', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='sourcecomment', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='srcauxdata', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/genericsource/migrations/__init__.py b/genericsource/migrations/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/genericsource/models.py b/genericsource/models.py new file mode 100755 index 0000000..f2d245d --- /dev/null +++ b/genericsource/models.py @@ -0,0 +1,127 @@ +from django.db import models +from django.contrib.auth.models import User +from astropy.coordinates import SkyCoord +from polymorphic.models import PolymorphicModel +import datetime + +class GenericCatalog(PolymorphicModel): + """ Base class for eRosita daily source list """ + name = models.CharField(max_length=51, blank=True) + """ Unique ObsID """ + nsource = models.IntegerField(default=0) + """ Number of sources in catalog """ + created = models.DateTimeField(auto_now_add=True) + """ Date and time of creation. Fully automatic. """ + modified = models.DateTimeField(auto_now=True) + """ Date and time of the last modification. Fully automatic. """ + start = models.DateTimeField(blank=True) + """ Start date and time """ + stop = models.DateTimeField(blank=True) + """ Stop date and time """ + mjd_start = models.FloatField(default=0.0) + """ MJD start """ + mjd_stop = models.FloatField(default=0.0) + """ MJD stop """ + ordering = ['-mjd_start',] + tfield = models.FloatField(default=0.0) + +class GenericCatalogsGroup(models.Model): + name = models.CharField(max_length=51, blank=True) + catalog = models.ManyToManyField(GenericCatalog) #, on_delete=models.CASCADE) + + +class GenericSource(PolymorphicModel): + """ + General class for a source, described with two sets of spherical coordinates (assumed fk5 and Gal) + some overhead, appended here is softxflux - which is useless for sources, obtained not in X-ray flux + but very helpfull for X-ray transient search + """ + catalog = models.ManyToManyField(GenericCatalog) #, on_delete=models.CASCADE) + healpix = models.BigIntegerField(db_index=True, default=0) + healpix_plate = models.IntegerField(db_index=True, default=0) + src_class = models.CharField(max_length=21, default="") + ra = models.FloatField(default=0.0) + """ Right Ascension of the Source """ + dec = models.FloatField(default=0.0) + """ Declination of the Source """ + lii = models.FloatField(default=0.0) + """ Galactic Longitude """ + bii = models.FloatField(default=0.0) + """ Galactic Latitude """ + error_radius = models.FloatField(default=0.0) + """ Positional Uncertainty (arcsec) """ + sxflux = models.FloatField(default=0.0) + """ x-ray flux in 0.5-2 keV (required for transients search)""" + mjdobs = models.FloatField(default=0.0) + """ observation date in mjd """ + connections = models.ManyToManyField("self", through="GenericConnection", + symmetrical=False) + + name = models.CharField(max_length=50,blank=True, default='') + """ Source name """ + def description(self): + return self.__class__.__name__ + + def get_absolute_url(self): + return "/genericsource/%i/source" % (self.pk) + + def add_neighbours(self, obj): + sc = SkyCoord(self.ra, self.dec, unit="deg", frame="fk5") + if issubclass(type(obj), GenericSource): obj = [obj,] + for src in obj: + scother = SkyCoord(src.ra, src.dec, unit="deg", frame="fk5") + GenericConnection(connectto=self, connected=src, + separation=sc.separation(scother).arcsec).save() + GenericConnection(connectto=src, connected=self, + separation=sc.separation(scother).arcsec).save() + +class GenericConnection(models.Model): + connectto = models.ForeignKey(GenericSource, on_delete=models.CASCADE, related_name="connectto") + connected = models.ForeignKey(GenericSource, on_delete=models.CASCADE, related_name="connected") + separation = models.FloatField(default=0.) + probability = models.FloatField(default=1.) + class Meta: + unique_together = ('connectto', 'connected') + +class SourceComment(models.Model): + src = models.ForeignKey(GenericSource, on_delete=models.CASCADE) + author = models.ForeignKey(User, on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + text = models.TextField(max_length=1000) + class Meta: + ordering = ["created", ] + +class ConnectionComment(models.Model): + connection = models.ForeignKey(GenericConnection, on_delete=models.CASCADE) + text = models.TextField(max_length=1000) + +class CatalogComment(models.Model): + catalog = models.ForeignKey(GenericCatalog, on_delete=models.CASCADE) + text = models.TextField(max_length=10000) + +class SrcAuxDATA(models.Model): + src = models.OneToOneField(GenericSource, on_delete=models.CASCADE) + spec = models.CharField(max_length=101, default="") + specimg = models.CharField(max_length=101, default="") + arf = models.CharField(max_length=101, default="") + rmf = models.CharField(max_length=101, default="") + bkg = models.CharField(max_length=101, default="") + img = models.CharField(max_length=101, default="") + reg = models.CharField(max_length=101, default="") + log = models.CharField(max_length=101, default="") + xcm = models.CharField(max_length=101, default="") + lcfits = models.CharField(max_length=101, default="") + lcpdf = models.CharField(max_length=101, default="") + gamma = models.FloatField(default=0.) + gammaeu = models.FloatField(default=0.) + gammael = models.FloatField(default=0.) + nh = models.FloatField(default=0.) + norm = models.FloatField(default=0.) + + +class GSBibliogrpahy(models.Model): + title = models.CharField(max_length=1000, default="") + code = models.CharField(max_length=100, default="") + addedby = models.ForeignKey(User, on_delete=models.CASCADE) + addedat = models.DateTimeField(auto_now_add=True) + srcs = models.ManyToManyField(GenericSource) diff --git a/genericsource/tests.py b/genericsource/tests.py new file mode 100755 index 0000000..7ce503c --- /dev/null +++ b/genericsource/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/genericsource/urls.py b/genericsource/urls.py new file mode 100644 index 0000000..d441ff7 --- /dev/null +++ b/genericsource/urls.py @@ -0,0 +1,8 @@ +from django.urls import path +from . import views + +app_name = "genericsource" + +urlpatterns = [path('/source', views.show_source, name='show source'), + path('add_comment', views.add_comment, name='gs add comment') + ] diff --git a/genericsource/views.py b/genericsource/views.py new file mode 100755 index 0000000..abbd5c7 --- /dev/null +++ b/genericsource/views.py @@ -0,0 +1,34 @@ +from django.shortcuts import render +from srglib.utils import srg_group_auth, srg_group_auth_stats, status_code +from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden +from genericsource.models import GenericSource, SourceComment + +def add_comment(request): + if request.method == 'POST': + text = request.POST['text'] + if "cid" in request.POST: + s = SourceComment.objects.get(id=request.POST["cid"]) + if text == "": + s.delete() + else: + s.text = text + s.save() + else: + SourceComment.objects.create(author=request.user, text=text, src=GenericSource.objects.get(id=request.POST["srcid"])) + return HttpResponse("comment added") + + +def show_source(request, source_id): + status=srg_group_auth_stats(request, "srg-erosita-transients") + if status != 0: + return HttpResponse(status_code[status]) + + try: + src=GenericSource.objects.get(id=source_id) + except: + return HttpResponse("This genericsource was not found %s " % source_id) + + cat = src.catalog.order_by("created").first() + res = {'src':src, "cat":cat} + return render(request,'erositadailycatalogues/viewsource.html', res) + diff --git a/monthplan/__init__.py b/monthplan/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monthplan/admin.py b/monthplan/admin.py new file mode 100644 index 0000000..46e09de --- /dev/null +++ b/monthplan/admin.py @@ -0,0 +1,15 @@ +from django.contrib import admin +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, MonthPlanUpload, FlightPlan, DataDump, SurveyHealpixPlate + +# Register your models here. +admin.site.register(Head) +admin.site.register(Observation) +admin.site.register(Seance) +admin.site.register(Correction) +admin.site.register(Scan) +admin.site.register(Survey) +admin.site.register(MonthPlanUpload) +admin.site.register(FlightPlan) +admin.site.register(DataDump) +admin.site.register(SurveyHealpixPlate) +#admin.site.register(ScanPath) diff --git a/monthplan/apps.py b/monthplan/apps.py new file mode 100644 index 0000000..6e34666 --- /dev/null +++ b/monthplan/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MonthplanConfig(AppConfig): + name = 'monthplan' diff --git a/monthplan/forms.py b/monthplan/forms.py new file mode 100644 index 0000000..2301fd6 --- /dev/null +++ b/monthplan/forms.py @@ -0,0 +1,4 @@ +from django import forms + +class ObsIDSearchForm(forms.Form): + obsid = forms.CharField(label='obsid', max_length=11) diff --git a/monthplan/management/commands/00_load_surveypath.py b/monthplan/management/commands/00_load_surveypath.py new file mode 100644 index 0000000..488370e --- /dev/null +++ b/monthplan/management/commands/00_load_surveypath.py @@ -0,0 +1,29 @@ +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import User +from django.core import mail +from django.template.loader import render_to_string +from django.utils.html import strip_tags +from django.core.mail import send_mail + +import time +import re +import glob +import logging + +from monthplan.utils import load_surveypath_data + + +class Command(BaseCommand): + help = 'Initiates data base' + + def handle(self, *args, **options): + + logging.getLogger().setLevel(logging.INFO) + + path = "/export/django/srg/data/npol/DMV/ARJ-SCAN/20??????_??????_???????????.iki" + ikifiles = [] + for file in glob.glob(path): + ikifiles.append(file) + + for file in ikifiles: + load_surveypath_data(file,100,notify=True) diff --git a/monthplan/management/commands/00_watchdog.py b/monthplan/management/commands/00_watchdog.py new file mode 100644 index 0000000..235e155 --- /dev/null +++ b/monthplan/management/commands/00_watchdog.py @@ -0,0 +1,76 @@ +from django.core.management.base import BaseCommand, CommandError +from django.contrib.auth.models import User +from django.core import mail +from django.template.loader import render_to_string +from django.utils.html import strip_tags +from django.core.mail import send_mail + +import time +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler +from watchdog.events import PatternMatchingEventHandler + +class MyHandler(PatternMatchingEventHandler): + patterns = ["*gyro.fits",] + + def process(self, event): + """ + event.event_type + 'modified' | 'created' | 'moved' | 'deleted' + event.is_directory + True | False + event.src_path + path/to/observed/file + """ + # the file will be processed there + + + if not event.event_type == 'created': + return + + print("Reading {} ty[e: {}".format(event.src_path, event.event_type)) + + #if not 'L1b' in event.src_path: + # return + + users = User.objects.filter(groups__name='srg-artxc-survey') + to_emails=[] + for user in users: + to_emails.append(user.email) + + print (event.src_path, event.event_type) # print now only for degug + + count=1 + subject = "New SRG data" + html_message = render_to_string('monthplan/data_loaded_email.html', {'event':event, 'count':count}) + plain_message = strip_tags(html_message) + from_email = 'Roman Krivonos ' + + mail.send_mail(subject, plain_message, from_email, ['krivonos@cosmos.ru','kate@cosmos.ru',], html_message=html_message) + + def on_modified(self, event): + self.process(event) + + def on_created(self, event): + self.process(event) + +class EventHandler(FileSystemEventHandler): + def on_created(self, event): + print (event) + +class Command(BaseCommand): + help = 'Initiates data base' + + def handle(self, *args, **options): + + + path = "/srg/work/oper/data/2020/" + observer = Observer() + observer.schedule(MyHandler(), path, recursive=True) + observer.start() + try: + while True: + time.sleep(10) + except KeyboardInterrupt: + observer.stop() + observer.join() diff --git a/monthplan/management/commands/01_init_monthplan.py b/monthplan/management/commands/01_init_monthplan.py new file mode 100644 index 0000000..fc49cb9 --- /dev/null +++ b/monthplan/management/commands/01_init_monthplan.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from logbook.models import LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate +import datetime +from django.utils import timezone +import astropy +import pandas as pd + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + self.stdout.write(self.style.SUCCESS('Successfully initiated "%s"' % 100)) diff --git a/monthplan/management/commands/02_load_monthplan.py b/monthplan/management/commands/02_load_monthplan.py new file mode 100644 index 0000000..6535e2d --- /dev/null +++ b/monthplan/management/commands/02_load_monthplan.py @@ -0,0 +1,123 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey +from monthplan.utils import load_monthplan_fits, load_monthplan_dir + +from astropy.io import fits + +import datetime +from django.utils import timezone + + +from astropy.io import fits +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile +#import xlrd +from astropy.table import Table + +import math + +def load_data(file): + print('Load data (dummy)') + fitsfile=INPUT_DATA_DIR+'/'+file + table = Table.read(fitsfile) + df = table.to_pandas() + print("Column headings:") + print(df.columns.str.encode('utf-8')) + hdul = fits.open(fitsfile) + hdul.info() + head=Head() + head.start=hdul[1].header['START'] + head.stop=hdul[1].header['STOP'] + head.gentime=hdul[1].header['GENTIME'] + head.version=hdul[1].header['VERSION'] + head.author=hdul[1].header['AUTHOR'] + head.file=file + head.nrows=df.shape[0] + head.save() + for i in df.index: + row=(i+1) + if(df['TYPE'][i].decode().strip() == 'SEANCE'): + print('SEA') + print(df['START'][i].decode().strip()) + seance=Seance(head=head, row=row) + seance.start=df['START'][i].decode().strip() + seance.stop=df['STOP'][i].decode().strip() + seance.stations=df['STATIONS'][i].decode().strip() + seance.guid=df['GUID'][i].decode().strip() + seance.save() + if(df['TYPE'][i].decode().strip() == 'OBSERVATION'): + obs=Observation(head=head, row=row) + obs.start=df['START'][i].decode().strip() + obs.stop=df['STOP'][i].decode().strip() + obs.target=df['TARGET'][i].decode().strip() + obs.experiment=df['EXPERIMENT'][i].decode().strip() + obs.ra=df['RA'][i] + obs.dec=df['DEC'][i] + obs.ra=df['RA'][i] + obs.roll_angle=df['ROLL_ANGLE'][i] + obs.sun_xoz_angle=df['SUN_XOZ_ANGLE'][i] + obs.guid=df['GUID'][i].decode().strip() + obs.save() + if(df['TYPE'][i].decode().strip() == 'CORRECTION'): + cor=Correction(head=head, row=row) + cor.start=df['START'][i].decode().strip() + cor.stop=df['STOP'][i].decode().strip() + cor.impstart=df['IMPSTART'][i].decode().strip() + cor.guid=df['GUID'][i].decode().strip() + cor.save() + if(df['TYPE'][i].decode().strip() == 'SCAN'): + print('SCAN') + scan=Scan(head=head, row=row) + scan.start=df['START'][i].decode().strip() + scan.stop=df['STOP'][i].decode().strip() + scan.target=df['TARGET'][i].decode().strip() + scan.experiment=df['EXPERIMENT'][i].decode().strip() + scan.ra=df['RA'][i] + scan.dec=df['DEC'][i] + scan.roll_angle=df['ROLL_ANGLE'][i] + scan.sun_xoz_angle=df['SUN_XOZ_ANGLE'][i] + scan.template=int(df['TEMPLATE'][i].decode().strip()) + scan.guid=df['GUID'][i].decode().strip() + scan.save() + if(df['TYPE'][i].decode().strip() == 'SURVEY'): + sur=Scan(head=head, row=row) + sur.start=df['START'][i].decode().strip() + sur.stop=df['STOP'][i].decode().strip() + sur.target=df['TARGET'][i].decode().strip() + sur.experiment=df['EXPERIMENT'][i].decode().strip() + sur.ra_p=df['RA_P'][i] + sur.dec_p=df['DEC_P'][i] + sur.ra_z0=df['RA_Z0'][i] + sur.dec_z0=df['DEC_Z0'][i] + sur.ra_zk=df['RA_ZK'][i] + sur.dec_zk=df['DEC_ZK'][i] + sur.z_speed=df['Z_SPEED'][i] + sur.guid=df['GUID'][i].decode().strip() + sur.save() + #myplan = LogBookPlan(daynumber=myday.daynumber, description=df['plan'][i], date=df['date'][i]) + #myplan.save() + #myday.plan.add(myplan) + #myday.save() + print(df['TYPE'][i]) + pass + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + #load_data('RG_MonthPlan_2019-07_v08.fits') + #load_data('RG_MonthPlan_2019-08_v14.fits') + #load_monthplan_fits('/export/django/srg/data/npol/fits/RG_MonthPlan_2020-06_v07.fits') + load_monthplan_dir('/export/django/srg/data/npol/fits') + self.stdout.write(self.style.SUCCESS("Done")) diff --git a/monthplan/management/commands/03_load_flightplan.py b/monthplan/management/commands/03_load_flightplan.py new file mode 100644 index 0000000..0fd6cd9 --- /dev/null +++ b/monthplan/management/commands/03_load_flightplan.py @@ -0,0 +1,127 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, FlightPlan + +from astropy.io import fits + +from django.utils import timezone + + +from astropy.io import fits +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile +#import xlrd +from astropy.table import Table + +import math + +import astropy.units as u +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch +from datetime import datetime + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +# 2019-08-01T02:28:00+03:00 +TIMESTR_FORMAT = '%Y-%m-%dT%H:%M:%S+03:00' + +def str2date(timestr, tzone): + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + + +def date2mission(dtime): + mjdref = Time(MJDREF, format='mjd') + dtime = Time(dtime , format='datetime') + return (dtime - mjdref).sec + + +def mission2date(timesec, tzone): + mjdref = Time(MJDREF, format='mjd') + delta = TimeDelta(timesec, format='sec') + dtime = mjdref + delta + 3*u.hour + return dtime.to_datetime() + +def print_obsplan_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + print(dt, '-->', ts) + +def print_pz_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_UTC) + ts = date2mission(dt) + print(dt, '-->', ts) + +def load_data(file): + print('Load data (dummy)') + fitsfile=INPUT_DATA_DIR+'/'+file + table = Table.read(fitsfile) + df = table.to_pandas() + print("Column headings:") + print(df.columns.str.encode('utf-8')) + hdul = fits.open(fitsfile) + hdul.info() + for i in df.index: + dtstr=df['START'][i].decode().strip() + dt = str2date(dtstr, tzone=TZ_MSK) + start_ts = date2mission(dt) + + dtstr=df['STOP'][i].decode().strip() + dt = str2date(dtstr, tzone=TZ_MSK) + stop_ts = date2mission(dt) + #print(dt, '-->', ts) + + obs=FlightPlan() + obs.start=df['START'][i].decode().strip() + obs.stop=df['STOP'][i].decode().strip() + obs.experiment=df['EXPERIMENT'][i].decode().strip() + obs.ra=df['RA'][i] + obs.dec=df['DEC'][i] + obs.q1=df['Q1'][i] + obs.q2=df['Q2'][i] + obs.q3=df['Q3'][i] + obs.q4=df['Q4'][i] + obs.start_npol = df['START_NPOL'][i].decode().strip() + obs.stop_npol = df['STOP_NPOL'][i].decode().strip() + obs.load_stamp = df['LOAD_STAMP'][i].decode().strip() + obs.mjd_start = df['MJD_START'][i] + obs.mjd_stop = df['MJD_STOP'][i] + obs.obt_start = start_ts # df['OBT_START'][i] + obs.obt_stop = stop_ts # df['OBT_STOP'][i] + obs.save() + print(df['EXPERIMENT'][i]) + pass + +def validate_flightplan(): + list = FlightPlan.objects.all() + for pz in list: + if (pz.experiment == '00000000000' and pz.valid == True): + pz.valid = False + pz.save() + continue + first = FlightPlan.objects.filter(experiment__exact=pz.experiment).order_by("-load_stamp")[0] + first.valid = True + first.save() + pass + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + pz = FlightPlan.objects.all() + pz.delete() + load_data('pz.fits') + validate_flightplan() diff --git a/monthplan/management/commands/04_load_datadump.py b/monthplan/management/commands/04_load_datadump.py new file mode 100644 index 0000000..8630fb2 --- /dev/null +++ b/monthplan/management/commands/04_load_datadump.py @@ -0,0 +1,126 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, FlightPlan, DataDump + +from astropy.io import fits + +from datetime import datetime +from django.utils import timezone + + +from astropy.io import fits +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile +import xlrd +from astropy.table import Table + +import astropy.units as u +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from django.db.models import Q +import math + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +TIMESTR_FORMAT = '%Y-%m-%d %H:%M:%S' + +def str2date(timestr, tzone): + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + + +def date2mission(dtime): + mjdref = Time(MJDREF, format='mjd') + dtime = Time(dtime , format='datetime') + return (dtime - mjdref).sec + + +def mission2date(timesec, tzone): + mjdref = Time(MJDREF, format='mjd') + delta = TimeDelta(timesec, format='sec') + dtime = mjdref + delta + 3*u.hour + return dtime.to_datetime() + + +def print_obsplan_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + print(dt, '-->', ts) + + +def print_pz_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_UTC) + ts = date2mission(dt) + + print(dt, '-->', ts) + + +def load_data(file): + print('Load data (dummy)') + csvfile=INPUT_DATA_DIR+'/'+file + df = pd.read_csv(csvfile,names=['name', 'path', 'filename', 'tstart','tstop']) + print(df) + for i in df.index: + dd = DataDump() + dd.start=mission2date(int(df['tstart'][i]), TZ_MSK) + dd.stop=mission2date(int(df['tstop'][i]), TZ_MSK) + dd.tstart=int(df['tstart'][i]) + dd.tstop=int(df['tstop'][i]) + dd.name=df['name'][i] + dd.filename=df['filename'][i] + dd.path=df['path'][i] + dd.save() + observations = Observation.objects.filter(Q(start__gte=dd.start) & Q(start__lte=dd.stop)) + scans = Scan.objects.filter(Q(start__gte=dd.start) & Q(start__lte=dd.stop)) + for observation in observations: + dd.observations.add(observation) + for scan in scans: + dd.scans.add(scan) + observations = Observation.objects.filter(Q(stop__gte=dd.start) & Q(stop__lte=dd.stop)) + scans = Scan.objects.filter(Q(stop__gte=dd.start) & Q(stop__lte=dd.stop)) + for observation in observations: + dd.observations.add(observation) + for scan in scans: + dd.scans.add(scan) + dd.save() + print(mission2date(int(df['tstart'][i]), TZ_MSK)) + print(df['name'][i]) + pass + +def validate_flightplan(): + list = FlightPlan.objects.all() + for pz in list: + if (pz.experiment == '00000000000' and pz.valid == True): + pz.valid = False + pz.save() + continue + first = FlightPlan.objects.filter(experiment__exact=pz.experiment).order_by("-load_stamp")[0] + first.valid = True + first.save() + pass + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + dds = DataDump.objects.all() + dds.delete() + load_data('read_seance.csv') + print_pz_dt_sec('2019-08-21 22:00:00') + print_pz_dt_sec('2019-08-21 23:00:00') + + #validate_flightplan() diff --git a/monthplan/management/commands/05_load_survey_track.py b/monthplan/management/commands/05_load_survey_track.py new file mode 100644 index 0000000..cf22a94 --- /dev/null +++ b/monthplan/management/commands/05_load_survey_track.py @@ -0,0 +1,239 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, FlightPlan, DataDump, SurveyPath +from monthplan.models import SurveyHealpixPlate, NSIDE_PLATES, ORDER + +from astropy.io import fits + +from datetime import datetime +from django.utils import timezone + + +from astropy.io import fits +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile +#import xlrd +from astropy.table import Table + +import astropy.units as u +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from django.db.models import Q +from math import pi, cos, sin +from Quaternion import Quat +from scipy.spatial.transform import Rotation, Slerp +import numpy as np + +from srglib.utils import slerp, quat_to_pol_and_roll +import math +from Quaternion import Quat + +from os import listdir +from astropy_healpix import HEALPix +from astropy import units as u +from astropy.coordinates import SkyCoord, RangeError +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import Angle + + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +TIMESTR_FORMAT = '%Y.%m.%d %H:%M:%S' + +def str2date(timestr, tzone): + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + + +def date2mission(dtime): + mjdref = Time(MJDREF, format='mjd') + dtime = Time(dtime , format='datetime') + #print(dtime.mjd) + return (dtime - mjdref).sec + + +def mission2date(timesec, tzone): + mjdref = Time(MJDREF, format='mjd') + delta = TimeDelta(timesec, format='sec') + dtime = mjdref + delta + 3*u.hour + return dtime.to_datetime() + + +def print_obsplan_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + print(dt, '-->', ts, ts/14400) + + +def print_pz_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_UTC) + ts = date2mission(dt) + + print(dt, '-->', ts) + + +def quat_to_pol_and_roll(qfin): + """ + it is assumed that quaternion is acting on the sattelite coordinate system + in order to orient in in icrs coordinates + opaxis - define dirrection of the main axis (x axis [1, 0, 0] coinside with optical axis) + we assume that north is oriented along z coordinate, we also name this coordinate + north for detectors + """ + opaxis=[1, 0, 0] + north=[0, 0, 1] + opticaxis = qfin.apply(opaxis) + print(opticaxis) + dec = np.arctan(opticaxis[2]/np.sqrt(opticaxis[1]**2 + opticaxis[0]**2)) + ra = np.arctan2(opticaxis[1], opticaxis[0])%(2.*pi) + print(ra/pi*180,dec/pi*180) + return + yzprojection = np.cross(opticaxis, north) + vort = np.cross(north, opaxis) + + rollangle = np.arctan2(np.sum(yzprojection*qfin.apply(vort), axis=1), + np.sum(yzprojection*qfin.apply(north), axis=1)) + print(ra, dec, rollangle) + return ra, dec, rollangle + +def logic(index,nskip): + if index % nskip == 0: + return False + return True + +""" +# Skip rows from based on condition like skip every 3rd line +usersDf = pd.read_csv('users.csv', skiprows= lambda x: logic(x) ) +print('Contents of the Dataframe created by skipping every 3rd row from csv file ') +print(usersDf) +""" + +def load_data(file,obsid,nskip): + print("Load data for %s" % file) + try: + survey=Survey.objects.get(experiment__exact=obsid) + except: + print("This survey is not found") + return + + # remove all attached SurveyPath to this survey: + spath=survey.surveypath_set.all() + spath.delete() + + csvfile=INPUT_DATA_DIR+'/'+file + df = pd.read_csv(csvfile,names=['date', 'time', 'q1', 'q2','q3','q4','dummy'], header=None, delim_whitespace=True, skipfooter=1, engine='python', skiprows = lambda x: logic(x, nskip)) + + tolerance = 10.0 + hp = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5()) + + for i in df.index: + dtstr="%s %s" % (df['date'][i], df['time'][i]) + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + eroday=ts/14400 + dtime = Time(dt , format='datetime') + mjd=dtime.mjd + q1=df['q1'][i] + q2=df['q2'][i] + q3=df['q3'][i] + q4=df['q4'][i] + quat=Quat(attitude=[q2,q3,q4,q1]) + sp=SurveyPath(survey=survey,dtime=dt,mjd=mjd,obt=ts,eroday=eroday, + q1=q1,q2=q2,q3=q3,q4=q4,ra=quat.ra,dec=quat.dec,roll=quat.roll) + sp.save() + + + print('Mark SurveyHealpixPlate') + surveypaths = survey.surveypath_set.all() + for i in range(len(surveypaths)-1): + delta=(surveypaths[i+1].dtime-surveypaths[i].dtime).total_seconds() + if(abs(int(delta)) > tolerance): + nstep = int(abs(delta) / tolerance) + delta_arr = np.linspace(0,1,nstep) + + quat = slerp([surveypaths[i].q2, surveypaths[i].q3, surveypaths[i].q4, surveypaths[i].q1], + [surveypaths[i+1].q2, surveypaths[i+1].q3, surveypaths[i+1].q4, surveypaths[i+1].q1], + delta_arr) + for q in quat: + qfin=Quat(attitude=q) + crd = SkyCoord(qfin.ra, qfin.dec, frame="fk5", unit="deg") + heal = hp.skycoord_to_healpix(crd) + try: + plate = SurveyHealpixPlate.objects.get(healpix=heal) + except: + print('Error: SurveyHealpixPlate not found, run ./manage.py init_survey_healpix_plates first') + break + queryset = survey.surveyhealpixplate_set.all() + if not queryset.filter(pk=plate.pk).exists(): + plate.survey.add(survey) + plate.save() + + + pass + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ + load_data('ARJ-SCAN/kvater22_23-09.rez','10000000100',100) + load_data('ARJ-SCAN/kvater23_24-09.rez','10000000200',100) + + load_data('ARJ-SCAN/kvater14-11_13_35_00.rez','10000100100',100) + load_data('ARJ-SCAN/kvater15-11_14_35_00.rez','10000100200',100) + + + load_data('ARJ-SCAN/20191203_150000_10000200100.iki','10000200100',100) + load_data('ARJ-SCAN/20191208_170000_10000300100.iki','10000300100',100) + load_data('ARJ-SCAN/20191210_223000_10000400100.iki','10000400100',100) + load_data('ARJ-SCAN/20191212_003000_11000000100.iki','11000000100',100) + load_data('ARJ-SCAN/20191217_003000_11000000200.iki','11000000200',100) + load_data('ARJ-SCAN/20191222_003000_11000000300.iki','11000000300',100) + load_data('ARJ-SCAN/20191227_003000_11000000400.iki','11000000400',100) + load_data('ARJ-SCAN/20200101_003000_11000000500.iki','11000000500',100) + """ + + """ Jan 2020 """ + """ + load_data('ARJ-SCAN/20200102_163000_11000100100.iki','11000100100',1) + load_data('ARJ-SCAN/20200106_163000_11000100200.iki','11000100200',100) + load_data('ARJ-SCAN/20200110_163000_11000100300.iki','11000100300',100) + load_data('ARJ-SCAN/20200114_163000_11000100400.iki','11000100400',100) + load_data('ARJ-SCAN/20200118_163000_11000100500.iki','11000100500',100) + load_data('ARJ-SCAN/20200122_163000_11000100600.iki','11000100600',100) + load_data('ARJ-SCAN/20200126_163000_11000100700.iki','11000100700',100) + + load_data('ARJ-SCAN/20200130_220000_11000200100.iki','11000200100',100) + """ + """ Feb 2020 """ + + #load_data('ARJ-SCAN/20200202_220000_11000300100.iki','11000300100',100) + #load_data('ARJ-SCAN/20200206_220000_11000300200.iki','11000300200',100) + #load_data('ARJ-SCAN/20200210_220000_11000300300.iki','11000300300',100) + #load_data('ARJ-SCAN/20200214_220000_11000400100.iki','11000400100',100) + load_data('ARJ-SCAN/20200218_220000_11000400200.iki','11000400200',100) + load_data('ARJ-SCAN/20200222_220000_11000400300.iki','11000400300',100) + load_data('ARJ-SCAN/20200226_220000_11000400400.iki','11000400400',100) + + # invalid after eRosita reset + #load_data('ARJ-SCAN/20200214_220000_11000300400.iki','11000300400',100) + #load_data('ARJ-SCAN/20200218_220000_11000300500.iki','11000300500',100) + #load_data('ARJ-SCAN/20200222_220000_11000300600.iki','11000300600',100) + #load_data('ARJ-SCAN/20200226_220000_11000300700.iki','11000300700',100) + + diff --git a/monthplan/management/commands/05_load_survey_track.py.bak b/monthplan/management/commands/05_load_survey_track.py.bak new file mode 100644 index 0000000..aaee3f8 --- /dev/null +++ b/monthplan/management/commands/05_load_survey_track.py.bak @@ -0,0 +1,181 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, FlightPlan, DataDump, SurveyPath + +from astropy.io import fits + +from datetime import datetime +from django.utils import timezone + + +from astropy.io import fits +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile +#import xlrd +from astropy.table import Table + +import astropy.units as u +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from django.db.models import Q +from math import pi, cos, sin +from Quaternion import Quat +from scipy.spatial.transform import Rotation, Slerp +import numpy as np + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +TIMESTR_FORMAT = '%Y.%m.%d %H:%M:%S' + +def str2date(timestr, tzone): + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + + +def date2mission(dtime): + mjdref = Time(MJDREF, format='mjd') + dtime = Time(dtime , format='datetime') + #print(dtime.mjd) + return (dtime - mjdref).sec + + +def mission2date(timesec, tzone): + mjdref = Time(MJDREF, format='mjd') + delta = TimeDelta(timesec, format='sec') + dtime = mjdref + delta + 3*u.hour + return dtime.to_datetime() + + +def print_obsplan_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + print(dt, '-->', ts, ts/14400) + + +def print_pz_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_UTC) + ts = date2mission(dt) + + print(dt, '-->', ts) + + +def quat_to_pol_and_roll(qfin): + """ + it is assumed that quaternion is acting on the sattelite coordinate system + in order to orient in in icrs coordinates + opaxis - define dirrection of the main axis (x axis [1, 0, 0] coinside with optical axis) + we assume that north is oriented along z coordinate, we also name this coordinate + north for detectors + """ + opaxis=[1, 0, 0] + north=[0, 0, 1] + opticaxis = qfin.apply(opaxis) + print(opticaxis) + dec = np.arctan(opticaxis[2]/np.sqrt(opticaxis[1]**2 + opticaxis[0]**2)) + ra = np.arctan2(opticaxis[1], opticaxis[0])%(2.*pi) + print(ra/pi*180,dec/pi*180) + return + yzprojection = np.cross(opticaxis, north) + vort = np.cross(north, opaxis) + + rollangle = np.arctan2(np.sum(yzprojection*qfin.apply(vort), axis=1), + np.sum(yzprojection*qfin.apply(north), axis=1)) + print(ra, dec, rollangle) + return ra, dec, rollangle + +def logic(index,nskip): + if index % nskip == 0: + return False + return True + +def load_data(file,obsid,nskip): + print('Load data (dummy)') + try: + survey=Survey.objects.get(experiment__exact=obsid) + except: + print("This survey is not found") + return + + # remove all attached SurveyPath to this survey: + spath=survey.surveypath_set.all() + spath.delete() + + csvfile=INPUT_DATA_DIR+'/ARJ-SCAN/'+file + df = pd.read_csv(csvfile,names=['date', 'time', 'q1', 'q2','q3','q4','dummy'], header=None, delim_whitespace=True, skipfooter=1, skiprows = lambda x: logic(x, nskip)) + print(df) + for i in df.index: + dtstr="%s %s" % (df['date'][i], df['time'][i]) + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + eroday=ts/14400 + dtime = Time(dt , format='datetime') + mjd=dtime.mjd + q1=df['q1'][i] + q2=df['q2'][i] + q3=df['q3'][i] + q4=df['q4'][i] + quat=Quat(attitude=[q2,q3,q4,q1]) + sp=SurveyPath(survey=survey,dtime=dt,mjd=mjd,obt=ts,eroday=eroday,q1=q1,q2=q2,q3=q3,q4=q4,ra=quat.ra,dec=quat.dec,roll=quat.roll) + sp.save() + pass + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + + load_data('kvater22_23-09.rez','10000000100',50) + load_data('kvater23_24-09.rez','10000000200',50) + + load_data('kvater14-11_13_35_00.rez','10000100100',50) + load_data('kvater15-11_14_35_00.rez','10000100200',50) + + + load_data('20191203_150000_10000200100.iki','10000200100',50) + load_data('20191208_170000_10000300100.iki','10000300100',50) + load_data('20191210_223000_10000400100.iki','10000400100',50) + load_data('20191212_003000_11000000100.iki','11000000100',50) + return + + # + # Experiments: + # + +# .407947163597554 +# .826585167542578 ! 327.89199999 +# -.077996859000543 ! -34.35000000 +# -.379805953741339 ! .000 + print('327.89199999 -34.35000000') + arr=[.826585167542578, -.077996859000543, -.379805953741339, .407947163597554] + #qfin = Rotation.from_quat(arr) + #qfin = Rotation.from_quat([0, 0, np.sin(np.pi/4), np.cos(np.pi/4)]) + #print(qfin.as_euler('zyx', degrees=True)) +#qadd=Rotation([0, 0, 0, 1]) + #qfin=Rotation(arr) + #a = quat_to_pol_and_roll(qfin) + #print(a) + #print_obsplan_dt_sec('2019.10.16 10:42:04') + + #print_obsplan_dt_sec('2019.10.17 00:47:42') +# # print_pz_dt_sec('2019.08.21 23:00:00') + + print('***') + quat = Quat(attitude=arr) + print(quat.ra, quat.dec, quat.roll) + + #validate_flightplan() diff --git a/monthplan/management/commands/06_load_scan_track.py b/monthplan/management/commands/06_load_scan_track.py new file mode 100644 index 0000000..d9bd813 --- /dev/null +++ b/monthplan/management/commands/06_load_scan_track.py @@ -0,0 +1,280 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, FlightPlan, ScanPath + +from astropy.io import fits + +from datetime import datetime +from django.utils import timezone + + +from astropy.io import fits +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile +#import xlrd +from astropy.table import Table + +import astropy.units as u +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from django.db.models import Q +from math import pi, cos, sin +from Quaternion import Quat +from scipy.spatial.transform import Rotation, Slerp +import numpy as np + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +TIMESTR_FORMAT = '%Y.%m.%d %H:%M:%S' + +def str2date(timestr, tzone): + #print(timestr) + if timestr.find('60') > 0: + timestr=timestr.replace('60', '00') + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + + +def date2mission(dtime): + mjdref = Time(MJDREF, format='mjd') + dtime = Time(dtime , format='datetime') + #print(dtime.mjd) + return (dtime - mjdref).sec + + +def mission2date(timesec, tzone): + mjdref = Time(MJDREF, format='mjd') + delta = TimeDelta(timesec, format='sec') + dtime = mjdref + delta + 3*u.hour + return dtime.to_datetime() + + +def print_obsplan_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + print(dt, '-->', ts, ts/14400) + + +def print_pz_dt_sec(dtstr): + dt = str2date(dtstr, tzone=TZ_UTC) + ts = date2mission(dt) + + print(dt, '-->', ts) + + +def quat_to_pol_and_roll(qfin): + """ + it is assumed that quaternion is acting on the sattelite coordinate system + in order to orient in in icrs coordinates + opaxis - define dirrection of the main axis (x axis [1, 0, 0] coinside with optical axis) + we assume that north is oriented along z coordinate, we also name this coordinate + north for detectors + """ + opaxis=[1, 0, 0] + north=[0, 0, 1] + opticaxis = qfin.apply(opaxis) + print(opticaxis) + dec = np.arctan(opticaxis[2]/np.sqrt(opticaxis[1]**2 + opticaxis[0]**2)) + ra = np.arctan2(opticaxis[1], opticaxis[0])%(2.*pi) + print(ra/pi*180,dec/pi*180) + return + yzprojection = np.cross(opticaxis, north) + vort = np.cross(north, opaxis) + + rollangle = np.arctan2(np.sum(yzprojection*qfin.apply(vort), axis=1), + np.sum(yzprojection*qfin.apply(north), axis=1)) + print(ra, dec, rollangle) + return ra, dec, rollangle + +def logic(index,nskip): + if index % nskip == 0: + return False + return True + +def load_data(file,obsid,nskip): + print('Load data for ',obsid) + try: + scan=Scan.objects.get(experiment__exact=obsid) + except: + print("This scan is not found ",obsid) + try: + scans=Scan.objects.filter(experiment__exact=obsid) + except: + print("Multiple scans were also not found ",obsid) + print('Remove some of the invalid scans below:') + for scan in scans: + print(scan.experiment,scan.start,scan.stop,scan.head) + return + + # remove all attached ScanPath to this scan: + spath=scan.scanpath_set.all() + spath.delete() + + csvfile=INPUT_DATA_DIR+'/ARJ-SCAN/'+file + df = pd.read_csv(csvfile,names=['date', 'time', 'q1', 'q2','q3','q4','dummy'], header=None, delim_whitespace=True, skipfooter=1, skiprows = lambda x: logic(x, nskip)) + print(df) + for i in df.index: + dtstr="%s %s" % (df['date'][i], df['time'][i]) + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + eroday=ts/14400 + dtime = Time(dt , format='datetime') + mjd=dtime.mjd + q1=df['q1'][i] + q2=df['q2'][i] + q3=df['q3'][i] + q4=df['q4'][i] + quat=Quat(attitude=[q2,q3,q4,q1]) + sp=ScanPath(scan=scan,dtime=dt,mjd=mjd,obt=ts,eroday=eroday,q1=q1,q2=q2,q3=q3,q4=q4,ra=quat.ra,dec=quat.dec,roll=quat.roll) + sp.save() + pass + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + # note: delete 00003017001 scan from August plan + """ + load_data('kvater01-09_02_15_00.rez','00003017001') + load_data('kvater01-09_10_17_58.rez','00003017002') + load_data('kvater01-10_01_40_00.rez','00003044001') + load_data('kvater02-09_02_40_00.rez','00003031001') + load_data('kvater02-09_10_42_58.rez','00003031002') + load_data('kvater02-10_01_10-00.rez','00003045001') + load_data('kvater03-09_04_06_00.rez','00003018001') + load_data('kvater03-09_11_22_58.rez','00003018002') + load_data('kvater03-11_05_25_50.rez','30000700100') + load_data('kvater04-09_04_06_00.rez','00003019001') + load_data('kvater04-09_11_22_58.rez','00003019002') + load_data('kvater04-11_07_05_52.rez','30000800100') + load_data('kvater05-09_04_06_00.rez','00003020001') + load_data('kvater05-09_11_22_58.rez','00003020002') + load_data('kvater05-11_08_45_54.rez','30000900100') + load_data('kvater06-09_04_06_00.rez','00003021001') + load_data('kvater06-09_11_22_58.rez','00003021002') + load_data('kvater06-11_10_25_56.rez','30001000100') + load_data('kvater07-09_04_06_00.rez','00003022001') + load_data('kvater07-09_11_22_58.rez','00003022002') + load_data('kvater08-09_01_40_00.rez','00003023001') + load_data('kvater08-10_19_11_10.rez','30000500100') + load_data('kvater09-09_01_40_00.rez','00003024001') + load_data('kvater09-10_12_20_02.rez','30000600100') + load_data('kvater10-09_01_30_00.rez','00003025001') + load_data('kvater11-09_01_40_00.rez','00003026001') + load_data('kvater12-09_04_06_00.rez','00003027001') + load_data('kvater12-09_11_22_58.rez','00003027002') + load_data('kvater13-09_04_06_00.rez','00003028001') + load_data('kvater13-09_11_22_58.rez','00003028002') + load_data('kvater13-10_08_59_10.rez','50000600100') + load_data('kvater13-10_23_48_18.rez','50000700100') + load_data('kvater14-08_12_50_00.rez','00003021001') + load_data('kvater14-09_04_06_00.rez','00003029001') + load_data('kvater14-09_11_22_58.rez','00003029002') + load_data('kvater15-09_03_03_20.rez','70004000100') + load_data('kvater15-10_19_52_56.rez','50000800100') + load_data('kvater16-10_10_42_04.rez','50000900100') + load_data('kvater17-08_02_00_00.rez','00003001001') + load_data('kvater17-08_11_01_00.rez','00003002001') + load_data('kvater17-08_22_10_00.rez','00003003001') + load_data('kvater17-09_04_06_00.rez','00003032001') + load_data('kvater17-09_11-22_58.rez','00003032002') + load_data('kvater18-08_03_58_19.rez','00003003002') + load_data('kvater18-08_09_46_38.rez','00003003003') + load_data('kvater18-08_22_10_00.rez','00003004001') + load_data('kvater18-09_04_06_00.rez','00003033001') + load_data('kvater18-09_11_22_58.rez','00003033002') + load_data('kvater18-10_03_16_32.rez','30001600100') + load_data('kvater19-08_01_38_28.rez','00003004002') + load_data('kvater19-08_05_06_56.rez','00003004003') + load_data('kvater19-08_08_35_24.rez','00003004004') + load_data('kvater19-08_12_03_52.rez','00003004005') + load_data('kvater19-08_22_10_00.rez','00003005001') + load_data('kvater19-09_01_40_00.rez','00003034001') + load_data('kvater20-08_01_29_18.rez','00003005002') + load_data('kvater20-08_06_48_36.rez','00003005003') + load_data('kvater20-08_11_07_54.rez','00003005004') + load_data('kvater20-08_22_10_00.rez','00003006001') + load_data('kvater20-09_01_40_00.rez','00003035001') + load_data('kvater20-09_06_18_00.rez','00003036001') + load_data('kvater20-09_10_56_00.rez','00003037001') + load_data('kvater21-08_02_29_18.rez','00003006002') + load_data('kvater21-08_06_48_36.rez','00003006003') + load_data('kvater21-08_11_07_54.rez','00003006004') + load_data('kvater21-09_01_27_00.rez','00003030001') + load_data('kvater21-09_08_43_58.rez','00003030002') + load_data('kvater22-08_22_10_00.rez','00003007001') + load_data('kvater23-08_01_38_28.rez','00003007002') + load_data('kvater23-08_05_06_56.rez','00003007003') + load_data('kvater23-08_08_35_24.rez','00003007004') + load_data('kvater23-08_12_03_52.rez','00003007005') + load_data('kvater23-08_22_10_00.rez','00003008001') + load_data('kvater24-08_02_47_38.rez','00003008002') + load_data('kvater24-08_07_25_16.rez','00003008003') + load_data('kvater24-08_12_02_54.rez','00003008004') + load_data('kvater24-08_22_10_00.rez','00003009001') + load_data('kvater24-09_03_30_00.rez','00003041001') + load_data('kvater24-09_11_07_09.rez','00003041001') + load_data('kvater24-09_13_05_00.rez','00003042001') + load_data('kvater24-09_15_02_09.rez','00003042002') + load_data('kvater25-08_02_47_38.rez','00003009002') + load_data('kvater25-08_07_25_16.rez','00003009003') + load_data('kvater25-08_12_02_54.rez','00003009004') + load_data('kvater25-08_22_10_00.rez','00003010001') + load_data('kvater25-09_01_40_00.rez','00003038001') + load_data('kvater26-08_02_47_38.rez','00003010002') + load_data('kvater26-08_12_02_54.rez','00003010004') + load_data('kvater26-08_23_40_00.rez','00003011001') + load_data('kvater26-09_01_40_00.rez','00003039001') + load_data('kvater26_08_07_25_16.rez','00003010003') + load_data('kvater27-08_04_17_38.rez','00003011002') + load_data('kvater27-08_08_55_16.rez','00003011003') + load_data('kvater27-08_13_32_54.rez','00003011004') + load_data('kvater27-08_23_40_00.rez','00003012001') + load_data('kvater27-09_01_40_00.rez','00003040001') + load_data('kvater28-08_04_17_38.rez','00003012002') + load_data('kvater28-08_08_55_16.rez','00003012003') + load_data('kvater28-08_13_32_54.rez','00003012004') + load_data('kvater28-08_23_40_00.rez','00003013001') + load_data('kvater28-09_01_40_00.rez','00003043001') + load_data('kvater29-08_04_17_38.rez','00003013002') + load_data('kvater29-08_08_55_16.rez','00003013003') + load_data('kvater30-08_02_40_00.rez','00003015001') + load_data('kvater30-08_10_42_58.rez','00003015002') + load_data('kvater31-08_02_40_00.rez','00003016001') + load_data('kvater31-08_10_42_58.rez','00003016002') + load_data('kvater12-11_03_04_38.rez','50001500100') + load_data('kvater12-11_21_50_31.rez','50001600100') + load_data('kvater13-11_11_06_34.rez','50001700100') + load_data('kvater14-11_00_22_37.rez','50001800100') + load_data('kvater16-11_07_42_50.rez','50001900100') + load_data('kvater16-11_16_47_48.rez','50002000100') + """ + + """ + load_data('70019900100.iki','70019900100') + load_data('70020000100.iki','70020000100') + """ + + load_data('70020100100.iki','70020100100',10) + load_data('20191204_041000_70020200100.iki','70020200100',10) + load_data('20191204_134001_50002600100.iki','50002600100',10) + load_data('20191205_005349_50002700100.iki','50002700100',10) + load_data('20191205_120737_50002800100.iki','50002800100',10) + load_data('20191205_232125_50002900100.iki','50002900100',10) + load_data('20191206_103513_50003000100.iki','50003000100',10) + load_data('20191207_071532_70020400100.iki','70020400100',10) + load_data('20191206_222021_70020300100.iki','70020300100',10) diff --git a/monthplan/management/commands/init_survey_healpix_plates.py b/monthplan/management/commands/init_survey_healpix_plates.py new file mode 100644 index 0000000..a423f2e --- /dev/null +++ b/monthplan/management/commands/init_survey_healpix_plates.py @@ -0,0 +1,36 @@ +from django.core.management.base import BaseCommand, CommandError +from monthplan.models import SurveyHealpixPlate, NSIDE_PLATES, ORDER + +from os import listdir +from astropy_healpix import HEALPix +from astropy import units as u +from astropy.coordinates import SkyCoord, RangeError +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import Angle + + +def init_healpix_plates(nside, order): + hp = HEALPix(nside=nside, order=order, frame=FK5()) + print("Make Healpix Plates",hp.npix,hp.pixel_area.value,hp.pixel_resolution.value) + + try: + print('Delete existing Healpix Plates') + plates = SurveyHealpixPlate.objects.all() + plates.delete() + except: + print('Abort') + exit() + for heal in range(hp.npix): + sc = hp.healpix_to_skycoord(heal) + plate = SurveyHealpixPlate(healpix=heal, ra=float(sc.ra.degree), dec=float(sc.dec.degree)) + plate.save() + pass + +class Command(BaseCommand): + + def handle(self, *args, **options): + init_healpix_plates(NSIDE_PLATES, ORDER) + + + + diff --git a/monthplan/management/commands/init_survey_healpix_sub_plates.py b/monthplan/management/commands/init_survey_healpix_sub_plates.py new file mode 100644 index 0000000..44920d5 --- /dev/null +++ b/monthplan/management/commands/init_survey_healpix_sub_plates.py @@ -0,0 +1,46 @@ +from django.core.management.base import BaseCommand, CommandError +from monthplan.models import SurveyHealpixPlate, SurveyHealpixSubPlate, NSIDE_PLATES, NSIDE_SUBPLATES, ORDER + +from os import listdir +from astropy_healpix import HEALPix +from astropy import units as u +from astropy.coordinates import SkyCoord, RangeError +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import Angle + + +def init_healpix_sub_plates(nside_plates, nside_subplates, order): + hp_plates = HEALPix(nside=nside_plates, order=order, frame=FK5()) + hp_subplates = HEALPix(nside=nside_subplates, order=order, frame=FK5()) + + print("Make Healpix Plates",hp_plates.npix,hp_plates.pixel_area.value,hp_plates.pixel_resolution.value) + print("Make Healpix SubPlates",hp_subplates.npix,hp_subplates.pixel_area.value,hp_subplates.pixel_resolution.value) + try: + print('Delete existing Healpix Plates') + subplates = SurveyHealpixSubPlate.objects.all() + subplates.delete() + except: + print('Abort') + exit() + + for heal in range(hp_subplates.npix): + sc = hp_subplates.healpix_to_skycoord(heal) + crd = SkyCoord(float(sc.ra.degree), float(sc.dec.degree), frame="fk5", unit="deg") + heal = hp_plates.skycoord_to_healpix(crd) + try: + plate = SurveyHealpixPlate.objects.get(healpix=heal) + except: + print('Error: SurveyHealpixPlate not found, run ./manage.py init_survey_healpix_plates first') + return + subplate = SurveyHealpixSubPlate(healpix=heal, ra=float(sc.ra.degree), dec=float(sc.dec.degree), plate=plate) + subplate.save() + pass + +class Command(BaseCommand): + + def handle(self, *args, **options): + init_healpix_sub_plates(NSIDE_PLATES, NSIDE_SUBPLATES, ORDER) + + + + diff --git a/monthplan/management/commands/mark_survey_healpix_plates.py b/monthplan/management/commands/mark_survey_healpix_plates.py new file mode 100644 index 0000000..c8f1104 --- /dev/null +++ b/monthplan/management/commands/mark_survey_healpix_plates.py @@ -0,0 +1,54 @@ +from django.core.management.base import BaseCommand, CommandError +from monthplan.models import SurveyHealpixPlate, NSIDE_PLATES, ORDER, MAPS_DIR, Survey, SurveyPath + +from os import listdir +from astropy_healpix import HEALPix +from astropy import units as u +from astropy.coordinates import SkyCoord, RangeError +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import Angle + +import numpy as np + +from srglib.utils import slerp, quat_to_pol_and_roll +import math +from Quaternion import Quat + +class Command(BaseCommand): + + + def handle(self, *args, **options): + tolerance = 10.0 + hp = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5()) + surveys=Survey.objects.all() + for survey in surveys: + print(survey.experiment) + surveypaths = survey.surveypath_set.all() + for i in range(len(surveypaths)-1): + delta=(surveypaths[i+1].dtime-surveypaths[i].dtime).total_seconds() + if(abs(int(delta)) > tolerance): + nstep = int(abs(delta) / tolerance) + delta_arr = np.linspace(0,1,nstep) + + quat = slerp([surveypaths[i].q2, surveypaths[i].q3, surveypaths[i].q4, surveypaths[i].q1], + [surveypaths[i+1].q2, surveypaths[i+1].q3, surveypaths[i+1].q4, surveypaths[i+1].q1], + delta_arr) + for q in quat: + qfin=Quat(attitude=q) + crd = SkyCoord(qfin.ra, qfin.dec, frame="fk5", unit="deg") + heal = hp.skycoord_to_healpix(crd) + try: + plate = SurveyHealpixPlate.objects.get(healpix=heal) + except: + print('Error: SurveyHealpixPlate not found, run ./manage.py init_survey_healpix_plates first') + break + queryset = survey.surveyhealpixplate_set.all() + if not queryset.filter(pk=plate.pk).exists(): + plate.survey.add(survey) + plate.save() + + + + + + diff --git a/monthplan/management/commands/real_time_track.py b/monthplan/management/commands/real_time_track.py new file mode 100644 index 0000000..10ed87d --- /dev/null +++ b/monthplan/management/commands/real_time_track.py @@ -0,0 +1,91 @@ +from django.core.management.base import BaseCommand, CommandError +from datetime import datetime as dt + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate + +from srglib.utils import make_source_name + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, FlightPlan + +from astropy.io import fits + +import datetime +from django.utils import timezone + + +from astropy.io import fits +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile +#import xlrd +from astropy.table import Table + +import math + +from django.db.models import Q + +def find_closest(now, closest_greater_qs, closest_less_qs): + try: + try: + closest_greater = closest_greater_qs[0] + except IndexError: + closest = closest_less_qs[0] + + try: + closest_less = closest_less_qs[0] + except IndexError: + closest = closest_greater_qs[0] + except IndexError: + raise self.model.DoesNotExist("There is no closest object" + " because there are no objects.") + if closest_greater.dtime - now > now - closest_less.dtime: + closest = closest_less + else: + closest = closest_greater + return closest.ra, closest.dec + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + name = make_source_name('SRG A',0.87,45.0) + print(name) + return + now = timezone.localtime(timezone.now()) + print(now) + ra='n/a' + dec='n/a' + request_path='load' + try: + surveys = Survey.objects.filter(Q(start__lte=now) & Q(stop__gte=now)) + # take first element only: + for survey in surveys: + print(survey.ra_p) + target = survey.target + obsid = survey.experiment + url = survey.get_absolute_url() + ra="%.4f" % survey.ra_p + dec="%.4f" % survey.dec_p + mode='Survey' + if request_path == "load": + path=list(survey.surveypath_set.all().values('ra', 'dec'))[::500] + + closest_greater_qs = survey.surveypath_set.filter(dtime__gt=now).order_by('dtime') + closest_less_qs = survey.surveypath_set.filter(dtime__lt=now).order_by('-dtime') + closest_ra, closest_dec = find_closest(now, closest_greater_qs, closest_less_qs) + ra="%.4f" % closest_ra + dec="%.4f" % closest_dec + print("*** closest found ***") + break + except NameError as error: + print('pass',error) + pass + + print(ra, dec) diff --git a/monthplan/management/commands/show_user_emails.py b/monthplan/management/commands/show_user_emails.py new file mode 100644 index 0000000..5aca7bb --- /dev/null +++ b/monthplan/management/commands/show_user_emails.py @@ -0,0 +1,34 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +from django.utils import dateparse +from logbook.models import LogBookPlan, LogBookDay +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate +from django.contrib.auth.models import User + +from monthplan.models import Head, Observation, Seance, Correction, Scan, Survey, FlightPlan + +from astropy.io import fits + +import datetime +from django.utils import timezone + + + +class Command(BaseCommand): + help = 'Initiates data base' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + user = User.objects.get(id=2) + user_email = user.email + print(user_email) + emails = [] + users = User.objects.all() + for user in users: + emails.append(user.email) + print(emails) + diff --git a/monthplan/migrations/0001_initial.py b/monthplan/migrations/0001_initial.py new file mode 100644 index 0000000..9b26489 --- /dev/null +++ b/monthplan/migrations/0001_initial.py @@ -0,0 +1,206 @@ +# Generated by Django 2.2.6 on 2019-12-09 18:37 + +import django.core.files.storage +from django.db import migrations, models +import django.db.models.deletion +import monthplan.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('plan', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='FlightPlan', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('experiment', models.CharField(max_length=11)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('q1', models.FloatField(default=0.0)), + ('q2', models.FloatField(default=0.0)), + ('q3', models.FloatField(default=0.0)), + ('q4', models.FloatField(default=0.0)), + ('start_npol', models.CharField(max_length=25)), + ('stop_npol', models.CharField(max_length=25)), + ('load_stamp', models.DateTimeField(blank=True)), + ('mjd_start', models.FloatField(default=0.0)), + ('mjd_stop', models.FloatField(default=0.0)), + ('obt_start', models.FloatField(default=0.0)), + ('obt_stop', models.FloatField(default=0.0)), + ('valid', models.BooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='Head', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(blank=True, default='', max_length=36)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('title', models.CharField(blank=True, default='', max_length=128)), + ('version', models.IntegerField(default=0)), + ('gentime', models.DateTimeField(blank=True)), + ('author', models.CharField(max_length=80)), + ('nrows', models.IntegerField(default=0)), + ('archived', models.BooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='Scan', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(max_length=36)), + ('experiment', models.CharField(max_length=11)), + ('target', models.CharField(default='', max_length=50)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('roll_angle', models.FloatField(default=0.0)), + ('sun_xoz_angle', models.FloatField(default=0.0)), + ('template', models.IntegerField(default=0)), + ('row', models.IntegerField(default=0)), + ('ra_obj', models.FloatField(default=0.0)), + ('dec_obj', models.FloatField(default=0.0)), + ('name_obj', models.CharField(default='', max_length=30)), + ('head', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.Head')), + ], + ), + migrations.CreateModel( + name='Survey', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(max_length=36)), + ('experiment', models.CharField(max_length=11)), + ('target', models.CharField(default='', max_length=50)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('ra_p', models.FloatField(default=0.0)), + ('dec_p', models.FloatField(default=0.0)), + ('ra_z0', models.FloatField(default=0.0)), + ('dec_z0', models.FloatField(default=0.0)), + ('ra_zk', models.FloatField(default=0.0)), + ('dec_zk', models.FloatField(default=0.0)), + ('z_speed', models.FloatField(default=0.0, help_text='deg/day')), + ('row', models.IntegerField(default=0)), + ('head', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.Head')), + ], + ), + migrations.CreateModel( + name='SurveyPath', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dtime', models.DateTimeField(blank=True)), + ('obt', models.FloatField(default=0.0)), + ('eroday', models.FloatField(default=0.0)), + ('mjd', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('roll', models.FloatField(default=0.0)), + ('q1', models.FloatField(default=0.0)), + ('q2', models.FloatField(default=0.0)), + ('q3', models.FloatField(default=0.0)), + ('q4', models.FloatField(default=0.0)), + ('survey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.Survey')), + ], + ), + migrations.CreateModel( + name='Seance', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(max_length=36)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('stations', models.CharField(max_length=36)), + ('row', models.IntegerField(default=0)), + ('head', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.Head')), + ], + ), + migrations.CreateModel( + name='ScanPath', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dtime', models.DateTimeField(blank=True)), + ('obt', models.FloatField(default=0.0)), + ('eroday', models.FloatField(default=0.0)), + ('mjd', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('roll', models.FloatField(default=0.0)), + ('q1', models.FloatField(default=0.0)), + ('q2', models.FloatField(default=0.0)), + ('q3', models.FloatField(default=0.0)), + ('q4', models.FloatField(default=0.0)), + ('scan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.Scan')), + ], + ), + migrations.CreateModel( + name='Observation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(max_length=36)), + ('experiment', models.CharField(max_length=11)), + ('target', models.CharField(default='', max_length=50)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('roll_angle', models.FloatField(default=0.0)), + ('sun_xoz_angle', models.FloatField(default=0.0)), + ('row', models.IntegerField(default=0)), + ('ra_obj', models.FloatField(default=0.0)), + ('dec_obj', models.FloatField(default=0.0)), + ('name_obj', models.CharField(default='', max_length=30)), + ('head', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.Head')), + ], + ), + migrations.CreateModel( + name='MonthPlanUpload', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(blank=True, max_length=128)), + ('filename', models.CharField(max_length=256)), + ('filefield', models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/srv/srg-plan/srg/Uploads/monthplan'), upload_to=monthplan.models.upload_filename, validators=[monthplan.models.validate_upload_file])), + ('uploaded', models.DateTimeField(auto_now_add=True)), + ('head', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='upload', to='monthplan.Head', verbose_name='month plan')), + ('owner', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='MonthPlanOwner', to='plan.UserProfile', verbose_name='Author')), + ], + ), + migrations.CreateModel( + name='DataDump', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=36)), + ('start', models.DateTimeField(blank=True, null=True)), + ('stop', models.DateTimeField(blank=True, null=True)), + ('tstart', models.BigIntegerField(default=0)), + ('tstop', models.BigIntegerField(default=0)), + ('path', models.CharField(max_length=120)), + ('filename', models.CharField(max_length=50)), + ('mjd_start', models.FloatField(default=0.0)), + ('mjd_stop', models.FloatField(default=0.0)), + ('observations', models.ManyToManyField(to='monthplan.Observation')), + ('scans', models.ManyToManyField(to='monthplan.Scan')), + ], + ), + migrations.CreateModel( + name='Correction', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(max_length=36)), + ('start', models.DateTimeField(blank=True)), + ('stop', models.DateTimeField(blank=True)), + ('impstart', models.DateTimeField(blank=True)), + ('row', models.IntegerField(default=0)), + ('head', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.Head')), + ], + ), + ] diff --git a/monthplan/migrations/0002_auto_20191209_2233.py b/monthplan/migrations/0002_auto_20191209_2233.py new file mode 100644 index 0000000..b3ee3df --- /dev/null +++ b/monthplan/migrations/0002_auto_20191209_2233.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.6 on 2019-12-09 22:33 + +import django.core.files.storage +from django.db import migrations, models +import monthplan.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='monthplanupload', + name='filefield', + field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/export/django/srg/Uploads/monthplan'), upload_to=monthplan.models.upload_filename, validators=[monthplan.models.validate_upload_file]), + ), + migrations.CreateModel( + name='SurveyHealpixPlate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('survey', models.ManyToManyField(to='monthplan.Survey')), + ], + ), + ] diff --git a/monthplan/migrations/0003_auto_20200210_1503.py b/monthplan/migrations/0003_auto_20200210_1503.py new file mode 100644 index 0000000..2cbe5b8 --- /dev/null +++ b/monthplan/migrations/0003_auto_20200210_1503.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.6 on 2020-02-10 12:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0002_auto_20191209_2233'), + ] + + operations = [ + migrations.AlterModelOptions( + name='surveypath', + options={'ordering': ['obt']}, + ), + ] diff --git a/monthplan/migrations/0004_survey_loaded.py b/monthplan/migrations/0004_survey_loaded.py new file mode 100644 index 0000000..f3f2a3b --- /dev/null +++ b/monthplan/migrations/0004_survey_loaded.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-03-06 07:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0003_auto_20200210_1503'), + ] + + operations = [ + migrations.AddField( + model_name='survey', + name='loaded', + field=models.BooleanField(default=False), + ), + ] diff --git a/monthplan/migrations/0005_auto_20200306_1022.py b/monthplan/migrations/0005_auto_20200306_1022.py new file mode 100644 index 0000000..0dba9f3 --- /dev/null +++ b/monthplan/migrations/0005_auto_20200306_1022.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.6 on 2020-03-06 07:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0004_survey_loaded'), + ] + + operations = [ + migrations.AddField( + model_name='survey', + name='eroday_start', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='survey', + name='eroday_stop', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='survey', + name='mjd_start', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='survey', + name='mjd_stop', + field=models.FloatField(default=0.0), + ), + ] diff --git a/monthplan/migrations/0006_surveyhealpixsubplate.py b/monthplan/migrations/0006_surveyhealpixsubplate.py new file mode 100644 index 0000000..9fe4b6a --- /dev/null +++ b/monthplan/migrations/0006_surveyhealpixsubplate.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.6 on 2020-03-21 17:57 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0005_auto_20200306_1022'), + ] + + operations = [ + migrations.CreateModel( + name='SurveyHealpixSubPlate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('plate', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='monthplan.SurveyHealpixPlate')), + ], + ), + ] diff --git a/monthplan/migrations/0007_auto_20211229_1259.py b/monthplan/migrations/0007_auto_20211229_1259.py new file mode 100644 index 0000000..dcd3a20 --- /dev/null +++ b/monthplan/migrations/0007_auto_20211229_1259.py @@ -0,0 +1,78 @@ +# Generated by Django 3.2.10 on 2021-12-29 09:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0006_surveyhealpixsubplate'), + ] + + operations = [ + migrations.AlterField( + model_name='correction', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='datadump', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='flightplan', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='head', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='monthplanupload', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='observation', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='scan', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='scanpath', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='seance', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='survey', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='surveyhealpixplate', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='surveyhealpixsubplate', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='surveypath', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/monthplan/migrations/__init__.py b/monthplan/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monthplan/models.py b/monthplan/models.py new file mode 100644 index 0000000..9ea1baa --- /dev/null +++ b/monthplan/models.py @@ -0,0 +1,524 @@ +from django.db import models +from django.core.files.storage import FileSystemStorage +from datetime import date, datetime +from datetime import timedelta +from django.utils.timezone import localdate, localtime + +from logbook.models import INPUT_DATA_DIR +from plan.models import UserProfile + +from django import forms + +import hashlib +import os, os.path as osp +from django.utils.translation import gettext_lazy +import time + +# Create your models here. + +# [head] +# // SRG Month plan for Month Year +# // All times are given in UTC+3 +# GUID = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# PROJECT = RG +# PLANNING_TERM = month +# START = dd.mm.yyyy +# STOP = dd.mm.yyyy +# VERSION = vv +# GENTIME = dd.mm.yyyy hh:mm:ss +# AUTHOR = Name, E-mail + + + +MONTH_PLAN_UPLOAD_DIR = '/export/django/srg/Uploads/monthplan' + +FS = FileSystemStorage(location=MONTH_PLAN_UPLOAD_DIR) + +def upload_filename(instance, filename): + _, ext = osp.splitext(filename) + + tohash = filename + str(time.clock()) + new_fname = hashlib.md5(tohash.encode('utf-8')).hexdigest() + return new_fname + ext + +class Head(models.Model): + """ Main class for month plan """ + guid = models.CharField(max_length=36,blank=True, default='') + """ GUID (mostly not used) """ + start = models.DateTimeField(blank=True) + """ Start DateTime Field """ + stop = models.DateTimeField(blank=True) + """ Stop DateTime Field """ + title = models.CharField(max_length=128,default='', blank=True) + """ Title """ + version = models.IntegerField(default=0) + """ Version """ + gentime = models.DateTimeField(blank=True) + """ Generation time """ + author = models.CharField(max_length=80) + """ Author """ + #file = models.FileField(storage=FS, blank=True, max_length=100) + nrows = models.IntegerField(default=0) + archived = models.BooleanField(default=False) + """ Archived or not """ + def __str__(self): + dt = self.get_datetime() + return 'MonthPlan: ' + "%d-%02d" % (dt.year, dt.month) + def get_absolute_url(self): + return "/monthplan/%i/table" % (self.pk) + def get_absolute_url_seance(self): + return "/monthplan/%i/seance" % (self.pk) + def get_absolute_url_share(self): + return "/monthplan/%i/share" % (self.pk) + def get_absolute_url_observation(self): + return "/monthplan/%i/observation" % (self.pk) + def get_absolute_url_aladin(self): + return "/monthplan/%i/aladin" % (self.pk) + def get_absolute_url_scan(self): + return "/monthplan/%i/scan" % (self.pk) + def get_absolute_url_survey(self): + return "/monthplan/%i/survey" % (self.pk) + def get_absolute_url_correction(self): + return "/monthplan/%i/correction" % (self.pk) + def get_absolute_url_delete(self): + return "/monthplan/%i/delete" % (self.pk) + @property + def get_datetime_middle(self): + return self.start+timedelta(days=10) + def get_datetime(self): + return self.start+timedelta(days=10) + +class FlightPlan(models.Model): + """ Fligh plan or Peletnoe Zadanie """ + experiment = models.CharField(max_length=11) + """ ObsID """ + ra = models.FloatField(default=0.0) + """ RA """ + dec = models.FloatField(default=0.0) + """ Dec """ + start = models.DateTimeField(blank=True) + """ Start """ + stop = models.DateTimeField(blank=True) + """ Stop """ + q1 = models.FloatField(default=0.0) + """ Quat 1 """ + q2 = models.FloatField(default=0.0) + """ Quat 2 """ + q3 = models.FloatField(default=0.0) + """ Quat 3 """ + q4 = models.FloatField(default=0.0) + """ Quat 4 """ + start_npol = models.CharField(max_length=25) + """ Original start time from NPOL """ + stop_npol = models.CharField(max_length=25) + """ Original stop time from NPOL """ + load_stamp = models.DateTimeField(blank=True) + """ When uploaded to SRG """ + mjd_start = models.FloatField(default=0.0) + """ MJD start """ + mjd_stop = models.FloatField(default=0.0) + """ MJD stop """ + obt_start = models.FloatField(default=0.0) + """ On-board time start """ + obt_stop = models.FloatField(default=0.0) + """ On-board time start """ + valid = models.BooleanField(default=False) + """ Valid or not """ + def __str__(self): + return "%s, %s (UTC), %s" % (self.experiment, self.load_stamp, self.valid) + @property + def is_past_due(self): + return localtime() > self.stop + @property + def is_future(self): + return localtime() < self.start + @property + def is_current(self): + return (localtime() > self.start and localtime() < self.stop) + def get_absolute_url(self): + return "/monthplan/%i/getpz" % (self.pk) + +VALID_EXTENSIONS = ['.pdf', '.ppt', '.pptx', '.png', '.txt', '.dat', '.xlsx','.docx','.fits','.gz','.jpg','.jpeg',] + +def validate_upload_file(value): + """ Validate uploaded file for extention """ + file = value.file + + _, ext = osp.splitext(file.name) + if ext not in VALID_EXTENSIONS: + raise forms.ValidationError( + 'Extension %s not supported.' % ext + ) + +class MonthPlanUpload(models.Model): + """ Class for uploaded month-plan file """ + owner = models.ForeignKey(UserProfile, + on_delete=models.CASCADE, + default=None, + unique=False, + verbose_name="Author", + related_name="MonthPlanOwner") + #head = models.ForeignKey(Head, unique=True, verbose_name="month plan", on_delete=models.CASCADE) + head = models.OneToOneField(Head, + verbose_name="month plan", + on_delete=models.CASCADE, + null=True, blank=True, + related_name='upload') + """ Related month plan :class:`Head` class """ + title = models.CharField(max_length=128,blank=True) + """ Title of the file (not actually used) """ + filename = models.CharField(max_length=256) + """ Filename """ + filefield = models.FileField(upload_to=upload_filename, storage=FS, validators=[validate_upload_file]) + """ Django FileField class """ + uploaded = models.DateTimeField(auto_now_add=True) + """ When uploaded """ + def get_filename(self): + return self.filename + def get_absolute_url(self): + return "/uploads/monthplan/%s" % (self.filefield) + def get_delete_url(self): + return self.head.get_absolute_url() + "/upload/%d/delete" % (self.id) + def delete(self, remove_file=True): + if remove_file: + FS.delete(str(self.filefield)) + return models.Model.delete(self) + def __str__(self): + return 'MonthPlan '+self.title + +class MonthPlanUploadForm(forms.ModelForm): + """ Form for file upload """ + class Meta: + model = MonthPlanUpload + fields = ('filefield',) + +# widgets = { +# 'title': forms.TextInput(attrs={'style': 'width: 620px;', 'class': 'title'}), +# } + +# [observation] +# GUID = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# EXPERIMENT = AAAAAAAAA +# RA = XXX.XXXXXXXX // J2000 +# DEC = zYY.YYYYYYYY // J2000 +# START = dd.mm.yyyy hh:mm:ss +# STOP = dd.mm.yyyy hh:mm:ss +# ROLL_ANGLE = zXXX.X // Reference value +# SUN_XOZ_ANGLE = zXX.X + +class Observation(models.Model): + """Observation class + + [observation] + GUID = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + EXPERIMENT = AAAAAAAAA + RA = XXX.XXXXXXXX // J2000 + DEC = zYY.YYYYYYYY // J2000 + START = dd.mm.yyyy hh:mm:ss + STOP = dd.mm.yyyy hh:mm:ss + ROLL_ANGLE = zXXX.X // Reference value + SUN_XOZ_ANGLE = zXX.X + """ + head = models.ForeignKey(Head, on_delete=models.CASCADE) + """ Related month plan :class:`Head` class """ + guid = models.CharField(max_length=36) + """ GUID (not actually used) """ + experiment = models.CharField(max_length=11) + """ ObsID """ + target = models.CharField(max_length=50, default='') + """ Target """ + ra = models.FloatField(default=0.0) + """ RA """ + dec = models.FloatField(default=0.0) + """ Dec """ + start = models.DateTimeField(blank=True) + """ Start """ + stop = models.DateTimeField(blank=True) + """ Stop """ + roll_angle = models.FloatField(default=0.0) + """ Roll angle """ + sun_xoz_angle = models.FloatField(default=0.0) + """ SUN XOZ ANGLE """ + row = models.IntegerField(default=0) + """ Row """ + ra_obj = models.FloatField(default=0.0) + """ RA of the Object """ + dec_obj = models.FloatField(default=0.0) + """ Dec of the Object """ + name_obj = models.CharField(max_length=30, default='') + """ Name of the Object """ + def get_absolute_url(self): + return "/monthplan/%i/getobs" % (self.pk) + def get_absolute_url_logbook_start(self): + t = localtime(self.start) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + def get_absolute_url_logbook_stop(self): + t = localtime(self.stop) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + @property + def is_past_due(self): + return localtime() > self.stop + @property + def is_future(self): + return localtime() < self.start + @property + def is_current(self): + return (localtime() > self.start and localtime() < self.stop) + def __str__(self): + return self.experiment+': '+self.target + + +# //SRG Scan Template Request +# //Created by Pavel Gureev at 10-07-2018 +# SCAN_AXIS=Z // Z or Y +# SPAN=+5.0 // deg +# STEP=+0.1 // deg +# N_PASS=26 // 1 pass is 2*SPAN+2*STEP +# SPEED=0.0017 // deg/s + +#class Template(models.Model): +# AXIS_CHOICES = ( +# ('Z', 'Z axis'), +# ('Y', 'Y axis'), +# ) +# scan_axis = models.CharField(max_length=1, choices=AXIS_CHOICES, help_text='Z or Y') +# span = models.FloatField(default=0.0, help_text='deg') +# step = models.FloatField(default=0.0, help_text='deg') +# n_pass = models.IntegerField(default=0, help_text='1 pass is 2*SPAN+2*STEP') +# speed = models.FloatField(default=0.0, help_text='deg/s') +# ref = models.IntegerField(default=0, help_text='reference number') + +# [scan] +# // Work #1 for dd.mm.yyyy - Scan +# // Scan Template NNN based on Template Request MMM +# // Template duration is HH hours +# // SCAN_AXIS=Z, SPAN=zX.Xdeg, STEP=zY.YYdeg, N_PASS=PP, SPEED=X.XXXXdeg/s +# GUID = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# EXPERIMENT = AAAAAAAAA +# RA = XXX.XXXXXXXX // J2000 +# DEC = zYY.YYYYYYYY // J2000 +# START = dd.mm.yyyy hh:mm:ss +# STOP = dd.mm.yyyy hh:mm:ss +# ROLL_ANGLE = zXXX.X // Reference value +# SUN_XOZ_ANGLE = zXX.X +# TEMPLATE = XXX + +class Scan(models.Model): + head = models.ForeignKey(Head, on_delete=models.CASCADE) + guid = models.CharField(max_length=36) + experiment = models.CharField(max_length=11) + target = models.CharField(max_length=50, default='') + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + start = models.DateTimeField(blank=True) + stop = models.DateTimeField(blank=True) + roll_angle = models.FloatField(default=0.0) + sun_xoz_angle = models.FloatField(default=0.0) + #template = models.ForeignKey(Template, on_delete=models.CASCADE) + template = models.IntegerField(default=0) + row = models.IntegerField(default=0) + ra_obj = models.FloatField(default=0.0) + dec_obj = models.FloatField(default=0.0) + name_obj = models.CharField(max_length=30, default='') + def get_absolute_url(self): + return "/monthplan/%i/getscan" % (self.pk) + def get_download_csv_url(self): + return "/monthplan/%i/getscan_csv" % (self.pk) + def get_absolute_url_logbook_start(self): + t = localtime(self.start) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + def get_absolute_url_logbook_stop(self): + t = localtime(self.stop) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + @property + def is_past_due(self): + return localtime() > self.stop + @property + def is_future(self): + return localtime() < self.start + @property + def is_current(self): + return (localtime() > self.start and localtime() < self.stop) + def __str__(self): + return self.experiment+': '+self.target + +class ScanPath(models.Model): + scan = models.ForeignKey(Scan, on_delete=models.CASCADE) + dtime = models.DateTimeField(blank=True) + obt = models.FloatField(default=0.0) + eroday = models.FloatField(default=0.0) + mjd = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + roll = models.FloatField(default=0.0) + q1 = models.FloatField(default=0.0) + q2 = models.FloatField(default=0.0) + q3 = models.FloatField(default=0.0) + q4 = models.FloatField(default=0.0) + def __str__(self): + return "%s %.6f" % (self.scan.experiment, self.eroday) + +# [survey] +# // Survey for January-April 2018 +# GUID = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# EXPERIMENT = AAAAAAAAA +# START = dd.mm.yyyy hh:mm:ss +# STOP = dd.mm.yyyy hh:mm:ss // Reference value +# RA_P = XXX.XXXXX +# DEC_P = zXX.XXXXX +# RA_Z0 = XXX.XXXXX +# DEC_Z0 = +XX.XXXXX +# RA_ZK = XXX.XXXXX // Reference value +# DEC_ZK = zXX.XXXXX // Reference value +# Z_SPEED = X.XXXX // deg/day + +class Survey(models.Model): + head = models.ForeignKey(Head, on_delete=models.CASCADE) + guid = models.CharField(max_length=36) + experiment = models.CharField(max_length=11) + target = models.CharField(max_length=50, default='') + start = models.DateTimeField(blank=True) + stop = models.DateTimeField(blank=True) + eroday_start = models.FloatField(default=0.0) + eroday_stop = models.FloatField(default=0.0) + mjd_start = models.FloatField(default=0.0) + mjd_stop = models.FloatField(default=0.0) + ra_p = models.FloatField(default=0.0) + dec_p = models.FloatField(default=0.0) + ra_z0 = models.FloatField(default=0.0) + dec_z0 = models.FloatField(default=0.0) + ra_zk = models.FloatField(default=0.0) + dec_zk = models.FloatField(default=0.0) + z_speed = models.FloatField(default=0.0, help_text='deg/day') + row = models.IntegerField(default=0) + loaded = models.BooleanField(default=False) + def get_path_filtered(self): + return self.surveypath_set.all() + def get_absolute_url(self): + t = localtime(self.start) + return "/monthplan/%i/getsurvey" % (self.pk) + def get_absolute_url_logbook_start(self): + t = localtime(self.start) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + def get_absolute_url_logbook_stop(self): + t = localtime(self.stop) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + @property + def is_past_due(self): + return localtime() > self.stop + @property + def is_future(self): + return localtime() < self.start + @property + def is_current(self): + return (localtime() > self.start and localtime() < self.stop) + def __str__(self): + return self.experiment + +class SurveyPath(models.Model): + survey = models.ForeignKey(Survey, on_delete=models.CASCADE) + dtime = models.DateTimeField(blank=True) + obt = models.FloatField(default=0.0) + eroday = models.FloatField(default=0.0) + mjd = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + roll = models.FloatField(default=0.0) + q1 = models.FloatField(default=0.0) + q2 = models.FloatField(default=0.0) + q3 = models.FloatField(default=0.0) + q4 = models.FloatField(default=0.0) + class Meta: + ordering = ['obt'] + def __str__(self): + return "%s %.6f" % (self.survey.experiment, self.eroday) + +NSIDE_PLATES=32 +NSIDE_SUBPLATES=256 +ORDER='nested' +MAPS_DIR='/data/srg/eRosita/plates/' + +class SurveyHealpixPlate(models.Model): + healpix = models.BigIntegerField(db_index=True, default=0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + survey = models.ManyToManyField('Survey') + def __str__(self): + return "HealpixPlate %s" % (self.healpix) + +class SurveyHealpixSubPlate(models.Model): + plate = models.ForeignKey(SurveyHealpixPlate, on_delete=models.CASCADE) + healpix = models.BigIntegerField(db_index=True, default=0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + def __str__(self): + return "HealpixSubPlate %s" % (self.healpix) + +# [seance] +# // Ground contact for dd.mm.yyyy +# GUID = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# START = dd.mm.yyyy hh:mm:ss +# STOP = dd.mm.yyyy hh:mm:ss +# STATIONS = station1, station2 + +class Seance(models.Model): + head = models.ForeignKey(Head, on_delete=models.CASCADE) + guid = models.CharField(max_length=36) + start = models.DateTimeField(blank=True) + stop = models.DateTimeField(blank=True) + stations = models.CharField(max_length=36) + row = models.IntegerField(default=0) + def get_absolute_url_logbook_start(self): + t = localtime(self.start) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + def get_absolute_url_logbook_stop(self): + t = localtime(self.stop) + return "/logbook/date/%i/%i/%i" % (t.year, t.month, t.day) + @property + def is_past_due(self): + return localtime() > self.stop + @property + def is_future(self): + return localtime() < self.start + @property + def is_current(self): + return (localtime() > self.start and localtime() < self.stop) + +# [correction] +# // Orbital correction for dd.mm.yyyy +# GUID = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# START = dd.mm.yyyy hh:mm:ss +# STOP = dd.mm.yyyy hh:mm:ss +# IMPSTART = dd.mm.yyyy hh:mm:ss + +class Correction(models.Model): + head = models.ForeignKey(Head, on_delete=models.CASCADE) + guid = models.CharField(max_length=36) + start = models.DateTimeField(blank=True) + stop = models.DateTimeField(blank=True) + impstart = models.DateTimeField(blank=True) + row = models.IntegerField(default=0) + @property + def is_past_due(self): + return date.today() > date(self.stop.year, self.stop.month, self.stop.day) + @property + def is_future(self): + return date.today() < date(self.start.year, self.start.month, self.start.day) + @property + def is_current(self): + return (date.today() > date(self.start.year, self.start.month, self.start.day) and date.today() < date(self.stop.year, self.stop.month, self.stop.day)) + +class DataDump(models.Model): + name = models.CharField(max_length=36) + start = models.DateTimeField(blank=True,null=True) + stop = models.DateTimeField(blank=True,null=True) + tstart = models.BigIntegerField(default=0) + tstop = models.BigIntegerField(default=0) + path = models.CharField(max_length=120) + filename = models.CharField(max_length=50) + mjd_start = models.FloatField(default=0.0) + mjd_stop = models.FloatField(default=0.0) + observations = models.ManyToManyField('Observation') + scans = models.ManyToManyField('Scan') + def __str__(self): + return self.name diff --git a/monthplan/templatetags/__init__.py b/monthplan/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monthplan/templatetags/monthplan_templates.py b/monthplan/templatetags/monthplan_templates.py new file mode 100644 index 0000000..6de3aa7 --- /dev/null +++ b/monthplan/templatetags/monthplan_templates.py @@ -0,0 +1,29 @@ +from django import template + +register = template.Library() + +@register.filter(name='has_group') +def has_group(user, group_name): + return user.groups.filter(name=group_name).exists() + +@register.filter +def next(some_list, current_index): + """ + Returns the next element of the list using the current index if it exists. + Otherwise returns an empty string. + """ + try: + return some_list[int(current_index) + 1] # access the next element + except: + return '' # return empty string in case of exception + +@register.filter +def previous(some_list, current_index): + """ + Returns the previous element of the list using the current index if it exists. + Otherwise returns an empty string. + """ + try: + return some_list[int(current_index) - 1] # access the previous element + except: + return '' # return empty string in case of exception diff --git a/monthplan/tests.py b/monthplan/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/monthplan/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/monthplan/urls.py b/monthplan/urls.py new file mode 100644 index 0000000..b514f25 --- /dev/null +++ b/monthplan/urls.py @@ -0,0 +1,44 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.index, name='monthplan index'), + path('scans', views.show_scans, name="list of scans"), + path('surveys', views.show_surveys, name="list of surveys"), + path('observations', views.show_observations, name="list of observations"), + path('time', views.time, name="AJAX time"), + path('wherenow', views.wherenow, name="AJAX currently observing"), + path('admin', views.index_admin, name='index_admin'), + path('load', views.load_monthplan, name='load plan'), + path('flightplan', views.index_flightplan, name='index_pz'), + path('tracking', views.show_tracking, name='tracking'), + path('flightplan/valid', views.index_flightplan_valid, name='index_pz'), + path('flightplan/invalid', views.index_flightplan_invalid, name='index_pz'), + path('', views.show_monthplan, name='show monthplan'), + path('/table', views.show_monthplan_table, name='show monthplan'), + path('/aladin', views.show_monthplan_aladin, name='show aladin'), + path('/seance', views.show_monthplan_seance, name='show monthplan seance'), + path('/share', views.share_monthplan, name='share monthplan'), + path('/observation', views.show_monthplan_observation, name='show monthplan seance'), + path('/getobs', views.show_observation_info, name='show observation info'), + path('/getpz', views.show_flightplan_info, name='show flight info'), + path('/getscan', views.show_scan_info, name='show scan info'), + path('/getscan_csv', views.scan_download_csv, name='download scan csv'), + path('/getsurvey', views.show_survey_info, name='show survey info'), + path('/scan', views.show_monthplan_scan, name='show monthplan seance'), + path('/survey', views.show_monthplan_survey, name='show monthplan survey'), + path('/correction', views.show_monthplan_correction, name='show monthplan seance'), + path('/delete', views.delete_monthplan, name='delete monthplan'), + path('upload', views.upload, name='upload month plan'), + path('obsid', views.get_obsid_search, name='get_obsid_search'), + #path('day//update', views.update, name='update'), + #path('day//upload//delete', views.delete_upload, name='delete'), + #path('day/', views.show_day, name='show'), + #path('login', views.login_username, name='login'), + #path('logout', views.logout_view, name='logout'), + #path('uploads', views.show_uploads, name='uploads'), + #path('info', views.show_info, name='info'), + #path('category//show', views.show_category, name='showCategory'), +] + diff --git a/monthplan/utils.py b/monthplan/utils.py new file mode 100644 index 0000000..2d3f3cc --- /dev/null +++ b/monthplan/utils.py @@ -0,0 +1,275 @@ +import numpy as np +from math import pi, cos, sin +from Quaternion import Quat +from dateutil.parser import parse +from datetime import timedelta +import glob +import re +import fnmatch +import os +import logging + +from scipy.spatial.transform import Rotation, Slerp +import pandas as pd +from pandas import ExcelWriter +from pandas import ExcelFile + +from astropy_healpix import HEALPix +from astropy import units as u +from astropy.coordinates import SkyCoord, RangeError +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy.coordinates import Angle +import astropy.units as u +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch +from astropy.io import fits +from astropy.table import Table + +from django.contrib.auth.models import User +from django.core import mail +from django.template.loader import render_to_string +from django.utils.html import strip_tags +from django.core.mail import send_mail + +from monthplan.models import Survey, SurveyPath, NSIDE_PLATES, ORDER, SurveyHealpixPlate +from monthplan.models import Head, Observation, Seance, Correction, Scan + +from srglib.utils import date2mission, slerp, str2date, TZ_MSK + + +#MJDREF = 51543.875 +#TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +#TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +def logic(index,nskip): + if index % nskip == 0: + return False + return True + +def load_surveypath_data(file, nskip=100, forced=False, notify=False): + match = '20??????_??????_???????????.iki' + filename=os.path.basename(file) + if fnmatch.fnmatch(filename, match): + logging.debug("Match filename: Read {}".format(file)) + else: + logging.debug("Match filename: Skip {}".format(file)) + + obsid = re.findall("\d+", file)[2] + + if(obsid == '11000400200'): + """ + Merge two files for 11000400200: + + cat + /mnt/nfs/npol/orientation/DMV/ARJ-SCAN/20200218_220000_11000300500.iki + /mnt/nfs/npol/orientation/DMV/ARJ-SCAN/20200219_180000_11000400200.iki + > /srv/srg-plan/srg/data/ARJ-SCAN/20200219_180000_11000400200.actual.iki + """ + file='/export/django/srg/data/npol/ARJ-SCAN/20200219_180000_11000400200.actual.iki' + logging.debug("Special case for 11000400200, load correct file {}".format(file)) + + try: + survey=Survey.objects.get(experiment__exact=obsid) + except: + logging.debug("Survey {} is not found".format(obsid)) + return + + """ Update eroday start/stop """ + survey.eroday_start=date2mission(survey.start)/14400 + survey.eroday_stop=date2mission(survey.stop)/14400 + survey.mjd_start = Time(survey.start , format='datetime').mjd + survey.mjd_stop = Time(survey.stop , format='datetime').mjd + survey.save() + + if(forced == True): + survey.loaded=False + survey.save() + + if(survey.loaded == True): + logging.debug("Survey {} is already loaded, skip.".format(obsid)) + return + + logging.info("Load {}, for Survey {}".format(file,obsid)) + + spath=survey.surveypath_set.all() + spath.delete() + + csvfile=file + df = pd.read_csv(csvfile,names=['date', 'time', 'q1', 'q2','q3','q4','dummy'], header=None, delim_whitespace=True, skipfooter=1, engine='python', skiprows = lambda x: logic(x, nskip)) + + for i in df.index: + dtstr="%s %s" % (df['date'][i], df['time'][i]) + dt = str2date(dtstr, tzone=TZ_MSK) + ts = date2mission(dt) + eroday=ts/14400 + dtime = Time(dt , format='datetime') + mjd=dtime.mjd + q1=df['q1'][i] + q2=df['q2'][i] + q3=df['q3'][i] + q4=df['q4'][i] + quat=Quat(attitude=[q2,q3,q4,q1]) + sp=SurveyPath(survey=survey,dtime=dt,mjd=mjd,obt=ts,eroday=eroday,q1=q1,q2=q2,q3=q3,q4=q4,ra=quat.ra,dec=quat.dec,roll=quat.roll) + sp.save() + + survey.loaded=True + """ Mark survey as loaded """ + survey.save() + + + tolerance=10 + hp = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5()) + surveypaths = survey.surveypath_set.all() + for i in range(len(surveypaths)-1): + delta=(surveypaths[i+1].dtime-surveypaths[i].dtime).total_seconds() + if(abs(int(delta)) > tolerance): + nstep = int(abs(delta) / tolerance) + delta_arr = np.linspace(0,1,nstep) + + quat = slerp([surveypaths[i].q2, surveypaths[i].q3, surveypaths[i].q4, surveypaths[i].q1], + [surveypaths[i+1].q2, surveypaths[i+1].q3, surveypaths[i+1].q4, surveypaths[i+1].q1], + delta_arr) + for q in quat: + qfin=Quat(attitude=q) + crd = SkyCoord(qfin.ra, qfin.dec, frame="fk5", unit="deg") + heal = hp.skycoord_to_healpix(crd) + try: + plate = SurveyHealpixPlate.objects.get(healpix=heal) + except: + logging.error('Error: SurveyHealpixPlate not found, run ./manage.py init_survey_healpix_plates first') + break + queryset = survey.surveyhealpixplate_set.all() + if not queryset.filter(pk=plate.pk).exists(): + plate.survey.add(survey) + plate.save() + + + if(notify == True): + users = User.objects.filter(groups__name='srg-iki-admin') + to_emails=[] + for user in users: + to_emails.append(user.email) + + subject = "SRG-ArXiv: New SRG flight plan {}".format(obsid) + html_message = render_to_string('monthplan/surveypath_loaded_email.html', {'survey':survey,}) + plain_message = strip_tags(html_message) + from_email = 'Roman Krivonos ' + + for email in to_emails: + mail.send_mail(subject, plain_message, from_email, [email], html_message=html_message) + + +def load_monthplan_fits(filename): + table = Table.read(filename) + df = table.to_pandas() + #print("Column headings:") + #print(df.columns.str.encode('utf-8')) + hdul = fits.open(filename) + #hdul.info() + version=int(hdul[1].header['VERSION']) + + """ delete existing month plan at this month """ + dt = parse(hdul[1].header['START']) + middle=dt+timedelta(days=10) + heads = Head.objects.all() + for h in heads: + dt = h.get_datetime() + if(dt.year == middle.year and dt.month == middle.month): + print("Found {} v{} (new v{})".format(h,h.version,version)) + if(h.version < version): + print("Delete {} v{} (new v{})".format(h,h.version,version)) + h.delete() + else: + print("Skip loading, already exists: {} v{}".format(h,h.version)) + return + print("Loading ",filename) + + head=Head() + head.start=hdul[1].header['START'] + head.stop=hdul[1].header['STOP'] + head.gentime=hdul[1].header['GENTIME'] + head.version=hdul[1].header['VERSION'] + head.author=hdul[1].header['AUTHOR'] + head.file=os.path.basename(filename) + head.nrows=df.shape[0] + head.save() + for i in df.index: + row=(i+1) + if(df['TYPE'][i].decode().strip() == 'SEANCE'): + """ print(df['START'][i].decode().strip()) """ + seance=Seance(head=head, row=row) + seance.start=df['START'][i].decode().strip() + seance.stop=df['STOP'][i].decode().strip() + seance.stations=df['STATIONS'][i].decode().strip() + seance.guid=df['GUID'][i].decode().strip() + seance.save() + if(df['TYPE'][i].decode().strip() == 'OBSERVATION'): + obs=Observation(head=head, row=row) + obs.start=df['START'][i].decode().strip() + obs.stop=df['STOP'][i].decode().strip() + obs.target=df['TARGET'][i].decode().strip() + obs.experiment=df['EXPERIMENT'][i].decode().strip() + obs.ra=df['RA'][i] + obs.dec=df['DEC'][i] + obs.ra=df['RA'][i] + obs.roll_angle=df['ROLL_ANGLE'][i] + obs.sun_xoz_angle=df['SUN_XOZ_ANGLE'][i] + obs.guid=df['GUID'][i].decode().strip() + obs.save() + if(df['TYPE'][i].decode().strip() == 'CORRECTION'): + cor=Correction(head=head, row=row) + cor.start=df['START'][i].decode().strip() + cor.stop=df['STOP'][i].decode().strip() + cor.impstart=df['IMPSTART'][i].decode().strip() + cor.guid=df['GUID'][i].decode().strip() + cor.save() + if(df['TYPE'][i].decode().strip() == 'SCAN'): + scan=Scan(head=head, row=row) + scan.start=df['START'][i].decode().strip() + scan.stop=df['STOP'][i].decode().strip() + scan.target=df['TARGET'][i].decode().strip() + scan.experiment=df['EXPERIMENT'][i].decode().strip() + scan.ra=df['RA'][i] + scan.dec=df['DEC'][i] + scan.roll_angle=df['ROLL_ANGLE'][i] + scan.sun_xoz_angle=df['SUN_XOZ_ANGLE'][i] + scan.template=int(df['TEMPLATE'][i].decode().strip()) + scan.guid=df['GUID'][i].decode().strip() + scan.save() + if(df['TYPE'][i].decode().strip() == 'SURVEY'): + sur=Survey(head=head, row=row) + sur.start=df['START'][i].decode().strip() + sur.stop=df['STOP'][i].decode().strip() + sur.target=df['TARGET'][i].decode().strip() + sur.experiment=df['EXPERIMENT'][i].decode().strip() + sur.ra_p=df['RA_P'][i] + sur.dec_p=df['DEC_P'][i] + sur.ra_z0=df['RA_Z0'][i] + sur.dec_z0=df['DEC_Z0'][i] + sur.ra_zk=df['RA_ZK'][i] + sur.dec_zk=df['DEC_ZK'][i] + sur.z_speed=df['Z_SPEED'][i] + sur.guid=df['GUID'][i].decode().strip() + sur.save() + + +def load_monthplan_dir(dir): + """ Loads monthplan from dir + + + Parameters + ---------- + dir : str + Absolute path to the input directory + + + :Author: + Roman Krivonos + """ + # /export/django/srg/data/npol/fits/RG_MonthPlan_2020-06_v07.fits + logging.info("Loading monthplan from {}".format(dir)) + files = glob.glob("{}/RG_MonthPlan_202?-??_v??.fits".format(dir)) + files.sort(key=os.path.getmtime) + for filename in files: + #print(filename) + load_monthplan_fits(filename) diff --git a/monthplan/views.py b/monthplan/views.py new file mode 100644 index 0000000..25efaff --- /dev/null +++ b/monthplan/views.py @@ -0,0 +1,553 @@ +from django.shortcuts import render +from monthplan.models import Head +from monthplan.models import Observation, Seance, Correction, Scan +from monthplan.models import Survey, MonthPlanUpload, MonthPlanUploadForm +from monthplan.models import FlightPlan, DataDump, ScanPath +from monthplan.forms import ObsIDSearchForm +from plan.models import LaunchDate +from datetime import datetime as dt +from django.utils import timezone + +from django.http import HttpResponse +from django.http import HttpResponseRedirect +from django.http import HttpResponseForbidden +import csv +from django.utils.encoding import smart_str + +from django.contrib.auth.models import User +from django.core.mail import send_mail + +import pandas as pd +from astropy.table import Table +from astropy.io import fits + +import math +from Quaternion import Quat +from django.http import JsonResponse + +from django.db.models import Q + +def find_closest(now, closest_greater_qs, closest_less_qs): + try: + try: + closest_greater = closest_greater_qs[0] + except IndexError: + closest = closest_less_qs[0] + + try: + closest_less = closest_less_qs[0] + except IndexError: + closest = closest_greater_qs[0] + except IndexError: + raise self.model.DoesNotExist("There is no closest object" + " because there are no objects.") + if closest_greater.dtime - now > now - closest_less.dtime: + closest = closest_less + else: + closest = closest_greater + return closest.ra, closest.dec + +# Create your views here. + +def time(request): + """ AJAX time procedure """ + now = timezone.localtime(timezone.now()) + + data = { + 'time': now.strftime('%d %b %Y %H:%M:%S MSK'), + } + return JsonResponse(data) + +def wherenow(request): + request_path = request.GET.get('request_path', None) + now = timezone.localtime(timezone.now()) # timezone.now() # dt.now() + target = 'None' + obsid = 'None' + url = '#' + ra = 'n/a' + dec = 'n/a' + mode='' + msg='' + path=list() + try: + #surveys=Survey.objects.filter(experiment__contains="10000100100") + surveys = Survey.objects.filter(Q(start__lte=now) & Q(stop__gte=now)) + # take first element only: + for survey in surveys: + target = survey.target + obsid = survey.experiment + url = survey.get_absolute_url() + ra="%.4f" % survey.ra_p + dec="%.4f" % survey.dec_p + mode='Survey' + """ + if request_path == "load": + path=list(survey.surveypath_set.all().values('ra', 'dec'))[::200] + """ + closest_greater_qs = survey.surveypath_set.filter(dtime__gt=now).order_by('dtime') + closest_less_qs = survey.surveypath_set.filter(dtime__lt=now).order_by('-dtime') + closest_ra, closest_dec = find_closest(now, closest_greater_qs, closest_less_qs) + ra="%.4f" % closest_ra + dec="%.4f" % closest_dec + break + except NameError as error: + pass + try: + scans = Scan.objects.filter(Q(start__lte=now) & Q(stop__gte=now)) + # take first element only + for scan in scans: + target = scan.target + obsid = scan.experiment + url = scan.get_absolute_url() + ra="%.4f" % scan.ra + dec="%.4f" % scan.dec + mode='Scan' + if request_path == "load": + path=list(scan.scanpath_set.all().values('ra', 'dec'))[::50] + closest_greater_qs = scan.scanpath_set.filter(dtime__gt=now).order_by('dtime') + closest_less_qs = scan.scanpath_set.filter(dtime__lt=now).order_by('-dtime') + try: + try: + closest_greater = closest_greater_qs[0] + except IndexError: + closest = closest_less_qs[0] + + try: + closest_less = closest_less_qs[0] + except IndexError: + closest = closest_greater_qs[0] + except IndexError: + raise self.model.DoesNotExist("There is no closest object" + " because there are no objects.") + if closest_greater.dtime - now > now - closest_less.dtime: + closest = closest_less + else: + closest = closest_greater + # update ra, dec + ra="%.4f" % closest.ra + dec="%.4f" % closest.dec + break + except NameError as error: + pass + try: + observations = Observation.objects.filter(Q(start__lte=now) & Q(stop__gte=now)) + # take first element only: + for obs in observations: + target = obs.target + obsid = obs.experiment + url = obs.get_absolute_url() + ra="%.4f" % obs.ra + dec="%.4f" % obs.dec + mode='Pointing' + break + except NameError as error: + pass + data = { + 'time': now.strftime('%d %b %Y %H:%M:%S MSK'), + 'target': target, + 'obsid': obsid, + 'url': url, + 'msg': msg, + 'ra': ra, 'dec': dec, + #'observations': list(observations.values()) , + #'scans': list(scans.values()) , + #'surveys': list(surveys.values()), + 'path':path, + 'mode':mode, + } + return JsonResponse(data) + +def index(request): + heads = Head.objects.all().order_by('-start') + return render(request,'monthplan/index.html', {'heads': heads,'today':dt.now(),}) + +def index_flightplan(request): + flightplans = FlightPlan.objects.all() + return render(request,'monthplan/index_flightplan.html', {'flightplans': flightplans,'today':dt.now(),}) + +def index_flightplan_valid(request): + flightplans = FlightPlan.objects.filter(valid__exact=True) + return render(request,'monthplan/index_flightplan.html', {'flightplans': flightplans,'today':dt.now(),}) + +def index_flightplan_invalid(request): + flightplans = FlightPlan.objects.filter(valid__exact=False) + return render(request,'monthplan/index_flightplan.html', {'flightplans': flightplans,}) + +def index_admin(request): + heads = Head.objects.all() + return render(request,'monthplan/index.html', {'heads': heads,'admin':True,}) + +def show_scans(request): + scans = Scan.objects.all().order_by('start') + return render(request,'monthplan/show_targets.html', {'scans':scans,}) + +def show_surveys(request): + surveys = Survey.objects.all().order_by('start') + return render(request,'monthplan/show_targets.html', {'surveys':surveys,}) + +def show_observations(request): + observations = Observation.objects.all().order_by('start') + return render(request,'monthplan/show_targets.html', {'observations': observations,}) + +def load_monthplan(request): + heads = Head.objects.all() + return render(request,'monthplan/index.html', {'heads': heads,'admin':True,}) + + +def show_monthplan(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponse("This month plan was not found") + seances = head.seance_set.all() + observations = head.observation_set.all() + corrections = head.correction_set.all() + scans = head.scan_set.all() + surveys = head.survey_set.all() + seq = range(1,head.nrows+1) + return render(request,'monthplan/show_monthplan.html', {'head': head, + 'seances':seances, + 'observations':observations, + 'corrections':corrections, + 'scans':scans, + 'surveys':surveys, + 'seq':seq, + 'today':dt.now(),}) + +def share_monthplan(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponse("This month plan was not found") + seances = head.seance_set.all() + observations = head.observation_set.all() + corrections = head.correction_set.all() + scans = head.scan_set.all() + surveys = head.survey_set.all() + seq = range(1,head.nrows+1) + + emails = [] + users = User.objects.all() + for user in users: + emails.append(user.email) + + mid=head.get_datetime() + email_subject = "SRG-IKI month plan for %s" % (mid.strftime("%B %Y")) + email_body = u"The following SRG month plan for %s has been added/updated: http://193.232.10.38/%s\n\nThis SRG-IKI service email is sent to all registered users on srg.cosmos.ru. This is an automatic email, please do not reply. If you do not want to receive this kind of notifications, please drop a message to Roman Krivonos (krivonos@cosmos.ru)." % (mid.strftime("%B %Y"), head.get_absolute_url()) + for email in emails: + send_mail(email_subject,email_body,'krivonos@cosmos.ru',[email]) + + return render(request,'monthplan/show_monthplan.html', {'head': head, + 'seances':seances, + 'observations':observations, + 'corrections':corrections, + 'scans':scans, + 'surveys':surveys, + 'seq':seq, + 'today':dt.now(),}) + +def show_monthplan_table(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponse("This month plan was not found") + seances = head.seance_set.all() + observations = head.observation_set.all() + corrections = head.correction_set.all() + scans = head.scan_set.all() + surveys = head.survey_set.all() + seq = range(1,head.nrows+1) + return render(request,'monthplan/show_monthplan_table.html', {'head': head, + 'seances':seances, + 'observations':observations, + 'corrections':corrections, + 'scans':scans, + 'surveys':surveys, + 'seq':seq, + 'today':dt.now(),}) + +def show_monthplan_aladin(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponse("This month plan was not found") + seances = head.seance_set.all() + observations = head.observation_set.all() + corrections = head.correction_set.all() + scans = head.scan_set.all() + surveys = head.survey_set.all() + surveypaths = [] + for survey in surveys: + surveypaths.extend(list(survey.surveypath_set.all().order_by('obt'))) + seq = range(1,head.nrows+1) + return render(request,'monthplan/show_monthplan_aladin.html', {'surveypaths':surveypaths, 'head': head, + 'seances':seances, + 'observations':observations, + 'corrections':corrections, + 'scans':scans, + 'surveys':surveys, + 'seq':seq, + 'today':dt.now(),}) + + + +def delete_monthplan(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponse("This month plan was not found") + #head.upload.delete() + head.delete() + heads = Head.objects.all() + return render(request,'monthplan/index.html', {'heads': heads,'today':dt.now(),'admin':True,}) + +def show_observation_info(request, observation_id): + try: + observation=Observation.objects.get(pk=observation_id) + except Observation.DoesNotExist: + return HttpResponseRedirect('/monthplan') + flightplans=FlightPlan.objects.filter(experiment__contains=observation.experiment) + datadumps=observation.datadump_set.all() + return render(request,'monthplan/show_observation_info.html', {'head': observation.head, 'observation': observation,'flightplans':flightplans,'datadumps':datadumps,'today':dt.now(),}) + +def show_flightplan_info(request, flightplan_id): + try: + pz=FlightPlan.objects.get(pk=flightplan_id) + except FlightPlan.DoesNotExist: + return HttpResponseRedirect('/monthplan') + observations=Observation.objects.filter(experiment__exact=pz.experiment) + scans=Scan.objects.filter(experiment__exact=pz.experiment) + surveys=Survey.objects.filter(experiment__exact=pz.experiment) + return render(request,'monthplan/show_flightplan_info.html', {'observations': observations, 'scans':scans, 'surveys':surveys, 'obs':pz, 'today':dt.now(),}) + +def show_scan_info(request, scan_id): + try: + scan=Scan.objects.get(pk=scan_id) + except Scan.DoesNotExist: + return HttpResponseRedirect('/monthplan') + flightplans=FlightPlan.objects.filter(experiment__contains=scan.experiment) + datadumps=scan.datadump_set.all() + scanpaths=scan.scanpath_set.all() + return render(request,'monthplan/show_scan_info.html', {'head':scan.head, 'scan': scan,'flightplans':flightplans,'scanpaths':scanpaths,'datadumps':datadumps,'today':dt.now(),}) + +def scan_download_csv(request, scan_id): + try: + scan=Scan.objects.get(pk=scan_id) + except Scan.DoesNotExist: + return HttpResponseRedirect('/monthplan') + scanpaths=scan.scanpath_set.all()#[::50] + response = HttpResponse(content_type='text/csv') + #decide the file name + response['Content-Disposition'] = 'attachment; filename='+"scan_%s.csv" % (scan.experiment,) + writer = csv.writer(response, csv.excel) + #response.write(u'\ufeff'.encode('utf8')) + response.write(u'# '.encode('utf8')) + + #write the headers + writer.writerow([ + smart_str(u"MJD"), + smart_str(u"EroDay"), + smart_str(u"RA"), + smart_str(u"Dec"), + ]) + + for path in scanpaths: + writer.writerow([ + smart_str(path.mjd), + smart_str(path.eroday), + smart_str(path.ra), + smart_str(path.dec), + ]) + return response + +# fsock = open('/path/to/file.mp3', 'r') +# response = HttpResponse(fsock, content_type='audio/mpeg') +# response['Content-Disposition'] = "attachment; filename=%s - %s.mp3" % \ +# (song.artist, song.title) +# return response + +def show_tracking(request): + return render(request,'monthplan/show_tracking.html') + +def show_survey_info(request, survey_id): + try: + survey=Survey.objects.get(pk=survey_id) + except Survey.DoesNotExist: + return HttpResponseRedirect('/monthplan') + flightplans=FlightPlan.objects.filter(experiment__contains=survey.experiment) + surveypaths=survey.surveypath_set.all().order_by('obt') + plates = survey.surveyhealpixplate_set.all().order_by('healpix') + return render(request,'monthplan/show_survey_info.html', {'head':survey.head, 'surveypaths':surveypaths, 'survey': survey,'flightplans':flightplans,'plates':plates,'today':dt.now(),}) + + +def show_monthplan_seance(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponseRedirect('/monthplan') + seances = head.seance_set.all() + + return render(request,'monthplan/show_monthplan.html', {'head': head,'seances':seances,'today':dt.now(),}) + +def show_monthplan_observation(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except Head.DoesNotExist: + return HttpResponseRedirect('/monthplan') + observations = head.observation_set.all() + return render(request,'monthplan/show_monthplan.html', {'head': head,'observations':observations,'today':dt.now(),}) + +def show_monthplan_correction(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponseRedirect('/monthplan') + corrections = head.correction_set.all() + return render(request,'monthplan/show_monthplan.html', {'head': head,'corrections':corrections,'today':dt.now(),}) + +def show_monthplan_scan(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponseRedirect('/monthplan') + scans = head.scan_set.all() + return render(request,'monthplan/show_monthplan.html', {'head': head,'scans':scans,'today':dt.now(),}) + +def show_monthplan_survey(request, monthplan_id): + try: + head=Head.objects.get(pk=monthplan_id) + except: + return HttpResponseRedirect('/monthplan') + surveys = head.survey_set.all() + return render(request,'monthplan/show_monthplan.html', {'head': head,'surveys':surveys,'today':dt.now(),}) + +def upload(request): + + if not request.user.is_authenticated: + html = "You have no rights for this operation." + return HttpResponseForbidden(html) + + try: + user_profile=request.user.profile + except: + html = "You have no user profile." + return HttpResponseForbidden(html) + + if request.method == 'POST': + form = MonthPlanUploadForm(request.POST, request.FILES) + if form.is_valid(): + upload = form.save(commit=False) + upload.owner = user_profile + upload.filename = request.FILES['filefield'].name.encode('utf-8') + upload.save() + table = Table.read(upload.filefield.path) + df = table.to_pandas() + hdul = fits.open(upload.filefield.path) + hdul.info() + head=Head() + head.title=upload.title + head.start=hdul[1].header['START'] + head.stop=hdul[1].header['STOP'] + head.gentime=hdul[1].header['GENTIME'] + head.version=hdul[1].header['VERSION'] + head.author=hdul[1].header['AUTHOR'] + head.nrows=df.shape[0] + head.save() + upload.head=head + upload.save() + for i in df.index: + row=(i+1) + if(df['TYPE'][i].decode().strip() == 'SEANCE'): + print('SEA') + print(df['START'][i].decode().strip()) + seance=Seance(head=head, row=row) + seance.start=df['START'][i].decode().strip() + seance.stop=df['STOP'][i].decode().strip() + seance.stations=df['STATIONS'][i].decode().strip() + seance.guid=df['GUID'][i].decode().strip() + seance.save() + if(df['TYPE'][i].decode().strip() == 'OBSERVATION'): + obs=Observation(head=head, row=row) + obs.start=df['START'][i].decode().strip() + obs.stop=df['STOP'][i].decode().strip() + obs.target=df['TARGET'][i].decode().strip() + obs.experiment=df['EXPERIMENT'][i].decode().strip() + obs.ra=df['RA'][i] + obs.dec=df['DEC'][i] + if('RA_OBJ' in df): + obs.ra_obj=df['RA_OBJ'][i] + if('DEC_OBJ' in df): + obs.dec_obj=df['DEC_OBJ'][i] + if('OBJECT' in df): + obs.name_obj=df['OBJECT'][i].decode().strip() + obs.roll_angle=df['ROLL_ANGLE'][i] + obs.sun_xoz_angle=df['SUN_XOZ_ANGLE'][i] + obs.guid=df['GUID'][i].decode().strip() + obs.save() + if(df['TYPE'][i].decode().strip() == 'CORRECTION'): + cor=Correction(head=head, row=row) + cor.start=df['START'][i].decode().strip() + cor.stop=df['STOP'][i].decode().strip() + cor.impstart=df['IMPSTART'][i].decode().strip() + cor.guid=df['GUID'][i].decode().strip() + cor.save() + if(df['TYPE'][i].decode().strip() == 'SCAN'): + print('SCAN') + scan=Scan(head=head, row=row) + scan.start=df['START'][i].decode().strip() + scan.stop=df['STOP'][i].decode().strip() + scan.target=df['TARGET'][i].decode().strip() + scan.experiment=df['EXPERIMENT'][i].decode().strip() + scan.ra=df['RA'][i] + scan.dec=df['DEC'][i] + scan.roll_angle=df['ROLL_ANGLE'][i] + scan.sun_xoz_angle=df['SUN_XOZ_ANGLE'][i] + scan.template=int(df['TEMPLATE'][i].decode().strip()) + scan.guid=df['GUID'][i].decode().strip() + scan.save() + if(df['TYPE'][i].decode().strip() == 'SURVEY'): + sur=Survey(head=head, row=row) + sur.start=df['START'][i].decode().strip() + sur.stop=df['STOP'][i].decode().strip() + sur.target=df['TARGET'][i].decode().strip() + sur.experiment=df['EXPERIMENT'][i].decode().strip() + sur.ra_p=df['RA_P'][i] + sur.dec_p=df['DEC_P'][i] + sur.ra_z0=df['RA_Z0'][i] + sur.dec_z0=df['DEC_Z0'][i] + sur.ra_zk=df['RA_ZK'][i] + sur.dec_zk=df['DEC_ZK'][i] + sur.z_speed=df['Z_SPEED'][i] + sur.guid=df['GUID'][i].decode().strip() + sur.save() + + return HttpResponseRedirect(head.get_absolute_url()) + else: + form = MonthPlanUploadForm() + + return render(request,'monthplan/upload.html', + {'UserProfile':user_profile, 'form': form,} + ) + + +def get_obsid_search(request): + # if this is a POST request we need to process the form data + if request.method == 'POST': + # create a form instance and populate it with data from the request: + form = ObsIDSearchForm(request.POST) + # check whether it's valid: + if form.is_valid(): + obsid = form.cleaned_data['obsid'] + observations=Observation.objects.filter(experiment__contains=obsid) + scans=Scan.objects.filter(experiment__contains=obsid) + surveys=Survey.objects.filter(experiment__contains=obsid) + flightplans=FlightPlan.objects.filter(experiment__contains=obsid) + #ordered=phots.order_by('separation') + return render(request, 'monthplan/obsid_search.html', {'form': form, 'observations':observations,'scans':scans,'surveys':surveys,'flightplans':flightplans,}) + + # if a GET (or any other method) we'll create a blank form + else: + form = ObsIDSearchForm() + + #MyModel.objects.all().annotate(mycolumn=Value('xxx', output_field=CharField())) + return render(request, 'monthplan/obsid_search.html', {'form': form}) diff --git a/plan/__init__.py b/plan/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plan/admin.py b/plan/admin.py new file mode 100644 index 0000000..1e6bf70 --- /dev/null +++ b/plan/admin.py @@ -0,0 +1,11 @@ +from django.contrib import admin + +# Register your models here. +from plan.models import Day, SessionBearLakes, SessionUssuriysk, Upload, UserProfile + +admin.site.register(Day) +admin.site.register(Upload) +admin.site.register(UserProfile) +admin.site.register(SessionBearLakes) +admin.site.register(SessionUssuriysk) + diff --git a/plan/apps.py b/plan/apps.py new file mode 100644 index 0000000..7cf7268 --- /dev/null +++ b/plan/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PlanConfig(AppConfig): + name = 'plan' diff --git a/plan/migrations/0001_initial.py b/plan/migrations/0001_initial.py new file mode 100644 index 0000000..26964f5 --- /dev/null +++ b/plan/migrations/0001_initial.py @@ -0,0 +1,86 @@ +# Generated by Django 2.2.6 on 2019-12-09 18:12 + +from django.conf import settings +import django.core.files.storage +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import plan.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Day', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=100)), + ('Notes', models.TextField(blank=True, max_length=2000, null=True, validators=[django.core.validators.MaxLengthValidator(1600)], verbose_name='Notes')), + ('date', models.DateField(blank=True, null=True)), + ('DayNumber', models.IntegerField()), + ('modified', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='UserProfile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='profile', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Upload', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=128)), + ('filename', models.CharField(max_length=256)), + ('filefield', models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/srv/srg-plan/srg/Uploads'), upload_to=plan.models.upload_filename, validators=[plan.models.validate_upload_file])), + ('uploaded', models.DateTimeField(auto_now_add=True)), + ('day', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='plan.Day', verbose_name='Day')), + ('owner', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='plan.UserProfile', verbose_name='Author')), + ], + ), + migrations.CreateModel( + name='SessionUssuriysk', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(blank=True, max_length=100)), + ('TimeBegin_hh', models.IntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(24)])), + ('TimeEnd_hh', models.IntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(24)])), + ('TimeBegin_mm', models.IntegerField(blank=True, default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(60)])), + ('TimeEnd_mm', models.IntegerField(blank=True, default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(60)])), + ('sysSessionRequest', models.IntegerField(default=0)), + ('eroSessionRequest', models.IntegerField(default=0)), + ('artSessionRequest', models.IntegerField(default=0)), + ('sysNotes', models.TextField(blank=True, max_length=2000, null=True, verbose_name='System Notes')), + ('eroNotes', models.TextField(blank=True, max_length=2000, null=True, verbose_name='eRosita Notes')), + ('artNotes', models.TextField(blank=True, max_length=2000, null=True, verbose_name='ART-XC Notes')), + ('Day', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='UssuriyskDay', to='plan.Day', verbose_name='Day')), + ], + ), + migrations.CreateModel( + name='SessionBearLakes', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(blank=True, max_length=100)), + ('TimeBegin_hh', models.IntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(24)])), + ('TimeEnd_hh', models.IntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(24)])), + ('TimeBegin_mm', models.IntegerField(blank=True, default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(60)])), + ('TimeEnd_mm', models.IntegerField(blank=True, default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(60)])), + ('sysSessionRequest', models.IntegerField(default=0)), + ('eroSessionRequest', models.IntegerField(default=0)), + ('artSessionRequest', models.IntegerField(default=0)), + ('sysNotes', models.TextField(blank=True, max_length=2000, null=True, validators=[django.core.validators.MaxLengthValidator(1600)], verbose_name='System Notes')), + ('eroNotes', models.TextField(blank=True, max_length=2000, null=True, validators=[django.core.validators.MaxLengthValidator(1600)], verbose_name='eRosita Notes')), + ('artNotes', models.TextField(blank=True, max_length=2000, null=True, validators=[django.core.validators.MaxLengthValidator(1600)], verbose_name='ART-XC Notes')), + ('Day', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='BearLakesDay', to='plan.Day', verbose_name='Day')), + ], + ), + ] diff --git a/plan/migrations/0002_auto_20211229_1259.py b/plan/migrations/0002_auto_20211229_1259.py new file mode 100644 index 0000000..07d3b96 --- /dev/null +++ b/plan/migrations/0002_auto_20211229_1259.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.10 on 2021-12-29 09:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('plan', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='day', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='sessionbearlakes', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='sessionussuriysk', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='upload', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='userprofile', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/plan/migrations/__init__.py b/plan/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plan/models.py b/plan/models.py new file mode 100644 index 0000000..56059cd --- /dev/null +++ b/plan/models.py @@ -0,0 +1,262 @@ +import logging +from django.db import models +from django import forms +from django.core.exceptions import ValidationError +import time +import datetime +import re +from django.core.validators import MaxLengthValidator, MinValueValidator, MaxValueValidator +from django.utils import timezone +from django.core.exceptions import ValidationError +from django.utils.translation import ugettext as _, ugettext_noop as _noop + +from django.contrib.auth.models import User + +import hashlib +import os, os.path as osp +from django.utils.translation import gettext_lazy + +# Create your models here. +LaunchDate = datetime.date(2019, 7, 13) +root='' +INPUT_DATA_DIR='/export/django/srg/data' + +from django.core.files.storage import FileSystemStorage +from django.core.validators import EmailValidator +from django.core.validators import MaxLengthValidator + +PRESENTATIONS_PATH = '/srv/srg-plan/srg/Uploads' + +email_notification=['krivonos@iki.rssi.ru',] + +readme_txt='/srv/srg-plan/srg/README.txt' + +def get_session_duration_in_min(begin_hh, begin_mm, end_hh, end_mm): + if (end_hh > begin_hh): + return (end_hh-begin_hh)*60+end_mm-begin_mm + else: + return (24-begin_hh+end_hh)*60+end_mm-begin_mm + + +class Day(models.Model): + title = models.CharField(max_length=100) + Notes = models.TextField("Notes",max_length=2000, null=True, blank=True, validators=[MaxLengthValidator(1600)]) + date = models.DateField(blank=True, null=True) + DayNumber = models.IntegerField() + modified = models.DateTimeField(auto_now=True) + def __unicode__(self): + return 'Day: ' + "%i" % (self.DayNumber) + def __str__(self): + return 'Day: ' + "%i" % (self.DayNumber) + def get_upload_url(self): + return "/plan/day/%i/upload" % (self.pk) + def get_absolute_url(self): + return "/plan/day/%i" % (self.pk) + +class SessionBearLakes(models.Model): + Day = models.OneToOneField(Day, on_delete=models.CASCADE,verbose_name="Day", related_name="BearLakesDay",null=True) + title = models.CharField(max_length=100,blank=True) + TimeBegin_hh = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(24)]) + TimeEnd_hh = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(24)]) + TimeBegin_mm = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(60)],blank=True) + TimeEnd_mm = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(60)],blank=True) + sysSessionRequest = models.IntegerField(default=0) + eroSessionRequest = models.IntegerField(default=0) + artSessionRequest = models.IntegerField(default=0) + sysNotes = models.TextField("System Notes",max_length=2000, null=True, blank=True, validators=[MaxLengthValidator(1600)]) + eroNotes = models.TextField("eRosita Notes",max_length=2000, null=True, blank=True, validators=[MaxLengthValidator(1600)]) + artNotes = models.TextField("ART-XC Notes",max_length=2000, null=True, blank=True, validators=[MaxLengthValidator(1600)]) + def clean(self, *args, **kwargs): + # add custom validation here + + session_duration_min=get_session_duration_in_min(self.TimeBegin_hh,self.TimeBegin_mm, self.TimeEnd_hh,self.TimeEnd_mm) + request_min=(self.sysSessionRequest+self.eroSessionRequest+self.artSessionRequest) + if (request_min>session_duration_min): + msg="Total requested time (sys:%i + ero:%i + art:%i = %i min) cannot be longer than session duration: %i min" % (self.sysSessionRequest, + self.eroSessionRequest, + self.artSessionRequest, + request_min, + session_duration_min) + raise forms.ValidationError(msg) + super(SessionBearLakes, self).clean(*args, **kwargs) + + def save(self, *args, **kwargs): + self.full_clean() + super(SessionBearLakes, self).save(*args, **kwargs) + + def __unicode__(self): + return self.title + def __str__(self): + return 'Session: Day ' + "%i" % (self.Day.DayNumber) + def get_absolute_url(self): + return root+"/plan/BearLakes/%i/show" % (self.Day.DayNumber) + def get_update_url(self): + return root+"/plan/BearLakes/%i/update" % (self.Day.DayNumber) + def get_rest(self): + session_duration_min=get_session_duration_in_min(self.TimeBegin_hh,self.TimeBegin_mm, self.TimeEnd_hh,self.TimeEnd_mm) + request_min=(self.sysSessionRequest+self.eroSessionRequest+self.artSessionRequest) + return "%i" % (session_duration_min - request_min) + + +class SessionBearLakesForm(forms.ModelForm): + class Meta: + model=SessionBearLakes + fields=('title','TimeBegin_hh','TimeBegin_mm','TimeEnd_hh','TimeEnd_mm','sysSessionRequest','eroSessionRequest','artSessionRequest','sysNotes','eroNotes','artNotes') + widgets = { + 'title': forms.TextInput(attrs={'style': 'width: 620px;', 'class': 'title'}), + 'TimeBegin_hh': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'TimeBegin_mm': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'TimeEnd_hh': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'TimeEnd_mm': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'sysSessionRequest': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'eroSessionRequest': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'artSessionRequest': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'sysNotes': forms.Textarea(attrs={'style': 'width: 620px;', 'rows': 10}), + 'eroNotes': forms.Textarea(attrs={'style': 'width: 620px;', 'rows': 10}), + 'artNotes': forms.Textarea(attrs={'style': 'width: 620px;', 'rows': 10}), + } + +class SessionUssuriysk(models.Model): + Day = models.OneToOneField(Day, on_delete=models.CASCADE,verbose_name="Day",related_name="UssuriyskDay",null=True) + title = models.CharField(max_length=100,blank=True) + TimeBegin_hh = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(24)]) + TimeEnd_hh = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(24)]) + TimeBegin_mm = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(60)],blank=True) + TimeEnd_mm = models.IntegerField(default=0,validators=[MinValueValidator(0),MaxValueValidator(60)],blank=True) + sysSessionRequest = models.IntegerField(default=0) + eroSessionRequest = models.IntegerField(default=0) + artSessionRequest = models.IntegerField(default=0) + sysNotes = models.TextField("System Notes",max_length=2000, null=True, blank=True) #,validators=[MaxLengthValidator(1600)]) + eroNotes = models.TextField("eRosita Notes",max_length=2000, null=True, blank=True) #,validators=[MaxLengthValidator(1600)]) + artNotes = models.TextField("ART-XC Notes",max_length=2000, null=True, blank=True) #,validators=[MaxLengthValidator(1600)]) + def clean(self, *args, **kwargs): + # add custom validation here + session_duration_min=get_session_duration_in_min(self.TimeBegin_hh,self.TimeBegin_mm, self.TimeEnd_hh,self.TimeEnd_mm) + request_min=(self.sysSessionRequest+self.eroSessionRequest+self.artSessionRequest) + if (request_min>session_duration_min): + msg="Total requested time (sys:%i + ero:%i + art:%i = %i min) cannot be longer than session duration: %i min" % (self.sysSessionRequest, + self.eroSessionRequest, + self.artSessionRequest, + request_min, + session_duration_min) + raise forms.ValidationError(msg) + super(SessionUssuriysk, self).clean(*args, **kwargs) + + def save(self, *args, **kwargs): + self.full_clean() + super(SessionUssuriysk, self).save(*args, **kwargs) + + def __unicode__(self): + return self.title + def __str__(self): + return 'Session: Day ' + "%i" % (self.Day.DayNumber) + def get_absolute_url(self): + return root+"/plan/Ussuriysk/%i/show" % (self.Day.DayNumber) + def get_update_url(self): + return root+"/plan/Ussuriysk/%i/update" % (self.Day.DayNumber) + def get_rest(self): + session_duration_min=get_session_duration_in_min(self.TimeBegin_hh,self.TimeBegin_mm, self.TimeEnd_hh,self.TimeEnd_mm) + request_min=(self.sysSessionRequest+self.eroSessionRequest+self.artSessionRequest) + return "%i" % (session_duration_min - request_min) + +class SessionUssuriyskForm(forms.ModelForm): + class Meta: + model=SessionUssuriysk + fields=('title','TimeBegin_hh','TimeBegin_mm','TimeEnd_hh','TimeEnd_mm','sysSessionRequest','eroSessionRequest','artSessionRequest','sysNotes','eroNotes','artNotes') + widgets = { + 'title': forms.TextInput(attrs={'style': 'width: 620px;', 'class': 'title'}), + 'TimeBegin_hh': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'TimeBegin_mm': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'TimeEnd_hh': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'TimeEnd_mm': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'sysSessionRequest': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'eroSessionRequest': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'artSessionRequest': forms.TextInput(attrs={'style': 'width: 70px;', 'class': 'title'}), + 'sysNotes': forms.Textarea(attrs={'style': 'width: 620px;', 'rows': 10}), + 'eroNotes': forms.Textarea(attrs={'style': 'width: 620px;', 'rows': 10}), + 'artNotes': forms.Textarea(attrs={'style': 'width: 620px;', 'rows': 10}), + } + +FS = FileSystemStorage(location=PRESENTATIONS_PATH) + +VALID_CONTENT_TYPES = ['application/pdf', 'image/png', 'application/vnd.ms-powerpoint','application/octet-stream', + 'application/nappdf','application/x-pdf','application/x-bzpdf', + 'application/x-gzpdf','application/force-download','application/x-file-download', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation' +] + +VALID_EXTENSIONS = ['.pdf', '.ppt', '.pptx', '.png', '.txt', '.dat'] + +def validate_upload_file(value): + file = value.file + + _, ext = osp.splitext(file.name) + if ext not in VALID_EXTENSIONS: + raise forms.ValidationError( + 'Extension %s not supported.' % ext + ) + + if file.content_type not in VALID_CONTENT_TYPES: + raise forms.ValidationError( + 'Filetype %s not supported.' % file.content_type + ) + + +def upload_filename(instance, filename): + _, ext = osp.splitext(filename) + + tohash = filename + str(time.clock()) + #new_fname = hashlib.md5(tohash).hexdigest() + new_fname = hashlib.md5(tohash.encode('utf-8')).hexdigest() + return new_fname + ext + +class UserProfile(models.Model): + user = models.OneToOneField(User, related_name="profile", on_delete=models.CASCADE) + #target_list = models.ManyToManyField(Target, blank=True) + #docker_list = models.ManyToManyField(DockerProxy, blank=True) + + def __unicode__(self): + return self.user.email + def __str__(self): + return self.user.email + def get_objects(self): + return self.object_set.all() + +class LoginUsernameForm(forms.Form): + username = forms.CharField(max_length=75, label="User name", widget=forms.TextInput(attrs={'class':'form-control', 'style': 'width: 450px;'})) + password = forms.CharField( widget=forms.PasswordInput(attrs={'class':'form-control', 'style': 'width: 450px;'}), label="Your Password" ) + def clean_username(self): + username = self.cleaned_data.get('username') + try: + User.objects.get(username=username) + except User.DoesNotExist: + raise forms.ValidationError('This username is no such username.') + return username + +class Upload(models.Model): + owner = models.ForeignKey(UserProfile, on_delete=models.CASCADE, default=None, unique=False, verbose_name="Author") + day = models.ForeignKey(Day, unique=False, verbose_name="Day", on_delete=models.CASCADE) + title = models.CharField(max_length=128) + filename = models.CharField(max_length=256) + filefield = models.FileField(upload_to=upload_filename, storage=FS, validators=[validate_upload_file]) + uploaded = models.DateTimeField(auto_now_add=True) + #read_only = models.BooleanField("Read only", default=False) + def get_delete_url(self): + return self.day.get_absolute_url() + "/upload/%d/delete" % (self.id) + + def delete(self, remove_file=True): + if remove_file: + FS.delete(str(self.filefield)) + return models.Model.delete(self) + def __str__(self): + return '['+"%d" % (self.day.DayNumber)+'] '+self.title + + +class UploadForm(forms.ModelForm): + class Meta: + model = Upload + fields = ('title', 'filefield') + widgets = { + 'title': forms.TextInput(attrs={'style': 'width: 620px;', 'class': 'title'}), + } + diff --git a/plan/tests.py b/plan/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/plan/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/plan/urls.py b/plan/urls.py new file mode 100644 index 0000000..87dd6fb --- /dev/null +++ b/plan/urls.py @@ -0,0 +1,17 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.index, name='index'), + path('BearLakes//show', views.show_SessionBearLakes, name='showBearLakes'), + path('BearLakes//update', views.update_SessionBearLakes, name='showBearLakes'), + path('Ussuriysk//show', views.show_SessionUssuriysk, name='showUssuriysk'), + path('Ussuriysk//update', views.update_SessionUssuriysk, name='showUssuriysk'), + path('day//upload', views.upload, name='upload'), + path('day//upload//delete', views.delete_upload, name='delete'), + path('day/', views.show_day, name='show'), + path('login', views.login_username, name='login'), + path('logout', views.logout_view, name='logout'), +] + diff --git a/plan/views.py b/plan/views.py new file mode 100644 index 0000000..119ffa9 --- /dev/null +++ b/plan/views.py @@ -0,0 +1,196 @@ +from django.shortcuts import render +from django.http import HttpResponse +from datetime import date +from plan.models import Day, SessionBearLakes, SessionBearLakesForm, SessionUssuriysk, SessionUssuriyskForm, LoginUsernameForm, UserProfile, UploadForm, Upload +from plan.models import INPUT_DATA_DIR +from plan.models import LaunchDate +from django.http import HttpResponseRedirect +import datetime +from django.utils import timezone +import astropy +import pandas as pd + +from django.utils import timezone +from django.contrib.auth import authenticate, login, logout +from django.contrib.auth.models import User +from django.core.files.storage import FileSystemStorage + +# Create your views here. Berkeley + +def index(request): + if request.user.is_authenticated: + try: + user_profile=request.user.profile + except: + return HttpResponse("docker: no UserProfile.") + else: + return HttpResponse("docker: not authenticated.") + all_bearlakes = SessionBearLakes.objects.all() + return render(request,'plan/show_plan.html', {'bearlakes': all_bearlakes,'LaunchDate':LaunchDate}) + + +def show_day(request, day_id): + if request.user.is_authenticated: + try: + user_profile=request.user.profile + except: + return HttpResponse("docker: no UserProfile.") + else: + return HttpResponse("docker: not authenticated.") + try: + myday=Day.objects.get(pk=day_id) + except: + return HttpResponse("This Day was not found") + uploads = myday.upload_set.all() + return render(request,'plan/show_day.html', {'uploads': uploads, 'day':myday,}) + +def show_SessionBearLakes(request,day): + try: + session=SessionBearLakes.objects.get(Day__DayNumber__exact=day) + except: + return HttpResponse("This Day was not found") + return render(request,'plan/show_session_bearlakes.html', {'session': session,}) + +def show_SessionUssuriysk(request,day): + try: + session=SessionUssuriysk.objects.get(Day__DayNumber__exact=day) + except: + return HttpResponse("This Day was not found") + session_lakes=session.Day.BearLakesDay + return render(request,'plan/show_session_bearlakes.html', {'session': session_lakes,}) + +def update_SessionBearLakes(request,day): + try: + session=SessionBearLakes.objects.get(Day__DayNumber__exact=day) + except: + return HttpResponse("This Day was not found") + if request.method == 'POST': # If the form has been submitted... + form = SessionBearLakesForm(request.POST,instance=session) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save() + return HttpResponseRedirect(new_form.get_absolute_url()) + else: + return render(request,'plan/update_session_bearlakes.html', {'form':form,'session': session}) + else: + form = SessionBearLakesForm(instance=session) + return render(request,'plan/update_session_bearlakes.html', {'form':form,'session': session}) + +def update_SessionUssuriysk(request,day): + try: + session=SessionUssuriysk.objects.get(Day__DayNumber__exact=day) + except: + return HttpResponse("This Day was not found") + if request.method == 'POST': # If the form has been submitted... + form = SessionUssuriyskForm(request.POST,instance=session) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + new_form=form.save() + return HttpResponseRedirect(new_form.get_absolute_url()) + else: + return render(request,'plan/update_session_ussuriysk.html', {'form':form,'session': session}) + else: + form = SessionUssuriyskForm(instance=session) + return render(request,'plan/update_session_ussuriysk.html', {'form':form,'session': session}) + +def login_username(request): + if request.user.is_authenticated: + try: + user_profile=request.user.profile + except: + return HttpResponse("login: authenticated but no UserProfile.") + + if request.method == 'POST': # If the form has been submitted... + form = LoginUsernameForm(request.POST) # A form bound to the POST data + if form.is_valid(): # All validation rules pass + username=form.clean_username() + password=form.cleaned_data["password"] + user = authenticate(username=username, password=password) + if user is not None: + if user.is_active: + login(request, user) + try: + user_profile=user.profile + except: + form = UserProfileForm() # An unbound form + return HttpResponse("login: authenticated but no UserProfile.") + else: + return HttpResponse("login: user account is not active.") + else: + form = LoginUsernameForm() # An unbound form + return render(request,'plan/login.html', {'form':form, 'invalid_login': True}) + else: + return HttpResponse("login: form is not valid.") + else: + form = LoginUsernameForm() # An unbound form + return render(request,'plan/login.html', {'form':form}) + + all_bearlakes = SessionBearLakes.objects.all() + return render(request,'plan/show_plan.html', {'bearlakes': all_bearlakes,'LaunchDate':LaunchDate}) + + +def logout_view(request): + if request.user.is_authenticated: + logout(request) + return HttpResponse('Login') + +def delete_upload(request, day_id, upload_id): + if not request.user.is_authenticated: + html = "You have no rights to upload/delete documents. Please login<>." + return HttpResponseForbidden(html) + try: + user_profile=request.user.profile + except: + html = "You have no user profile." + return HttpResponseForbidden(html) + try: + myday=Day.objects.get(pk=day_id) + except Day.DoesNotExist: + raise Http404 + +# if mytarget.owner != user_profile: +# html = "You trying to delete presentation for someone else's target." +# return HttpResponseForbidden(html) + + try: + upload=Upload.objects.get(pk=upload_id) + except Upload.DoesNotExist: + raise Http404 + + if upload.day != myday: + html = "Sorry, wrong day/upload pair." + return HttpResponseForbidden(html) + + upload.delete() + return HttpResponseRedirect(myday.get_absolute_url()) + +def upload(request, day_id): + + if not request.user.is_authenticated: + html = "You have no rights for this operation." + return HttpResponseForbidden(html) + + try: + user_profile=request.user.profile + except: + html = "You have no user profile." + return HttpResponseForbidden(html) + + try: + myday=Day.objects.get(pk=day_id) + except Day.DoesNotExist: + raise Http404 + + if request.method == 'POST': + form = UploadForm(request.POST, request.FILES) + if form.is_valid(): + upload = form.save(commit=False) + upload.day = myday + upload.owner = user_profile + upload.filename = request.FILES['filefield'].name.encode('utf-8') + upload.save() + return HttpResponseRedirect(myday.get_absolute_url()) + else: + form = UploadForm() + + return render(request,'plan/upload.html', + {'UserProfile':user_profile, 'form': form, 'day':myday} + ) diff --git a/srgcat/__init__.py b/srgcat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/srgcat/admin.py b/srgcat/admin.py new file mode 100644 index 0000000..1f087a4 --- /dev/null +++ b/srgcat/admin.py @@ -0,0 +1,8 @@ +from django.contrib import admin +from srgcat.models import ArtCat, SrgDataDump, Selection + +# Register your models here. + +admin.site.register(ArtCat) +admin.site.register(SrgDataDump) +admin.site.register(Selection) diff --git a/srgcat/apps.py b/srgcat/apps.py new file mode 100644 index 0000000..ca3e239 --- /dev/null +++ b/srgcat/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class SrgcatConfig(AppConfig): + name = 'srgcat' diff --git a/srgcat/forms.py b/srgcat/forms.py new file mode 100644 index 0000000..ba3a816 --- /dev/null +++ b/srgcat/forms.py @@ -0,0 +1,32 @@ +from django import forms +from srgcat.models import Selection, UserSessionParameters +#import urllib + +class SourceNameForm(forms.Form): + ra = forms.FloatField(label='RA') + dec = forms.FloatField(label='Dec') + +class LookupTargetForm(forms.Form): + ra = forms.FloatField(label='RA') + dec = forms.FloatField(label='Dec') + load_surveypath = forms.BooleanField(required=False, initial=False) + +class SkyMapForm(forms.Form): + skymap = forms.IntegerField(label='SkyMap') + + +class SelectionForm(forms.ModelForm): + name = forms.CharField(label='Long name') + slug = forms.CharField(label='Short name (slug)') + public = forms.BooleanField(required=False, initial=True) + class Meta: + model = Selection + fields = ('name', 'slug', 'public') + +class UserSessionParametersForm(forms.ModelForm): + sign_threshold = forms.FloatField(required=False) + target_ra = forms.FloatField(required=False) + target_dec = forms.FloatField(required=False) + class Meta: + model = UserSessionParameters + fields = ('sign_threshold','target_ra','target_dec') diff --git a/srgcat/management/commands/00_init_skymaps.py b/srgcat/management/commands/00_init_skymaps.py new file mode 100644 index 0000000..4776f8c --- /dev/null +++ b/srgcat/management/commands/00_init_skymaps.py @@ -0,0 +1,183 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix, neighbours +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from astropy.io import fits +from django.db.models import Q + +from srgcat.models import SkyMaps +from monthplan.models import SurveyHealpixPlate, NSIDE_PLATES, ORDER + +# XMMU JHHMMSS.s+/-DDMMSS. +# www.cosmos.esa.int/web/xmm-newton/source-naming-convention +def make_source_name(key, ra, dec): + c = SkyCoord(ra, dec, frame=FK5(), unit="deg") + str1 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=1).split() + str2 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=0).split() + name = key+" J%s%s" % (str1[0].replace('h','').replace('m','').replace('s',''), + str2[1].replace('d','').replace('m','').replace('s','')) + + return name + + +def load_skymaps(filename): + #hdul = fits.open(filename) + #hdul.info() + + table = Table.read(filename) + df = table.to_pandas() + for i in df.index: + skymap = SkyMaps( + SMAPNR = int(df['SMAPNR'][i]), + RA_MIN = float(df['RA_MIN'][i]), + RA_MAX = float(df['RA_MAX'][i]), + DE_MIN = float(df['DE_MIN'][i]), + DE_MAX = float(df['DE_MAX'][i]), + RA_CEN = float(df['RA_CEN'][i]), + DE_CEN = float(df['DE_CEN'][i]), + ELON_CEN = float(df['ELON_CEN'][i]), + ELAT_CEN = float(df['ELAT_CEN'][i]), + GLON_CEN = float(df['GLON_CEN'][i]), + GLAT_CEN = float(df['GLAT_CEN'][i]), + X_MIN = float(df['X_MIN'][i]), + Y_MIN = float(df['Y_MIN'][i]), + OWNER = int(df['OWNER'][i]), + N_NBRS = int(df['N_NBRS'][i]), + FIELD1 = int(df['FIELD1'][i]), + FIELD2 = int(df['FIELD2'][i]), + FIELD3 = int(df['FIELD3'][i]), + FIELD4 = int(df['FIELD4'][i]), + FIELD5 = int(df['FIELD5'][i]), + FIELD6 = int(df['FIELD6'][i]), + FIELD7 = int(df['FIELD7'][i]), + FIELD8 = int(df['FIELD8'][i]), + FIELD9 = int(df['FIELD9'][i])) + skymap.save() + pass + +def find_neighbours(): + skymaps = SkyMaps.objects.all() + for skymap in skymaps: + skymap.neighbours.clear() + try: + if(skymap.FIELD1 > 0): + n1 = SkyMaps.objects.get(SMAPNR=skymap.FIELD1) + skymap.neighbours.add(n1) + if(skymap.FIELD2 > 0): + n2 = SkyMaps.objects.get(SMAPNR=skymap.FIELD2) + skymap.neighbours.add(n2) + if(skymap.FIELD3 > 0): + n3 = SkyMaps.objects.get(SMAPNR=skymap.FIELD3) + skymap.neighbours.add(n3) + if(skymap.FIELD4 > 0): + n4 = SkyMaps.objects.get(SMAPNR=skymap.FIELD4) + skymap.neighbours.add(n4) + if(skymap.FIELD5 > 0): + n5 = SkyMaps.objects.get(SMAPNR=skymap.FIELD5) + skymap.neighbours.add(n5) + if(skymap.FIELD6 > 0): + n6 = SkyMaps.objects.get(SMAPNR=skymap.FIELD6) + skymap.neighbours.add(n6) + if(skymap.FIELD7 > 0): + n7 = SkyMaps.objects.get(SMAPNR=skymap.FIELD7) + skymap.neighbours.add(n7) + if(skymap.FIELD8 > 0): + n8 = SkyMaps.objects.get(SMAPNR=skymap.FIELD8) + skymap.neighbours.add(n8) + if(skymap.FIELD9 > 0): + n9 = SkyMaps.objects.get(SMAPNR=skymap.FIELD9) + skymap.neighbours.add(n9) + except: + print(skymap," neighbour not found") + skymap.save() + + print(skymap) + +def find_plates(): + + skymaps = SkyMaps.objects.all() + for skymap in skymaps: + skymap.survey_healpix_plate.clear() + + plates = SurveyHealpixPlate.objects.all() + for plate in plates: + ra = plate.ra + dec = plate.dec + + try: + skymap = SkyMaps.objects.get(Q(RA_MIN__lte=ra) & Q(RA_MAX__gt=ra) & Q(DE_MIN__lte=dec) & Q(DE_MAX__gt=dec)) + except Exception as e: + print("%s (%s)" % (e.message, type(e))) + return + + skymap.survey_healpix_plate.add(plate) + skymap.save() + + around = neighbours(plate.healpix, NSIDE_PLATES, order=ORDER) + for index in around: + if(index<0): + continue + try: + pl = SurveyHealpixPlate.objects.get(healpix=index) + except: + print("Healpix index {} not found, exit".format(index)) + return + skymap.survey_healpix_plate.add(pl) + skymap.save() + + + + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + #skymaps = SkyMaps.objects.all() + #skymaps.delete() + + #load_skymaps('/data/erosita/SKYMAPS.fits') + + #find_neighbours() + + find_plates() + return + """ + some checks: + """ + + skymaps = SkyMaps.objects.all()[:10] + for skymap in skymaps: + print(skymap, skymap.FIELD1, + skymap.FIELD2, + skymap.FIELD3, + skymap.FIELD4, + skymap.FIELD5, + skymap.FIELD6, + skymap.FIELD7, + skymap.FIELD8, + skymap.FIELD9) + n = skymap.survey_healpix_plate.all() + for nn in n: + print(nn) + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/00_load_art_gc.py b/srgcat/management/commands/00_load_art_gc.py new file mode 100644 index 0000000..cbb24d1 --- /dev/null +++ b/srgcat/management/commands/00_load_art_gc.py @@ -0,0 +1,90 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +# XMMU JHHMMSS.s+/-DDMMSS. +# www.cosmos.esa.int/web/xmm-newton/source-naming-convention +def make_source_name(key, ra, dec): + c = SkyCoord(ra, dec, frame=FK5(), unit="deg") + str1 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=1).split() + str2 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=0).split() + name = key+" J%s%s" % (str1[0].replace('h','').replace('m','').replace('s',''), + str2[1].replace('d','').replace('m','').replace('s','')) + + return name + + +def load_gc_table(filename, name, e1, e2): + datafile='/export/django/srg/data/Rodion_GC/'+filename + + cat = ArtCat(name=name, e1=e1, e2=e2) + cat.save() + + table = Table.read(datafile, format='ascii') + # ra dec sig cnts/s erg/s/cm^2 + for item in table: + ra=item['ra'] + dec=item['dec'] + src_name = make_source_name('IKIa', ra, dec) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + print(ra,dec,crd.galactic.l,crd.galactic.b) + src = ArtSource(catalog=cat, + name=src_name, + radec_error=30.0, + ra=item['ra'], + dec=item['dec'], + rate=item['cnts/s'], + flux=item['erg/s/cm^2'], + sign=item['sig']) + src.save() + print('--> Successfully loaded "%s"' % name) + pass + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + # create sqlalchemy engine +# engine = create_engine("mysql+pymysql://{user}:{pw}@localhost/{db}" +# .format(user="heauser", +# pw="srg2019@L2_heasarc", +# db="heasarc_db")) + + cats = ArtCat.objects.all() + cats.delete() + + load_gc_table('src.list', "ArtCat GC e04to12keV v1.0", 4.0, 12.0) + load_gc_table('src_10_20.list', "ArtCat GC e10to20keV v1.0", 10.0, 20.0) + load_gc_table('src_20_30.list', "ArtCat GC e20to30keV v1.0", 20.0, 30.0) + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + # update healpix indexes + srcs = ArtSource.objects.all() + for src in srcs: + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + src.healpix = hp.skycoord_to_healpix(crd) + src.save() + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/00_load_gyro.py b/srgcat/management/commands/00_load_gyro.py new file mode 100644 index 0000000..f3cdf06 --- /dev/null +++ b/srgcat/management/commands/00_load_gyro.py @@ -0,0 +1,144 @@ +from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone +import numpy as np +from srglib.utils import slerp, quat_to_pol_and_roll +from Quaternion import Quat + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from astropy.io import fits +from django.db.models import Q +from datetime import datetime +from astropy.time import Time + +from srgcat.models import SkyMaps, SrgDataDump, Gyro +from monthplan.models import SurveyHealpixPlate, Survey +from srglib.utils import TZ_UTC, TZ_MSK, load_gyro_file, mission2date +import glob +import os +import logging + +def find_closest(now, closest_greater_qs, closest_less_qs): + + try: + try: + closest_greater = closest_greater_qs[0] + except IndexError: + closest = closest_less_qs[0] + + try: + closest_less = closest_less_qs[0] + except IndexError: + closest = closest_greater_qs[0] + except IndexError: + raise self.model.DoesNotExist("There is no closest object because there are no objects.") + print("closest_less:",closest_less.ra,closest_less.dec) + print("closest_greater:",closest_greater.ra,closest_greater.dec) + + + delta = (closest_greater.dtime-now).total_seconds()/(closest_greater.dtime-closest_less.dtime).total_seconds() + #print((now-closest_greater.dtime).total_seconds()) + #print((closest_greater.dtime-closest_less.dtime).total_seconds()) + print('delta ',delta) + ra = np.interp(delta, [0.0, 1.0], [closest_less.ra, closest_greater.ra]) + dec = np.interp(delta, [0.0, 1.0], [closest_less.dec, closest_greater.dec]) + #quat = slerp([1,0,0,0], [0,0,0,1], np.arange(0, 1, 0.001)) + + quat = slerp([closest_less.q2, closest_less.q3, closest_less.q4, closest_less.q1], + [closest_greater.q2, closest_greater.q3, closest_greater.q4, closest_greater.q1], + [delta]) + qfin=Quat(attitude=quat[0]) + ra = qfin.ra + dec=qfin.dec + roll=qfin.roll + + return ra, dec + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + logging.getLogger().setLevel(logging.INFO) + + gyros = Gyro.objects.all() + #print(gyros.count()) + + + + #filename='/srg/a1/work/oper/data/2020/srg_20201125_182700/L0/srg_20201125_182700_000_gyro.fits' + #load_gyro_file(filename) + + #now_utc = timezone.now() + #survey = Survey.objects.filter(Q(start__lte=now_utc) & Q(stop__gte=now_utc))[0] + #closest_greater_qs = survey.surveypath_set.filter(dtime__gt=now_utc).order_by('dtime') + #closest_less_qs = survey.surveypath_set.filter(dtime__lt=now_utc).order_by('-dtime') + #closest_ra, closest_dec = find_closest(now_utc, closest_greater_qs, closest_less_qs) + #print(closest_ra, closest_dec) + + #path = survey.surveypath_set.all() + #print(path.count()) + + #for p in path: + # mjd=Time(p.dtime, format='datetime') + # #print(p.dtime,mjd) + # break + + for g in gyros: + dtime=Time(mission2date(g.obt), format='datetime').datetime + mjd=Time(g.mjd, format='mjd') - 3*u.hour + msk=mjd.to_datetime(timezone=TZ_MSK) + utc=mjd.to_datetime(timezone=TZ_UTC) + + try: + survey = Survey.objects.filter(Q(start__lte=utc) & Q(stop__gte=utc))[0] + except: + continue + closest_greater_qs = survey.surveypath_set.filter(dtime__gt=utc).order_by('dtime') + closest_less_qs = survey.surveypath_set.filter(dtime__lt=utc).order_by('-dtime') + try: + try: + closest_greater = closest_greater_qs[0] + except IndexError: + closest = closest_less_qs[0] + + try: + closest_less = closest_less_qs[0] + except IndexError: + closest = closest_greater_qs[0] + except IndexError: + raise self.model.DoesNotExist("There is no closest object because there are no objects.") + + closest_ra, closest_dec = find_closest(utc, closest_greater_qs, closest_less_qs) + + crd_path = SkyCoord(closest_ra, closest_dec, frame="fk5", unit="deg") + crd_gyro = SkyCoord(g.ra, g.dec, frame="fk5", unit="deg") + print(utc) + print(g.ra) + print('1 ',closest_greater.dtime) + print('1 ',closest_greater.ra) + print('2 ',closest_less.dtime) + print('2 ',closest_less.ra) + print('interp -->',closest_ra) + print(crd_gyro.separation(crd_path).arcmin) + break + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/01_load_galcenter_for_artsurvey.py b/srgcat/management/commands/01_load_galcenter_for_artsurvey.py new file mode 100644 index 0000000..f2f7381 --- /dev/null +++ b/srgcat/management/commands/01_load_galcenter_for_artsurvey.py @@ -0,0 +1,90 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import logging +from astropy.io import fits +from django.db.models import Q +from datetime import datetime +from astropy.time import Time +from astropy.io import fits + +from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource +from monthplan.models import SurveyHealpixPlate +from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources +from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir +from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed +from srglib.utils import load_skymap_sources_for_artsurvey + +import glob +import os +import logging + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + logging.getLogger().setLevel(logging.DEBUG) + + #path='/export/django/srg/data/Rodion_GC/18Apr2022' + #load_skymap_sources_for_artsurvey("{}/src_list_0.fits".format(path),'GC.E0',force=True) + #return + + #path='/export/django/srg/data/Andy_L20' + #load_skymap_sources_for_artsurvey("{}/cat_lp20.fits".format(path),'L20.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/lp20_20220307.fits".format(path),'L20.E0',force=True) + + #path='/export/django/srg/data/Galplane_l20' + #load_skymap_sources_for_artsurvey("{}/src_list_0.fits".format(path),'L20rb.E0',force=True) + + #path='/export/django/srg/data/Galplane_l20' + #load_skymap_sources_for_artsurvey("{}/src_list_0.fits".format(path),'L20rb_ml.E0',force=True) + + #path='/srg/a1/work/oper/staff/kate/skymap/catalog_artdays/' + #load_skymap_sources_for_artsurvey("{}/22y_cat_0.fits".format(path),'22y.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/22y_cat_5.fits".format(path),'22y.E5',force=True) + + path='/srg/a1/work/kate/skymap_artdays/catalog/' + load_skymap_sources_for_artsurvey("{}/22y_cat_5.fits".format(path),'22y.E5',force=True) + + + #load_skymap_sources_for_artsurvey("{}/scan_tot_cat.fits".format(path),'L20.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/scan_0_cat.fits".format(path),'L20p1.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/scan_1_cat.fits".format(path),'L20p2.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/scan_2_cat.fits".format(path),'L20p3.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/scan_3_cat.fits".format(path),'L20p4.E0',force=True) + + + force=True + ebands=['0','1','2','3','4'] + #fields=['040','025','020','015','010','005','000','355','350','345'] + #fields=['070','075','080','085','090',] + #fields=['270','275','280','285','290','295','300','305','310','215','320','325'] + #fields=['315'] + fields=['330','335','340'] + path='/srg/a1/work/rodion/SRG/art-xc/data/galmap/field_data/' + """ + for field in fields: + for eband in ebands: + load_skymap_sources_for_artsurvey("{}/{}/1g/cat_1g_{}.fits".format(path,field,eband), + "GP{}.E{}".format(field,eband), + force=force) + """ + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/01_load_skymap_erosita.py b/srgcat/management/commands/01_load_skymap_erosita.py new file mode 100644 index 0000000..9295fa1 --- /dev/null +++ b/srgcat/management/commands/01_load_skymap_erosita.py @@ -0,0 +1,50 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import logging +from astropy.io import fits +from django.db.models import Q +from datetime import datetime +from astropy.time import Time +from astropy.io import fits + +from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource +from monthplan.models import SurveyHealpixPlate +from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources +from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir +from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed +from srglib.utils import load_skymap_sources_erosita + +import glob +import os +import logging +import sys + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + logger = logging.getLogger('django') + logger.setLevel(logging.INFO) + + load_skymap_sources_erosita('/data/artxc/erosita/srg_20200322_000000_000.csv') + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/01_load_skymap_for_artsurvey.py b/srgcat/management/commands/01_load_skymap_for_artsurvey.py new file mode 100644 index 0000000..90e6c2a --- /dev/null +++ b/srgcat/management/commands/01_load_skymap_for_artsurvey.py @@ -0,0 +1,100 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import logging +from astropy.io import fits +from django.db.models import Q +from datetime import datetime +from astropy.time import Time +from astropy.io import fits + +from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource +from monthplan.models import SurveyHealpixPlate +from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources +from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir +from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed +from srglib.utils import load_skymap_sources_for_artsurvey +from srglib.utils import update_skymap_sources_for_artsurvey + +import glob +import os +import logging + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + logging.getLogger().setLevel(logging.DEBUG) + + + path='/srg/work/oper/staff/kate/skymap/catalog' + + """ + load_skymap_sources_for_artsurvey("{}/1y_cat_0.fits".format(path),'S12.E0',force=True) + load_skymap_sources_for_artsurvey("{}/1y_cat_1.fits".format(path),'S12.E1',force=True) + load_skymap_sources_for_artsurvey("{}/1y_cat_2.fits".format(path),'S12.E2',force=True) + load_skymap_sources_for_artsurvey("{}/1y_cat_3.fits".format(path),'S12.E3',force=True) + load_skymap_sources_for_artsurvey("{}/1y_cat_4.fits".format(path),'S12.E4',force=True) + + load_skymap_sources_for_artsurvey("{}/1h_cat_0.fits".format(path),'S1.E0',force=True) + load_skymap_sources_for_artsurvey("{}/1h_cat_1.fits".format(path),'S1.E1',force=True) + load_skymap_sources_for_artsurvey("{}/1h_cat_2.fits".format(path),'S1.E2',force=True) + load_skymap_sources_for_artsurvey("{}/1h_cat_3.fits".format(path),'S1.E3',force=True) + load_skymap_sources_for_artsurvey("{}/1h_cat_4.fits".format(path),'S1.E4',force=True) + + load_skymap_sources_for_artsurvey("{}/2h_cat_0.fits".format(path),'S2.E0',force=True) + load_skymap_sources_for_artsurvey("{}/2h_cat_1.fits".format(path),'S2.E1',force=True) + load_skymap_sources_for_artsurvey("{}/2h_cat_2.fits".format(path),'S2.E2',force=True) + load_skymap_sources_for_artsurvey("{}/2h_cat_3.fits".format(path),'S2.E3',force=True) + load_skymap_sources_for_artsurvey("{}/2h_cat_4.fits".format(path),'S2.E4',force=True) + """ + + """ + update_skymap_sources_for_artsurvey("{}/1y_cat_0.fits".format(path),'S12.E0') + update_skymap_sources_for_artsurvey("{}/1y_cat_1.fits".format(path),'S12.E1') + update_skymap_sources_for_artsurvey("{}/1y_cat_2.fits".format(path),'S12.E2') + update_skymap_sources_for_artsurvey("{}/1y_cat_3.fits".format(path),'S12.E3') + update_skymap_sources_for_artsurvey("{}/1y_cat_4.fits".format(path),'S12.E4') + + update_skymap_sources_for_artsurvey("{}/1h_cat_0.fits".format(path),'S1.E0') + update_skymap_sources_for_artsurvey("{}/1h_cat_1.fits".format(path),'S1.E1') + update_skymap_sources_for_artsurvey("{}/1h_cat_2.fits".format(path),'S1.E2') + update_skymap_sources_for_artsurvey("{}/1h_cat_3.fits".format(path),'S1.E3') + update_skymap_sources_for_artsurvey("{}/1h_cat_4.fits".format(path),'S1.E4') + + update_skymap_sources_for_artsurvey("{}/2h_cat_0.fits".format(path),'S2.E0') + update_skymap_sources_for_artsurvey("{}/2h_cat_1.fits".format(path),'S2.E1') + update_skymap_sources_for_artsurvey("{}/2h_cat_2.fits".format(path),'S2.E2') + update_skymap_sources_for_artsurvey("{}/2h_cat_3.fits".format(path),'S2.E3') + update_skymap_sources_for_artsurvey("{}/2h_cat_4.fits".format(path),'S2.E4') + """ + + #load_skymap_sources_for_artsurvey("{}/15y_cat_0.fits".format(path),'S123.E0',force=True) + + #load_skymap_sources_for_artsurvey("{}/2y_cat_0.fits".format(path),'S1-4.E0',force=True) + + #load_skymap_sources_for_artsurvey("{}/3h_cat_0.fits".format(path),'S3.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/4h_cat_0.fits".format(path),'S4.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/34h_cat_0.fits".format(path),'S34.E0',force=True) + #load_skymap_sources_for_artsurvey("{}/5h_cat_0.fits".format(path),'S5.E0',force=True) + load_skymap_sources_for_artsurvey("{}/22y_cat_0.fits".format(path),'S1-5.E0',force=True) + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/01_load_skymap_sources.py b/srgcat/management/commands/01_load_skymap_sources.py new file mode 100644 index 0000000..e0ebbaf --- /dev/null +++ b/srgcat/management/commands/01_load_skymap_sources.py @@ -0,0 +1,80 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import logging +from astropy.io import fits +from django.db.models import Q +from datetime import datetime +from astropy.time import Time +from astropy.io import fits + +from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource +from monthplan.models import SurveyHealpixPlate +from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources +from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir +from srglib.utils import mark_new_skymap_sources_in_latest, clean_skymap_sources, update_allsky_missed + +import glob +import os +import logging + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + logging.getLogger().setLevel(logging.INFO) + + load_skymap_sources_dir('/srg/work/oper/staff/kate/skymap/catalog') + + update_allsky_missed() + + #print('mark_new_skymap_sources_in_latest') + #mark_new_skymap_sources_in_latest() + #print('update_allsky_missed') + #update_allsky_missed() + + """ + skymap = SkyMaps.objects.get(SMAPNR=230147) + dump=SrgDataDump.objects.get(name='srg_20200310_163523_000') + print(skymap) + srcs=skymap.skymapsource_set.all().filter(dump=dump).order_by('-rate') + clean_skymap_sources(srcs) + """ + + """ + skymap = SkyMaps.objects.get(SMAPNR=273024) + dump=SrgDataDump.objects.get(name='srg_20200311_164642_000') + print(skymap) + srcs=skymap.skymapsource_set.all().filter(dump=dump).order_by('-rate') + for s in srcs: + print(s) + """ + + + + #load_simbad_sources(srcs,minrad=60,maxdist=120) + #clean_skymap_sources(srcs) + + return + + load_skymap_sources_file('/srg/work/oper/staff/kate/skymap/catalog/srg_20200302_165649_000.fits') + """ update_all_skymap_sources_counterparts() """ + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/01_load_skymaps.py b/srgcat/management/commands/01_load_skymaps.py new file mode 100644 index 0000000..e062181 --- /dev/null +++ b/srgcat/management/commands/01_load_skymaps.py @@ -0,0 +1,47 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from astropy.io import fits +from django.db.models import Q +from datetime import datetime +from astropy.time import Time + +from srgcat.models import SkyMaps, SrgDataDump +from monthplan.models import SurveyHealpixPlate +from srglib.utils import TZ_MSK, load_srg_data_dumps +import glob +import os +import logging + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + + logging.getLogger().setLevel(logging.INFO) + + #dumps = SrgDataDump.objects.all() + #dumps.delete() + #return + + load_srg_data_dumps('/srg/work/oper/staff/kate/skymap') + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/01_update_allsky_missed.py b/srgcat/management/commands/01_update_allsky_missed.py new file mode 100644 index 0000000..15d59c6 --- /dev/null +++ b/srgcat/management/commands/01_update_allsky_missed.py @@ -0,0 +1,47 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import logging +from astropy.io import fits +from django.db.models import Q +from datetime import datetime +from astropy.time import Time +from astropy.io import fits + +from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource, SelectAllskyMissed +from monthplan.models import SurveyHealpixPlate +from srglib.utils import TZ_MSK, load_srg_data_dumps, make_source_name, load_simbad_sources +from srglib.utils import find_heasarc_counterparts, update_all_skymap_sources_counterparts, load_skymap_sources_dir +from srglib.utils import update_allsky_missed + +import glob +import os +import logging + + +class Command(BaseCommand): + help = 'Initiates data dase' + + def handle(self, *args, **options): + logging.getLogger().setLevel(logging.INFO) + + update_allsky_missed() + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/run_find_heasarc_counterparts.py b/srgcat/management/commands/run_find_heasarc_counterparts.py new file mode 100644 index 0000000..9b6fb60 --- /dev/null +++ b/srgcat/management/commands/run_find_heasarc_counterparts.py @@ -0,0 +1,47 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase +from srgcat.models import ArtCat, ArtSource, SkyMaps, SkyMapSource +from astropy.table import Table +from astropy_healpix import HEALPix, neighbours +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from srglib.utils import find_heasarc_counterparts + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ + cats = ArtCat.objects.all() + for cat in cats: + cat.source_set = cat.artsource_set + find_heasarc_counterparts(cat) + """ + + skymaps = SkyMaps.objects.all() + for cat in skymaps: + ntot = cat.skymapsource_set.count() + if (ntot): + cat.source_set = cat.skymapsource_set + find_heasarc_counterparts(cat) + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/run_find_simbad_counterparts.py b/srgcat/management/commands/run_find_simbad_counterparts.py new file mode 100644 index 0000000..f9c7e50 --- /dev/null +++ b/srgcat/management/commands/run_find_simbad_counterparts.py @@ -0,0 +1,55 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase +from srgcat.models import ArtCat, ArtSource, SkyMaps, SkyMapSource, SrgDataDump +from astropy.table import Table +from astropy_healpix import HEALPix, neighbours +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from srglib.utils import find_heasarc_counterparts +from srglib.utils import load_simbad_for_skymap_sources + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + """ + cats = ArtCat.objects.all() + for cat in cats: + cat.source_set = cat.artsource_set + find_heasarc_counterparts(cat) + + + skymaps = SkyMaps.objects.all() + for cat in skymaps: + ntot = cat.skymapsource_set.count() + if (ntot): + cat.source_set = cat.skymapsource_set + find_heasarc_counterparts(cat) + """ + try: + dump = SrgDataDump.objects.get(name='srg_20200530_231807_000') + except: + return + srcs = dump.skymapsource_set.all() + load_simbad_for_skymap_sources(srcs,minrad=30,maxdist=120) + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/simple_healpix_match.py b/srgcat/management/commands/simple_healpix_match.py new file mode 100644 index 0000000..531ea3b --- /dev/null +++ b/srgcat/management/commands/simple_healpix_match.py @@ -0,0 +1,75 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix, neighbours +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + + radius_cut=30.0 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + + srcs = ArtSource.objects.all() # get(name__contains="J175524.3-260347") + + for src in srcs: + print("***",src.name,src.healpix) + src.heasarc.clear() + c = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + """ + Add central pixel: + """ + try: + heasarcs = HeasarcBase.objects.filter(healpix=src.healpix)#.distinct('srcid') + except: + print("healpix not found ", src.healpix) + continue + + for heasarc in heasarcs: + #print(heasarc.name) + c0 = SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg") + sep=c.separation(c0) + if(sep.arcsecond < radius_cut): + src.heasarc.add(heasarc) + src.save() + + """ + Add neighbour pixels: + """ + around = neighbours(src.healpix, NSIDE_SOURCES, order=ORDER) + for index in around: + try: + heasarcs = HeasarcBase.objects.filter(healpix=index)#.distinct('srcid') + except: + print("healpix not found ", src.healpix) + continue + for heasarc in heasarcs: + #print(heasarc.name) + c0 = SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg") + sep=c.separation(c0) + if(sep.arcsecond < radius_cut): + src.heasarc.add(heasarc) + src.save() + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/management/commands/testme_artcat.py b/srgcat/management/commands/testme_artcat.py new file mode 100644 index 0000000..a9dbc2f --- /dev/null +++ b/srgcat/management/commands/testme_artcat.py @@ -0,0 +1,63 @@ +from django.core.management.base import BaseCommand, CommandError + +from datetime import date +import datetime +from django.utils import timezone +import astropy +from astropy.io import ascii +import pandas as pd +import pymysql +from sqlalchemy import create_engine + +from heasarc.tdat import tDat +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase +from srgcat.models import ArtCat, ArtSource +from astropy.table import Table +from astropy_healpix import HEALPix, neighbours +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u + +from astropy.config import reload_config +from astropy.config import get_config_dir + +class Command(BaseCommand): + help = 'Initiates data dase' + +# def add_arguments(self, parser): +# parser.add_argument('poll_id', nargs='+', type=int) + + + def handle(self, *args, **options): + + print(get_config_dir()) + return + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + + try: + src = ArtSource.objects.get(name='NRTA J175524.3-260347') + except: + print('Not Found') + return + + print('SRC:',src,src.healpix) + + try: + refs = HeasarcXMMSSC.objects.filter(name__contains="J175525.1-260403") + except: + print("no XMMSSC found") + print("***") + #print("NRTA J175524.3-260347",src.healpix,src.ra, src.dec) + around = neighbours(src.healpix, NSIDE_SOURCES, order=ORDER) + for index in around: + sc=hp.healpix_to_skycoord(index) + print(index,sc.ra.deg, sc.dec.deg) + + for ref in refs: + print("ref ***") + print(ref.name,ref.healpix,src.ra, src.dec) + + + self.stdout.write(self.style.SUCCESS('Done')) diff --git a/srgcat/migrations/0001_initial.py b/srgcat/migrations/0001_initial.py new file mode 100644 index 0000000..8341fd0 --- /dev/null +++ b/srgcat/migrations/0001_initial.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.6 on 2020-01-21 10:21 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('heasarc', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ArtCat', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, default='', max_length=120, unique=True)), + ('e1', models.FloatField(default=0)), + ('e2', models.FloatField(default=0)), + ], + ), + migrations.CreateModel( + name='ArtSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('lii', models.FloatField(default=0.0)), + ('bii', models.FloatField(default=0.0)), + ('radec_error', models.FloatField(default=0.0)), + ('name', models.CharField(blank=True, default='', max_length=36)), + ('sign', models.FloatField(default=0.0)), + ('rate', models.FloatField(default=0.0)), + ('flux', models.FloatField(default=0.0)), + ('catalog', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='srgcat.ArtCat')), + ('heasarc', models.ManyToManyField(to='heasarc.HeasarcBase')), + ], + ), + ] diff --git a/srgcat/migrations/0002_skymaps.py b/srgcat/migrations/0002_skymaps.py new file mode 100644 index 0000000..08c2eaa --- /dev/null +++ b/srgcat/migrations/0002_skymaps.py @@ -0,0 +1,44 @@ +# Generated by Django 2.2.6 on 2020-02-03 14:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='SkyMaps', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('SMAPNR', models.PositiveIntegerField(db_index=True, default=0)), + ('RA_MIN', models.FloatField(default=0.0)), + ('RA_MAX', models.FloatField(default=0.0)), + ('DE_MIN', models.FloatField(default=0.0)), + ('DE_MAX', models.FloatField(default=0.0)), + ('RA_CEN', models.FloatField(default=0.0)), + ('DE_CEN', models.FloatField(default=0.0)), + ('ELON_CEN', models.FloatField(default=0.0)), + ('ELAT_CEN', models.FloatField(default=0.0)), + ('GLON_CEN', models.FloatField(default=0.0)), + ('GLAT_CEN', models.FloatField(default=0.0)), + ('X_MIN', models.FloatField(default=0.0)), + ('Y_MIN', models.FloatField(default=0.0)), + ('OWNER', models.PositiveIntegerField(default=0)), + ('N_NBRS', models.PositiveIntegerField(default=0)), + ('FIELD1', models.PositiveIntegerField(default=0)), + ('FIELD2', models.PositiveIntegerField(default=0)), + ('FIELD3', models.PositiveIntegerField(default=0)), + ('FIELD4', models.PositiveIntegerField(default=0)), + ('FIELD5', models.PositiveIntegerField(default=0)), + ('FIELD6', models.PositiveIntegerField(default=0)), + ('FIELD7', models.PositiveIntegerField(default=0)), + ('FIELD8', models.PositiveIntegerField(default=0)), + ('FIELD9', models.PositiveIntegerField(default=0)), + ('healpix_plate', models.PositiveIntegerField(default=0)), + ], + ), + ] diff --git a/srgcat/migrations/0003_remove_skymaps_healpix_plate.py b/srgcat/migrations/0003_remove_skymaps_healpix_plate.py new file mode 100644 index 0000000..cc85258 --- /dev/null +++ b/srgcat/migrations/0003_remove_skymaps_healpix_plate.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.6 on 2020-02-03 14:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0002_skymaps'), + ] + + operations = [ + migrations.RemoveField( + model_name='skymaps', + name='healpix_plate', + ), + ] diff --git a/srgcat/migrations/0004_skymaps_survey_healpix_plate.py b/srgcat/migrations/0004_skymaps_survey_healpix_plate.py new file mode 100644 index 0000000..233d909 --- /dev/null +++ b/srgcat/migrations/0004_skymaps_survey_healpix_plate.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-02-03 14:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('monthplan', '0002_auto_20191209_2233'), + ('srgcat', '0003_remove_skymaps_healpix_plate'), + ] + + operations = [ + migrations.AddField( + model_name='skymaps', + name='survey_healpix_plate', + field=models.ManyToManyField(to='monthplan.SurveyHealpixPlate'), + ), + ] diff --git a/srgcat/migrations/0005_skymaps_neighbours.py b/srgcat/migrations/0005_skymaps_neighbours.py new file mode 100644 index 0000000..5d97c2a --- /dev/null +++ b/srgcat/migrations/0005_skymaps_neighbours.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-03 15:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0004_skymaps_survey_healpix_plate'), + ] + + operations = [ + migrations.AddField( + model_name='skymaps', + name='neighbours', + field=models.ManyToManyField(blank=True, related_name='_skymaps_neighbours_+', to='srgcat.SkyMaps'), + ), + ] diff --git a/srgcat/migrations/0006_srgdatadump.py b/srgcat/migrations/0006_srgdatadump.py new file mode 100644 index 0000000..3fbdef3 --- /dev/null +++ b/srgcat/migrations/0006_srgdatadump.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.6 on 2020-02-03 16:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0005_skymaps_neighbours'), + ] + + operations = [ + migrations.CreateModel( + name='SrgDataDump', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=23)), + ('date', models.DateTimeField()), + ('start', models.DateTimeField(blank=True, null=True)), + ('stop', models.DateTimeField(blank=True, null=True)), + ('tstart', models.BigIntegerField(default=0)), + ('tstop', models.BigIntegerField(default=0)), + ('mjd_start', models.FloatField(default=0.0)), + ('mjd_stop', models.FloatField(default=0.0)), + ('created', models.DateTimeField(auto_now_add=True)), + ('skymap', models.ManyToManyField(to='srgcat.SkyMaps')), + ], + ), + ] diff --git a/srgcat/migrations/0007_srgdatadump_version.py b/srgcat/migrations/0007_srgdatadump_version.py new file mode 100644 index 0000000..f4085ab --- /dev/null +++ b/srgcat/migrations/0007_srgdatadump_version.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-02-03 19:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0006_srgdatadump'), + ] + + operations = [ + migrations.AddField( + model_name='srgdatadump', + name='version', + field=models.PositiveIntegerField(default=0), + ), + ] diff --git a/srgcat/migrations/0008_skymapsource.py b/srgcat/migrations/0008_skymapsource.py new file mode 100644 index 0000000..c882b05 --- /dev/null +++ b/srgcat/migrations/0008_skymapsource.py @@ -0,0 +1,44 @@ +# Generated by Django 2.2.6 on 2020-02-10 12:23 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('heasarc', '0005_heasarcrass2rxs_detection_likelihood'), + ('srgcat', '0007_srgdatadump_version'), + ] + + operations = [ + migrations.CreateModel( + name='SkyMapSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name_orig', models.CharField(max_length=20)), + ('healpix', models.BigIntegerField(db_index=True, default=0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('lii', models.FloatField(default=0.0)), + ('bii', models.FloatField(default=0.0)), + ('radec_error', models.FloatField(default=0.0)), + ('name', models.CharField(blank=True, default='', max_length=36)), + ('x', models.FloatField(default=0.0)), + ('y', models.FloatField(default=0.0)), + ('cnts', models.FloatField(default=0.0)), + ('cnts_err', models.FloatField(default=0.0)), + ('cnts_bg', models.FloatField(default=0.0)), + ('exptime', models.FloatField(default=0.0)), + ('rate', models.FloatField(default=0.0)), + ('rate_err', models.FloatField(default=0.0)), + ('flux', models.FloatField(default=0.0)), + ('flux_err', models.FloatField(default=0.0)), + ('sig', models.FloatField(default=0.0)), + ('nfalse', models.FloatField(default=0.0)), + ('created', models.DateTimeField(auto_now_add=True)), + ('heasarc', models.ManyToManyField(to='heasarc.HeasarcBase')), + ('skymap', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='srgcat.SkyMaps')), + ], + ), + ] diff --git a/srgcat/migrations/0009_skymapsource_simbad.py b/srgcat/migrations/0009_skymapsource_simbad.py new file mode 100644 index 0000000..4d1d6e1 --- /dev/null +++ b/srgcat/migrations/0009_skymapsource_simbad.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-02-28 09:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0035_auto_20200225_1625'), + ('srgcat', '0008_skymapsource'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='simbad', + field=models.ManyToManyField(to='astrobasis.Simbad'), + ), + ] diff --git a/srgcat/migrations/0010_skymapsource_gaia.py b/srgcat/migrations/0010_skymapsource_gaia.py new file mode 100644 index 0000000..db869f4 --- /dev/null +++ b/srgcat/migrations/0010_skymapsource_gaia.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-03-04 07:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('astrobasis', '0035_auto_20200225_1625'), + ('srgcat', '0009_skymapsource_simbad'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='gaia', + field=models.ManyToManyField(to='astrobasis.GAIADR2'), + ), + ] diff --git a/srgcat/migrations/0011_skymapsource_dump.py b/srgcat/migrations/0011_skymapsource_dump.py new file mode 100644 index 0000000..89599b5 --- /dev/null +++ b/srgcat/migrations/0011_skymapsource_dump.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-03-07 09:12 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0010_skymapsource_gaia'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='dump', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='srgcat.SrgDataDump'), + ), + ] diff --git a/srgcat/migrations/0012_srgdatadump_catalog_loaded.py b/srgcat/migrations/0012_srgdatadump_catalog_loaded.py new file mode 100644 index 0000000..bbe2d8b --- /dev/null +++ b/srgcat/migrations/0012_srgdatadump_catalog_loaded.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-03-07 09:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0011_skymapsource_dump'), + ] + + operations = [ + migrations.AddField( + model_name='srgdatadump', + name='catalog_loaded', + field=models.BooleanField(default=False), + ), + ] diff --git a/srgcat/migrations/0013_skymapsource_new.py b/srgcat/migrations/0013_skymapsource_new.py new file mode 100644 index 0000000..80a83ce --- /dev/null +++ b/srgcat/migrations/0013_skymapsource_new.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-03-07 19:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0012_srgdatadump_catalog_loaded'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='new', + field=models.BooleanField(default=False), + ), + ] diff --git a/srgcat/migrations/0014_skymapsource_bad.py b/srgcat/migrations/0014_skymapsource_bad.py new file mode 100644 index 0000000..6e6fbdf --- /dev/null +++ b/srgcat/migrations/0014_skymapsource_bad.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-03-12 09:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0013_skymapsource_new'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='bad', + field=models.BooleanField(default=False), + ), + ] diff --git a/srgcat/migrations/0015_skymapsource_healpix_plate.py b/srgcat/migrations/0015_skymapsource_healpix_plate.py new file mode 100644 index 0000000..c11f637 --- /dev/null +++ b/srgcat/migrations/0015_skymapsource_healpix_plate.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-03-12 10:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0014_skymapsource_bad'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='healpix_plate', + field=models.IntegerField(db_index=True, default=0), + ), + ] diff --git a/srgcat/migrations/0016_skymapsource_good.py b/srgcat/migrations/0016_skymapsource_good.py new file mode 100644 index 0000000..5a53dbb --- /dev/null +++ b/srgcat/migrations/0016_skymapsource_good.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-03-12 10:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0015_skymapsource_healpix_plate'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='good', + field=models.BooleanField(default=False), + ), + ] diff --git a/srgcat/migrations/0017_selectallskymissed.py b/srgcat/migrations/0017_selectallskymissed.py new file mode 100644 index 0000000..538a751 --- /dev/null +++ b/srgcat/migrations/0017_selectallskymissed.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.6 on 2020-03-18 19:31 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0016_skymapsource_good'), + ] + + operations = [ + migrations.CreateModel( + name='SelectAllskyMissed', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dump', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='srgcat.SrgDataDump')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='srgcat.SkyMapSource')), + ], + ), + ] diff --git a/srgcat/migrations/0018_selection.py b/srgcat/migrations/0018_selection.py new file mode 100644 index 0000000..ba9975b --- /dev/null +++ b/srgcat/migrations/0018_selection.py @@ -0,0 +1,26 @@ +# Generated by Django 2.2.6 on 2020-03-25 13:25 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('srgcat', '0017_selectallskymissed'), + ] + + operations = [ + migrations.CreateModel( + name='Selection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(default='', max_length=36)), + ('public', models.BooleanField(default=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('skymap_sources', models.ManyToManyField(to='srgcat.SkyMapSource')), + ], + ), + ] diff --git a/srgcat/migrations/0019_delete_selection.py b/srgcat/migrations/0019_delete_selection.py new file mode 100644 index 0000000..731edc9 --- /dev/null +++ b/srgcat/migrations/0019_delete_selection.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.6 on 2020-03-25 13:33 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0018_selection'), + ] + + operations = [ + migrations.DeleteModel( + name='Selection', + ), + ] diff --git a/srgcat/migrations/0020_selection.py b/srgcat/migrations/0020_selection.py new file mode 100644 index 0000000..7fb1b79 --- /dev/null +++ b/srgcat/migrations/0020_selection.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.6 on 2020-03-25 13:33 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('srgcat', '0019_delete_selection'), + ] + + operations = [ + migrations.CreateModel( + name='Selection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(default='', max_length=36)), + ('slug', models.SlugField()), + ('public', models.BooleanField(default=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('skymap_sources', models.ManyToManyField(to='srgcat.SkyMapSource')), + ], + ), + ] diff --git a/srgcat/migrations/0021_auto_20200325_2157.py b/srgcat/migrations/0021_auto_20200325_2157.py new file mode 100644 index 0000000..1772cb6 --- /dev/null +++ b/srgcat/migrations/0021_auto_20200325_2157.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-03-25 18:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0020_selection'), + ] + + operations = [ + migrations.AlterField( + model_name='selection', + name='skymap_sources', + field=models.ManyToManyField(blank=True, to='srgcat.SkyMapSource'), + ), + ] diff --git a/srgcat/migrations/0022_skymapsource_catalog.py b/srgcat/migrations/0022_skymapsource_catalog.py new file mode 100644 index 0000000..98dbf34 --- /dev/null +++ b/srgcat/migrations/0022_skymapsource_catalog.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.6 on 2020-03-27 11:32 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0021_auto_20200325_2157'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='catalog', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='srgcat.ArtCat'), + ), + ] diff --git a/srgcat/migrations/0023_erositamatch.py b/srgcat/migrations/0023_erositamatch.py new file mode 100644 index 0000000..2ab6d4d --- /dev/null +++ b/srgcat/migrations/0023_erositamatch.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.6 on 2020-03-27 15:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0022_skymapsource_catalog'), + ] + + operations = [ + migrations.CreateModel( + name='eRositaMatch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('exp', models.FloatField(default=0.0)), + ('sep', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('lkh', models.FloatField(default=0.0)), + ('cts', models.FloatField(default=0.0)), + ('flux', models.FloatField(default=0.0)), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='srgcat.SkyMapSource')), + ], + ), + ] diff --git a/srgcat/migrations/0024_remove_erositamatch_source.py b/srgcat/migrations/0024_remove_erositamatch_source.py new file mode 100644 index 0000000..2e06dd2 --- /dev/null +++ b/srgcat/migrations/0024_remove_erositamatch_source.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.6 on 2020-03-27 15:52 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0023_erositamatch'), + ] + + operations = [ + migrations.RemoveField( + model_name='erositamatch', + name='source', + ), + ] diff --git a/srgcat/migrations/0025_delete_erositamatch.py b/srgcat/migrations/0025_delete_erositamatch.py new file mode 100644 index 0000000..9fbfff0 --- /dev/null +++ b/srgcat/migrations/0025_delete_erositamatch.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.6 on 2020-03-27 15:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0024_remove_erositamatch_source'), + ] + + operations = [ + migrations.DeleteModel( + name='eRositaMatch', + ), + ] diff --git a/srgcat/migrations/0026_erositamatch.py b/srgcat/migrations/0026_erositamatch.py new file mode 100644 index 0000000..1495603 --- /dev/null +++ b/srgcat/migrations/0026_erositamatch.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.6 on 2020-03-27 15:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0025_delete_erositamatch'), + ] + + operations = [ + migrations.CreateModel( + name='eRositaMatch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('exp', models.FloatField(default=0.0)), + ('sep', models.FloatField(default=0.0)), + ('ra', models.FloatField(default=0.0)), + ('dec', models.FloatField(default=0.0)), + ('lkh', models.FloatField(default=0.0)), + ('cts', models.FloatField(default=0.0)), + ('flux', models.FloatField(default=0.0)), + ('source', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='srgcat.SkyMapSource')), + ], + ), + ] diff --git a/srgcat/migrations/0027_usersessionparameters.py b/srgcat/migrations/0027_usersessionparameters.py new file mode 100644 index 0000000..9303f71 --- /dev/null +++ b/srgcat/migrations/0027_usersessionparameters.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.6 on 2020-05-03 11:09 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('srgcat', '0026_erositamatch'), + ] + + operations = [ + migrations.CreateModel( + name='UserSessionParameters', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('snr_threshold', models.FloatField(blank=True, default=0.0)), + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='session', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/srgcat/migrations/0028_auto_20200503_1438.py b/srgcat/migrations/0028_auto_20200503_1438.py new file mode 100644 index 0000000..410b3fe --- /dev/null +++ b/srgcat/migrations/0028_auto_20200503_1438.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-05-03 11:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0027_usersessionparameters'), + ] + + operations = [ + migrations.AlterField( + model_name='usersessionparameters', + name='snr_threshold', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + ] diff --git a/srgcat/migrations/0029_auto_20200505_0947.py b/srgcat/migrations/0029_auto_20200505_0947.py new file mode 100644 index 0000000..f8ca369 --- /dev/null +++ b/srgcat/migrations/0029_auto_20200505_0947.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.6 on 2020-05-05 06:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0028_auto_20200503_1438'), + ] + + operations = [ + migrations.AddField( + model_name='usersessionparameters', + name='target_dec', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + migrations.AddField( + model_name='usersessionparameters', + name='target_ra', + field=models.FloatField(blank=True, default=0.0, null=True), + ), + ] diff --git a/srgcat/migrations/0030_auto_20200505_1338.py b/srgcat/migrations/0030_auto_20200505_1338.py new file mode 100644 index 0000000..33e3bf5 --- /dev/null +++ b/srgcat/migrations/0030_auto_20200505_1338.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-05-05 10:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0029_auto_20200505_0947'), + ] + + operations = [ + migrations.RenameField( + model_name='usersessionparameters', + old_name='snr_threshold', + new_name='sign_threshold', + ), + ] diff --git a/srgcat/migrations/0031_trackstats.py b/srgcat/migrations/0031_trackstats.py new file mode 100644 index 0000000..e871c07 --- /dev/null +++ b/srgcat/migrations/0031_trackstats.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.6 on 2020-05-15 12:53 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('srgcat', '0030_auto_20200505_1338'), + ] + + operations = [ + migrations.CreateModel( + name='TrackStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ip_address', models.GenericIPAddressField()), + ('url_name', models.CharField(blank=True, default='', max_length=50)), + ('path_info', models.CharField(blank=True, default='', max_length=50)), + ('created', models.DateTimeField(auto_now_add=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='trackstats', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/srgcat/migrations/0032_auto_20200515_1706.py b/srgcat/migrations/0032_auto_20200515_1706.py new file mode 100644 index 0000000..02d1a38 --- /dev/null +++ b/srgcat/migrations/0032_auto_20200515_1706.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-05-15 14:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0031_trackstats'), + ] + + operations = [ + migrations.AlterField( + model_name='trackstats', + name='path_info', + field=models.CharField(blank=True, default='', max_length=150), + ), + ] diff --git a/srgcat/migrations/0033_trackstats_group.py b/srgcat/migrations/0033_trackstats_group.py new file mode 100644 index 0000000..21debb5 --- /dev/null +++ b/srgcat/migrations/0033_trackstats_group.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.6 on 2020-05-15 16:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0011_update_proxy_permissions'), + ('srgcat', '0032_auto_20200515_1706'), + ] + + operations = [ + migrations.AddField( + model_name='trackstats', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='trackstats', to='auth.Group'), + ), + ] diff --git a/srgcat/migrations/0034_skymapsource_simbad_failed.py b/srgcat/migrations/0034_skymapsource_simbad_failed.py new file mode 100644 index 0000000..73698c7 --- /dev/null +++ b/srgcat/migrations/0034_skymapsource_simbad_failed.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-05-30 17:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0033_trackstats_group'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='simbad_failed', + field=models.BooleanField(default=False), + ), + ] diff --git a/srgcat/migrations/0035_auto_20200601_2317.py b/srgcat/migrations/0035_auto_20200601_2317.py new file mode 100644 index 0000000..1acdd4e --- /dev/null +++ b/srgcat/migrations/0035_auto_20200601_2317.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-01 20:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0034_skymapsource_simbad_failed'), + ] + + operations = [ + migrations.AlterField( + model_name='skymapsource', + name='simbad_failed', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/srgcat/migrations/0036_skymapsource_simbad_notfound.py b/srgcat/migrations/0036_skymapsource_simbad_notfound.py new file mode 100644 index 0000000..e2d0589 --- /dev/null +++ b/srgcat/migrations/0036_skymapsource_simbad_notfound.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-06-02 06:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0035_auto_20200601_2317'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='simbad_notfound', + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/srgcat/migrations/0037_skymapsource_artsurvey_source.py b/srgcat/migrations/0037_skymapsource_artsurvey_source.py new file mode 100644 index 0000000..1caf2cf --- /dev/null +++ b/srgcat/migrations/0037_skymapsource_artsurvey_source.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.6 on 2020-06-19 15:03 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('artsurvey', '0001_initial'), + ('srgcat', '0036_skymapsource_simbad_notfound'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='artsurvey_source', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='artsurvey.ArtSurveySource'), + ), + ] diff --git a/srgcat/migrations/0038_auto_20210311_1313.py b/srgcat/migrations/0038_auto_20210311_1313.py new file mode 100644 index 0000000..8178cf6 --- /dev/null +++ b/srgcat/migrations/0038_auto_20210311_1313.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-11 10:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0037_skymapsource_artsurvey_source'), + ] + + operations = [ + migrations.AlterField( + model_name='srgdatadump', + name='skymap', + field=models.ManyToManyField(blank=True, to='srgcat.SkyMaps'), + ), + ] diff --git a/srgcat/migrations/0039_skymapsource_ext.py b/srgcat/migrations/0039_skymapsource_ext.py new file mode 100644 index 0000000..efadfb9 --- /dev/null +++ b/srgcat/migrations/0039_skymapsource_ext.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-11 10:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0038_auto_20210311_1313'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='ext', + field=models.FloatField(default=0.0), + ), + ] diff --git a/srgcat/migrations/0040_skymapsource_cnts_tot.py b/srgcat/migrations/0040_skymapsource_cnts_tot.py new file mode 100644 index 0000000..68c9b1b --- /dev/null +++ b/srgcat/migrations/0040_skymapsource_cnts_tot.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-11 10:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0039_skymapsource_ext'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='cnts_tot', + field=models.FloatField(default=0.0), + ), + ] diff --git a/srgcat/migrations/0041_auto_20210311_1338.py b/srgcat/migrations/0041_auto_20210311_1338.py new file mode 100644 index 0000000..a761753 --- /dev/null +++ b/srgcat/migrations/0041_auto_20210311_1338.py @@ -0,0 +1,48 @@ +# Generated by Django 3.0.8 on 2021-03-11 10:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0040_skymapsource_cnts_tot'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='cnts0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_bg0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_er0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_t0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='exptime0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux0', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux_err0', + field=models.FloatField(default=0.0), + ), + ] diff --git a/srgcat/migrations/0042_auto_20210311_1340.py b/srgcat/migrations/0042_auto_20210311_1340.py new file mode 100644 index 0000000..005655d --- /dev/null +++ b/srgcat/migrations/0042_auto_20210311_1340.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-03-11 10:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0041_auto_20210311_1338'), + ] + + operations = [ + migrations.RenameField( + model_name='skymapsource', + old_name='flux_err0', + new_name='flux_er0', + ), + ] diff --git a/srgcat/migrations/0043_auto_20210311_1342.py b/srgcat/migrations/0043_auto_20210311_1342.py new file mode 100644 index 0000000..bc1b92e --- /dev/null +++ b/srgcat/migrations/0043_auto_20210311_1342.py @@ -0,0 +1,153 @@ +# Generated by Django 3.0.8 on 2021-03-11 10:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0042_auto_20210311_1340'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='cnts1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_bg1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_bg2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_bg3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_bg4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_er1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_er2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_er3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_er4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_t1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_t2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_t3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='cnts_t4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='exptime1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='exptime2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='exptime3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='exptime4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux_er1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux_er2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux_er3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux_er4', + field=models.FloatField(default=0.0), + ), + ] diff --git a/srgcat/migrations/0044_auto_20210315_1542.py b/srgcat/migrations/0044_auto_20210315_1542.py new file mode 100644 index 0000000..f3cf07d --- /dev/null +++ b/srgcat/migrations/0044_auto_20210315_1542.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.8 on 2021-03-15 12:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0043_auto_20210311_1342'), + ] + + operations = [ + migrations.AlterField( + model_name='skymapsource', + name='skymap', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='srgcat.SkyMaps'), + ), + ] diff --git a/srgcat/migrations/0045_srgdatadump_gyro_loaded.py b/srgcat/migrations/0045_srgdatadump_gyro_loaded.py new file mode 100644 index 0000000..0101903 --- /dev/null +++ b/srgcat/migrations/0045_srgdatadump_gyro_loaded.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-04-26 14:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0044_auto_20210315_1542'), + ] + + operations = [ + migrations.AddField( + model_name='srgdatadump', + name='gyro_loaded', + field=models.BooleanField(default=False), + ), + ] diff --git a/srgcat/migrations/0046_gyro.py b/srgcat/migrations/0046_gyro.py new file mode 100644 index 0000000..24e2a84 --- /dev/null +++ b/srgcat/migrations/0046_gyro.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.8 on 2021-04-26 14:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0045_srgdatadump_gyro_loaded'), + ] + + operations = [ + migrations.CreateModel( + name='Gyro', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dump', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='srgcat.SrgDataDump')), + ], + ), + ] diff --git a/srgcat/migrations/0047_auto_20210426_1746.py b/srgcat/migrations/0047_auto_20210426_1746.py new file mode 100644 index 0000000..9197f11 --- /dev/null +++ b/srgcat/migrations/0047_auto_20210426_1746.py @@ -0,0 +1,53 @@ +# Generated by Django 3.0.8 on 2021-04-26 14:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0046_gyro'), + ] + + operations = [ + migrations.AddField( + model_name='gyro', + name='dec', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='mjd', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='obt', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='q1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='q2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='q3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='q4', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='ra', + field=models.FloatField(default=0.0), + ), + ] diff --git a/srgcat/migrations/0048_auto_20210426_1805.py b/srgcat/migrations/0048_auto_20210426_1805.py new file mode 100644 index 0000000..568c89a --- /dev/null +++ b/srgcat/migrations/0048_auto_20210426_1805.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.8 on 2021-04-26 15:05 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0047_auto_20210426_1746'), + ] + + operations = [ + migrations.RemoveField( + model_name='gyro', + name='q1', + ), + migrations.RemoveField( + model_name='gyro', + name='q2', + ), + migrations.RemoveField( + model_name='gyro', + name='q3', + ), + migrations.RemoveField( + model_name='gyro', + name='q4', + ), + ] diff --git a/srgcat/migrations/0049_gyro_healpix.py b/srgcat/migrations/0049_gyro_healpix.py new file mode 100644 index 0000000..44e72d1 --- /dev/null +++ b/srgcat/migrations/0049_gyro_healpix.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-04-26 15:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0048_auto_20210426_1805'), + ] + + operations = [ + migrations.AddField( + model_name='gyro', + name='healpix', + field=models.BigIntegerField(db_index=True, default=0), + ), + ] diff --git a/srgcat/migrations/0050_auto_20210426_1836.py b/srgcat/migrations/0050_auto_20210426_1836.py new file mode 100644 index 0000000..3ff86d0 --- /dev/null +++ b/srgcat/migrations/0050_auto_20210426_1836.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.8 on 2021-04-26 15:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0049_gyro_healpix'), + ] + + operations = [ + migrations.AddField( + model_name='gyro', + name='q1', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='q2', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='q3', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='gyro', + name='q4', + field=models.FloatField(default=0.0), + ), + ] diff --git a/srgcat/migrations/0051_skymapsource_ext_id.py b/srgcat/migrations/0051_skymapsource_ext_id.py new file mode 100644 index 0000000..b049f94 --- /dev/null +++ b/srgcat/migrations/0051_skymapsource_ext_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-05-24 09:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0050_auto_20210426_1836'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='ext_id', + field=models.CharField(blank=True, default='', max_length=100), + ), + ] diff --git a/srgcat/migrations/0052_auto_20210615_0831.py b/srgcat/migrations/0052_auto_20210615_0831.py new file mode 100644 index 0000000..a0ad6e4 --- /dev/null +++ b/srgcat/migrations/0052_auto_20210615_0831.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.8 on 2021-06-15 05:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0051_skymapsource_ext_id'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='detlike', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='flux_uplim', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='fluxerr_hi', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='skymapsource', + name='fluxerr_lo', + field=models.FloatField(default=0.0), + ), + ] diff --git a/srgcat/migrations/0053_auto_20210615_0902.py b/srgcat/migrations/0053_auto_20210615_0902.py new file mode 100644 index 0000000..4638305 --- /dev/null +++ b/srgcat/migrations/0053_auto_20210615_0902.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.8 on 2021-06-15 06:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0052_auto_20210615_0831'), + ] + + operations = [ + migrations.RenameField( + model_name='skymapsource', + old_name='fluxerr_hi', + new_name='fluxerr_up', + ), + ] diff --git a/srgcat/migrations/0054_auto_20211229_1259.py b/srgcat/migrations/0054_auto_20211229_1259.py new file mode 100644 index 0000000..9fa4eb8 --- /dev/null +++ b/srgcat/migrations/0054_auto_20211229_1259.py @@ -0,0 +1,68 @@ +# Generated by Django 3.2.10 on 2021-12-29 09:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0053_auto_20210615_0902'), + ] + + operations = [ + migrations.AlterField( + model_name='artcat', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='artsource', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='erositamatch', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='gyro', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='selectallskymissed', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='selection', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='skymaps', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='skymapsource', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='srgdatadump', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='trackstats', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='usersessionparameters', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/srgcat/migrations/0055_remove_skymapsource_gaia.py b/srgcat/migrations/0055_remove_skymapsource_gaia.py new file mode 100644 index 0000000..e139f29 --- /dev/null +++ b/srgcat/migrations/0055_remove_skymapsource_gaia.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.10 on 2022-01-12 09:01 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0054_auto_20211229_1259'), + ] + + operations = [ + migrations.RemoveField( + model_name='skymapsource', + name='gaia', + ), + ] diff --git a/srgcat/migrations/0056_alter_skymapsource_name_orig.py b/srgcat/migrations/0056_alter_skymapsource_name_orig.py new file mode 100644 index 0000000..ebb321e --- /dev/null +++ b/srgcat/migrations/0056_alter_skymapsource_name_orig.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-02 16:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0055_remove_skymapsource_gaia'), + ] + + operations = [ + migrations.AlterField( + model_name='skymapsource', + name='name_orig', + field=models.CharField(max_length=22), + ), + ] diff --git a/srgcat/migrations/0057_auto_20220303_1826.py b/srgcat/migrations/0057_auto_20220303_1826.py new file mode 100644 index 0000000..ec809e4 --- /dev/null +++ b/srgcat/migrations/0057_auto_20220303_1826.py @@ -0,0 +1,263 @@ +# Generated by Django 3.2.12 on 2022-03-03 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0056_alter_skymapsource_name_orig'), + ] + + operations = [ + migrations.AlterField( + model_name='skymapsource', + name='cnts0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_bg', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_bg0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_bg1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_bg2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_bg3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_bg4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_er0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_er1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_er2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_er3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_er4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_t0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_t1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_t2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_t3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_t4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='cnts_tot', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='detlike', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='exptime', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='exptime0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='exptime1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='exptime2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='exptime3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='exptime4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='ext', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux_er0', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux_er1', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux_er2', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux_er3', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux_er4', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='flux_uplim', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='fluxerr_lo', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='fluxerr_up', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='nfalse', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='rate', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='rate_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AlterField( + model_name='skymapsource', + name='sig', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/srgcat/migrations/0058_alter_skymapsource_ext_id.py b/srgcat/migrations/0058_alter_skymapsource_ext_id.py new file mode 100644 index 0000000..e26aa19 --- /dev/null +++ b/srgcat/migrations/0058_alter_skymapsource_ext_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-03 15:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0057_auto_20220303_1826'), + ] + + operations = [ + migrations.AlterField( + model_name='skymapsource', + name='ext_id', + field=models.CharField(blank=True, default='', max_length=100, null=True), + ), + ] diff --git a/srgcat/migrations/0059_auto_20230516_1536.py b/srgcat/migrations/0059_auto_20230516_1536.py new file mode 100644 index 0000000..e4d2ab4 --- /dev/null +++ b/srgcat/migrations/0059_auto_20230516_1536.py @@ -0,0 +1,63 @@ +# Generated by Django 3.2.13 on 2023-05-16 12:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('srgcat', '0058_alter_skymapsource_ext_id'), + ] + + operations = [ + migrations.AddField( + model_name='skymapsource', + name='ml_dec', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_detlike', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_exp', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_flux', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_flux_err', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_nfalse', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_ra', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_radec_err_90', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_radec_err_98', + field=models.FloatField(default=0.0, null=True), + ), + migrations.AddField( + model_name='skymapsource', + name='ml_sig', + field=models.FloatField(default=0.0, null=True), + ), + ] diff --git a/srgcat/migrations/__init__.py b/srgcat/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/srgcat/models.py b/srgcat/models.py new file mode 100644 index 0000000..ea0c9ff --- /dev/null +++ b/srgcat/models.py @@ -0,0 +1,367 @@ +from django.db import models +from django.contrib.auth.models import User +from django.contrib.auth.models import Group + +from heasarc.models import NSIDE_SOURCES, ORDER +from heasarc.models import HeasarcBase, HeasarcSwiftBAT105m, Heasarc4XMMDR9, HeasarcRASS2RXS, HeasarcIntRefCat +import os +import math +from astropy import units as u +from astropy.coordinates import SkyCoord +from astrobasis.models import Simbad +#from astrobasis.models import GAIADR2 +from artsurvey.models import ArtSurveySource + +#from srghub.models import Connection + +# Create your models here. + +#SKYMAP_DIR='/srg/work/oper/staff/kate/skymap' +SKYMAP_DIR='/data/kate/skymap' + +class SkyMaps(models.Model): + """ SkyMaps eRosita plates """ + SMAPNR = models.PositiveIntegerField(db_index=True, default=0) + RA_MIN = models.FloatField(default=0.0) + RA_MAX = models.FloatField(default=0.0) + DE_MIN = models.FloatField(default=0.0) + DE_MAX = models.FloatField(default=0.0) + RA_CEN = models.FloatField(default=0.0) + DE_CEN = models.FloatField(default=0.0) + ELON_CEN = models.FloatField(default=0.0) + ELAT_CEN = models.FloatField(default=0.0) + GLON_CEN = models.FloatField(default=0.0) + GLAT_CEN = models.FloatField(default=0.0) + X_MIN = models.FloatField(default=0.0) + Y_MIN = models.FloatField(default=0.0) + OWNER = models.PositiveIntegerField(default=0) + N_NBRS = models.PositiveIntegerField(default=0) + FIELD1 = models.PositiveIntegerField(default=0) + FIELD2 = models.PositiveIntegerField(default=0) + FIELD3 = models.PositiveIntegerField(default=0) + FIELD4 = models.PositiveIntegerField(default=0) + FIELD5 = models.PositiveIntegerField(default=0) + FIELD6 = models.PositiveIntegerField(default=0) + FIELD7 = models.PositiveIntegerField(default=0) + FIELD8 = models.PositiveIntegerField(default=0) + FIELD9 = models.PositiveIntegerField(default=0) + survey_healpix_plate = models.ManyToManyField('monthplan.SurveyHealpixPlate') + neighbours = models.ManyToManyField("self", blank=True) + def __str__(self): + return "%06d" % (self.SMAPNR) + def get_absolute_url(self): + return "/srgcat/%i/skymap" % (self.pk) + + def get_absolute_path(self): + return "10.5.2.26:{}/{:06d}".format(SKYMAP_DIR, self.SMAPNR) + @property + def is_L3_valid(self): + return os.path.exists(self.get_absolute_path()+'/L3') + + def get_neighbours(self): + return self.neighbours.order_by("DE_CEN") + + +class SrgDataDump(models.Model): + name = models.CharField(max_length=23) + date = models.DateTimeField() + start = models.DateTimeField(blank=True,null=True) + stop = models.DateTimeField(blank=True,null=True) + tstart = models.BigIntegerField(default=0) + tstop = models.BigIntegerField(default=0) + mjd_start = models.FloatField(default=0.0) + mjd_stop = models.FloatField(default=0.0) + skymap = models.ManyToManyField('SkyMaps', blank=True) + catalog_loaded = models.BooleanField(default=False) + gyro_loaded = models.BooleanField(default=False) + version = models.PositiveIntegerField(default=0) + created = models.DateTimeField(auto_now_add=True) + """ Date and time of creation. Fully automatic. """ + def __str__(self): + return self.name + def get_absolute_url(self): + """Class method docstrings go here.""" + return "/srgcat/%i/ddump" % (self.pk) + + def get_update_catalog_url(self): + return "/srgcat/%i/ddump/update" % (self.pk) + + def get_catalog_absolute_path(self): + return "{}/catalog/{}.fits".format(SKYMAP_DIR,self.name) + +class ArtCat(models.Model): + """ ArtCat makes high-level description of ART-XC catalogue. """ + name = models.CharField(max_length=120,blank=True, unique=True, default='') + e1 = models.FloatField(default=0) + e2 = models.FloatField(default=0) + def __str__(self): + return self.name + def get_absolute_url(self): + """Class method docstrings go here.""" + return "/srgcat/erosita/%i/list" % (self.pk) + def get_erosita_matched_url(self): + return "/srgcat/erosita/%i/matched" % (self.pk) + def get_erosita_nomatch_url(self): + return "/srgcat/erosita/%i/nomatch" % (self.pk) + +class Gyro(models.Model): + dump = models.ForeignKey(SrgDataDump, null=True, on_delete=models.CASCADE) + healpix = models.BigIntegerField(db_index=True, default=0) + ra = models.FloatField(default=0.0) + """ RA """ + dec = models.FloatField(default=0.0) + """ Dec """ + q1 = models.FloatField(default=0.0) + """ Quat 1 """ + q2 = models.FloatField(default=0.0) + """ Quat 2 """ + q3 = models.FloatField(default=0.0) + """ Quat 3 """ + q4 = models.FloatField(default=0.0) + """ Quat 4 """ + mjd = models.FloatField(default=0.0) + """ MJD """ + obt = models.FloatField(default=0.0) + def __str__(self): + return self.dump.name + +class SkyMapSource(models.Model): + name_orig = models.CharField(max_length=22) + skymap = models.ForeignKey(SkyMaps, null=True, on_delete=models.CASCADE) + dump = models.ForeignKey(SrgDataDump, null=True, on_delete=models.CASCADE) + artsurvey_source = models.ForeignKey(ArtSurveySource, null=True, on_delete=models.SET_NULL) + catalog = models.ForeignKey(ArtCat, null=True, on_delete=models.CASCADE) + heasarc = models.ManyToManyField(HeasarcBase) + simbad = models.ManyToManyField(Simbad) + simbad_failed = models.BooleanField(default=False, null=True) + """ Simbad failed to respond """ + simbad_notfound = models.BooleanField(default=False, null=True) + """ No astronomical object found """ + #gaia = models.ManyToManyField(GAIADR2) + healpix = models.BigIntegerField(db_index=True, default=0) + healpix_plate = models.IntegerField(db_index=True, default=0) + new = models.BooleanField(default=False) + bad = models.BooleanField(default=False) + good = models.BooleanField(default=False) + """ New in this data dump """ + ra = models.FloatField(default=0.0) + """ Right Ascension of the Source """ + dec = models.FloatField(default=0.0) + """ Declination of the Source """ + lii = models.FloatField(default=0.0) + """ Galactic Longitude """ + bii = models.FloatField(default=0.0) + """ Galactic Latitude """ + radec_error = models.FloatField(default=0.0) + """ Positional Uncertainty (arcsec) + + Must be exactly radec_error to be consistent with EroTransSource and EroSurveySource + """ + name = models.CharField(max_length=36,blank=True, default='') # source name + x = models.FloatField(default=0.0) + y = models.FloatField(default=0.0) + + cnts = models.FloatField(default=0.0) + cnts_err = models.FloatField(default=0.0,null=True) + cnts_bg = models.FloatField(default=0.0,null=True) + cnts_tot = models.FloatField(default=0.0,null=True) + exptime = models.FloatField(default=0.0,null=True) + rate = models.FloatField(default=0.0,null=True) + rate_err = models.FloatField(default=0.0,null=True) + flux = models.FloatField(default=0.0,null=True) + flux_err = models.FloatField(default=0.0,null=True) + """ obsolete """ + fluxerr_lo = models.FloatField(default=0.0,null=True) + fluxerr_up = models.FloatField(default=0.0,null=True) + flux_uplim = models.FloatField(default=0.0,null=True) + """ + 33 CNTS0 1E cnts + 34 CNTS_ER0 1E cnts + 35 CNTS_T0 1E cnts + 36 CNTS_BG0 1E cnts + 37 EXPTIME0 1E s + 38 FLUX0 1E erg/s/cm^2 + 39 FLUX_ER0 1E erg/s/cm^2 + """ + cnts0 = models.FloatField(default=0.0,null=True) + cnts_er0 = models.FloatField(default=0.0,null=True) + cnts_bg0 = models.FloatField(default=0.0,null=True) + cnts_t0 = models.FloatField(default=0.0,null=True) + exptime0 = models.FloatField(default=0.0,null=True) + flux0 = models.FloatField(default=0.0,null=True) + flux_er0 = models.FloatField(default=0.0,null=True) + + cnts1 = models.FloatField(default=0.0,null=True) + cnts_er1 = models.FloatField(default=0.0,null=True) + cnts_bg1 = models.FloatField(default=0.0,null=True) + cnts_t1 = models.FloatField(default=0.0,null=True) + exptime1 = models.FloatField(default=0.0,null=True) + flux1 = models.FloatField(default=0.0,null=True) + flux_er1 = models.FloatField(default=0.0,null=True) + + cnts2 = models.FloatField(default=0.0,null=True) + cnts_er2 = models.FloatField(default=0.0,null=True) + cnts_bg2 = models.FloatField(default=0.0,null=True) + cnts_t2 = models.FloatField(default=0.0,null=True) + exptime2 = models.FloatField(default=0.0,null=True) + flux2 = models.FloatField(default=0.0,null=True) + flux_er2 = models.FloatField(default=0.0,null=True) + + cnts3 = models.FloatField(default=0.0,null=True) + cnts_er3 = models.FloatField(default=0.0,null=True) + cnts_bg3 = models.FloatField(default=0.0,null=True) + cnts_t3 = models.FloatField(default=0.0,null=True) + exptime3 = models.FloatField(default=0.0,null=True) + flux3 = models.FloatField(default=0.0,null=True) + flux_er3 = models.FloatField(default=0.0,null=True) + + cnts4 = models.FloatField(default=0.0,null=True) + cnts_er4 = models.FloatField(default=0.0,null=True) + cnts_bg4 = models.FloatField(default=0.0,null=True) + cnts_t4 = models.FloatField(default=0.0,null=True) + exptime4 = models.FloatField(default=0.0,null=True) + flux4 = models.FloatField(default=0.0,null=True) + flux_er4 = models.FloatField(default=0.0,null=True) + + sig = models.FloatField(default=0.0,null=True) + nfalse = models.FloatField(default=0.0,null=True) + detlike = models.FloatField(default=0.0,null=True) + ext = models.FloatField(default=0.0,null=True) + ext_id = models.CharField(max_length=100,blank=True, default='',null=True) + created = models.DateTimeField(auto_now_add=True) + + ml_sig = models.FloatField(default=0.0,null=True) + ml_detlike = models.FloatField(default=0.0,null=True) + ml_nfalse = models.FloatField(default=0.0,null=True) + ml_ra = models.FloatField(default=0.0,null=True) + ml_dec = models.FloatField(default=0.0,null=True) + ml_radec_err_90 = models.FloatField(default=0.0,null=True) + ml_radec_err_98 = models.FloatField(default=0.0,null=True) + ml_flux = models.FloatField(default=0.0,null=True) + ml_flux_err = models.FloatField(default=0.0,null=True) + ml_exp = models.FloatField(default=0.0,null=True) + + """ Date and time of creation. Fully automatic. """ + def __str__(self): + return self.name + def get_absolute_url(self): + return "/srgcat/%i/skymapsource" % (self.pk) + def get_new_artsurvey_url(self): + return "/artsurvey/%i/new" % (self.pk) + + def get_last_data_dump(self): + dump = self.skymap.srgdatadump_set.latest(field_name='date') + return dump.date + + def get_swiftbat_count(self): + swiftbat = HeasarcBase.objects.instance_of(HeasarcSwiftBAT105m).filter(healpix=self.healpix) + return swiftbat.count() + + def get_4xmm_count(self): + return HeasarcBase.objects.instance_of(Heasarc4XMMDR9).filter(healpix=self.healpix).count() + + def get_rosat_count(self): + return HeasarcBase.objects.instance_of(HeasarcRASS2RXS).filter(healpix=self.healpix).count() + + def get_integral_count(self): + return HeasarcBase.objects.instance_of(HeasarcIntRefCat).filter(healpix=self.healpix).count() + + def get_log10_nfalse(self): + log10 = -100.0 + if(self.nfalse > 0.0): + log10 = math.log10(self.nfalse) + return log10 + + def get_erosita_ratio(self): + ratio=-1.0 + if(hasattr(self, 'erositamatch')): + ratio = float(self.flux/self.erositamatch.flux) + return ratio + +class eRositaMatch(models.Model): + source = models.OneToOneField(SkyMapSource, on_delete=models.CASCADE) + exp = models.FloatField(default=0.0) + sep = models.FloatField(default=0.0) + ra = models.FloatField(default=0.0) + dec = models.FloatField(default=0.0) + lkh = models.FloatField(default=0.0) + cts = models.FloatField(default=0.0) + flux = models.FloatField(default=0.0) + def __str__(self): + return self.source.name + + +class SelectAllskyMissed(models.Model): + dump = models.ForeignKey(SrgDataDump, on_delete=models.CASCADE) + source = models.ForeignKey(SkyMapSource, on_delete=models.CASCADE) + + + +class Selection(models.Model): + owner = models.ForeignKey(User, on_delete=models.CASCADE) + name = models.CharField(max_length=36,blank=False, default='') + slug = models.SlugField() + skymap_sources = models.ManyToManyField(SkyMapSource, blank=True) + public = models.BooleanField(default=True) + + def __str__(self): + return self.name + def get_absolute_url(self): + return "/srgcat/%i/selection" % (self.pk) + def get_edit_url(self): + return "/srgcat/%i/selection/edit" % (self.pk) + def get_delete_url(self): + return "/srgcat/%i/selection/delete" % (self.pk) + def get_download_url(self): + return "/srgcat/%i/selection/download" % (self.pk) + + def is_selected(self, skymapsource): + if self.skymap_sources.filter(pk=skymapsource.pk).exists(): + return True + else: + return False + +class TrackStats(models.Model): + user = models.ForeignKey(User, related_name="trackstats", on_delete=models.CASCADE) + group = models.ForeignKey(Group, related_name="trackstats", null=True, blank=True, on_delete=models.SET_NULL) + ip_address = models.GenericIPAddressField() + url_name = models.CharField(max_length=50, blank=True, default='') + path_info = models.CharField(max_length=150, blank=True, default='') + created = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return self.url_name + +class UserSessionParameters(models.Model): + owner = models.OneToOneField(User, related_name="session", on_delete=models.CASCADE) + sign_threshold = models.FloatField(default=0.0, blank=True, null=True) + target_ra = models.FloatField(default=0.0, blank=True, null=True) + target_dec = models.FloatField(default=0.0, blank=True, null=True) + +class ArtSource(models.Model): + catalog = models.ForeignKey(ArtCat, on_delete=models.CASCADE) + heasarc = models.ManyToManyField(HeasarcBase) + + healpix = models.BigIntegerField(db_index=True, default=0) + ra = models.FloatField(default=0.0) + """ Right Ascension of the Source """ + dec = models.FloatField(default=0.0) + """ Declination of the Source """ + lii = models.FloatField(default=0.0) + """ Galactic Longitude """ + bii = models.FloatField(default=0.0) + """ Galactic Latitude """ + radec_error = models.FloatField(default=0.0) + """ Positional Uncertainty (arcsec) + + Must be exactly radec_error to be consistent with EroTransSource and EroSurveySource + """ + name = models.CharField(max_length=36,blank=True, default='') # source name + sign = models.FloatField(default=0.0) + rate = models.FloatField(default=0.0) + flux = models.FloatField(default=0.0) + def __str__(self): + return "%s" % (self.name) + def get_absolute_url(self): + return "/srgcat/%i/artsource" % (self.pk) + diff --git a/srgcat/templatetags/__init__.py b/srgcat/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/srgcat/templatetags/srgcat_templates.py b/srgcat/templatetags/srgcat_templates.py new file mode 100644 index 0000000..df5a47e --- /dev/null +++ b/srgcat/templatetags/srgcat_templates.py @@ -0,0 +1,10 @@ +from django import template + +register = template.Library() + +@register.simple_tag +def is_skymapsource_in_selection(selection, source): + return selection.is_selected(source) + + + diff --git a/srgcat/tests.py b/srgcat/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/srgcat/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/srgcat/urls.py b/srgcat/urls.py new file mode 100644 index 0000000..fe6cb25 --- /dev/null +++ b/srgcat/urls.py @@ -0,0 +1,66 @@ +from django.urls import path, re_path, register_converter, include + +from . import views +from . import utils + +""" +https://www.webforefront.com/django/accessurlparamstemplates.html +""" +register_converter(utils.FloatConverter, 'float') + +urlpatterns = [ + path('', views.index, name='art-xc survey index'), + path('srga', views.get_name, name='srga'), + path('aladin', views.show_aladin_healpix, name='art-xc aladin healpix'), + path('stats', views.show_stats, name='show stats'), + path('stats//group', views.show_stats_by_group, name='show stats by group'), + path('gaia', views.show_gaia, name='skymap sources gaia'), + path('heasarc', views.show_heasarc, name='skymap sources heasarc'), + path('simbad', views.show_simbad, name='skymap sources simbad'), + path('integral2020', views.show_integral2020, name='skymap sources integral2020'), + path('intrefcat', views.show_intrefcat, name='skymap sources intrefcat'), + path('intrefcat_csv', views.get_intrefcat_csv, name='skymap sources intrefcat csv'), + path('xss', views.show_xss, name='skymap sources xss'), + path('xss_csv', views.get_xss_csv, name='skymap sources xss csv'), + path('csv', views.get_csv, name='skymap sources csv'), + path('3maxi', views.show_3maxi, name='skymap sources 3maxi'), + path('3maxi_csv', views.get_3maxi_csv, name='skymap sources 3maxi csv'), + path('maxi_gsc', views.show_maxi_gsc, name='skymap sources maxi gsc'), + path('maxi_gsc_csv', views.get_maxi_gsc_csv, name='skymap sources maxi gsc csv'), + path('swiftbat105m', views.show_swiftbat105m, name='skymap sources swiftbat105m'), + path('swiftbat105m_csv', views.get_swiftbat105m_csv, name='skymap sources swiftbat105m csv'), + path('rass2rxs', views.show_rass2rxs, name='skymap sources rass2rxs'), + path('rass2rxs_csv', views.get_rass2rxs_csv, name='skymap sources rass2rxs csv'), + path('xmmsl2', views.show_xmmsl2, name='skymap sources xmmsl2'), + path('xmmsl2_csv', views.get_xmmsl2_csv, name='skymap sources xmmsl2 csv'), + path('exclude_allsky', views.show_exclude_allsky, name='skymap sources exclude allsky'), + path('exclude_allsky_csv', views.get_exclude_allsky, name='skymap sources exclude allsky csv'), + path('latest', views.show_latest, name='skymap sources latest'), + path('erosita', views.show_erosita_matched_catalogs, name='skymap sources erosita catalogs'), + path('erosita//list', views.show_erosita_matched_catalog, name='skymap sources erosita catalog list'), + path('erosita//matched', views.show_erosita_matched, name='skymap sources erosita matched'), + path('erosita//nomatch', views.show_erosita_nomatch, name='skymap sources erosita nomatch'), + path('target', views.get_target_survey_post, name='target survey POST'), + path('target_get', views.get_target_survey_get, name='target survey GET'), + path('ddumps', views.show_srg_data_dumps, name='srg data dumps'), + path('/info', views.show_cat, name='show info'), + path('/ddump', views.show_data_dump, name='show data dump'), + path('/ddump/update', views.update_catalog_data_dump, name='update catalog data dump'), + path('create_new_selection', views.create_new_selection, name='create new selection'), + path('session', views.user_session_parameters, name='user session parameters'), + path('edit_all_selections', views.edit_all_selections, name='edit all selections'), + path('/selection', views.show_selection, name='show selection'), + path('/selection/edit', views.edit_selection, name='edit selection'), + path('/selection/delete', views.delete_selection, name='delete selection'), + path('/selection/download', views.download_selection, name='download selection'), + path('/selection//add', views.add_skymapsource_to_selection, name='add to selection'), + path('/selection//delete', views.delete_skymapsource_in_selection, name='delete in selection'), + path('//skymap', views.show_dump_skymap, name='show dump sky map'), + path('/skymap', views.show_skymap, name='show sky map'), + path('/skymap_radec///', views.show_skymap_radec, name='show sky map radec'), + path('/field_radec///', views.show_field_radec, name='show field radec'), + path('/name_skymap', views.show_skymap_by_name, name='show sky map by name'), + path('skymap', views.show_skymap_form, name='show sky map form'), + path('/artsource', views.show_artsource, name='show art source'), + path('/skymapsource', views.show_skymapsource, name='show skymap source'), +] diff --git a/srgcat/utils.py b/srgcat/utils.py new file mode 100644 index 0000000..c9537dd --- /dev/null +++ b/srgcat/utils.py @@ -0,0 +1,45 @@ +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK4, FK5 +from astropy.coordinates import Angle, Latitude, Longitude +import astropy.units as u + +class FloatConverter: + regex = '[-?\d\.\d]+' + + def to_python(self, value): + return float(value) + + def to_url(self, value): + return '{}'.format(value) + + +def make_source_name(key, ra, dec): + """ Makes source name in format JHHMMSS.s+/-DDMMSS based on input RA and Dec. """ + try: + c = SkyCoord(ra, dec, frame=FK5(), unit="deg") + str1 = c.to_string('hmsdms',alwayssign=False,pad=False,precision=1).split() + str2 = c.to_string('hmsdms',alwayssign=True,pad=False,precision=0).split() + name = key+" J%s%s" % (str1[0].replace('h','').replace('m','').replace('s',''), + str2[1].replace('d','').replace('m','').replace('s','')) + return(name) + except: + return('None') + +status_code = {0:"No error", + 1:'SRG arXiv: User is not authenticated', + 2:'SRG arXiv: User has no user profile', + 3:'SRG arXiv: User has no appropriate group to access this page',} + +def srg_group_auth(user, group): + + if not user.is_authenticated: + return 1 + try: + user_profile=user.profile + except: + return 2 + + if not user.groups.filter(name=group).exists(): + return 3 + + return 0 diff --git a/srgcat/views.py b/srgcat/views.py new file mode 100644 index 0000000..b4fa332 --- /dev/null +++ b/srgcat/views.py @@ -0,0 +1,1150 @@ +from django.shortcuts import render +from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden +from django.db.models import DecimalField, Value +from django.db.models import F +from django.db.models import Q +from django.utils.encoding import smart_str +from django.urls import reverse +from django.urls import resolve +from django.contrib.auth.models import Group +from django.shortcuts import get_object_or_404 + +from heasarc.models import HeasarcXrayMaster +from heasarc.models import HeasarcXMMSSC +from heasarc.models import HeasarcBase +from heasarc.models import HeasarcSwiftBAT105m +from heasarc.models import HeasarcIntegral9 +from heasarc.models import HeasarcIntRefCat +from heasarc.models import HeasarcRASS2RXS +from heasarc.models import HeasarcMAXIGSCHGL +from heasarc.models import HeasarcXTEASSCAT +from heasarc.models import Heasarc4XMMDR9 +from heasarc.models import HeasarcXMMSL2 +from heasarc.models import Heasarc3MAXI + +from srgcat.models import ArtCat, ArtSource +from srglib.utils import srg_auth, srg_group_auth, srg_group_auth_stats, status_code +from srgcat.models import SkyMaps, SrgDataDump, SkyMapSource, SelectAllskyMissed +from srgcat.models import Selection, UserSessionParameters +from srgcat.models import TrackStats + +from monthplan.models import Survey +from monthplan.models import SurveyHealpixPlate, NSIDE_PLATES, ORDER, MAPS_DIR, Survey, SurveyPath + +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.coordinates import ICRS, Galactic, FK5 +from astropy_healpix import HEALPix + +from .utils import make_source_name +from .forms import SourceNameForm, LookupTargetForm, SkyMapForm +from .forms import SelectionForm, UserSessionParametersForm + +import csv +from math import isfinite + +# Create your views here. + +def HttpResponseCSV(srcs, dump=None, key='', subclass=None): + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename='+"{}_{}.csv".format(key,dump) + writer = csv.writer(response, csv.excel) + #response.write(u'\ufeff'.encode('utf8')) + + response.write(u'# '.encode('utf8')) + """ CNTS,CNTS_ERR,CNTS_BG,EXPTIME,FLUX,FLUX_ERR,SIG,NFALSE """ + writer.writerow([ + smart_str(u"NAME"), + smart_str(u"FIELD"), + smart_str(u"RA"), + smart_str(u"Dec"), + smart_str(u"CNTS"), + smart_str(u"CNTS_ERR"), + smart_str(u"CNTS_BG"), + smart_str(u"EXPTIME"), + smart_str(u"FLUX"), + smart_str(u"FLUX_ERR"), + smart_str(u"SIG"), + smart_str(u"Simbad(offset, type, redshift, name, ra, dec)"), + smart_str(u"4XMM(offset, name, ra, dec, ep_5_flux)"), + smart_str(subclass.__name__) if subclass else None, + ]) + + for src in srcs: + if(subclass): + subcatalog = src.heasarc.instance_of(subclass).first() + + crd = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + + xmm = src.heasarc.instance_of(Heasarc4XMMDR9) + xmm_name=[] + xmm_separation=[] + xmm_ra=[] + xmm_dec=[] + xmm_ep_5_flux=[] + xmm_ep_5_flux_err=[] + + for xmmsrc in xmm: + xmm_name.append(xmmsrc.name) + xmm_ra.append(xmmsrc.ra) + xmm_dec.append(xmmsrc.dec) + xmm_ep_5_flux.append(xmmsrc.EP_5_FLUX) + xmm_ep_5_flux_err.append(xmmsrc.EP_5_FLUX_ERR) + xmm_separation.append(crd.separation(SkyCoord(xmmsrc.ra, xmmsrc.dec, frame="fk5", unit="deg")).arcsecond) + + xmm_szip = sorted(zip(xmm_separation, xmm_name, xmm_ra, xmm_dec, xmm_ep_5_flux, xmm_ep_5_flux_err)) + + simbad = src.simbad.all() + simbad_name=[] + simbad_type=[] + simbad_redshift=[] + simbad_separation=[] + simbad_ra=[] + simbad_dec=[] + for simsrc in simbad: + simbad_ra.append(simsrc.ra) + simbad_dec.append(simsrc.dec) + simbad_name.append(simsrc.main_id) + simbad_type.append(simsrc.obj_class) + if isfinite(simsrc.z): + simbad_redshift.append(simsrc.z) + else: + simbad_redshift.append(0.0) + simbad_separation.append(crd.separation(SkyCoord(simsrc.ra, simsrc.dec, frame="fk5", unit="deg")).arcsecond) + simbad_szip = sorted(zip(simbad_separation, simbad_type, simbad_redshift, simbad_name, simbad_ra, simbad_dec)) + + writer.writerow([ + smart_str(src.name_orig), + smart_str(src.skymap), + smart_str(src.ra), + smart_str(src.dec), + smart_str(src.cnts), + smart_str(src.cnts_err), + smart_str(src.cnts_bg), + smart_str(src.exptime), + smart_str(src.flux), + smart_str(src.flux_err), + smart_str(src.sig), + smart_str([simbad_name for simbad_name in simbad_szip]), + smart_str([xmm_name for xmm_name in xmm_szip]), + smart_str(subcatalog.name) if subclass else None, + ]) + return response + +def get_skymap_sources(dump, owner): + try: + session = UserSessionParameters.objects.get(owner=owner) + if not session.sign_threshold is None: + skymap_sources = dump.skymapsource_set.all().filter(sig__gt=session.sign_threshold).order_by('ra') + else: + skymap_sources = dump.skymapsource_set.all().order_by('ra') + except UserSessionParameters.DoesNotExist: + skymap_sources = dump.skymapsource_set.all().order_by('ra') + pass + + return skymap_sources + +def show_stats(request): + status=srg_auth(request.user) + if status != 0: + return HttpResponse(status_code[status]) + groups=Group.objects.all() + data = TrackStats.objects.all().order_by('-created')[:100] + return render(request,'srgcat/stats.html', {'data': data,'groups':groups,}) + +def show_stats_by_group(request, group_id): + status=srg_auth(request.user) + if status != 0: + return HttpResponse(status_code[status]) + + try: + group = Group.objects.get(pk=group_id) + except: + return HttpResponse("This group was not found") + groups=Group.objects.all() + data = TrackStats.objects.all().filter(group=group).order_by('-created') + return render(request,'srgcat/stats.html', {'data': data,'groups':groups,}) + +def show_aladin_healpix(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + + + selections = Selection.objects.filter(owner=request.user) + artcats = ArtCat.objects.all() + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #dump = SrgDataDump.objects.latest('date') + + skymap_sources = get_skymap_sources(dump, request.user) + ntotal=skymap_sources.count() + #skymap_sources = SkyMapSource.objects.all().order_by('created') + try: + survey = Survey.objects.get(Q(start__lt=dump.date) & Q(stop__gt=dump.date)) + surveypaths = survey.surveypath_set.all() + except: + survey=None + surveypaths=None + pass + return render(request,'srgcat/healpix.html', {'artcats': artcats, + 'selections':selections, + 'skymap_sources':skymap_sources, + 'ntotal':ntotal, + 'srcs':skymap_sources, + 'dump':dump, + 'survey':survey, + 'surveypaths':surveypaths,}) + +def index(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + print("srgcat index") + + selections = Selection.objects.filter(owner=request.user) + artcats = ArtCat.objects.all() + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #dump = SrgDataDump.objects.latest('date') + + skymap_sources = get_skymap_sources(dump, request.user) + ntotal=skymap_sources.count() + #skymap_sources = SkyMapSource.objects.all().order_by('created') + try: + survey = Survey.objects.get(Q(start__lt=dump.date) & Q(stop__gt=dump.date)) + surveypaths = survey.surveypath_set.all() + except: + survey=None + surveypaths=None + pass + print("srgcat index render") + return render(request,'srgcat/index.html', {'artcats': artcats, + 'selections':selections, + 'skymap_sources':skymap_sources, + 'ntotal':ntotal, + 'srcs':skymap_sources, + 'dump':dump, + 'survey':survey, + 'surveypaths':surveypaths,}) +def show_erosita_matched_catalogs(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + selections = Selection.objects.filter(owner=request.user) + artcats = ArtCat.objects.all() + + return render(request,'srgcat/erosita_matched_catalogs.html', {'artcats': artcats,'selections':selections,}) + +def show_erosita_matched_catalog(request, catalog_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + try: + catalog=ArtCat.objects.get(pk=catalog_id) + except: + return HttpResponse("This catalog was not found") + artcats = ArtCat.objects.all() + skymap_sources=catalog.skymapsource_set.all().order_by("-ra") + + selections = Selection.objects.filter(owner=request.user) + artcats = ArtCat.objects.all() + + return render(request,'srgcat/erosita_matched_catalogs.html', {'artcats': artcats, + 'catalog':catalog, + 'selections':selections, + 'skymap_sources':skymap_sources,}) +def show_erosita_matched(request, catalog_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + try: + catalog=ArtCat.objects.get(pk=catalog_id) + except: + return HttpResponse("This catalog was not found") + artcats = ArtCat.objects.all() + skymap_sources=catalog.skymapsource_set.filter(erositamatch__isnull=False).order_by("-ra") + + selections = Selection.objects.filter(owner=request.user) + artcats = ArtCat.objects.all() + + return render(request,'srgcat/erosita_matched_catalogs.html', {'artcats': artcats, + 'catalog':catalog, + 'selections':selections, + 'skymap_sources':skymap_sources,}) +def show_erosita_nomatch(request, catalog_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + try: + catalog=ArtCat.objects.get(pk=catalog_id) + except: + return HttpResponse("This catalog was not found") + artcats = ArtCat.objects.all() + skymap_sources=catalog.skymapsource_set.filter(erositamatch__isnull=True).order_by("-ra") + + selections = Selection.objects.filter(owner=request.user) + artcats = ArtCat.objects.all() + + return render(request,'srgcat/erosita_matched_catalogs.html', {'artcats': artcats, + 'catalog':catalog, + 'selections':selections, + 'skymap_sources':skymap_sources,}) + +def show_selection(request, selection_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + selections = Selection.objects.filter(owner=request.user) + try: + selection = Selection.objects.get(pk=selection_id) + except: + return HttpResponse("This Selection was not found") + skymap_sources = selection.skymap_sources.all() + ntotal=skymap_sources.count() + + return render(request,'srgcat/index.html', {'selection':selection, + 'ntotal':ntotal, + 'skymap_sources':skymap_sources, + 'selections':selections,}) + +def add_skymapsource_to_selection(request, selection_id, skymapsource_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + selections = Selection.objects.filter(owner=request.user) + try: + selection = Selection.objects.get(pk=selection_id) + except: + return HttpResponse("This Selection was not found") + + try: + src=SkyMapSource.objects.get(pk=skymapsource_id) + except: + return HttpResponse("This SkyMapSource was not found") + + if not selection.skymap_sources.filter(pk=src.pk).exists(): + selection.skymap_sources.add(src) + selection.save() + + skymap_sources = selection.skymap_sources.all() + ntotal=skymap_sources.count() + + return render(request,'srgcat/index.html', {'selection':selection, + 'ntotal':ntotal, + 'skymap_sources':skymap_sources, + 'selections':selections,}) + +def delete_skymapsource_in_selection(request, selection_id, skymapsource_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + selections = Selection.objects.filter(owner=request.user) + try: + selection = Selection.objects.get(pk=selection_id) + except: + return HttpResponse("This Selection was not found") + + try: + src=SkyMapSource.objects.get(pk=skymapsource_id) + except: + return HttpResponse("This SkyMapSource was not found") + + if selection.skymap_sources.filter(pk=src.pk).exists(): + selection.skymap_sources.remove(src) + selection.save() + + skymap_sources = selection.skymap_sources.all() + ntotal=skymap_sources.count() + + return render(request,'srgcat/index.html', {'selection':selection, + 'ntotal':ntotal, + 'skymap_sources':skymap_sources, + 'selections':selections,}) + +def show_gaia(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + artcats = ArtCat.objects.all() + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.exclude(gaia=None) + skymap_sources = get_skymap_sources(dump, request.user).exclude(gaia=None) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + #skymap_sources = SkyMapSource.objects.exclude(gaia=None).order_by('created') + + return render(request,'srgcat/index.html', {'selections':selections, + 'artcats': artcats, + 'skymap_sources':skymap_sources, + 'ntotal':ntotal, + 'srcs':skymap_sources, + 'dump':dump,}) + +def show_heasarc(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.exclude(heasarc=None) + skymap_sources = get_skymap_sources(dump, request.user).exclude(heasarc=None) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + """ skymap_sources = SkyMapSource.objects.exclude(heasarc=None).order_by('created') """ + + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'ntotal':ntotal,'srcs':skymap_sources,'dump':dump,}) + +def show_xss(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcxteasscat__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcxteasscat__isnull=False) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'ntotal':ntotal,'srcs':skymap_sources,'dump':dump,}) +def get_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.all() + + return HttpResponseCSV(skymap_sources, dump, key='iki') + +def get_xss_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcxteasscat__isnull=False) + + return HttpResponseCSV(skymap_sources, dump, key='xss',subclass=HeasarcXTEASSCAT) + +def show_maxi_gsc(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcmaxigschgl__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcmaxigschgl__isnull=False) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'ntotal':ntotal,'srcs':skymap_sources,'dump':dump,}) +def get_maxi_gsc_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcmaxigschgl__isnull=False) + + return HttpResponseCSV(skymap_sources, dump, key='maxi_gsc',subclass=HeasarcMAXIGSCHGL) + +def show_intrefcat(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcintrefcat__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcintrefcat__isnull=False) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'srcs':skymap_sources,'ntotal':ntotal,'dump':dump,}) +def show_integral2020(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcintrefcat__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcintegral2020__isnull=False) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'srcs':skymap_sources,'ntotal':ntotal,'dump':dump,}) +def get_intrefcat_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcintrefcat__isnull=False) + + return HttpResponseCSV(skymap_sources, dump, key='intrefcat', subclass=HeasarcIntRefCat) + +def show_swiftbat105m(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcswiftbat105m__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcswiftbat105m__isnull=False) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'srcs':skymap_sources,'ntotal':ntotal,'dump':dump,}) +def get_swiftbat105m_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcswiftbat105m__isnull=False) + + return HttpResponseCSV(skymap_sources, dump, key='swiftbat105m', subclass=HeasarcSwiftBAT105m) + +def show_rass2rxs(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcrass2rxs__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcrass2rxs__isnull=False) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'srcs':skymap_sources,'ntotal':ntotal,'dump':dump,}) +def show_3maxi(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + cnt = dump.skymapsource_set.count() + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcxmmsl2__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarc3maxi__isnull=False).distinct() + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'srcs':skymap_sources,'ntotal':ntotal,'dump':dump,}) +def get_3maxi_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcxmmsl2__isnull=False).distinct() + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarc3maxi__isnull=False).distinct() + #print("get_xmmsl2_csv: {}".format(skymap_sources.count())) + return HttpResponseCSV(skymap_sources, dump, key='xmmsl2',subclass=Heasarc3MAXI) + +def show_xmmsl2(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + cnt = dump.skymapsource_set.count() + #print("show_xmmsl2: {} total".format(cnt)) + + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcxmmsl2__isnull=False) + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcxmmsl2__isnull=False).distinct() + ntotal=skymap_sources.count() + #print("show_xmmsl2: {}".format(ntotal)) + selections = Selection.objects.filter(owner=request.user) + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'srcs':skymap_sources,'ntotal':ntotal,'dump':dump,}) +def get_xmmsl2_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcxmmsl2__isnull=False).distinct() + skymap_sources = get_skymap_sources(dump, request.user).filter(heasarc__heasarcxmmsl2__isnull=False).distinct() + #print("get_xmmsl2_csv: {}".format(skymap_sources.count())) + return HttpResponseCSV(skymap_sources, dump, key='xmmsl2',subclass=None) + +def get_rass2rxs_csv(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(heasarc__heasarcrass2rxs__isnull=False) + return HttpResponseCSV(skymap_sources, dump, key='rass2rxs',subclass=HeasarcRASS2RXS) + +def show_exclude_allsky(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + + sign_threshold=0.0 + try: + session = UserSessionParameters.objects.get(owner=request.user) + if not session.sign_threshold is None: + sign_threshold = session.sign_threshold + except UserSessionParameters.DoesNotExist: + pass + + select_missed = SelectAllskyMissed.objects.filter(dump=dump) + ntotal=0 + skymap_sources = [] + for s in select_missed: + if s.source.sig >= sign_threshold: + skymap_sources.append(s.source) + ntotal=ntotal+1 + selections = Selection.objects.filter(owner=request.user) + return render(request,'srgcat/index.html', {'selections':selections, + 'skymap_sources':skymap_sources, + 'srcs':skymap_sources,'dump':dump,'ntotal':ntotal}) + +def get_exclude_allsky(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + select_missed = SelectAllskyMissed.objects.filter(dump=dump) + skymap_sources = [] + for s in select_missed: + skymap_sources.append(s.source) + return HttpResponseCSV(skymap_sources, dump, key='exclude_allsky_surveys_') + +def show_simbad(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + artcats = ArtCat.objects.all() + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + #skymap_sources = dump.skymapsource_set.exclude(simbad=None) + skymap_sources = get_skymap_sources(dump, request.user).exclude(simbad=None) + ntotal=skymap_sources.count() + selections = Selection.objects.filter(owner=request.user) + #skymap_sources = SkyMapSource.objects.exclude(simbad=None).order_by('created') + return render(request,'srgcat/index.html', {'selections':selections, + 'artcats': artcats, + 'skymap_sources':skymap_sources,'ntotal':ntotal,'srcs':skymap_sources,'dump':dump,}) + +def show_latest(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + artcats = ArtCat.objects.all() + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymaps = dump.skymap.all() + skymap_sources = dump.skymapsource_set.filter(new=True) + ntotal=skymap_sources.count() + survey = Survey.objects.get(Q(start__lt=dump.date) & Q(stop__gt=dump.date)) + surveypaths = survey.surveypath_set.all() + selections = Selection.objects.filter(owner=request.user) + return render(request,'srgcat/index.html', {'selections':selections, + 'artcats': artcats, + 'skymap_sources':skymap_sources, + 'ntotal':ntotal, + 'srcs':skymap_sources, + 'dump':dump, + 'survey':survey, + 'surveypaths':surveypaths,}) + + +def show_srg_data_dumps(request): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + ddumps = SrgDataDump.objects.all().order_by("-date") + ntotal = ddumps.count() + return render(request,'srgcat/ddumps.html', {'ddumps': ddumps,'ntotal':ntotal,}) + +def show_data_dump(request, dump_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + dump=SrgDataDump.objects.get(pk=dump_id) + except: + return HttpResponse("This data dump was not found") + + try: + survey = Survey.objects.get(Q(start__lt=dump.date) & Q(stop__gt=dump.date)) + surveypaths = survey.surveypath_set.all().order_by('obt') + except: + survey=None + surveypaths=None + pass + skymap_sources = dump.skymapsource_set.all() + return render(request,'srgcat/ddumps.html', {'dump':dump,'skymap_sources':skymap_sources,'surveypaths':surveypaths,}) + +def update_catalog_data_dump(request, dump_id): + status=srg_group_auth_stats(request, "srg-iki-admin") + if status != 0: + return HttpResponse(status_code[status]) + + dump = get_object_or_404(SrgDataDump, pk=dump_id) + dump.catalog_loaded = False + dump.save() + + ddumps = SrgDataDump.objects.all().order_by("-date") + ntotal = ddumps.count() + return render(request,'srgcat/ddumps.html', {'ddumps':ddumps,}) + +def show_skymap(request, skymap_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + skymap=SkyMaps.objects.get(pk=skymap_id) + except: + return HttpResponse("This sky map was not found") + + related_dumps = skymap.srgdatadump_set.all().order_by('-date') + + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(Q(skymap=skymap) & Q(bad=False)) + + skymaps = [] + for plate in skymap.survey_healpix_plate.all(): + skymaps.append(plate.healpix) + swiftbat = HeasarcBase.objects.instance_of(HeasarcSwiftBAT105m).filter(healpix_plate__in=skymaps) + intrefcat = HeasarcBase.objects.instance_of(HeasarcIntRefCat).filter(healpix_plate__in=skymaps) + rosat = HeasarcBase.objects.instance_of(HeasarcRASS2RXS).filter(healpix_plate__in=skymaps) + return render(request,'srgcat/skymap.html', {'intrefcat':intrefcat,'skymaps':skymaps, + 'rosat':rosat,'swiftbat':swiftbat, + 'skymap':skymap,'skymap_sources':skymap_sources, + 'related_dumps':related_dumps,}) + +def show_skymap_radec(request, skymap_id, ra, dec): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + try: + skymap=SkyMaps.objects.get(pk=skymap_id) + except: + return HttpResponse("This sky map was not found") + related_dumps = skymap.srgdatadump_set.all().order_by('-date') + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(Q(skymap=skymap) & Q(bad=False)) + + skymaps = [] + for plate in skymap.survey_healpix_plate.all(): + skymaps.append(plate.healpix) + swiftbat = HeasarcBase.objects.instance_of(HeasarcSwiftBAT105m).filter(healpix_plate__in=skymaps) + intrefcat = HeasarcBase.objects.instance_of(HeasarcIntRefCat).filter(healpix_plate__in=skymaps) + rosat = HeasarcBase.objects.instance_of(HeasarcRASS2RXS).filter(healpix_plate__in=skymaps) + return render(request,'srgcat/skymap.html', {'intrefcat':intrefcat,'skymaps':skymaps, + 'rosat':rosat,'swiftbat':swiftbat, + 'skymap':skymap,'skymap_sources':skymap_sources, + 'related_dumps':related_dumps,'target_ra':ra,'target_dec':dec}) +def show_field_radec(request, field, ra, dec): + """ the same as show_field_radec, but query field number rather than skymap.id """ + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + try: + skymap=SkyMaps.objects.get(SMAPNR=field) + except: + return HttpResponse("This sky map was not found") + related_dumps = skymap.srgdatadump_set.all().order_by('-date') + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + skymap_sources = dump.skymapsource_set.filter(Q(skymap=skymap) & Q(bad=False)) + + skymaps = [] + for plate in skymap.survey_healpix_plate.all(): + skymaps.append(plate.healpix) + swiftbat = HeasarcBase.objects.instance_of(HeasarcSwiftBAT105m).filter(healpix_plate__in=skymaps) + intrefcat = HeasarcBase.objects.instance_of(HeasarcIntRefCat).filter(healpix_plate__in=skymaps) + rosat = HeasarcBase.objects.instance_of(HeasarcRASS2RXS).filter(healpix_plate__in=skymaps) + return render(request,'srgcat/skymap.html', {'intrefcat':intrefcat,'skymaps':skymaps, + 'rosat':rosat,'swiftbat':swiftbat, + 'skymap':skymap,'skymap_sources':skymap_sources, + 'related_dumps':related_dumps,'target_ra':ra,'target_dec':dec}) + +def show_skymap_by_name(request, skymap_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + skymap=SkyMaps.objects.get(SMAPNR=skymap_id) + except: + return HttpResponse("This sky map was not found") + + related_dumps = skymap.srgdatadump_set.all().order_by('-date') + + return render(request,'srgcat/skymap.html', {'skymap':skymap,'related_dumps':related_dumps,}) + + +def show_dump_skymap(request, dump_id, skymap_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + dump=SrgDataDump.objects.get(pk=dump_id) + except: + return HttpResponse("This data dump was not found") + + try: + skymap=SkyMaps.objects.get(pk=skymap_id) + except: + return HttpResponse("This sky map was not found") + + related_dumps = skymap.srgdatadump_set.all().order_by('-date') + try: + survey = Survey.objects.get(Q(start__lt=dump.date) & Q(stop__gt=dump.date)) + except: + survey = None + + skymaps = [] + for plate in skymap.survey_healpix_plate.all(): + skymaps.append(plate.healpix) + + rosat = HeasarcBase.objects.instance_of(HeasarcRASS2RXS).filter(healpix_plate__in=skymaps) + intrefcat = HeasarcBase.objects.instance_of(HeasarcIntRefCat).filter(healpix_plate__in=skymaps) + swiftbat = HeasarcBase.objects.instance_of(HeasarcSwiftBAT105m).filter(healpix_plate__in=skymaps) + + return render(request,'srgcat/skymap.html', {'intrefcat':intrefcat, + 'rosat':rosat,'swiftbat':swiftbat, + 'survey':survey, + 'dump':dump, + 'skymap':skymap, + 'related_dumps':related_dumps,}) + +def download_selection(request, selection_id): + """ Delete new Selection """ + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + selection=Selection.objects.get(pk=selection_id) + except: + return HttpResponse("This selection was not found") + + if not (selection.public or selection.owner == request.user): + return HttpResponse("You are trying to download not your (and private) Selection") + + return HttpResponseCSV(selection.skymap_sources.all(), key="{}".format(selection.slug)) + + +def delete_selection(request, selection_id): + """ Delete new Selection """ + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + selection=Selection.objects.get(pk=selection_id) + except: + return HttpResponse("This selection was not found") + + if(selection.owner != request.user): + return HttpResponse("You are trying to delete not your Selection") + + selection.delete() + + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/edit_all_selections.html', {'selections': selections,}) + +def edit_all_selections(request): + """ Edit all Selection """ + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/edit_all_selections.html', {'selections': selections,}) + +def edit_selection(request, selection_id): + """ Create new Selection """ + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + selection=Selection.objects.get(pk=selection_id) + except: + return HttpResponse("This selection was not found") + + if(selection.owner != request.user): + return HttpResponse("You are trying to edit not your Selection") + + if request.method == 'POST': + form = SelectionForm(request.POST) + if form.is_valid(): + name=form.cleaned_data['name'] + slug=form.cleaned_data['slug'] + public=form.cleaned_data['public'] + if (selection.slug != slug): + """ if user changed slug, check it """ + if(Selection.objects.filter(owner=request.user).filter(slug=slug).exists()): + return render(request, + 'srgcat/selection.html', {'form': form, + 'message':"Short name \"{}\" for user {} is already exists".format(slug,request.user),}) + selection.slug=slug + selection.name=name + selection.public=public + selection.save() + #return HttpResponseRedirect(reverse('art-xc survey index')) + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/edit_all_selections.html', {'selections': selections,}) + else: + form = SelectionForm(instance=selection) + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/selection.html', {'form': form,'selections':selections,}) + +def user_session_parameters(request): + """ Edit user session parameters """ + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + if request.method == 'POST': + form = UserSessionParametersForm(request.POST) + if form.is_valid(): + sign_threshold=form.cleaned_data['sign_threshold'] + target_ra=form.cleaned_data['target_ra'] + target_dec=form.cleaned_data['target_dec'] + + try: + session = UserSessionParameters.objects.get(owner=request.user) + except UserSessionParameters.DoesNotExist: + session=form.save(commit=False) + session.owner=request.user + session.save() + + session.sign_threshold = sign_threshold + session.target_ra = target_ra + session.target_dec = target_dec + session.save() + + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/session.html', {'selections': selections,'form':form,'session':session,}) + else: + #form = UserSessionParametersForm() + pass + + try: + session = UserSessionParameters.objects.get(owner=request.user) + except UserSessionParameters.DoesNotExist: + session = UserSessionParameters() + session.owner=request.user + session.save() + + form = UserSessionParametersForm(instance=session) + + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/session.html', {'form': form,'selections':selections,'session':session,}) + +def create_new_selection(request): + """ Create new Selection """ + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + if request.method == 'POST': + form = SelectionForm(request.POST) + if form.is_valid(): + name=form.cleaned_data['name'] + slug=form.cleaned_data['slug'] + public=form.cleaned_data['public'] + if(Selection.objects.filter(owner=request.user).filter(slug=slug).exists()): + return render(request, + 'srgcat/selection.html', {'form': form, + 'message':"Short name \"{}\" for user {} is already exists".format(slug,request.user),}) + selection=form.save(commit=False) + selection.owner=request.user + selection.save() + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/selection.html', {'selections': selections,}) + else: + form = SelectionForm() + selections = Selection.objects.filter(owner=request.user) + return render(request, 'srgcat/selection.html', {'form': form,'selections':selections,}) + + +def get_name(request): + """ Get source name """ + if request.method == 'POST': + form = SourceNameForm(request.POST) + if form.is_valid(): + ra=float(form.cleaned_data['ra']) + dec=float(form.cleaned_data['dec']) + name=make_source_name('SRG',ra,dec) + return render(request, 'srgcat/source_name.html', {'form': form, 'name':name,}) + else: + form = SourceNameForm() + + return render(request, 'srgcat/source_name.html', {'form': form}) + +def show_skymap_form(request): + """ Get target in survey """ + # if this is a POST request we need to process the form data + if request.method == 'POST': + # create a form instance and populate it with data from the request: + form = SkyMapForm(request.POST) + # check whether it's valid: + if form.is_valid(): + # process the data in form.cleaned_data as required + # ... + # redirect to a new URL: + #return HttpResponseRedirect('/srgcat/srga') + skymap_id=int(form.cleaned_data['skymap']) + try: + skymap=SkyMaps.objects.get(SMAPNR=skymap_id) + except: + return HttpResponse("This sky map was not found") + return render(request, 'srgcat/skymap.html', {'form': form,'skymap':skymap,}) + + # if a GET (or any other method) we'll create a blank form + else: + form = SkyMapForm() + + return render(request, 'srgcat/skymap.html', {'form': form}) + + +def get_target_survey_post(request): + """ Get target in survey """ + # if this is a POST request we need to process the form data + if request.method == 'POST': + # create a form instance and populate it with data from the request: + form = LookupTargetForm(request.POST) + # check whether it's valid: + if form.is_valid(): + # process the data in form.cleaned_data as required + # ... + # redirect to a new URL: + #return HttpResponseRedirect('/srgcat/srga') + ra=float(form.cleaned_data['ra']) + dec=float(form.cleaned_data['dec']) + load_surveypath=form.cleaned_data['load_surveypath'] + #name=make_source_name('SRG',ra,dec) + hp = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5()) + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + heal = hp.skycoord_to_healpix(crd) + plate=SurveyHealpixPlate.objects.get(healpix=heal) + surveys = plate.survey.all() + skymaps = plate.skymaps_set.all().annotate(separation=Value(0.0, output_field=DecimalField())) + c = SkyCoord(ra, dec, frame="fk5", unit="deg") + for smap in skymaps: + c0 = SkyCoord(smap.RA_CEN, smap.DE_CEN, frame="fk5", unit="deg") + smap.separation=c.separation(c0).arcmin + return render(request, 'srgcat/target_survey.html', {'form': form, 'ra_cen':ra, 'dec_cen':dec, + 'load_surveypath':load_surveypath, + 'plate':plate,'surveys':surveys,'skymaps':skymaps,}) + + # if a GET (or any other method) we'll create a blank form + else: + form = LookupTargetForm() + + return render(request, 'srgcat/target_survey.html', {'form': form}) + +def get_target_survey_get(request): + """ Get target in survey """ + # if this is a GET request we need to process the form data + if request.method == 'GET': + # create a form instance and populate it with data from the request: + form = LookupTargetForm(request.GET) + # check whether it's valid: + if form.is_valid(): + # process the data in form.cleaned_data as required + # ... + # redirect to a new URL: + #return HttpResponseRedirect('/srgcat/srga') + ra=float(form.cleaned_data['ra']) + dec=float(form.cleaned_data['dec']) + load_surveypath=form.cleaned_data['load_surveypath'] + name=make_source_name('SRG',ra,dec) + hp = HEALPix(nside=NSIDE_PLATES, order=ORDER, frame=FK5()) + crd = SkyCoord(ra, dec, frame="fk5", unit="deg") + heal = hp.skycoord_to_healpix(crd) + plate=SurveyHealpixPlate.objects.get(healpix=heal) + surveys = plate.survey.all() + return render(request, 'srgcat/target_survey.html', {'form': form, + 'ra_cen':ra, + 'dec_cen':dec, + 'load_surveypath':load_surveypath, + 'name':name, + 'plate':plate, + 'surveys':surveys}) + + # if a GET (or any other method) we'll create a blank form + else: + form = LookupTargetForm() + + return render(request, 'srgcat/target_survey.html', {'form': form}) + + +def show_cat(request, cat_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + cat=ArtCat.objects.get(pk=cat_id) + except: + return HttpResponse("This catalog was not found") + artcats = ArtCat.objects.all() + srcs=cat.artsource_set.all().order_by("-sign") + return render(request,'srgcat/index.html', {'artcats': artcats,'cat':cat,'srcs':srcs,}) + +def show_artsource(request, artsource_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + src=ArtSource.objects.get(pk=artsource_id) + except: + return HttpResponse("This ArtSource was not found") + artcats = ArtCat.objects.all() + cat = src.catalog + heasarcs = src.heasarc.all().annotate(separation=Value(0.0, output_field=DecimalField())) + c = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + for heasarc in heasarcs: + c0 = SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg") + sep=c.separation(c0) + heasarc.separation=sep.arcsecond + return render(request,'srgcat/index.html', {'artcats': artcats,'cat':cat,'src':src,'heasarcs':heasarcs,}) + + +def show_skymapsource(request, skymapsource_id): + status=srg_group_auth_stats(request, "srg-artxc-survey") + if status != 0: + return HttpResponse(status_code[status]) + + try: + src=SkyMapSource.objects.get(pk=skymapsource_id) + except: + return HttpResponse("This SkyMapSource was not found") + + heasarcs = src.heasarc.all().annotate(separation=Value(0.0, output_field=DecimalField())) + c = SkyCoord(src.ra, src.dec, frame="fk5", unit="deg") + for heasarc in heasarcs: + heasarc.separation=c.separation(SkyCoord(heasarc.ra, heasarc.dec, frame="fk5", unit="deg")).arcsecond + + """ + gaia = src.gaia.all().annotate(separation=Value(0.0, output_field=DecimalField())) + for gaiasrc in gaia: + gaiasrc.separation=c.separation(SkyCoord(gaiasrc.ra, gaiasrc.dec, frame="fk5", unit="deg")).arcsecond + """ + + simbad = src.simbad.all().annotate(separation=Value(0.0, output_field=DecimalField())) + for simsrc in simbad: + simsrc.separation=c.separation(SkyCoord(simsrc.ra, simsrc.dec, frame="fk5", unit="deg")).arcsecond + + return render(request,'srgcat/index.html', {'src':src,'cat':src.skymap, + 'heasarcs':heasarcs, + 'simbad':simbad,}) + diff --git a/srglib/__init__.py b/srglib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/srglib/admin.py b/srglib/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/srglib/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/srglib/apps.py b/srglib/apps.py new file mode 100644 index 0000000..03fa9eb --- /dev/null +++ b/srglib/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class SrglibConfig(AppConfig): + name = 'srglib' diff --git a/srglib/check_transient.py b/srglib/check_transient.py new file mode 100644 index 0000000..c376da7 --- /dev/null +++ b/srglib/check_transient.py @@ -0,0 +1,92 @@ +import srglib.correlate_utils as correlate_utils +from heasarc.models import HeasarcTable, TableColumn, HeasarcObjectClass, NSIDE_SOURCES, ORDER, \ + HeasarcXrayMaster, HeasarcXMMSSC, HeasarcBase, HeasarcCSC, HeasarcRASS2RXS, Heasarc2SXPS +from django.contrib.contenttypes.models import ContentType +import numpy as np + +polctypes = {cat: ContentType.objects.get_for_model(model) for cat, model in + zip(["rxs2", "sxps2", "3xmm", "csc2"], [HeasarcRASS2RXS, Heasarc2SXPS, HeasarcXMMSSC, HeasarcCSC])} + +def check_rass_transient(erositasource): + """ + for provided erotranssource checks wether it has rosat counterparts in the RXS2 catalogue, + it it has, checks, whether the source is transient relative to all counterparts, if it does not, + check wether the source is expected to be 10 times brighter then detection limit for rass exposure + + as a result of the check switches roast_transient field of the source + + Parameters: + -------- + erositasource: django.queryset + a queryset of erosita sources + """ + correlate_utils.load_rassemap() + rasssources = erositasource.heasarc.filter(polymorphic_ctype=polctypes["rxs2"]) + if rasssources.count(): + rassfluxes = np.array([rass.count_rate for rass in rasssources]) + if np.any(erositasource.ML_FLUX_0 < rassfluxes*correlate_utils.FSCALES["2rxs"]/10.) or \ + np.any(erositasource.ML_FLUX_0 > rassfluxes*correlate_utils.FSCALES["2rxs"]*10.): + erositasource.rosat_transient = True + erositasource.has_rosat=True + else: + erositasource.rosat_transient = correlate_utils.check_rss_cts_individual(erositasource) + +def check_sxps_transients(erositasource): + """ + for provided erotranssource checks wether it has swift counterparts in the sxps2 catalog, + it it has, checks, whether the source is transient relative to all counterparts + + as a result of the check switches transient field of the source to False, if the erosita not a transient to at least one counterpart + + Parameters: + -------- + erositasource: django.queryset + a queryset of erosita sources + """ + sxpscounterparts = erositasource.heasarc.filter(polymorphic_ctype=polctypes["sxps2"]) + if sxpscounterparts.exists(): + flux = np.array([s.PeakRate_band1 + s.PeakRate_band2 for s in sxpscounterparts]) + transient = np.all(flux*10*correlate_utils.FSCALES["sxps2"] < erositasource.ML_FLUX_0) + print("sxps flux", flux, "eflux", erositasource.ML_FLUX_0*0.9e-12, "sxps transient", transient) + erositasource.transient = erositasource.transient & transient + +def check_xmm3_transients(erositasource): + """ + for provided erotranssource checks wether it has XMM-Newton counterparts in the 3xmm catalog, + it it has, checks, whether the source is transient relative to all counterparts + + as a result of the check switches transient field of the source to False, if the erosita not a transient to at least one counterpart + + Parameters: + -------- + erositasource: django.queryset + a queryset of erosita sources + """ + xmmcounterparts = erositasource.heasarc.filter(polymorphic_ctype=polctypes["3xmm"]) + if xmmcounterparts.exists(): + flux = np.array([s.ep_1_flux + s.ep_3_flux + s.ep_2_flux for s in xmmcounterparts]) + transient = np.all(flux*10*correlate_utils.FSCALES["3XMM"] < erositasource.ML_FLUX_0) + print("3xmm flux", flux, "eflux", erositasource.ML_FLUX_0*0.9e-12, "3xmm transient", transient) + erositasource.transient = erositasource.transient & transient + +def check_csc_transients(erositasource): + """ + for provided erotranssource checks wether it has chandra counterparts in the csc2 catalog, + it it has, checks, whether the source is transient relative to all counterparts + + as a result of the check switches transient field of the source to False, if the erosita not a transient to at least one counterpart + + Parameters: + -------- + erositasource: django.queryset + a queryset of erosita sources + """ + chancounterparts = erositasource.heasarc.filter(polymorphic_ctype=polctypes["csc2"]) + if chancounterparts.exists(): + flux = np.array([s.s_photflux_ap + s.m_photflux_ap for s in chancounterparts]) + transient = np.all(flux*10*correlate_utils.FSCALES["csc2"] < erositasource.ML_FLUX_0) + print("csc2 flux", flux, "eflux", erositasource.ML_FLUX_0*0.9e-12, "csc2 transient", transient) + erositasource.transient = erositasource.transient & transient + + +search_transients = {"sxps2": check_sxps_transients, "xmm3": check_xmm3_transients, "csc2": check_csc_transients} diff --git a/srglib/correlate_utils.py b/srglib/correlate_utils.py new file mode 100644 index 0000000..e90a831 --- /dev/null +++ b/srglib/correlate_utils.py @@ -0,0 +1,536 @@ +import pandas +from scipy.spatial import cKDTree +import numpy as np +import pickle +from astropy.io import fits +from astropy.table import Table +from math import sin, cos, pi, sqrt, log +from scipy.sparse import coo_matrix +from scipy.stats import norm as normdist +import healpy +import sys, os + + +""" +a note on who rosat-erosita scaling was obtained + +significant amount of rosat sources in rxs2 catalog have accumulated count below 10 photons, +therefore poisson statisitcs has to be applied for fitting. +first 5000 erosita sources were crossmatched with 2rxs, 1365 counterparts were found +along which 744 were one-to-one counterparts +we cat that to 596 sources, by excluding 0-0.1 and 0.9-1.0 quantiles of rxs2.flux/erosita.flux distribution +(we assume that those quantiles contain significant fraction of transients and small fraction of statistical outliers. + +for the rest of the sources we minimized lkl function L = -ln \prod_i \int N(x, eflux_i, esigma_i)*Poisson(rasscounts_i, x*rassexposure_i) dx +thus coefficient erosita.flux = 5.73*rxs2.countrate was obtained +""" + +FSCALES = {"3XMM": 1.24e+12, "2rxs": 5.74, "sxps2": 16.73, "csc2": 2.3e3} +CFLUX = {"3XMM": lambda x: (x.EP_1_FLUX + x.EP_2_FLUX + x.EP_3_FLUX).values, "2rxs": lambda x: x.RATE.values, "sxps2": lambda x: (x.Rate_band0 + x.Rate_band1 + x.Rate_band2).values} +EFLUX = {"3XMM": lambda x: np.sqrt(x.EP_1_FLUX_ERR**2. + x.EP_2_FLUX_ERR**2. + x.EP_3_FLUX_ERR**2.).values, "2rxs": lambda x: x.ERATE.values, "sxps2": lambda x: (x.Rate_band0_pos + x.Rate_band1_pos + x.Rate_band2_pos).values} +CATALOGUES = {"3XMM": "xmm3_2.pickle", "2rxs": "rxs2table.pickle", "sxps2": "sxps2.pickle", } +CATCOLORS = {"3XMM": "r", "2rxs": "g", "sxps2": "b", "csc2": "m"} +RASSEMAP = None + +def load_rassemap(): + global RASSEMAP + if RASSEMAP is None: + RASSEMAP = healpy.read_map("/data/rosat/RASS_EXP_1_2048.fits") + +def vectopol(cat1): + """ + from the catalouge in the form of pandas table, which has RA and DEC columns with values in degrees + produces 1 length vectors in cartesin space + Parameters: + catalogue with RA and DEC columns in degrees + + Returns: + vector of shape Nx3 where N - length of the catalogue, each column contains X, Y and Z cartesian coordinates + """ + vec = np.empty((cat1.index.size, 3), np.double) + vec[:, 0] = np.cos(cat1.dec.values*pi/180.)*np.sin(cat1.ra.values*pi/180.) + vec[:, 1] = np.cos(cat1.dec.values*pi/180.)*np.cos(cat1.ra.values*pi/180.) + vec[:, 2] = np.sin(cat1.dec.values*pi/180.) + return vec + + +def prepare_ecat(ecat): + ecat["error_radius"] = ecat.RADEC_ERR + ecat["fllux"] = ecat.ML_FLUX_0 + return makehealpixindex(ecat) + + +def pairwise_separation(cat1, cat2): + """ + with two pandas catalogues provided, produces one to one distances in arcseconds based ont the RA, DEC coorindates of each entry + """ + vec1 = vectopol(cat1) + vec2 = vectopol(cat2) + return np.arccos(np.minimum(np.sum(vec1*vec2, axis=1), 1))*180.*3600./pi + + +def makedistmat3d(cat1, cat2, angularseparation=10.): + """ + produces distance matrix between two catalogues, each must have ra and dec columns + + Parameters: + cat1 and cat2: two catalogues in the form of pandas table, which have RA and DEC columns in degrees + angularseparation: maximum accepted angular separation between objects in catalogues + + Returns: + distance sparce matrix of the separation lower then angularseparation between two input catalgoues + """ + + vec1 = vectopol(cat1) + vec2 = vectopol(cat2) + dd = 2.*sin(angularseparation*pi/180./3600./2.) + c1 = cKDTree(vec1) + c2 = cKDTree(vec2) + dmat = c1.sparse_distance_matrix(c2, dd).tocoo() + dmat.data = 2.*np.arcsin(dmat.data/2.)*180./pi*3600. #distance in arcseconds + sidx = np.argsort(dmat.row) + return coo_matrix((dmat.data[sidx], (dmat.row[sidx], dmat.col[sidx])), dmat.shape) + +def get_closest_unique(cat1, cat2, angularseparation=5): + dmat = makedistmat3d(cat1, cat2, angularseparation) + + idxd = np.argsort(dmat.data) + row = np.copy(dmat.row[idxd]) + col = np.copy(dmat.col[idxd]) + urow, uidx = np.unique(row, return_index=True) + ucol, uuidx, ucts = np.unique(col[uidx], return_counts=True, return_index=True) + masku = np.zeros(urow.size, np.bool) + masku[uuidx] = True + urow, ucol = urow[masku], col[uidx][masku] #one to one closest neighbours + return urow, ucol + + +def select_closest_pairs(dist, row, col): + sidx = np.argsort(dist) + dist = dist[sidx] + row = row[sidx] + col = col[sidx] + ur, uc = [], [] + while True: + urow, uidx = np.unique(row, return_index=True) + ucol, uuidx, ucts = np.unique(col[uidx], return_counts=True, return_index=True) + masku = np.zeros(urow.size, np.bool) + masku[uuidx] = True + urow, ucol = urow[masku], col[uidx][masku] #one to one closest neighbours + ur.append(urow) + uc.append(ucol) + maskrow = np.isin(row, urow, invert=True) + maskcol = np.isin(col, ucol, invert=True) + mask = maskrow & maskcol + if not np.any(mask): + break + row, col, dist = row[mask], col[mask], dist[mask] + return np.concatenate(ur).astype(np.int), np.concatenate(uc).astype(np.int) + + +def get_all_closest_unique(cat1, cat2, asep): + dmat = makedistmat3d(cat1.iloc[idx1], cat2.iloc[idx2], asep) + return select_closest_pairs(dmat.data, dmat.row, dmat.col) + + +def get_closest(scat, ccat): + ccat.error_radius.fillna(0., inplace=True) + dmat = makedistmat3d(scat, ccat, 1200) + sidx = np.argsort(dmat.data) + urow, uidx = np.unique(dmat.row[sidx], return_index=True) + ucol = dmat.col[sidx][uidx] + return urow, ucol + + +def cutfaraway(cat1, cat2, dmat, minrad=30, exclrad=0.): + """ + given distance matrix and two catalogues, cut all the elements of matrixes wich are + greater then triple common distance error for corresponding catalogues elements + catalogues should have error_radius column in it for this function to work + """ + mask = (dmat.data < np.maximum(np.sqrt(cat1.iloc[dmat.row].error_radius.values**2. + cat2.iloc[dmat.col].error_radius.values**2.)*3, minrad)) & (dmat.data > exclrad) + dmat = coo_matrix((dmat.data[mask], (dmat.row[mask], dmat.col[mask])), dmat.shape) + return dmat + +def makeposerrdist(cat1, cat2, maxdist=None, minrad=30, **kwargs): + """ + assuming a 2d gaussian probability distribution of the true sourcce position relative to catalogue positions + computes probaility of a common + """ + if maxdist is None: + maxdist = max(min(sqrt(cat1.error_radius.dropna().values.max()**2. + cat2.error_radius.dropna().values.max()**2.)*3, 1200), minrad) #search radius not more then 20' + print("used maxdist", maxdist) + dmat = makedistmat3d(cat1, cat2, maxdist) + return cutfaraway(cat1, cat2, dmat, minrad, **kwargs) + +def dmattodistprobmat(cat1, cat2, dmat): + pmat = coo_matrix((normdist.pdf(dmat.data/np.sqrt(cat1.iloc[dmat.row].error_radius.values**2. + cat2.iloc[dmat.col].error_radius.values**2.)), (dmat.row, dmat.col)), dmat.shape) + return pmat + +def makehealpixindex(cat, nside=1024, overwrite=False): + """ + produces healpix index for catalogue + + parameteres: + cat [mandatory]: a catalogue containing RA and DEC columns in degrees + nside : nside for healpix + overwrite[True]: overwrite healpix indexation if exists + """ + if not overwrite and isinstance(cat.index, pandas.MultiIndex) and "healpix" in cat.index.names: + print("already has healpix indexing") + return cat + cat.reset_index(drop=True, inplace=True) + hpix = healpy.ang2pix(nside, cat.RA.values, cat.DEC.values, lonlat=True) + cat.set_index(hpix, inplace=True) + cat.sort_index(inplace=True) + hpix = np.copy(cat.index.values) + cat.set_index(pandas.MultiIndex.from_arrays( + [hpix, np.arange(cat.index.size)], + sortorder=0, names=["healpix", "Num"]), inplace=True) + cat.sort_index(inplace=True) + return cat + +def merge_coo_matrixes(dmatlist): + """ + assuming that the are several fraction of sparse matrix produces common sparse distance matrix + + """ + rows = np.concatenate([d.row for d in dmatlist]) + cols = np.concatenate([d.col for d in dmatlist]) + data = np.concatenate([d.data for d in dmatlist]) + return coo_matrix((data, (rows, cols)), shape=np.max([d.shape for d in dmatlist], axis=0)) + +def distwithhealpix(cat1, cat2, nside=1024, maxarea=20, maxcsize=100000, **kwargs): + """ + find counterparts of cat1 relative to cat2 using healpixes + """ + print("kwargs", kwargs) + hpix = np.unique(cat1.index.get_level_values("healpix").values) + hpix2 = np.unique(cat2.index.get_level_values("healpix").values) + invert = False + + if hpix2.size < hpix.size: + cat1, cat2 = cat2, cat1 + invert = True + hpix = hpix2 + + print("run distwithhealpix, unique pix size in first catalogue", hpix.size) + catarea = hpix.size*healpy.nside2pixarea(nside) + print("catalogue surface area", catarea) + if catarea > maxarea or hpix.size > maxcsize: + print("catalogues area is gt then maximal allowed") + chank = int(max(catarea//maxarea + 1, hpix.size//maxcsize + 1)) + print("spit on %d parts" % chank) + step = (hpix.size + (chank - 1))//chank + dmats = [] + for i in range(chank): + #print("start %d chank, from %d pices" % (i, chank)) + dmats.append(distwithhealpix(cat1.loc[hpix[i:min(i + step, hpix.size)]], cat2, nside, maxarea + 1, **kwargs)) + return merge_coo_matrixes(dmats) + hpix = np.unique(np.concatenate([hpix, healpy.get_all_neighbours(256, hpix).ravel()])) + catsm = cat2.loc[hpix] + dmat = makeposerrdist(cat1, catsm, **kwargs) + rows = cat1.index.get_level_values("Num").values[dmat.row] + cols = catsm.index.get_level_values("Num").values[dmat.col] + + if invert: + rows, cols = cols, rows + + sidx = np.argsort(rows) + + dmat = coo_matrix((dmat.data[sidx], (rows[sidx], cols[sidx])), shape = (rows.max() + 1, cols.max() + 1)) + return dmat + +def get_brightest_counterpart(cat1, cat2, dmat, axis=0): + if axis == 0: + fluxcat2 = coo_matrix((cat2.flux.values[dmat.col], (dmat.row, dmat.col)), dmat.shape) + urow, idx1 = np.unique(dmat.row, return_index=True) + fmcol = np.asarray(np.argmax(fluxcat2, axis=1)[urow]).reshape(-1) + return urow, fmcol + if axis == 1: + fluxcat1 = coo_matrix((cat1.flux.values[dmat.row], (dmat.row, dmat.col)), dmat.shape) + ucol = np.unique(dmat.col) + fmrow = np.asarray(np.argmax(fluxcat1, axis=0)[:, ucol]).reshape(-1) + return ucol, fmrow + +def get_brighter_counterpart(cat1, cat2, dmat): + fluxcat1 = coo_matrix((cat1.flux.values[dmat.row], (dmat.row, dmat.col)), dmat.shape) + fluxcat2 = coo_matrix((cat2.flux.values[dmat.col], (dmat.row, dmat.col)), dmat.shape) + urow = np.unique(dmat.row) + fmcol = np.unique(np.asarray(np.argmax(fluxcat1, axis=1)[urow]).reshape(-1)) + fmrow = np.asarray(np.argmax(fluxcat2, axis=0)[:, fmcol]).reshape(-1) + return fmrow, fmcol + +def get_closest_counterpart(dmat, axis=0): + dmati = coo_matrix((1./np.maximum(dmat.data, 1e-9), (dmat.row, dmat.col)), dmat.shape) + if axis==0: + fmrow = np.unique(dmat.row) + fmcol = np.asarray(np.argmax(dmati, axis=1)[fmrow]).reshape(-1) + if axis==1: + fmcol = np.unique(dmat.col) + fmrow = np.asarray(np.argmax(dmati, axis=0)[:, fmcol]).reshape(-1) + return fmrow, fmcol + +def coo_mat_argmax(coomat, axis=0): + if axis==0: + fmrow = np.unique(coomat.row) + fmcol = np.asarray(np.argmax(coomat, axis=1)[fmrow]).reshape(-1) + return fmcol + if axis==1: + fmcol = np.unique(coomat.col) + fmrow = np.asarray(np.argmax(coomat, axis=0)[:, fmcol]).reshape(-1) + return fmrow + + +def argmax_plane_sorted_row_sparse_mat(smat): + """ + for a sparse matrix filled with some vlaues + produces 1d array of indexes cooresponding to the data structure stored in smat, which provides + argmax for each column of matrix + """ + urow, uiidx, ucts = np.unique(smat.row, return_counts=True, return_inverse=True) + ii = np.arange(uiidx.size) + uicts = np.empty(uiidx.size, np.int) + uicts[:] = ucts[uiidx] + ipos = np.empty(urow.size, np.int) + for i in range(1, ucts.max() + 1): + mask = ucts == i + if not np.any(mask): + continue + maskall = uicts == i + ms = mask.sum() + ipos[mask] = ii[maskall].reshape((-1, i))[np.arange(ms), smat.data[maskall].reshape((-1, i)).argmax(axis=1)] + return ipos + + +def get_all_neighbours_fluxes(fluxmat, axis=0): + ucol, uidx = np.return_index(fluxmat.row, return_counts=True) + sidx = np.argsort(fluxmat.row) + +def onetoonedistance(cat1, cat2): + vec1 = vectopol(cat1) + vec2 = vectopol(cat2) + return np.arccos(np.sum(vec1*vec2, axis=1))*180./pi*3600. + + +class CrossCorr(object): + def __init__(self, cat1, cat2): + self.cat1 = cat1 + self.cat2 = cat2 + self.dmat = distwithhealpix(cat1, cat2) + self.flux2 = self.cat2.flux.values[self.dmat.col] + self.flux2err = self.cat2.fluxerr.values[self.dmat.col] + self.flux1 = self.cat1.flux.values[self.dmat.row] + + self.pval = normdist.pdf(dmat.data/np.sqrt(self.cat1.iloc[dmat.row].error_radius.values**2. + \ + self.cat2.iloc[dmat.col].error_radius.values**2.)) + + self.urow, self.idx, self.iidx, self.ucts = np.unique(self.dmat.row, + return_index=True, return_inverse=True, return_counts=True) + + self.pidx = np.empty(self.ucts.size + 1, np.int) + self.pidx[1:] = np.cumsum(self.ucts) + self.pidx[0] = 0 + self.eidx = np.lib.stride_tricks.as_strided(self.pidx, + (self.pidx.size - 1, 2), + self.pidx.strides*2) + self.mflux = np.empty(self.urow.size, np.double) + self.maxflux = np.zeros(self.urow.size, np.double) + np.maximum.at(self.maxflux, self.iidx, self.flux2) + self.minflux = np.full(self.urow.size, np.inf, np.double) + np.minimum.at(self.minflux, self.iidx, self.flux2) + self.medflux = np.array([np.median(self.flux2[s:e]) for s, e in self.eidx]) + + self.uflux = self.cat1["flux"].iloc[self.urow] #flux1.data[self.idx] + + self.newmask = np.isin(self.cat1.index.get_level_values("Num").values, self.urow, assume_unique=True, invert=True) + self.hascandidates = np.logical_not(self.newmask) + self.new = self.cat1[self.newmask] + self.newidx = self.new.index.get_level_values("Num").values + self.analyse_fluxes("") + self.bestdmat = get_closest_counterpart(self.dmat)[1] + self.bestpmat = coo_mat_argmax(self.pmat) + self.bestdist = pairwise_separation(self.cat1.iloc[self.urow], self.cat2.iloc[self.bestdmat]) + self.bestdistprob = normdist.pdf(self.bestdist/np.sqrt(cat1.iloc[self.urow].error_radius.values**2. + cat2.iloc[self.bestdmat].error_radius.values**2.)) + self.bestdflux = self.cat2.iloc[self.bestdmat].flux.values + + def analyse_fluxes(self, IndexSlice): + for i in range(2, self.ucts.max() + 1): + mask = self.ucts == i + fl = np.array([self.flux2.data[self.pidx[mask] + j] for j in range(i)]) + flmax = np.max(fl, axis=0) + flmin = np.min(fl, axis=0) + flmed = np.median(fl, axis=0) + self.mflux[mask] = flmed + self.maxflux[mask] = flmax + self.minflux[mask] = flmin + #for k, uidx in enumerate(self.urow[mask]): + # print(uidx, flmax[k], flmin[k], flmed[k], i) + + def get_all_counterparts(self, idx): + idx = np.sort(np.asarray(idx)) + idxmask = self.hascandidates[idx] + urmask = np.isin(self.dmat.row, idx) + catidx = pandas.DataFrame.copy(self.cat2.iloc[self.dmat.col[urmask]]) + catidx["counterpart"] = self.dmat.row[urmask] + return catidx + + def get_flux_outliers_mask(self, fscale): + #print("uflux size", self.uflux.size) + mask = np.logical_or(self.uflux > self.maxflux*fscale*10., self.uflux < self.minflux*fscale/10.) + maskout = np.zeros(self.cat1.index.size, np.bool) + maskout[self.hascandidates] = mask + return maskout + + +#RASSEMAP = healpy.read_map("RASS_EXP_1_2048.fits") +#rets, rcval = pickle.load(open("rxsmediandetection.pickle", "rb")) +#rcval = np.array(rcval) + +def rxs_detection_limit(exposure): + """ + returns amount of counts for the source to be detected depending on exposure + """ + return 4.*(1. + (exposure/425)**4.)**(0.6/4.) + +def check_rss_cts(ecat): + expp = RASSEMAP[healpy.ang2pix(2048, ecat.RA.values, ecat.DEC.values, lonlat=True)] + idx = np.searchsorted(rets, expp) + alert = ecat.flux.values > rxs_detection_limit(expp)**FSCALES["2rxs"]*10./expp + return ecat.index.get_level_values("Num").values[alert] + +def check_rss_cts_individual(esrc): + expp = RASSEMAP[healpy.ang2pix(2048, esrc.lii, esrc.bii, lonlat=True)] + alert = False if expp == 0 else esrc.ML_FLUX_0 > rxs_detection_limit(expp)*FSCALES["2rxs"]*10/expp + return alert + + +#======================================================================================== +#to use with django HeasarcBase models + +def check_rass_transient(erositasource, rasssources): + rassfluxes = np.array([rass.count_rate for rass in rasssources]) + if np.any(erositasource.ML_FLUX_0 < rassfluxes*FSCALES["2rxs"]/10.) or \ + np.any(erositasource.ML_FLUX_0 > rassfluxes*FSCALES["2rxs"]*10.): + erositasource.rosat_transient = True + +def check_sxps_transients(erositasource, sxpscounterparts, transientstatus): + flux = np.array([s.PeakRate_band1 + s.PeakRate_band2 for s in sxpscounterparts]) + transient = np.all(flux*10*FSCALES["sxps2"] < erositasource.ML_FLUX_0) or np.all(flux*0.1*FSCALES["sxps2"] > erositasource.ML_FLUX_0) + return transientstatus & transient + +def check_xmm3_transients(erositasource, xmmcounterparts, transientstatus): + flux = np.array([s.ep_1_flux + s.ep_3_flux + s.ep_2_flux for s in xmmcounterparts]) + transient = np.all(flux*10*FSCALES["3XMM"] < erositasource.ML_FLUX_0) + return transientstatus & transient + +#====================================================================================== + + + +def run_all(ecat): + newsources = np.ones(ecat.index.size, np.bool) + transients = None #no information which of new sources are transients yet + transient_catalogues = {} + crosscat = {} + crclist = {} + rosat_outliers = {} + + #take a look 2rxs is last one run + for catalog in ["3XMM", "sxps2", "2rxs"]: + cattable = pickle.load(open(CATALOGUES[catalog], "rb")) + crc = CrossCorr(ecat, cattable) + """ + if catalog == "sxps2": + midx = dmat.col[np.argsort(dmat.row)] + mflx = (cattable.iloc[midx].PeakRate_band0.values + cattable.iloc[midx].PeakRate_band1.values)/1.9 + crc.maxflux[crc.dmat.row] = (cattable.iloc[crc.dmat.col].PeakRate_band0.values + cattable.iloc[crc.dmat.col].PeakRate_band1.values)/1.9 + """ + crclist[catalog] = crc + outliers = crc.get_flux_outliers_mask(FSCALES[catalog]) + transient_catalogues[catalog] = crc.get_all_counterparts(ecat.index.get_level_values("Num")[outliers]) + if transients is None: + """first candidates to be transients""" + transients = outliers + print("outliers indices", ecat.index.get_level_values("Num").values[outliers]) + print("%s catalog outliers" % catalog, outliers.sum()) + print("%s catalog newsources" % catalog, crc.newmask.sum()) + print("transient and new", np.sum(transients & crc.newmask)) + print("transient and outlier", np.sum(transients & outliers)) + transients = np.logical_or(transients & crc.newmask, transients & outliers) + newsources = np.logical_and(newsources, crc.newmask) + print("transients:", transients.sum()) + + + nonrxs2_outliers = check_rss_cts(ecat[crc.newmask]) + haslcs = np.logical_not(newsources[nonrxs2_outliers]) + transients = transients & outliers + transients[nonrxs2_outliers] = True + #transients = transients & nonrxs2_outliers + return transient_catalogues, transients, newsources, nonrxs2_outliers, crclist + + +def make_report(ecat, transients): + ntrans = transients.sum() + ndetect = newsources.size - newsources.sum() - transients.sum() + nnew = newsources.sum() + report = \ + f""" + total sources in erosita catalogue {ecat.index.size}, + number of detections: {ndetect}, + new sources: {nnew}, + """ + + + + +if __name__ == "__main__": + """ + idea to implement heare: + generally speaking we want to find potnential transients in the daily produced by erosita source catalogue + therefore in main I expected to find counterparts in several X-ray catalogues to the input erosita source catalogues + """ + + #fname = sys.argv[1] + #data = pickle.load(open("eruds2.pickle", "rb")) #fits.getdata(fname) + #data = pickle.load(open("/srg/work/medvedev/for_andrey/catalog_10000000101.pkl", "rb")) + print("\n\n\n\nstart %s analysis" % sys.argv[1]) + data = pickle.load(open(sys.argv[1], "rb")) + try: + ecat = pandas.DataFrame(data) + ecat[:10] + except ValueError: + dswp = data.newbyteorder("l") + dswp.byteswap(inplace=True) + data = dswp + ecat = pandas.DataFrame(data) + + ecat["error_radius"] = ecat.RADEC_ERR + ecat["flux"] = ecat.ML_FLUX_0 + #ecat.DEC += 0.001 + ecat = makehealpixindex(ecat) + + transient_catalogues, transients, newsources, nonrxs2_outliers, crclist = run_all(ecat) + ecat_transients = ecat[transients] + + + print("overall statistics, number of sources in catalogue", ecat.index.size, "newsources:", newsources.sum(), "transients:", transients.sum()) + #for row in ecat.transients: + # print(row[["RA", "DEC", "flux", "DET_LIKE_0"]]) + #for transient_catalogues + + #rssoutl = check_rss_cts(ecat) + #rssoutl = rssout[newsources[rssoutl] + + if transients.sum(): + erastranients = ecat[transients] + erastranients["Num"] = erastranients.index.get_level_values("Num").values + hdulist = [fits.PrimaryHDU(), fits.BinTableHDU(Table.from_pandas(erastranients), name="SRGE"),] + for catalogue in transient_catalogues: + cat = transient_catalogues[catalogue] + tcat = cat[transients[cat.counterpart.values]] + if tcat.index.size > 0: + hdulist.append(fits.BinTableHDU(Table.from_pandas(tcat), name=catalogue)) + hdulist[-1].header.update({"CATALOG": catalogue}) + obsid = os.path.basename(sys.argv[1]) + fits.HDUList(hdulist).writeto("transients_%s.fits" % obsid[8:-4], overwrite=True) diff --git a/srglib/genericsourceutils.py b/srglib/genericsourceutils.py new file mode 100644 index 0000000..8a51516 --- /dev/null +++ b/srglib/genericsourceutils.py @@ -0,0 +1,894 @@ +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +from ErositaDailySourceCatalogues.models import ErositaDailyCatalog, ErositaSource +from genericsource.models import GenericSource, GenericConnection, GenericCatalog, SrcAuxDATA +import pandas as pd +from astropy.io import fits +from astropy.time import Time, TimeDelta +from astropy_healpix import HEALPix, neighbours +from astropy import units as au +from django_pandas.io import read_frame +from srglib import correlate_utils, check_transient +from srglib.utils import make_source_name +import numpy as np +import re +import glob +import pickle + +from math import pi, cos, sin +from scipy.sparse import coo_matrix +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from erosurvey.models import NSIDE_PLATES, ORDER_PLATES +from heasarc.models import NSIDE_SOURCES, ORDER, HeasarcBase, GreenSNR, RosatExpMap +from auxcatalogues.models import SimbadSource, RXS2, SXPS2, CSC2, \ + XMM3DR8, ZTFSource, SDSSSpec, GAIASource, ZTFOid, ZTFlc, AllWISEgs, \ + NVSSgs +from django.core.exceptions import ObjectDoesNotExist +from django.db.models.functions.math import Sqrt, Cos, Power, Abs +from django.db.models import F, Q, Max, Subquery, OuterRef, Min, Count, Value +from django.contrib.contenttypes.models import ContentType + +import datetime +from time import ctime +import pytz +import subprocess +#xspec = {} +""" +import xspec +from gdpyc import GasMap +from plotly import graph_objects as go +""" + + +#============================================================== +import os +os.environ["HOME"] = "/tmp/djangoastropy/" +if not os.path.exists("/tmp/djangoastropy/.astropy"): os.makedirs("/tmp/djangoastropy/.astropy") +from astroquery.simbad import Simbad as aSimbad +from astroquery.vizier import Vizier + +import logging +import requests +import json +import pprint +import pandas + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*au.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*au.hour) + +TIMESTR_FORMAT = '%Y.%m.%d %H:%M:%S' + +xraycats = [ContentType.objects.get_for_model(m) for m in [RXS2, CSC2, XMM3DR8, SXPS2]] #, ErositaSource]] + +hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) +hp_plates = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, frame=FK5()) +def mission2date(timesec): + mjdref = Time(MJDREF, format='mjd') + delta = TimeDelta(timesec, format='sec') + dtime = mjdref + delta + 3*au.hour + return dtime.to_datetime(timezone=TZ_MSK) + +def str2date(timestr, tzone): + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + +def date2mission(dtime): + mjdref = Time(MJDREF, format='mjd') + dtime = Time(dtime , format='datetime') + return (dtime - mjdref).sec + +def check_catsources_transient(cat): + check_sources_transient(cat.erotranssource_set.all()) + cat.ntransients = cat.erotranssource_set.filter(Q(transient=True) | Q(rosat_transient=True)).count() + cat.save() + + +from astrobasis.models import GAIADR2 #container of ALL gaia sources DR2 + +def find_from_old_gaia_db(srcquery, neighsearch=1, **kwargs): + """ + for all sources in srcquery, which is assumed to have many-to-many relationtip to counterpatscat catalog, + performs search of sources in counterpart catalog within max(minrad[=30], 3*sqrt(src.poserr**2 + counterpart.poserr**2)) + + + Parameters: + ---------- + srcquery: django.srcquery - iterable providing access to catalog under consideration + + counterpartscat: catalog to search counterparts in + counterpartname: name of the many-to-many relationtip field in srcquery + **kwargs: additional arguments provided to makeposerrdist functions (see doc within) + """ + scat = read_frame(srcquery) + + uhealpix = np.unique(scat.healpix.values) + uhealpix = np.unique(np.concatenate([uhealpix, neighbours(uhealpix, NSIDE_SOURCES, order=ORDER).ravel()])) + + ccat = read_frame(GAIADR2.objects.filter(healpix__in=uhealpix)) + ccat.drop(columns=["filename", "id", "ra_error", "dec_error"], inplace=True) + if scat.index.size == 0 or ccat.index.size == 0: + return 0 + + gexist = GAIASource.objects.filter(healpix__in=uhealpix) + find_counterparts(srcquery, gexist, neighsearch, **kwargs) + mask = np.isin(ccat.source_id.values, gexist.values_list("source_id"), invert=True) + ccat = ccat[mask] + + ccat.error_radius.fillna(0., inplace=True) + dmat = correlate_utils.makeposerrdist(scat, ccat, **kwargs) + print(dmat.data) + uidx, ucts = np.unique(dmat.row, return_counts=True) + cs = np.empty(ucts.size + 1, np.int) + cs[0] = 0 + cs[1:] = ucts.cumsum() + ce = np.lib.stride_tricks.as_strided(cs, (cs.size - 1, 2), cs.strides*2) + glist = [] + gnewlist = [] + gaiacat = GenericCatalog.objects.get(name="GAIA DR2") + for sid, cid, d in zip(scat.iloc[dmat.row].id.values, dmat.col, dmat.data): + src1 = srcquery.get(id=sid) + src2 = GAIASource.objects.create(**ccat.iloc[cid]) + gaiacat.genericsource_set.add(src2) + glist.append(GenericConnection(connected=src1, connectto=src2, separation=d)) + glist.append(GenericConnection(connected=src2, connectto=src1, separation=d)) + GenericConnection.objects.bulk_create(glist, ignore_conflicts=True) + +def update_catalog_connections(newsrc): + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=CSC2), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SXPS2), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=XMM3DR8), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=RXS2), minrad=20., maxdist=80.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SDSSSpec), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(Q(instance_of=ErositaSource) & Q(erositasource__obsdate__lt=cat.stop - datetime.timedelta(days=10))), minrad=3, maxdist=20) + load_simbad_auxcat(newsrc) + load_ztf_auxcat(newsrc, maxdist=10, minrad=3) + load_allwise_auxcat(newsrc, maxdist=10, minrad=3) + load_NVSS_auxcat(newsrc, maxdist=10, minrad=3) + newsrc = newsrc.annotate(xct=Count("connections", filter=Q(connections__polymorphic_ctype__in=xraycats))) + hasxcounterparts = newsrc.filter(xct__gt=0) + hasxcounterparts.annotate(mcxflux=Max("connections__sxflux")).filter(sxflux__gt=F("mcxflux")*Value(10.)).update(transient=True) + GenericCatalog.objects.get(name="Erosita flux transients").genericsource_set.add(*hasxcounterparts) + hasnoxcounterparts = newsrc.filter(xct=0).annotate(rassexp=Subquery(RosatExpMap.objects.filter(healpix=OuterRef("healpix")).values("exposure")[:1])) + hasnoxcounterparts = hasnoxcounterparts.annotate(rasslim=Value(4.*5.76*0.89211e-12)*Power(Value(1.) + Power(F("rassexp")/Value(425), Value(4.)), Value(0.15))/F("rassexp")) + hasnoxcounterparts.filter(rassexp__gt=0, sxflux__gt=F("rasslim")*Value(10.)).update(rosat_transient=True) + load_simbad_auxcat(newsrc) + + +def read_trans_pickle(filename): + data = pandas.read_pickle(filename) + data.rename(columns={"RA":"ra", "DEC":"dec", "L":"lii", "B":"bii", "CTS":"ML_CTS_0", "LIKE": "DET_LIKE_0",}, inplace=True) + sc = SkyCoord(data.ra.values, data.dec.values, unit="deg", frame="fk5") + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + data["healpix"] = hp.skycoord_to_healpix(sc) + data["healpix_plate"] = hp_plate.skycoord_to_healpix(sc) + data["name"] = [make_source_name("SRGe", r, d) for r, d in zip(data.ra.values, data.dec.values)] + + return data.drop(columns=["LIKE_REF", "REF_FLUX", "D2D", "LIKE_REF", "ID_REF", "EXT_REF", \ + "CTS_REF", "EXP", "EXP_REF", "FLAG", "G_NH", "g_d2d", "g_s", \ + "g_id", "g_ra", "g_dec", "s_d2d", "s_id", "s_z", "s_otype", \ + "sdss_d2d", "sdss_id", "sdss_z", "FLUX", "ID_SRC", "RATIO"]) + +def read_esrc_pack(filename): + ftime = datetime.datetime.strptime(ctime(os.path.getctime(filename)), "%a %b %d %H:%M:%S %Y") + df = pandas.read_pickle(filename) + df.rename(columns={"RADEC_ERR": "error_radius", + "srcID": "srcid", "ML_EXP_1":'APE_EXP_1'}, inplace=True) + + if "RA_fin" in df.columns.values: + df.rename(columns={"RA_fin":"ra", "DEC_fin":"dec"}, inplace=True) + elif "RA_corr" in df.columns.values: + df.rename(columns={"RA_corr":"ra", "DEC_corr":"dec"}, inplace=True) + else: + df.rename(columns={"RA":"ra", "DEC":"dec"}, inplace=True) + + sc = SkyCoord(df.ra.values, df.dec.values, unit="deg", frame="fk5") + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + df["lii"] = sc.galactic.l.value + df["bii"] = sc.galactic.b.value + df["healpix"] = hp.skycoord_to_healpix(sc) + df["healpix_plate"] = hp_plate.skycoord_to_healpix(sc) + df["name"] = [make_source_name("SRGe", r, d) for r, d in zip(df.ra.values, df.dec.values)] + df["id"] = df.index.values + df["sxflux"] = df.ML_FLUX_0.values + df["obsdate"] = (Time(51543.87, format="mjd") + TimeDelta((df.TSTART.values + df.TSTOP.values)/2., format="sec")).to_datetime(timezone=pytz.UTC) #datetime.timezone.utc) + df["tstart"] = (Time(51543.87, format="mjd") + TimeDelta(df.TSTART.values, format="sec")).to_datetime(timezone=pytz.UTC) + df["tstop"] = (Time(51543.87, format="mjd") + TimeDelta(df.TSTOP.values, format="sec")).to_datetime(timezone=pytz.UTC) + df["detected"] = True + return df.drop(columns=df.columns.difference([f.name for f in ErositaSource._meta.get_fields()] + ["id",])), ftime + +def setup_xspec(xspec): + """Setup xspec parameters.""" + xspec.Fit.statMethod = 'cstat' + xspec.Fit.query = 'yes' + xspec.Fit.nIterations = 100 + xspec.Xset.abund = 'wilm' + xspec.Plot.area = True + xspec.Plot.xLog = True + xspec.Plot.yLog = True + xspec.Plot.xAxis = 'keV' + xspec.Plot.setRebin(1.5, 20) + +def fit_powerlaw_v2(fname, nH=0.006, G=2.0, norm=1e-4, fit_nH=True, pdfname=None): + # Initialization + dirpath = os.path.dirname(fname) + cwd = os.getcwd() + os.chdir(dirpath if dirpath else "./") + + #xspec.Xset.openLog(fname[:-4] + "log") + xspec.Xset.chatter = 0 + xspec.AllData.clear() + xspec.AllModels.clear() + xspec.AllChains.clear() + # Fitting + sp = xspec.AllData(fname) + xspec.AllData.ignore('**-0.3 8.-**') + xspec.AllData.ignore('bad') + model = xspec.Model('tbabs*pow') + model.TBabs.nH.values = [nH, 0.001, nH, nH, 20, 20] + model.powerlaw.PhoIndex.values = [G, 0.01, 0.1, 0.1, 10, 10] + model.powerlaw.norm.values = [norm, 1e-6, 1e-10, 1e-10, 100, 100] + + # first, fit the normalization + model.TBabs.nH.frozen = True + model.powerlaw.PhoIndex.frozen = True + xspec.Fit.perform() + model.TBabs.nH.frozen = False + model.powerlaw.PhoIndex.frozen = False + if not fit_nH: + model.TBabs.nH.frozen = True + # than make a full fit + xspec.Fit.perform() + xspec.Fit.error('stop 1000,1.,2') + xspec.Fit.perform() + xspec.Fit.error('stop 1000,1.,2') + xspec.Fit.perform() + if fit_nH: + xspec.Fit.error('stop 1000,1.,1') + xspec.Fit.perform() + #xspec.Fit.error('1. 1-3') + #bestfit = XContainer(srcID) + #bestfit.get_session() + #xspec.Xset.closeLog() + xspec.Plot.device = "/null" + xspec.Plot("eeuf") + x, y, xe, ye, mo = xspec.Plot.x(), xspec.Plot.y(), xspec.Plot.xErr(), xspec.Plot.yErr(), xspec.Plot.model() + return model, np.array([x, y, xe, ye, mo]) + +def add_aux_data(srcid, survey, coord=None, newsourceid=None, xspec=None): + if xspec is None: + import xspec + from gdpyc import GasMap + #from plotly import graph_objects as go + + specfile = glob.glob("/data/erosita/transients/products/src_%d_020_SourceSpec_00001_e%d.fits*" % (srcid, survey)) + bkgfile = glob.glob("/data/erosita/transients/products/src_%d_020_BackgrSpec_00001_e%d.fits*" % (srcid, survey)) + arffile = glob.glob("/data/erosita/transients/products/src_%d_020_ARF_00001_e%d.fits*" % (srcid, survey)) + rmffile = glob.glob("/data/erosita/transients/products/src_%d_020_RMF_00001_e%d.fits*" % (srcid, survey)) + specimg = glob.glob("/data/erosita/transients/products/src_%d_020_SourceSpec_00001_e%d.pdf" % (srcid, survey)) + phimg = glob.glob("/data/erosita/transients/products/trans_%d_e%d.png" % (srcid, survey)) + specmo = glob.glob("/data/erosita/transients/products/src_%d_020_SourceSpec_00001_e%d.pdf" % (srcid, survey)) + lcpdf = glob.glob("/data/erosita/transients/products/lc*_%d_e%d.pdf" % (srcid, survey)) + + res = {} + if specfile: res["spec"] = os.path.basename(specfile[0]) + if bkgfile: res["bkg"] = os.path.basename(bkgfile[0]) + if arffile: res["arf"] = os.path.basename(arffile[0]) + if rmffile: res["rmf"] = os.path.basename(rmffile[0]) + if specimg: res["specimg"] = os.path.basename(specimg[0]) + if phimg: res["img"] = os.path.basename(phimg[0]) + if specmo: res["xcm"] = os.path.basename(specmo[0]) + if lcpdf: res["lcpdf"] = os.path.basename(lcpdf[0]) + print(res, newsourceid) + cwd = os.getcwd() + os.chdir("/data/erosita/transients/products") + rname = "" if not "spec" in res else res["spec"] + if newsourceid: + dpath = "/data/products/erotrans/%d" % newsourceid + ppath = "/data/erosita/transients/products/" + if not os.path.exists(dpath): + os.mkdir(dpath) + if "spec" in res: subprocess.run(["cp", os.path.join(ppath, res["spec"]), os.path.join(dpath, res["spec"])]) + if "bkg" in res: subprocess.run(["cp", os.path.join(ppath, res["bkg"]), os.path.join(dpath, res["bkg"])]) + if "arf" in res: subprocess.run(["cp", os.path.join(ppath, res["arf"]), os.path.join(dpath, res["arf"])]) + if "rmf" in res: subprocess.run(["cp", os.path.join(ppath, res["rmf"]), os.path.join(dpath, res["rmf"])]) + if "specimg" in res: subprocess.run(["cp", os.path.join(ppath, res["specimg"]), os.path.join(dpath, res["specimg"])]) + if "img" in res: subprocess.run(["cp", os.path.join(ppath, res["img"]), os.path.join(dpath, res["img"])]) + if "xcm" in res: subprocess.run(["cp", os.path.join(ppath, res["xcm"]), os.path.join(dpath, res["xcm"])]) + if "lcpdf" in res: subprocess.run(["cp", os.path.join(ppath, res["lcpdf"]), os.path.join(dpath, res["lcpdf"])]) + os.chdir(dpath) + p = subprocess.Popen(["echo", "group min 5 & exit"], stdout=subprocess.PIPE) + if "spec" in res: + rname = res["spec"][:-5] + "_r5.fits" + subprocess.run(["grppha", res["spec"], rname], stdin=p.stdout) + + cts = 0 if not "spec" in res else fits.getheader(res["spec"], 1)["CTS"] + try: + if cts > 20 or coord is None: + model, pdata = fit_powerlaw_v2(rname) + else: + model, pdata = fit_powerlaw_v2(rname, fit_nH=False, nH=GasMap(coord, nhmap="HI4PI",hires=True).value/1e22) + except: + print("fail to fit") + else: + res["gamma"] = model.powerlaw.PhoIndex.values[0] + res["nh"] = model.TBabs.nH.values[0]*1e21 + res["norm"] = model.powerlaw.norm.values[0] + res["gammael"] = model.powerlaw.PhoIndex.error[0] + res["gammaeu"] = model.powerlaw.PhoIndex.error[1] + pickle.dump(pdata, open(os.path.join(dpath, 'pdata.pickle'), 'wb')) + x, y, xe, ye, mo = pdata + fig = go.Figure(data=[go.Scatter(x=x, y=y, error_x={"type":"data", "array":xe}, error_y={"type":"data", "array":ye}, mode="markers"), + go.Scatter(x=x, y=mo, mode="lines")], layout=dict(width=1000)) + fig.update_xaxes(type="log") + fig.update_yaxes(type="log") + open(os.path.join(dpath, "spec.html"), "w").write(fig.to_html(full_html=False)) + return res + +def load_erosita_trans_daily(filename, survey, apps=5, load_cat="Erosita surveys transients"): + ncat, ftime = read_esrc_pack(filename) + ncat["survey"] = survey + return load_erosita_trans_daily_cat(ncat, ftime, apps, load_cat) + + +def load_erosita_trans_daily_cat(ncat, ftime, apps=5, load_cat="Erosita surveys transients"): + import xspec + from gdpyc import GasMap + from plotly import graph_objects as go + """ + there is a new type of file, which contains only new transients between first and second survey + """ + cat = GenericCatalog.objects.get(name = load_cat) + if cat.stop == ftime: + print("latest version already loaded") + return None + catsrc = ErositaSource.objects.filter(id__in=cat.genericsource_set.all()) + ecat = read_frame(catsrc) + urow = np.empty(0, np.int) + ucol = np.empty(0, np.int) + if ecat.index.size: + maske = np.ones(ecat.index.size, np.bool) + maskn = np.ones(ncat.index.size, np.bool) + eidx = np.arange(ecat.index.size) + nidx = np.arange(ncat.index.size) + for i in range(21): + if not (np.any(maske) and np.any(maskn)): + break + row, col = correlate_utils.get_closest_unique(ecat[maske], ncat[maskn], apps) + if row.size == 0 or np.all(~maske) or np.all(~maskn): + break + urow = np.concatenate([urow, eidx[maske][row]]) + ucol = np.concatenate([ucol, nidx[maskn][col]]) + maske[urow] = False + maskn[ucol] = False + + if urow.size: + mask = np.array([abs((ecat.iloc[i].obsdate - ncat.iloc[j].obsdate).days) for i, j in zip(urow, ucol)]) < 3 + urow, ucol = urow[mask], ucol[mask] + + setup_xspec(xspec) + + ncatidd = ncat.drop(columns=["id",]) + print(urow.size, ncat.index.size) + print("list of existing sources", urow, ucol) + for i, j in zip(urow, ucol): + src = catsrc.filter(id__in=[ecat.iloc[i].id,]) + src.update(**ncatidd.iloc[j]) + src = src.first() + sc = SkyCoord(src.ra, src.dec, unit="deg", frame="fk5") + print(src.name, ncat.iloc[j].name) + try: + auxdata = add_aux_data(ncat.iloc[j].name, src.survey, sc, src.id, xspec) + except Exception: + print("warn: no spectum info for %d found" % ncat.iloc[j].id) + else: + print("auxdata", auxdata) + try: + adata = src.srcauxdata + print(adata) + except: + print(src) + print(src.name, src.id) + adata = SrcAuxDATA.objects.create(src=src) + + print("update source", auxdata) + for key, val in auxdata.items(): + adata.__setattr__(key, val) + print(key, val, getattr(adata, key)) + adata.save() + + """ + if urow.size != ecat.index.size: + mask = np.ones(ecat.index.size, np.bool) + mask[urow] = False + catr = catsrc.filter(id__in=ecat[mask].id.values) + cat.genericsource_set.remove(*catr) + GenericCatalog.objects.get(name="Erosita surveys transients, removed").genericsource_set.add(*catr) + """ + if ucol.size < ncat.index.size: + catsrc.filter(srcstatus="new").update(srcstatus="") + idx = np.arange(ncat.index.size) + idx = idx[np.isin(idx, ucol, invert=True, assume_unique=True)] + newsrc = [] + for i in idx: + src = ErositaSource.objects.create(**ncatidd.iloc[i], srcstatus="new") + sc = SkyCoord(src.ra, src.dec, unit="deg", frame="fk5") + print("processing", ncat.iloc[i].name, src.id) + try: + auxdata = add_aux_data(ncat.iloc[i].name, src.survey, sc, src.id, xspec) + except Exception: + print("warn: no spectum info found") + else: + pass + print('create new source', auxdata) + SrcAuxDATA.objects.create(src=src, **auxdata) + #SrcAuxDATA.objects.create(src=src, **auxdata) + newsrc.append(src) + + cat.genericsource_set.add(*newsrc) + newsrc = cat.genericsource_set.filter(id__in=[src.id for src in newsrc]) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=CSC2), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SXPS2), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=XMM3DR8), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=RXS2), minrad=20., maxdist=80.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SDSSSpec), minrad=3., maxdist=20.) + #find_from_old_gaia_db(newsrc, minrad=5, maxdist=20) + #load_allwise_auxcat(newsrc, maxdist=10, minrad=3) + #find_counterparts(newsrc, GenericSource.objects.filter(instance_of=AllWISEgs), minrad=3., maxdist=20.) + #load_NVSS_auxcat(newsrc, maxdist=10, minrad=3) + #find_counterparts(newsrc, GenericSource.objects.filter(instance_of=NVSSgs), minrad=3., maxdist=20.) + #find_counterparts(newsrc, GenericSource.objects.filter(Q(instance_of=ErositaSource) & Q(erositasource__obsdate__lt=cat.stop - datetime.timedelta(days=10))), minrad=3, maxdist=20) + #load_simbad_auxcat(newsrc) + #find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SimbadSource), minrad=3., maxdist=20.) + #load_ztf_auxcat(newsrc, maxdist=10, minrad=3) + #find_counterparts(newsrc, GenericSource.objects.filter(instance_of=ZTFOid), minrad=3., maxdist=20.) + + cat.stop = ftime + cat.save() + return newsrc + + +def load_erosita_daily_source(filename): + """ Loads catalog of eRosita sources to database. + + This procedure accepts filename in format 'catalog_OBSID.pkl', + where OBSID is 11-digit identificator. In case this OBSID is + already loaded to DB, the procedure exits. Otherwise, + catalog is loaded to DB. + Parameters + ---------- + filename : str + Absolute path to the input catalog file + + Returns + ------- + status : int + Status code {0:'Already in DB', ID:'Object ID or Primary key in DB'} + + + :Author: + Andrey Semena + + """ + #metadata = json.load(open(filename.split(".")[0] + ".json")) + + obsid = re.findall("\d+", filename)[0] + + logging.debug("Loading %s, ObsID: %s" % (filename,obsid,)) + + + check_repeat = ErositaDailyCatalog.objects.filter(eroday_start=obsid[:5]).first() + if check_repeat and check_repeat.obsid >= obsid: + print("obsid exists") + logging.info("ObsID %s is already loaded, skip" % obsid) + return 0 + + logging.info("Loading ObsID %s" % obsid) + + hp = HEALPix(nside=NSIDE_SOURCES, + order=ORDER, frame=FK5()) + + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + + datafile=filename + + eroday_start = int(obsid[0:5]) + eroday_stop = int(obsid[5:10]) + ts_start = eroday_start*14400 + ts_stop = eroday_stop*14400 + start = mission2date(ts_start) + stop = mission2date(ts_stop) + dtime_start = Time(start , format='datetime') + dtime_stop = Time(stop , format='datetime') + + survey = ErositaDailyCatalog(name="ErositaDaily", + obsid=obsid, + eroday_start=eroday_start, + eroday_stop=eroday_stop, + start=start, + stop=stop, + mjd_start=dtime_start.mjd, + mjd_stop=dtime_stop.mjd) + + survey.save() + + if check_repeat and check_repeat.obsid < obsid: + print("removing older version of the catalog") + logging.info("remove older version of the catalog: %s" % obsid) + check_repeat.delete() + + df=pd.read_pickle(datafile) + df.drop(columns=["tilenum", 'srcname', 'hpidx', "TSTART", "ID_SRC", + "TSTOP",], inplace=True, errors="ignore") + df.rename(columns={"RA":"ra", "DEC":"dec", "RADEC_ERR": "error_radius", + "srcID": "srcid", "ML_EXP_1":'APE_EXP_1'}, inplace=True) + df["flux"] = df.ML_FLUX_0.values*0.8911e-12 + df["sxflux"] = df.flux + sc = SkyCoord(df.ra.values, df.dec.values, unit="deg", frame="fk5") + df["lii"] = sc.galactic.l.value + df["bii"] = sc.galactic.b.value + sc = sc[df.lii.values < 180.] + df = df[df.lii.values < 180.] + df["healpix"] = hp.skycoord_to_healpix(sc) + df["name"] = [make_source_name("SRGe", row.ra, row.dec) for row in df[["ra", "dec"]].itertuples()] + df["healpix_plate"] = hp_plate.skycoord_to_healpix(sc) + + enew = [] + for idx, src in df.iterrows(): + enew.append(ErositaSource.objects.create(obsdate=survey.stop, **src)) + survey.genericsource_set.add(*enew) + #newsrc = ErositaSource.objects.filter(catalog=survey) #survey.genericsource_set.all() + #newsrc = survey.genericsource_set.all() + newsrc = ErositaSource.objects.filter(id__in=survey.genericsource_set.all()) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=CSC2), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SXPS2), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=XMM3DR8), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=RXS2), minrad=20., maxdist=80.) + find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SDSSSpec), minrad=3., maxdist=20.) + find_counterparts(newsrc, GenericSource.objects.filter(Q(instance_of=ErositaSource) & Q(erositasource__obsdate__lt=cat.stop - datetime.timedelta(days=10))), minrad=3, maxdist=20) + load_simbad_auxcat(newsrc) + newsrc = newsrc.annotate(xct=Count("connections", filter=Q(connections__polymorphic_ctype__in=xraycats))) + hasxcounterparts = newsrc.filter(xct__gt=0) + hasxcounterparts.annotate(mcxflux=Max("connections__sxflux")).filter(sxflux__gt=F("mcxflux")*Value(10.)).update(transient=True) + GenericCatalog.objects.get(name="Erosita flux transients").genericsource_set.add(*hasxcounterparts) + hasecounterparts = newsrc.annotate(ect=Count("connections", filter=Q(connections__polymorphic_ctype=ContentType.objects.get_for_model(ErositaSource)))) + hasecounterpartstr = hasecounterparts.filter(ect__gt=0, sxflux__gt=Value(10.)*Max("connections__sxflux", filter=Q(connections__polymorphic_ctype=ContentType.objects.get_for_model(ErositaSource)))) + hasnoxcounterparts = newsrc.filter(xct=0).annotate(rassexp=Subquery(RosatExpMap.objects.filter(healpix=OuterRef("healpix")).values("exposure")[:1])) + GenericCatalog.objects.get(name="Erosita flux transients").genericsource_set.add(*hasecounterpartstr.filter(xct=0)) + hasnoxcounterparts = hasnoxcounterparts.annotate(rasslim=Value(4.*5.76*0.89211e-12)*Power(Value(1.) + Power(F("rassexp")/Value(425), Value(4.)), Value(0.15))/F("rassexp")) + hasnoxcounterparts.filter(rassexp__gt=0, sxflux__gt=F("rasslim")*Value(10.)).update(rosat_transient=True) + GenericCatalog.objects.get(name="Erosita rosat expmap transient").genericsource_set.add(*hasnoxcounterparts.filter(rosat_transient=True)) + load_ztf_auxcat(newsrc, maxdist=10, minrad=3) + return newsrc + + +def find_counterparts(srcquery, counterpatscat, neighsearch=1, **kwargs): + """ + for all sources in srcquery, which is assumed to have many-to-many relationtip to counterpatscat catalog, + performs search of sources in counterpart catalog within max(minrad[=30], 3*sqrt(src.poserr**2 + counterpart.poserr**2)) + + + Parameters: + ---------- + srcquery: django.srcquery - iterable providing access to catalog under consideration + + counterpartscat: catalog to search counterparts in + counterpartname: name of the many-to-many relationtip field in srcquery + **kwargs: additional arguments provided to makeposerrdist functions (see doc within) + """ + scat = read_frame(srcquery) + + uhealpix = np.unique(scat.healpix.values) + uhealpix = np.unique(np.concatenate([uhealpix, neighbours(uhealpix, NSIDE_SOURCES, order=ORDER).ravel()])) + + ccat = read_frame(counterpatscat.filter(healpix__in=uhealpix)) + if scat.index.size == 0 or ccat.index.size == 0: + return 0 + + ccat.error_radius.fillna(0., inplace=True) + dmat = correlate_utils.makeposerrdist(scat, ccat, **kwargs) + uidx, ucts = np.unique(dmat.row, return_counts=True) + cs = np.empty(ucts.size + 1, np.int) + cs[0] = 0 + cs[1:] = ucts.cumsum() + ce = np.lib.stride_tricks.as_strided(cs, (cs.size - 1, 2), cs.strides*2) + glist = [] + for sid, cid, d in zip(scat.iloc[dmat.row].id.values, ccat.iloc[dmat.col].id.values, dmat.data): + if sid == cid: + continue + src1, src2 = srcquery.get(id=sid), counterpatscat.get(id=cid) + glist.append(GenericConnection(connected=src1, connectto=src2, separation=d)) + glist.append(GenericConnection(connected=src2, connectto=src1, separation=d)) + print("bulk create") + GenericConnection.objects.bulk_create(glist, ignore_conflicts=True) + +def find_closest_counterpart(srcquery, counterpatscat, neighsearch=1): + scat = read_frame(srcquery) + + #uhealpix = np.unique(scat.healpix.values) + #uhealpix = np.unique(np.concatenate([uhealpix, neighbours(uhealpix, NSIDE_SOURCES, order=ORDER).ravel()])) + + #ccat = read_frame(counterpatscat.filter(healpix__in=uhealpix)) + ccat = read_frame(counterpatscat) + + if scat.index.size == 0 or ccat.index.size == 0: + return 0 + + ccat.error_radius.fillna(0., inplace=True) + dmat = correlate_utils.makedistmat3d(scat, ccat, 1200) + sidx = np.argsort(dmat.data) + print(dmat.data.size) + urow, uidx = np.unique(dmat.row[sidx], return_index=True) + ucol = dmat.col[sidx][uidx] + + glist = [] + for sid, cid, d in zip(scat.iloc[urow].id.values, ccat.iloc[ucol].id.values, dmat.data[sidx][uidx]): + if sid == cid: + continue + src1, src2 = srcquery.get(id=sid), counterpatscat.get(id=cid) + glist.append(GenericConnection(connected=src1, connectto=src2, separation=d)) + glist.append(GenericConnection(connected=src2, connectto=src1, separation=d)) + GenericConnection.objects.bulk_create(glist, ignore_conflicts=True) + +def load_simbad_auxcat(genericsources, **kwargs): + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + srcs = read_frame(genericsources) + s1 = aSimbad() + s1.reset_votable_fields() + s1.add_votable_fields("z_value", "otype", "plx", "parallax", "plx_error", "plx_bibcode") + fails = [] + simbadgenericcat = GenericCatalog.objects.get(name="Simbad") + print(srcs.index.size) + for i in range((srcs.index.size + 4095)//4096): + eloc = srcs.iloc[i*4096:min((i + 1)*4096, srcs.index.size)] + try: + simbadc = s1.query_region(SkyCoord(eloc.ra.values, eloc.dec.values, unit="deg", frame="fk5"), radius=kwargs.get("maxdist", 60.)*au.arcsec).to_pandas() + except: + fails.append(eloc.id.values) + continue + print(simbadc.index.size) + sc = SkyCoord(simbadc.RA.values + " " + simbadc.DEC.values, unit=("hourangle", "deg"), frame="fk5") + simbadc["ra"] = sc.fk5.ra.value + simbadc["dec"] = sc.fk5.dec.value + simbadc["healpix"] = hp.skycoord_to_healpix(sc) + #simbadc['name'] = simbadc.MAIN_ID.str.decode("ascii") + #print(simbadc.name) + #simbadc["name"] = simbadc.name.astype("str") + simbadc["name"] = simbadc.MAIN_ID.astype("str") + print(simbadc.name) + simbadc['main_id'] = simbadc.name + simbadc['obj_class'] = simbadc.OTYPE.str.decode("ascii") + simbadc['coo_bibcode'] = simbadc.COO_BIBCODE.str.decode("ascii") + simbadc['coo_wavelength'] = simbadc.COO_WAVELENGTH.str.decode("ascii") + simbadc.PLX_BIBCODE = simbadc.PLX_BIBCODE.fillna(simbadc.PLX_BIBCODE_2) + simbadc['plx_bib'] = simbadc.PLX_BIBCODE.str.decode("ascii") + simbadc.PLX_VALUE = simbadc.PLX_VALUE.fillna(simbadc.PLX_VALUE_2) + simbadc.PLX_ERROR = simbadc.PLX_ERROR.fillna(simbadc.PLX_ERROR_2) + print(simbadc) + print(simbadc.name) + #return simbadc + simbadc.drop_duplicates('name', inplace=True) + simbadc.rename(columns={"COO_ERR_MINA": "error_radius", "Z_VALUE": "z", + "PLX_VALUE":"plx", "PLX_ERROR": "plx_err"}, inplace=True) + simbadc.error_radius.fillna(0., inplace=True) + + duplicates = SimbadSource.objects.filter(name__in=simbadc.name.values) + print(simbadc.index.size) + print(simbadc.name) + if duplicates.exists(): + duplicatesnames = np.array(duplicates.values_list("name"))[:, 0] + mask = np.isin(simbadc.name.values, duplicatesnames, invert=True) + simbadc = simbadc[mask] + print(simbadc.index.size) + simbadc.drop(columns=["RA", "DEC", "RA_PREC", "DEC_PREC", "COO_ERR_MAJA", + "COO_ERR_ANGLE", "COO_QUAL", "MAIN_ID", + "OTYPE", "COO_WAVELENGTH", "COO_BIBCODE", + "PLX_BIBCODE", "PLX_ERROR_2", "PLX_QUAL", + "OTYPE", "PLX_VALUE_2", "PLX_PREC", + "PLX_BIBCODE_2"], inplace=True) + + sobjlist = [] + for i, sobj in simbadc.iterrows(): + sobjlist.append(SimbadSource.objects.create(**sobj)) + simbadgenericcat.genericsource_set.add(*sobjlist) + + #ech = erosrcs.filter(id__in=eloc.id.values) + find_counterparts(genericsources, GenericSource.objects.filter(instance_of=SimbadSource), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + +def load_allwise_auxcat(genericsources, **kwargs): + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + srcs = read_frame(genericsources) + #s1.reset_votable_fields() + fails = [] + allwisegenericcat = GenericCatalog.objects.get(name="AllWISE") + for i in range((srcs.index.size + 4095)//4096): + eloc = srcs.iloc[i*4096:min((i + 1)*4096, srcs.index.size)] + print(eloc.index.size) + #allwisec = Vizier.query_region(SkyCoord(eloc.ra.values, eloc.dec.values, unit="deg", frame="fk5"), radius=kwargs.get("maxdist", 10.)*au.arcsec, catalog="II/328").to_pandas() + try: + allwisec = Vizier.query_region(SkyCoord(eloc.ra.values, eloc.dec.values, unit="deg", frame="fk5"), radius=kwargs.get("maxdist", 10.)*au.arcsec, catalog="II/328")[0].to_pandas() + except: + print("fail to load") + fails.append(eloc.id.values) + continue + print(allwisec.index.size) + allwisec.rename(columns={"RAJ2000":"ra", "DEJ2000":"dec", "W1mag":"w1mag", "W2mag":"w2mag","W3mag":"w3mag","W4mag":"w4mag", + "e_W1mag":"e_w1mag", "e_W2mag":"e_w2mag","e_W3mag":"e_w3mag","e_W4mag":"e_w4mag", "AllWISE": "name"}, inplace=True) + sc = SkyCoord(allwisec.ra.values, allwisec.dec.values, unit="deg", frame="fk5") + allwisec["lii"] = sc.galactic.l.value + allwisec["bii"] = sc.galactic.b.value + allwisec["name"] = allwisec.name.astype(str) + allwisec["healpix"] = hp.skycoord_to_healpix(sc) + allwisec.drop_duplicates('name', inplace=True) + allwisec["error_radius"] = np.zeros(allwisec.index.size) + duplicates = AllWISEgs.objects.filter(name__in=allwisec.name.values) + if duplicates.exists(): + duplicatesnames = np.array(duplicates.values_list("name"))[:, 0] + mask = np.isin(allwisec.name.values, duplicatesnames, invert=True) + allwisec = allwisec[mask] + allwisec.drop(columns=allwisec.columns.difference([f.name for f in AllWISEgs._meta.get_fields()]), inplace=True) + + sobjlist = [] + for i, sobj in allwisec.iterrows(): + sobjlist.append(AllWISEgs.objects.create(**sobj)) + allwisegenericcat.genericsource_set.add(*sobjlist) + + #ech = erosrcs.filter(id__in=eloc.id.values) + find_counterparts(genericsources, GenericSource.objects.filter(instance_of=AllWISEgs), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + +def load_NVSS_auxcat(genericsources, **kwargs): + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + srcs = read_frame(genericsources) + fails = [] + nvssgenericcat = GenericCatalog.objects.get(name="NVSS") + for i in range((srcs.index.size + 4095)//4096): + eloc = srcs.iloc[i*4096:min((i + 1)*4096, srcs.index.size)] + print(eloc.index.size) + #nvssc = Vizier.query_region(SkyCoord(eloc.ra.values, eloc.dec.values, unit="deg", frame="fk5"), radius=kwargs.get("maxdist", 10.)*au.arcsec, catalog="II/328").to_pandas() + try: + nvssc = Vizier.query_region(SkyCoord(eloc.ra.values, eloc.dec.values, unit="deg", frame="fk5"), radius=kwargs.get("maxdist", 10.)*au.arcsec, catalog="VIII/65")[0].to_pandas() + except: + print("fail to load") + fails.append(eloc.id.values) + continue + print(nvssc.index.size) + nvssc.rename(columns={"NVSS": "name", "S1.4":"s14", "e_S1.4":"s14e", "MajAxis":"maxjaxis", "MinAxis":"minaxis"}, inplace=True) + + sc = SkyCoord(nvssc.RAJ2000.values, nvssc.DEJ2000.values, unit=("hourangle", "deg"), frame="fk5") + nvssc["ra"] = sc.ra.value + nvssc["dec"] = sc.dec.value + nvssc["lii"] = sc.galactic.l.value + nvssc["bii"] = sc.galactic.b.value + nvssc["name"] = nvssc.name.astype(str) + nvssc["healpix"] = hp.skycoord_to_healpix(sc) + nvssc.drop_duplicates('name', inplace=True) + nvssc["error_radius"] = np.sqrt(nvssc.e_RAJ2000.values**2.*np.cos(np.deg2rad(nvssc.dec.values))**2. + nvssc.e_DEJ2000.values**2.) + duplicates = NVSSgs.objects.filter(name__in=nvssc.name.values) + if duplicates.exists(): + duplicatesnames = np.array(duplicates.values_list("name"))[:, 0] + mask = np.isin(nvssc.name.values, duplicatesnames, invert=True) + nvssc = nvssc[mask] + nvssc.drop(columns=nvssc.columns.difference([f.name for f in NVSSgs._meta.get_fields()]), inplace=True) + + sobjlist = [] + for i, sobj in nvssc.iterrows(): + sobjlist.append(NVSSgs.objects.create(**sobj)) + nvssgenericcat.genericsource_set.add(*sobjlist) + find_counterparts(genericsources, GenericSource.objects.filter(instance_of=NVSSgs), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + + +def load_ztf_auxcat(genericsources, **kwargs): + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + hp_plates = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, frame=FK5()) + srcs = read_frame(genericsources) + uhealpix = np.unique(srcs.healpix.values) + uhealpix = np.unique(np.concatenate([uhealpix, neighbours(uhealpix, NSIDE_SOURCES, order=ORDER).ravel()])) + + ztfurl = "https://mars.lco.global/" + params = {'format':'json', 'rb__gte':0.8,'nbad__lte':0,'fwhm__lte':5,'elong__lte':1.2} + ztfsrc = [] + ztfloaded = ZTFOid.objects.filter(healpix__in=uhealpix) + + for i, row in srcs.iterrows(): + print("%d out of %d, %d loaded, ra & dec: %.5f %.5f" % (i, srcs.index.size, len(ztfsrc), row.ra, row.dec)) + params["cone"] = "%.5f,%.5f,%.5f" % (row.ra, row.dec, kwargs.get("maxdist", 10)/3600.) + r = requests.get(url=ztfurl, params=params) + if not r.ok: continue + ztfsrc += r.json()["results"] + + + def unpack_mars_dict(d): + s = d["candidate"] + s["lco_id"] = d["lco_id"] + s["objectId"] = d["objectId"] + return pandas.Series(s) + + pd = pandas.DataFrame([unpack_mars_dict(z) for z in ztfsrc]) + pd = pd.drop_duplicates("candid") + pd.to_pickle("/srg/a1/work/andrey/rass/tmp_ztf_dumpy.pkl") + + sc = SkyCoord(pd.ra.values, pd.dec.values, unit="deg", frame="fk5") + pd["healpix"] = hp.skycoord_to_healpix(sc) + pd["healpix_plate"] = hp_plates.skycoord_to_healpix(sc) + format_string='%a, %d %b %Y %H:%M:%S %Z' + pd["wt"] = [datetime.datetime.strptime(t, format_string) for t in pd.wall_time.values] + pd.drop(columns=["wall_time"], inplace=True) + pd.rename(columns={"wt":"wall_time"}, inplace=True) + pd.drop(columns=pd.columns.difference(["objectId",] + [field.name for field in ZTFlc._meta.get_fields()]), inplace=True) + pdgrp = pd.groupby("objectId") + ztfurl = "https://ztf.alerce.online/query" + ztfcat = GenericCatalog.objects.get(name="ZTF") + ztffieldnames = [field.name for field in ZTFOid._meta.get_fields()] + for name, group in pdgrp: + try: + ztfobj = ZTFOid.objects.get(objectId=name) + except ObjectDoesNotExist: + r = requests.post(ztfurl, json={"total":1, "query_parameters":{"filters":{"oid":name}}}) + if r.ok and r.json()["result"] and name in r.json()["result"]: + ztfobj = r.json()["result"][name] + ztfobj["objectId"] = ztfobj["oid"] + ztfobj["ra"] = ztfobj["meanra"] + ztfobj["dec"] = ztfobj["meandec"] + sc = SkyCoord(ztfobj["meanra"], ztfobj["meandec"], unit="deg", frame="fk5") + ztfobj["healpix"] = hp.skycoord_to_healpix(sc) + ztfobj["healpix_plate"] = hp_plates.skycoord_to_healpix(sc) + ztfobj["name"] = ztfobj["objectId"] + print(ztfobj) + ztfobj = {k:v for k, v in ztfobj.items() if k in ztffieldnames} + if ztfobj["classrf"] is None: + ztfobj["classrf"] = "Not specified" + ztfobj["pclassrf"] = 1. + print(ztfobj) + ztfobj = ZTFOid.objects.create(**ztfobj) + ztfcat.genericsource_set.add(ztfobj) + else: + continue + clist = np.array([l[0] for l in ztfobj.ztflc_set.values_list("candid")]) + for i, row in group[~group.candid.isin(clist)].drop(columns=["objectId",]).iterrows(): + ZTFlc.objects.create(ztfobject=ztfobj, **row) + + find_counterparts(genericsources, GenericSource.objects.filter(instance_of=ZTFOid), maxdist=kwargs.get("maxdist", 10.), minrad=kwargs.get("minrad", 3.)) + +ztfclasses={ + None: "Not specified", + 'null':"Not specified", + "classified":"Classified", + "not classified":"Not classified", + 1:"Ceph", + 2:"DSCT", #Delta SCuTi, a class of pulsating variable stars named after Delta Scuti, the archetype for the class + 3:"EB", # Eclipsing Beta Lyrae, a class of eclipsing binary stars named after Beta Lyrae, the archetype for the class + 4:"LPV", #Long Period Variable, a type of variable star that changes in brightness slowly over time + 5:"RRL", #RR Lyrae, a class of pulsating variable stars named after RR Lyrae, the archetype of the class + 6:"SN", + 0:"Other", + 7:"AGN I", + 8:"Blazar", + 9:"CV/Nova", + 10:"SN Ia", + 11:"SN Ibc", + 12:"SN II", + 14:"SLSN", #A super-luminous supernova (SLSN, plural super luminous supernovae or SLSNe) is a type of stellar explosion with a luminosity 10 or more times higher than that of standard supernovae + 15:"EB/SD/D", # SD - subdwarf, stars fainter than main-sequence stars with the same colors; often used as a prefix to a star's spectral type + 16:"EB/C", + 17:"Periodic/Other", + 18:"AGN", + 19:"SN", + 20:"Variable Star", + 21:"Asteroid", + 22:"Bogus", + 23:"RS-CVn", + 24:"QSO-I"} + diff --git a/srglib/load_transinets.py b/srglib/load_transinets.py new file mode 100644 index 0000000..5116261 --- /dev/null +++ b/srglib/load_transinets.py @@ -0,0 +1,278 @@ +import pandas +import glob +import os +from srglib import correlate_utils +from srglib import genericsourceutils as gsu +from astropy.coordinates import SkyCoord +from django_pandas.io import read_frame +import numpy as np +from astropy.time import Time, TimeDelta +import datetime +from math import sin, cos, sqrt, pi +from ErositaDailySourceCatalogues.models import ErositaSource +import pytz +import subprocess +from genericsource.models import * +from ErositaDailySourceCatalogues import views as eviews +from ErositaDailySourceCatalogues.models import * +from auxcatalogues.models import SimbadSource, RXS2, SXPS2, CSC2, \ + XMM3DR8, ZTFSource, SDSSSpec, GAIASource, ZTFOid, ZTFlc, AllWISEgs, \ + NVSSgs + +northpole = np.array([-4.26367573e-16, -3.97777165e-01, 9.17482058e-01]) +surveystart = np.array([-7.43149503e-06, -4.00344939e-04, -1.73318355e-04]) +pangle = np.cross(northpole, surveystart) +pangle = pangle/np.sqrt(np.sum(pangle**2.)) + +SurveyCatName = {1: "1st survey transients", + 2: "2nd survey transients", + 3: "3rd survey transients",} + + +def add_aux_data(srcid, survey, coord=None, newsourceid=None): + #print("/data/erosita/transients/products/trans_%d_e%d-1.png" % (srcid, survey)) + phimg = glob.glob("/data/erosita/transients/products/trans_%d_e%d.png" % (srcid, survey)) + + res = {k: "" for k in ["spec", "specimg", "arf", "rmf", "bkg", "reg", "log", "lcfits", "lcpdf"]} + if phimg: res["img"] = os.path.basename(phimg[0]) + cwd = os.getcwd() + os.chdir("/data/erosita/transients/products") + rname = "" if not "spec" in res else res["spec"] + if newsourceid: + dpath = "/data/products/erotrans/%d" % newsourceid + ppath = "/data/erosita/transients/products/" + if not os.path.exists(dpath): + os.mkdir(dpath) + if "img" in res: subprocess.run(["cp", os.path.join(ppath, res["img"]), os.path.join(dpath, res["img"])]) + os.chdir(dpath) + return res + + +def get_closest_unique_withinapp(ecat, ncat, apps): + """ + find all closest unique counterparts within definde distance + """ + urow = np.empty(0, np.int) + ucol = np.empty(0, np.int) + if ecat.index.size: + maske = np.ones(ecat.index.size, np.bool) + maskn = np.ones(ncat.index.size, np.bool) + eidx = np.arange(ecat.index.size) + nidx = np.arange(ncat.index.size) + for i in range(21): + if not (np.any(maske) and np.any(maskn)): + break + row, col = correlate_utils.get_closest_unique(ecat[maske], ncat[maskn], apps) + print(row.size) + if row.size == 0 or np.all(~maske) or np.all(~maskn): + break + urow = np.concatenate([urow, eidx[maske][row]]) + ucol = np.concatenate([ucol, nidx[maskn][col]]) + maske[urow] = False + maskn[ucol] = False + + if False: #urow.size: + mask = np.array([abs((ecat.iloc[i].obsdate - ncat.iloc[j].obsdate).days) for i, j in zip(urow, ucol)]) < 3 + urow, ucol = urow[mask], ucol[mask] + return urow, ucol + + + +def load_nondetections(ecat, ncat, catsrc, apps, survey): + urow, ucol = get_closest_unique_withinapp(ecat, ncat, apps) + print(urow.size, ncat.index.size, ecat.index.size) + ncatidd = ncat.drop(columns=["id",]) + + for i, j in zip(urow, ucol): + src = catsrc.filter(id__in=[ecat.iloc[i].id,]) + src.update(**ncatidd.iloc[j]) + src = src.first() + sc = SkyCoord(src.ra, src.dec, unit="deg", frame="fk5") + auxdata = add_aux_data(ncat.iloc[j].id, survey, sc, src.id) + try: + auxdata = add_aux_data(ncat.iloc[j].id, survey, sc, src.id) + except Exception: + print("warn: no spectum info for %d found" % ncat.iloc[j].id) + else: + #print("auxdata", auxdata) + try: + adata = src.srcauxdata + except: + adata = SrcAuxDATA.objects.create(src=src) + + #print("update source", auxdata) + for key, val in auxdata.items(): + adata.__setattr__(key, val) + #print(key, val, getattr(adata, key)) + adata.save() + for key in ["spec", "specimg", "arf", "rmf", "bkg", "reg", "log", "lcfits", "lcpdf"]: + adata.__setattr__(key, "") + adata.save() + + newsrc = [] + if ucol.size < ncat.index.size: + idx = np.arange(ncat.index.size) + idx = idx[np.isin(idx, ucol, invert=True, assume_unique=True)] + print(idx.size) + for i in idx: + #print(ncatidd.iloc[i]) + src = ErositaSource.objects.create(**ncatidd.iloc[i], srcstatus="new") + sc = SkyCoord(src.ra, src.dec, unit="deg", frame="fk5") + print("processing", ncat.iloc[i].id, src.id) + auxdata = add_aux_data(ncat.iloc[i].id, survey, sc, src.id) + try: + auxdata = add_aux_data(ncat.iloc[i].id, survey, sc, src.id) + except Exception: + print("warn: no spectum info found") + else: + pass + #print('create new source', auxdata) + SrcAuxDATA.objects.create(src=src, **auxdata) + newsrc.append(src) + + #catsrc.genericsource_set.add(*newsrc) + return newsrc + + + +def update_columns(mtable, survey): + sc = SkyCoord(mtable.ra.values, mtable.dec.values, unit="deg", frame="fk5") + mtable["lii"] = sc.galactic.l.value + mtable["bii"] = sc.galactic.b.value + mtable["healpix"] = gsu.hp.skycoord_to_healpix(sc) + mtable["healpix_plate"] = gsu.hp_plates.skycoord_to_healpix(sc) + mtable["detected"] = False + mtable["name"] = [gsu.make_source_name("SRGe", r, d) for r, d in zip(mtable.ra.values, mtable.dec.values)] + mtable["sxflux"] = mtable["UPLIM_e%d" % survey] + mtable["id"] = mtable.index.values + + """ + vecs = np.empty((mtable.index.size, 3), np.double) + vecs[:, 0] = np.cos(mtable.dec.values*pi/180.)*np.cos(mtable.ra.values*pi/180.) + vecs[:, 1] = np.cos(mtable.dec.values*pi/180.)*np.sin(mtable.ra.values*pi/180.) + vecs[:, 2] = np.sin(mtable.dec.values*pi/180.) + + vecsp = vecs - northpole*np.sum(northpole*vecs, axis=1)[:, np.newaxis] + vecsp = vecsp/np.sqrt(np.sum(vecsp**2, axis=1))[:, np.newaxis] + + alpha = np.arctan2(np.sum(vecsp*pangle, axis=1), np.sum(vecsp*surveystart, axis=1)) + alpha[alpha < 0] = pi + alpha[alpha < 0.] + """ + + tstart = mtable["TSTART_e%d" % survey].values + tstop = mtable["TSTOP_e%d" % survey].values + + """ + mtable["obsdate"] = (Time(51543.87, format="mjd") + TimeDelta((tstart + tstop)/2., format="sec")).to_datetime(timezone=pytz.UTC) + mtable["tstart"] = (Time(51543.87, format="mjd") + TimeDelta(tstart, format="sec")).to_datetime(timezone=pytz.UTC) + mtable["tstop"] = (Time(51543.87, format="mjd") + TimeDelta(tstop, format="sec")).to_datetime(timezone=pytz.UTC) + """ + mtable["tstart"] = mtable["TSTART_e%d" % survey] + mtable["tstop"] = mtable["TSTOP_e%d" % survey] + mtable["obsdate"] = mtable["TSTART_e%d" % survey] + (mtable["TSTOP_e%d" % survey] - mtable["TSTART_e%d" % survey])/2. + + mtable.drop(columns=mtable.columns.difference([f.name for f in ErositaSource._meta.get_fields()] + ["id",]), inplace=True) + print(survey, mtable.index.size) + return mtable + + +def read_master_table(mastertablename, **kwargs): + mtable = pandas.read_pickle(mastertablename) + + newsrc = [] + for survey in [2,]: #np.arange(1, 4): + mel = mtable[mtable["CTS_e%d" % survey].isna() & ~mtable["TSTART_e%d" % survey].isna()].copy() + mel = update_columns(mel, survey) + ccat = ErositaSource.objects.filter(id__in=GenericCatalog.objects.get(name=SurveyCatName[survey]).genericsource_set.all()) + ecat = read_frame(ccat) + nset = load_nondetections(ecat, mel, ccat, 30, survey) + GenericCatalog.objects.get(name=SurveyCatName[survey]).genericsource_set.add(*nset) + newsrc += nset + + if len(newsrc) > 0: + newsrc = GenericSource.objects.filter(id__in=[src.id for src in newsrc]) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=CSC2), minrad=3., maxdist=20.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SXPS2), minrad=3., maxdist=20.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=XMM3DR8), minrad=3., maxdist=20.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=RXS2), minrad=20., maxdist=80.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SDSSSpec), minrad=3., maxdist=20.) + gsu.find_from_old_gaia_db(newsrc, minrad=5, maxdist=20) + #gsu.load_allwise_auxcat(newsrc, maxdist=10, minrad=3) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=AllWISEgs), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + #gsu.load_NVSS_auxcat(newsrc, maxdist=10, minrad=3) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=NVSSgs), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + #find_counterparts(newsrc, GenericSource.objects.filter(Q(instance_of=ErositaSource) & Q(erositasource__obsdate__lt=cat.stop - datetime.timedelta(days=10))), minrad=3, maxdist=20) + #gsu.load_simbad_auxcat(newsrc) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SimbadSource), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + #gsu.load_ztf_auxcat(newsrc, maxdist=10, minrad=3) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=ZTFOid), maxdist=kwargs.get("maxdist", 10.), minrad=kwargs.get("minrad", 3.)) + +def set_transient_data(ecat): + ecat = eviews.create_annotations(ecat) + ecat = eviews.addErositaSrcannotattion(ecat) + ecat = ecat.filter(ratio_to_maxxflux__gt=1.) + print(ecat.count()) + for src in ecat: + try: + ErositaTransient.objects.create(src=src, plx=src.plx, plxe=src.plxe, + z=src.ssz if src.ssz else src.sdssz, + ssep=src.ssep, + sname=src.sname if src.sname else "", + ztfcl=src.ztfcl if src.ztfcl else "", + ztfnm=src.ztfnm if src.ztfnm else "", + zsep=src.zsep, plx_sep=src.plx_sep, + plx_mag=src.plx_mag, + sclass=src.sclass if src.sclass else "", + ratio_to_maxxflux=src.ratio_to_maxxflux, + ratio_to_minxflux=src.ratio_to_minxflux) + except: + raise + print(src.name, " already has transints") + else: + print("transient for %s created" % src.name) + +def read_master_table_and_surveys(mastertablename, surveynames, **kwargs): + mtable = pandas.read_pickle(mastertablename) + mtable.rename(columns={"RA": "ra", "DEC": "dec"}, inplace=True) + elast = ErositaSource.objects.last() #order_by("created").last() + newsrc = [] + + for survey in np.arange(1, 4): + detectedsources, ftime = gsu.read_esrc_pack(surveynames % survey) + print("load survey", survey, surveynames % survey) + detectedsources["survey"] = survey + dmat = correlate_utils.makedistmat3d(detectedsources, mtable, 40.) + urow, ucol = correlate_utils.select_closest_pairs(dmat.data, dmat.row, dmat.col) + detectedsources = detectedsources.iloc[urow].set_index(mtable.index.values[ucol]) + ns = gsu.load_erosita_trans_daily_cat(detectedsources, ftime, 30, load_cat=SurveyCatName[survey]) + newsrc += list(ns) + + mel = mtable[mtable["CTS_e%d" % survey].isna() & ~mtable["TSTART_e%d" % survey].isna()].copy() + mel = update_columns(mel, survey) + ccat = ErositaSource.objects.filter(id__in=GenericCatalog.objects.get(name=SurveyCatName[survey]).genericsource_set.all()) + ecat = read_frame(ccat) + nset = load_nondetections(ecat, mel, ccat, 30, survey) + GenericCatalog.objects.get(name=SurveyCatName[survey]).genericsource_set.add(*nset) + newsrc += nset + + + allloaded = ErositaSource.objects.filter(created__gt=elast.created) + if len(newsrc) > 0: + newsrc = GenericSource.objects.filter(id__in=[src.id for src in newsrc]) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=CSC2), minrad=3., maxdist=20.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SXPS2), minrad=3., maxdist=20.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=XMM3DR8), minrad=3., maxdist=20.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=RXS2), minrad=20., maxdist=80.) + gsu.find_counterparts(newsrc, GenericSource.objects.filter(instance_of=SDSSSpec), minrad=3., maxdist=20.) + gsu.find_from_old_gaia_db(newsrc, minrad=5, maxdist=20) + gsu.load_allwise_auxcat(newsrc, maxdist=10, minrad=3) + gsu.find_counterparts(allloaded, GenericSource.objects.filter(instance_of=AllWISEgs), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + gsu.load_NVSS_auxcat(newsrc, maxdist=10, minrad=3) + gsu.find_counterparts(allloaded, GenericSource.objects.filter(instance_of=NVSSgs), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + gsu.load_simbad_auxcat(newsrc) + gsu.find_counterparts(allloaded, GenericSource.objects.filter(instance_of=SimbadSource), maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + gsu.load_ztf_auxcat(newsrc, maxdist=10, minrad=3) + gsu.find_counterparts(allloaded, GenericSource.objects.filter(instance_of=ZTFOid), maxdist=kwargs.get("maxdist", 10.), minrad=kwargs.get("minrad", 3.)) + set_transient_data(allloaded) + GenericCatalog.objects.get(name='transients 2').genericsource_set.add(*allloaded.exclude(erositatransient=None)) + diff --git a/srglib/migrations/__init__.py b/srglib/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/srglib/models.py b/srglib/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/srglib/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/srglib/tasks.py b/srglib/tasks.py new file mode 100644 index 0000000..c20078b --- /dev/null +++ b/srglib/tasks.py @@ -0,0 +1,104 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils import timezone + +from srglib.utils import load_erotrans_table, load_srg_data_dumps +from srglib.utils import mark_new_skymap_sources_in_latest, update_allsky_missed +from srglib.utils import load_skymap_sources_dir +from srglib.ztf import load_ztf_alerce +import pytz + +from erotrans.models import EroTrans +from monthplan.utils import load_surveypath_data, load_monthplan_dir + +import glob +from django.contrib.auth.models import User +from django.core import mail +from django.template.loader import render_to_string +from django.utils.html import strip_tags +from django.core.mail import send_mail + +from celery import shared_task +from srg.celery import app + +import logging + +@app.task +def load_monthplan_task(): + load_monthplan_dir('/export/django/srg/data/npol/fits') + +@app.task +def load_surveypath_task(): + path = "/export/django/srg/data/npol/DMV/ARJ-SCAN/20??????_??????_???????????.iki" + logging.info("load SRG survey path {}".format(path)) + ikifiles = [] + for file in glob.glob(path): + ikifiles.append(file) + + for file in ikifiles: + load_surveypath_data(file,100,notify=True) + +@app.task +def load_srg_data_dumps_task(): + dir='/srg/work/oper/staff/kate/skymap_v0' + load_srg_data_dumps(dir) + #load_skymap_sources_dir('/srg/work/oper/staff/kate/skymap/catalog') + #mark_new_skymap_sources_in_latest() + #update_allsky_missed() + +@app.task +def load_skymap_sources_task(): + load_skymap_sources_dir('/srg/work/oper/staff/kate/skymap/catalog') + #mark_new_skymap_sources_in_latest() + update_allsky_missed() + +@app.task +def load_ztf_alerce_task(): + logging.info("load ALeRCE ZTF alerts for last 2 days") + load_ztf_alerce(2) + +@app.task +def load_ztf_alerts_task(): + """ deprecated """ + logging.warning("load_ztf_alerts_task") + load_ztf_alerts() + +@app.task +def load_erosita_daily_source_list(): + """ Loads eRosita daily source catalog + + In /export/django/srg, use the following command to start celery.worker: + celery -A srg worker --pool=solo -l info --loglevel=DEBUG + + Use the following command to start celery.beat: + celery -A srg beat + """ + + users = User.objects.filter(groups__name='srg-erosita-transients') + to_emails=[] + for user in users: + to_emails.append(user.email) + + pklfiles = [] + for filename in glob.glob("/data/erosita/transients/srclist_*.pkl"): + pklfiles.append(file) + + + for filename in pklfiles: + res = load_erotrans_table(file) + if (res > 0): + try: + cat = EroTrans.objects.get(pk=res) + except: + logging.error("EroTrans with pk=%d not found", res) + continue + """ + Send email notification to group + """ + nomatch_count=cat.erotranssource_set.filter(heasarc=None).count() + subject = "New eRosita catalog %s" % cat.obsid + html_message = render_to_string('erotrans/erotrans_loaded_email.html', {'cat':cat,'nomatch_count':nomatch_count}) + plain_message = strip_tags(html_message) + from_email = 'Roman Krivonos ' + + #mail.send_mail(subject, plain_message, from_email, to_emails, html_message=html_message) diff --git a/srglib/tests.py b/srglib/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/srglib/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/srglib/update_erotrans_tables.py b/srglib/update_erotrans_tables.py new file mode 100644 index 0000000..384e7e4 --- /dev/null +++ b/srglib/update_erotrans_tables.py @@ -0,0 +1,44 @@ +from ErositaDailySourceCatalogues import views as eviews +from ErositaDailySourceCatalogues.models import * +from genericsource.models import * +from srglib import genericsourceutils as gsu +from srglib.load_transinets import SurveyCatName +from django.db.models import F, Q, Max, Subquery, OuterRef, Min, Count, Value +from functools import reduce + + + +def getalltransients(): + query = reduce(lambda a, b: a | b, [Q(id__in=GenericCatalog.objects.get(name=survname).genericsource_set.all()) for survname in SurveyCatName.values()]) + return ErositaSource.objects.filter(query) + + +def get_transients(cat, tcat): + cat = eviews.create_annotations(cat) + cat = eviews.addErositaSrcannotattion(cat) + cat = cat.filter(ratio_to_maxxflux > 1.1) + removed = ErositaSource.objects.filter(tcat.genericsource_set.exclude(id__in=cat)) + tcat.genericsource_set.remove(tcat.genericsource_set.all()) + + +def set_transient_data(ecat): + ecat = eviews.create_annotations(ecat) + ecat = eviews.addErositaSrcannotattion(ecat) + ecat = ecat.filter(ratio_to_maxxflux__gt=1.) + for src in ecat: + try: + ErositaTransient.objects.create(src=src, plx=src.plx, plxe=src.plxe, + z=src.ssz if src.ssz else src.sdssz, + ssep=src.ssep, + sname=src.sname if src.sname else "", + ztfcl=src.ztfcl if src.ztfcl else "", + ztfnm=src.ztfnm if src.ztfnm else "", + zsep=src.zsep, plx_sep=src.plx_sep, + plx_mag=src.plx_mag, + sclass=src.sclass if src.sclass else "", + ratio_to_maxxflux=src.ratio_to_maxxflux, + ratio_to_minxflux=src.ratio_to_minxflux) + except: + print(src.name, " already has transints") + else: + print("transient for %s created" % src.name) diff --git a/srglib/utils.py b/srglib/utils.py new file mode 100644 index 0000000..ed7d1ae --- /dev/null +++ b/srglib/utils.py @@ -0,0 +1,2458 @@ +import re +import json +import glob +import os +import sys +import logging +import requests +import json +import pprint +import time + +import pandas as pd +import numpy as np +from math import pi, cos, sin +from Quaternion import Quat +from scipy.spatial.transform import Rotation, Slerp +from scipy.sparse import coo_matrix +from datetime import datetime + +from django.db.models import Q +from django.urls import resolve +from django.contrib.auth.models import Group +from django_pandas.io import read_frame +from django.utils.timezone import localdate, localtime + +import astropy +from astropy.io import fits +from astropy.io import ascii +from astropy_healpix import HEALPix +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +from astropy_healpix import HEALPix, neighbours +from astropy import units as au +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from erotrans.models import EroTransSource, EroTrans +from erosurvey.models import NSIDE_PLATES, ORDER_PLATES + +from heasarc.models import NSIDE_SOURCES, ORDER, HeasarcBase, GreenSNR + +from astrobasis.models import GAIADR2 +from astrobasis.models import Flesch +from astrobasis.models import SDSSDR12Spec +from astrobasis.models import ZTFAlert +from astrobasis.models import Simbad +from astrobasis.models import AllWise +from astrobasis.models import NVSS +from astrobasis.models import FIRST +from astrobasis.models import SUMSS +from astrobasis.models import VLASSfromVizieR +from astrobasis.models import BJfromVizieR +from astrobasis.models import GAIADR3fromVizieR + +from srgcat.models import ArtCat, SkyMaps, SrgDataDump +from srgcat.models import SkyMapSource, SelectAllskyMissed +from srgcat.models import eRositaMatch +from srgcat.models import TrackStats +from srgcat.models import Gyro + +from monthplan.models import Survey +from monthplan.models import SurveyHealpixPlate + + +from srglib import correlate_utils, check_transient + + + +os.environ["HOME"] = "/tmp/djangoastropy/" +if not os.path.exists("/tmp/djangoastropy/.astropy"): os.makedirs("/tmp/djangoastropy/.astropy") +from astroquery.simbad import Simbad as aSimbad +from astroquery.vizier import Vizier as aVizier + + +MJDREF = 51543.875 +TZ_UTC = TimezoneInfo(utc_offset=0*u.hour) +TZ_MSK = TimezoneInfo(utc_offset=3*u.hour) + +TIMESTR_FORMAT = '%Y.%m.%d %H:%M:%S' + +def boolean_notnan(x): + var = x if not (np.isnan(x)) else None + if(var): + return True if (int(var) == 1) else False + else: + return False + +def notnan(val): + return val if not (np.isnan(val)) else None + +def mission2date(timesec): + mjdref = Time(MJDREF, format='mjd') + delta = TimeDelta(timesec, format='sec') + dtime = mjdref + delta + 3*u.hour + return dtime.to_datetime(timezone=TZ_MSK) + +def str2date(timestr, tzone): + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + +def str2dateT(timestr, tzone): + TIMESTR_FORMAT = '%Y-%m-%dT%H:%M:%S' + time_raw = datetime.strptime(timestr, TIMESTR_FORMAT) + time_zoned = time_raw.replace(tzinfo=tzone) + return time_zoned + +def date2mission(dtime): + mjdref = Time(MJDREF, format='mjd') + dtime = Time(dtime , format='datetime') + return (dtime - mjdref).sec + +def load_vizier_allwise(srcs, ssize=4096, **kwargs): + print("allwise") + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + av = aVizier() + av.ROW_LIMIT=-1 + av.TIMEOUT = 150 + minrad=kwargs.get("minrad", 30.) + for src in srcs: + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = av.query_region(sc0, radius=kwargs.get("maxdist", 120)*au.arcsec, catalog='II/328/allwise') + except Exception as e: + print("{} failed to load {}".format(src,e)) + #src.simbad_failed=True + #src.save() + + if(simreply): + df=simreply[0].to_pandas() + print("{} found {} simbad sources".format(src,df.index.size)) + for col in df.columns: + print("col: {}".format(col)) + #print(simreply[0]) + + for index, row in df.iterrows(): + name = row['AllWISE']#.decode("ascii") + ra = row['RAJ2000'] + dec = row['DEJ2000'] + print("{} --> {}".format(src,name)) + + try: + simsrc = AllWise.objects.get(name=name) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + print("Found in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find AllWise object for {}, {}".format(name,e)) + print("Create AllWise object {}".format(name)) + sc = SkyCoord(ra, dec, frame=FK5(), unit="deg") + #sc = SkyCoord(row['RA'] + " " + row['DEC'], unit=("hourangle", "deg"), frame="fk5") + simsrc = AllWise(name=name) + simsrc.ra = ra + simsrc.dec = dec + simsrc.healpix = hp.skycoord_to_healpix(sc) + simsrc.error_radius = 0.0 + simsrc.W1mag = row['W1mag'] + simsrc.e_W1mag = row['e_W1mag'] + simsrc.W2mag = row['W2mag'] + simsrc.e_W2mag = row['e_W2mag'] + simsrc.save() + print("Created in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + + print("Cross-match with loaded AllWise objects") + find_counterparts(srcs, AllWise.objects.all(), "allwise", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 40.)) + + +def load_vizier_nvss(srcs, ssize=4096, **kwargs): + print("allwise") + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + av = aVizier() + av.ROW_LIMIT=-1 + av.TIMEOUT = 150 + minrad=kwargs.get("minrad", 30.) + for src in srcs: + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = av.query_region(sc0, radius=kwargs.get("maxdist", 120)*au.arcsec, catalog='VIII/65/nvss') + except Exception as e: + print("{} failed to load {}".format(src,e)) + #src.simbad_failed=True + #src.save() + + + if(simreply): + df=simreply[0].to_pandas() + print("{} found {} NVSS sources".format(src,df.index.size)) + #for col in df.columns: + # print("col: {}".format(col)) + #print(simreply[0]) + + + for index, row in df.iterrows(): + name = row['NVSS']#.decode("ascii") + ra = row['RAJ2000'] + dec = row['DEJ2000'] + print("{} --> {}".format(src,name)) + + try: + simsrc = NVSS.objects.get(name=name) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + print("Found in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find NVSS object for {}, {}".format(name,e)) + print("Create NVSS object {}".format(name)) + #sc = SkyCoord(ra, dec, frame=FK5(), unit="deg") + sc = SkyCoord(row['RAJ2000'] + " " + row['DEJ2000'], unit=("hourangle", "deg"), frame="fk5") + simsrc = NVSS(name=name, ra=sc.fk5.ra.value, dec=sc.fk5.dec.value) + simsrc.healpix = hp.skycoord_to_healpix(sc) + simsrc.error_radius = (row['e_RAJ2000']+row['e_DEJ2000'])/2.0 + simsrc.e_ra = row['e_RAJ2000'] + simsrc.e_dec = row['e_DEJ2000'] + simsrc.S14 = row['S1.4'] + simsrc.e_S14 = row['e_S1.4'] + simsrc.save() + print("Created in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + + print("Cross-match with loaded AllWise objects") + find_counterparts(srcs, NVSS.objects.all(), "nvss", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 40.)) + +def load_vizier_first(srcs, ssize=4096, **kwargs): + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + av = aVizier() + av.ROW_LIMIT=-1 + av.TIMEOUT = 150 + minrad=kwargs.get("minrad", 30.) + for src in srcs: + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = av.query_region(sc0, radius=kwargs.get("maxdist", 120)*au.arcsec, catalog='VIII/92/first14') + except Exception as e: + print("{} failed to load {}".format(src,e)) + #src.simbad_failed=True + #src.save() + + + if(simreply): + df=simreply[0].to_pandas() + print("{} found {} NVSS sources".format(src,df.index.size)) + #for col in df.columns: + # print("col: {}".format(col)) + #print(simreply[0]) + + + for index, row in df.iterrows(): + name = row['FIRST']#.decode("ascii") + ra = row['RAJ2000'] + dec = row['DEJ2000'] + print("{} --> {}".format(src,name)) + + try: + simsrc = FIRST.objects.get(name=name) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + print("Found in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find FIRST object for {}, {}".format(name,e)) + print("Create FIRST object {}".format(name)) + #sc = SkyCoord(ra, dec, frame=FK5(), unit="deg") + sc = SkyCoord(row['RAJ2000'] + " " + row['DEJ2000'], unit=("hourangle", "deg"), frame="fk5") + simsrc = FIRST(name=name, ra=sc.fk5.ra.value, dec=sc.fk5.dec.value) + simsrc.healpix = hp.skycoord_to_healpix(sc) + simsrc.error_radius = (row['Maj']+row['Min'])/2.0 + simsrc.Fpeak = row['Fpeak'] + simsrc.Fint = row['Fint'] + simsrc.rms = row['Rms'] + simsrc.major_axis = row['Maj'] + simsrc.minor_axis = row['Min'] + simsrc.save() + print("Created in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + + print("Cross-match with loaded objects") + find_counterparts(srcs, FIRST.objects.all(), "first", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 40.)) + +def load_vizier_sumss(srcs, ssize=4096, **kwargs): + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + av = aVizier() + av.ROW_LIMIT=-1 + av.TIMEOUT = 150 + minrad=kwargs.get("minrad", 30.) + for src in srcs: + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = av.query_region(sc0, radius=kwargs.get("maxdist", 120)*au.arcsec, catalog='VIII/81B') + except Exception as e: + print("{} failed to load {}".format(src,e)) + + + if(simreply): + df=simreply[0].to_pandas() + print("{} found {} SUMSS sources".format(src,df.index.size)) + for col in df.columns: + print("col: {}".format(col)) + print(simreply[0]) + + + for index, row in df.iterrows(): + name = "{}, {}".format(row['RAJ2000'], row['DEJ2000']) + print("{} --> {}".format(src,name)) + + try: + simsrc = SUMSS.objects.get(name=name) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + print("Found in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find SUMSS object for {}, {}".format(name,e)) + print("Create SUMSS object {}".format(name)) + #sc = SkyCoord(ra, dec, frame=FK5(), unit="deg") + sc = SkyCoord(row['RAJ2000'] + " " + row['DEJ2000'], unit=("hourangle", "deg"), frame="fk5") + simsrc = SUMSS(name=name, ra=sc.fk5.ra.value, dec=sc.fk5.dec.value) + simsrc.healpix = hp.skycoord_to_healpix(sc) + simsrc.error_radius = (row['MajAxis']+row['MinAxis'])/2.0 + simsrc.Sp = row['Sp'] + simsrc.e_Sp = row['e_Sp'] + simsrc.St = row['St'] + simsrc.e_St = row['e_St'] + simsrc.e_ra = row['e_RAJ2000'] + simsrc.e_dec = row['e_DEJ2000'] + simsrc.ra_orig = row['RAJ2000']#.decode("ascii") + simsrc.dec_orig = row['DEJ2000']#.decode("ascii") + simsrc.major_axis = row['MajAxis'] + simsrc.minor_axis = row['MinAxis'] + simsrc.save() + print("Created in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + + print("Cross-match with loaded objects") + find_counterparts(srcs, SUMSS.objects.all(), "sumss", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 40.)) + +def load_vizier_vlass(srcs, ssize=4096, **kwargs): + + cols=['recno', + 'CompName', + 'CompId','_Glon','_Glat', + 'RAJ2000','e_RAJ2000', + 'DEJ2000','e_DEJ2000', + 'Ftot', + 'e_Ftot', + 'Fpeak', + 'e_Fpeak', + 'Islrms', + 'SCode', + 'DCMaj', + 'DCMin', + 'DCPA', + 'Subtile', + 'NVSSdist', + 'FIRSTdist', + 'PeakToRing', + 'DupFlag', + 'QualFlag', + 'NNdist', + 'MainSample', + 'QLcutout',] + + + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + av = aVizier(columns=cols) + av.ROW_LIMIT=-1 + av.TIMEOUT = 150 + minrad=kwargs.get("minrad", 30.) + for src in srcs: + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = av.query_region(sc0, radius=kwargs.get("maxdist", 120)*au.arcsec, catalog='J/ApJS/255/30/comp') + except Exception as e: + print("{} failed to load {}".format(src,e)) + #src.simbad_failed=True + #src.save() + + + if(simreply): + df=simreply[0].to_pandas() + print("{} found {} source(s)".format(src,df.index.size)) + """ + for col in df.columns: + print("col: {}".format(col)) + print(simreply) + """ + + + for index, row in df.iterrows(): + name = row['CompName']#.decode("ascii") + ra = row['RAJ2000'] + dec = row['DEJ2000'] + print("{} --> {}".format(src,name)) + + try: + simsrc = VLASSfromVizieR.objects.get(name=name) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + """ print("Found in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) """ + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find VLASS object for {}, {}".format(name,e)) + print("Create VLASS object {}".format(name)) + sc = SkyCoord(ra, dec, frame=FK5(), unit="deg") + #sc = SkyCoord(row['RAJ2000'] + " " + row['DEJ2000'], unit=("hourangle", "deg"), frame="fk5") + simsrc = VLASSfromVizieR(name=name, ra=sc.fk5.ra.value, dec=sc.fk5.dec.value) + simsrc.healpix = hp.skycoord_to_healpix(sc) + simsrc.error_radius = (row['e_RAJ2000']+row['e_DEJ2000'])/2.0 + simsrc.recno=int(row['recno']) + simsrc.glon=float(row['_Glon']) + simsrc.glat=float(row['_Glat']) + simsrc.e_ra=float(row['e_RAJ2000']) + simsrc.e_dec=float(row['e_DEJ2000']) + simsrc.ftot = float(row['Ftot']) + simsrc.e_ftot = float(row['e_Ftot']) + simsrc.fpeak = float(row['Fpeak']) + simsrc.e_fpeak = float(row['e_Fpeak']) + simsrc.dupflag = int(row['DupFlag']) + simsrc.qualflag = int(row['QualFlag']) + simsrc.save() + print("Created in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + + print("Cross-match with loaded objects") + find_counterparts(srcs, VLASSfromVizieR.objects.all(), "vlass", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 40.)) + + +def load_vizier_bj(srcs, ssize=4096, **kwargs): + + cols=['Source', + 'RA_ICRS', + 'DE_ICRS', + 'rgeo', + 'b_rgeo', + 'B_rgeo', + 'rpgeo', + 'b_rpgeo', + 'B_rpgeo', + 'Flag',] + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + av = aVizier(columns=cols) + av.ROW_LIMIT=-1 + av.TIMEOUT = 150 + minrad=kwargs.get("minrad", 30.) + for src in srcs: + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = av.query_region(sc0, radius=kwargs.get("maxdist", 120)*au.arcsec, catalog='I/352/gedr3dis') + except Exception as e: + print("{} failed to load {}".format(src,e)) + #src.simbad_failed=True + #src.save() + + + if(simreply): + df=simreply[0].to_pandas() + print("{} found {} source(s)".format(src,df.index.size)) + + """ + for col in df.columns: + print("col: {}".format(col)) + print(simreply) + sys.exit() + """ + + for index, row in df.iterrows(): + source_id = int(row['Source'])#.decode("ascii") + ra = float(row['RA_ICRS']) + dec = float(row['DE_ICRS']) + print("{} --> {:d}".format(src,source_id)) + + try: + simsrc = BJfromVizieR.objects.get(source_id=source_id) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + """ print("Found in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) """ + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find BJ object for {}, {}".format(source_id,e)) + print("Create BJ object {}".format(source_id)) + sc = SkyCoord(ra, dec, frame=FK5(), unit="deg") + #sc = SkyCoord(row['RAJ2000'] + " " + row['DEJ2000'], unit=("hourangle", "deg"), frame="fk5") + simsrc = BJfromVizieR(source_id=source_id, ra=sc.fk5.ra.value, dec=sc.fk5.dec.value) + simsrc.healpix = hp.skycoord_to_healpix(sc) + #simsrc.recno=int(row['recno']) + simsrc.r_med_geo=row['rgeo'] + simsrc.r_lo_geo=row['b_rgeo'] + simsrc.r_hi_geo=row['B_rgeo'] + simsrc.r_med_pgeo=row['rpgeo'] + simsrc.r_lo_pgeo=row['b_rpgeo'] + simsrc.r_hi_pgeo=row['B_rpgeo'] + #simsrc.flag=row['Flag'] + simsrc.save() + print("Created in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + + print("Cross-match with loaded objects") + find_counterparts(srcs, BJfromVizieR.objects.all(), "bj2021", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 40.)) + +def load_vizier_gaia_dr3(srcs, ssize=4096, **kwargs): + #all = GAIADR3fromVizieR.objects.all() + #all.delete() + #return + cols=['Source', + 'RA_ICRS', + 'DE_ICRS', + 'e_RA_ICRS', + 'e_DE_ICRS', + 'Plx', + 'e_Plx', + 'RPlx', + 'PM', + 'pmRA', + 'e_pmRA', + 'pmDE', + 'e_pmDE', + 'Gmag','e_Gmag','BPmag','e_BPmag','RPmag','e_RPmag'] + + """ + col: RA_ICRS col: DE_ICRS + col: Source col: e_RA_ICRS + col: e_DE_ICRS col: Plx col: e_Plx + col: PM col: pmRA col: e_pmRA col: pmDE col: e_pmDE + col: RUWE col: FG col: e_FG col: Gmag col: FBP col: e_FBP + col: BPmag col: FRP col: e_FRP col: RPmag + col: BP-RP col: RV col: e_RV col: Vbroad + col: GRVSmag col: QSO col: Gal col: NSS col: XPcont + col: XPsamp col: RVS col: EpochPh + col: EpochRV col: MCMCGSP + col: MCMCMSC col: And col: Teff col: logg col: __Fe_H_ col: Dist + col: A0 col: HIP col: PS1 col: SDSS13 + col: SKYM2 col: TYC2 + col: URAT1 col: AllWISE + col: APASS9 col: GSC23 col: RAVE5 col: _2MASS col: RAVE6 col: RAJ2000 col: DEJ2000 + """ + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + av = aVizier(columns=cols) + #av = aVizier() + av.ROW_LIMIT=-1 + av.TIMEOUT = 150 + minrad=kwargs.get("minrad", 30.) + for src in srcs: + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = av.query_region(sc0, radius=kwargs.get("maxdist", 120)*au.arcsec, catalog='I/355/gaiadr3') + except Exception as e: + print("{} failed to load {}".format(src,e)) + #src.simbad_failed=True + #src.save() + + + if(simreply): + df=simreply[0].to_pandas() + print("{} found {} source(s)".format(src,df.index.size)) + + """ + for col in df.columns: + print("col: {}".format(col)) + print(simreply) + sys.exit() + """ + + for index, row in df.iterrows(): + source_id = int(row['Source'])#.decode("ascii") + ra = float(row['RA_ICRS']) + dec = float(row['DE_ICRS']) + print("{} --> {:d}".format(src,source_id)) + + try: + simsrc = GAIADR3fromVizieR.objects.get(source_id=source_id) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + """ print("Found in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) """ + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find GAIADR3 object for {}, {}".format(source_id,e)) + print("Create BJ object {}".format(source_id)) + sc = SkyCoord(ra, dec, frame=FK5(), unit="deg") + #sc = SkyCoord(row['RAJ2000'] + " " + row['DEJ2000'], unit=("hourangle", "deg"), frame="fk5") + simsrc = GAIADR3fromVizieR(source_id=source_id, ra=sc.fk5.ra.value, dec=sc.fk5.dec.value) + simsrc.healpix = hp.skycoord_to_healpix(sc) + simsrc.e_ra = row['e_RA_ICRS'] + simsrc.e_dec = row['e_DE_ICRS'] + simsrc.plx = row['Plx'] + simsrc.e_plx = row['e_Plx'] + simsrc.rplx = row['RPlx'] + simsrc.pm = row['PM'] + simsrc.pmra = row['pmRA'] + simsrc.e_pmra = row['e_pmRA'] + simsrc.pmde = row['pmDE'] + simsrc.e_pmde = row['e_pmDE'] + simsrc.gmag= row['Gmag'] + simsrc.e_gmag= row['e_Gmag'] + simsrc.bpmag= row['BPmag'] + simsrc.e_bpmag= row['e_BPmag'] + simsrc.rpmag = row['RPmag'] + simsrc.e_rpmag = row['e_RPmag'] + simsrc.save() + print("Created in DB: {} offset {:.2f} arcsec".format(simsrc,sc0.separation(sc).arcsec)) + + print("Cross-match with loaded objects") + find_counterparts(srcs, GAIADR3fromVizieR.objects.all(), "gaiadr3", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 40.)) + + +def load_simbad_sources(erosrcs, ssize=4096, **kwargs): + """ + for the set of sources queries simbad in 1 arcmin vicinity, cahces them and after that searches + for cross matches with provided sources in specified zone + """ + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + esrc = read_frame(erosrcs) + aSimbad.TIMEOUT = 150 + s1 = aSimbad() + s1.reset_votable_fields() + s1.add_votable_fields("z_value", "otype") + fails = [] + print("sources to correlate", esrc.index.size) + for i in range((esrc.index.size + ssize - 1)//ssize): + eloc = esrc.iloc[i*ssize:min((i + 1)*ssize, esrc.index.size)] + print("load simbad for first %d objects" % eloc.index.size) + try: + simbadc = s1.query_region(SkyCoord(eloc.ra.values, eloc.dec.values, unit="deg", frame="fk5"), radius=kwargs.get("maxdist", 60)*au.arcsec).to_pandas() + except Exception as exp: + print(exp) + print("ahtung failed to load") + fails.append(eloc.id.values) + continue + print("found %d simbad sources" % simbadc.index.size) + sc = SkyCoord(simbadc.RA.values + " " + simbadc.DEC.values, unit=("hourangle", "deg"), frame="fk5") + simbadc["ra"] = sc.fk5.ra.value + simbadc["dec"] = sc.fk5.dec.value + simbadc["healpix"] = hp.skycoord_to_healpix(sc) + simbadc.rename(columns={"COO_ERR_MINA": "error_radius", "Z_VALUE": "z"}, inplace=True) + simbadc.error_radius.fillna(0., inplace=True) + simbadc['main_id'] = simbadc.MAIN_ID.str.decode("ascii") + simbadc['obj_class'] = simbadc.OTYPE.str.decode("ascii") + simbadc['coo_bibcode'] = simbadc.COO_BIBCODE.str.decode("ascii") + simbadc['coo_wavelength'] = simbadc.COO_WAVELENGTH.str.decode("ascii") + """ + simbad.rename({"COO_ERR_MINA":"error_radius", + "MAIN_ID": "main_id", + "OTYPE": "obj_class", + "COO_BIBCODE": "coo_bibcode", + "COO_WAVELENGTH": "coo_wavelength", + "Z_VALUE": "z"}, inplace=True) + """ + simbadc.drop_duplicates('main_id', inplace=True) + simbadc.drop(columns=["RA", "DEC", "RA_PREC", "DEC_PREC", "COO_ERR_MAJA", + "COO_ERR_ANGLE", "COO_QUAL", "MAIN_ID", + "OTYPE", "COO_WAVELENGTH", "COO_BIBCODE"], inplace=True) + print("loaded", simbadc.index.size) + sobj = [Simbad(**o) for i, o in simbadc.iterrows()] + Simbad.objects.bulk_create(sobj, ignore_conflicts=True) + ech = erosrcs.filter(id__in=eloc.id.values) + print("correlating with %d sources" % ech.count()) + find_counterparts(ech, Simbad.objects.all(), "simbad", maxdist=kwargs.get("maxdist", 20.), minrad=kwargs.get("minrad", 3.)) + efail = erosrcs.filter(id__in=np.concatenate(fails)) if fails else erosrcs.filter(id__in=[]) + return efail + +def load_simbad_for_skymap_sources(srcs, **kwargs): + """ + for the set of sources queries simbad in 1 arcmin vicinity, cahces them and after that searches + for cross matches with provided sources in specified zone + """ + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + + aSimbad.TIMEOUT = 150 + s1 = aSimbad() + s1.reset_votable_fields() + s1.add_votable_fields("z_value", "otype") + + minrad=kwargs.get("minrad", 30.) + for src in srcs: + simbad_found_minrad=False + """ within minrad """ + + src.simbad_failed=False + src.simbad_notfound=False + src.save() + sc0 = SkyCoord(src.ra, src.dec, frame=FK5(), unit="deg") + try: + simreply = s1.query_region(SkyCoord(src.ra, src.dec, unit="deg", frame="fk5"), radius=kwargs.get("maxdist", 120)*au.arcsec) + except Exception as e: + print("{} failed to load {}".format(src,e)) + src.simbad_failed=True + src.save() + + if(simreply): + df=simreply.to_pandas() + print("{} found {} simbad sources".format(src,df.index.size)) + #for col in simbadc.columns: + # print(col) + + for index, row in df.iterrows(): + main_id = row['MAIN_ID']#.decode("ascii") + print("{} --> {}".format(src,main_id)) + try: + simsrc = Simbad.objects.get(main_id=main_id) + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + print("Found in DB: {} offset {:.2f} arcmin".format(simsrc,sc0.separation(sc).arcsec)) + #simsrc.delete() + #continue + except Exception as e: + print("Failed to find Simbad object for {}, {}".format(main_id,e)) + print("Create Simbad object {}".format(main_id)) + sc = SkyCoord(row['RA'] + " " + row['DEC'], unit=("hourangle", "deg"), frame="fk5") + simsrc = Simbad(main_id=main_id) + simsrc.ra = sc.fk5.ra.value + simsrc.dec = sc.fk5.dec.value + simsrc.healpix = hp.skycoord_to_healpix(sc) + simsrc.error_radius = 0.0 + simsrc.obj_class = row['OTYPE']#.decode("ascii") + simsrc.coo_bibcode = row['COO_BIBCODE']#.decode("ascii") + simsrc.coo_wavelength = row['COO_WAVELENGTH'] + simsrc.z = row['Z_VALUE'] + simsrc.save() + print("Created in DB: {} offset {:.2f} arcmin".format(simsrc,sc0.separation(sc).arcsec)) + + sc = SkyCoord(simsrc.ra, simsrc.dec, frame=FK5(), unit="deg") + if(sc0.separation(sc).arcsec < minrad): + print("Take {} to cross-match".format(src)) + simbad_found_minrad=True + + else: + print("{} No astronomical object found".format(src)) + src.simbad_notfound=True + src.save() + + if not (simbad_found_minrad): + print("Mark {} as simbad_notfound".format(src)) + src.simbad_notfound=True + src.save() + + print("Cross-match with loaded Simbad objects") + find_counterparts(srcs, Simbad.objects.all(), "simbad", maxdist=kwargs.get("maxdist", 120.), minrad=kwargs.get("minrad", 30.)) + + +def make_source_name(key, ra, dec): + """ Makes source name in format JHHMMSS.s+/-DDMMSS based on input RA and Dec. """ + try: + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + name = '{0} J{1}{2}'.format(key, + crd.ra.to_string(unit=u.hourangle, sep='', precision=1, pad=True), + crd.dec.to_string(sep='', precision=0, alwayssign=True, pad=True)) + return(name) + except BaseException as e: + print('Failed: '+ str(e)) + return('None') + +def srg_record_stats(request, group): + """ Save user access statistics """ + try: + grp = Group.objects.get(name=group) + except: + grp=None + + current_url = resolve(request.path_info).url_name + x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + + if x_forwarded_for: + ipaddress = x_forwarded_for.split(',')[-1].strip() + else: + ipaddress = request.META.get('REMOTE_ADDR') + + rec = TrackStats() + rec.ip_address= ipaddress + rec.url_name = current_url + rec.path_info = request.path_info + rec.user = request.user + rec.group = grp + rec.save() + pass + +status_code = {0:"No error", + 1:'

User is not authenticated, please login

', + 2:'

User has no Triton User Profile

', + 3:'

User has no appropriate group to access this page

',} + +def srg_group_auth(user, group): + """ Check whether User has appropriate group for access. + + Parameters + ---------- + user : User object + User object from request + group : str + Requested group name + + Returns + ------- + status : int + Status code {0:'No error', 1:'User is not authenticated', 2:'User has no user profile', 3:'User has no appropriate group to access this page',} + + + :Author: + Roman Krivonos + + """ + if not user.is_authenticated: + return 1 + try: + user_profile=user.profile + except: + return 2 + + if not user.groups.filter(name=group).exists(): + return 3 + + return 0 + +def srg_group_auth_stats(request, group): + """ check user group and write record """ + status = srg_group_auth(request.user, group) + if status == 0: + srg_record_stats(request, group) + return status + +def srg_auth(user): + """ Check User authorization and profile. + + Parameters + ---------- + user : User object + User object from request + + Returns + ------- + status : int + Status code {0:'No error', 1:'User is not authenticated', 2:'User has no user profile',} + + + :Author: + Roman Krivonos + + """ + + if not user.is_authenticated: + return 1 + try: + user_profile=user.profile + except: + return 2 + return 0 + +def check_sources_transient(srclistquery): + """ + check whether the objects in the provided queryset have fluxes 10 times different from their heasarc counterparts + + Parameters + ---------- + srclistquery: queryset + a quueryset of the erosita sources which have ML_FLUX_0 field, corresponding to 0.5--2 keV band + + + :Author: + Semena Andrey + """ + srctoupdate = [] + for src in srclistquery.prefetch_related("heasarc"): + check_transient.check_rass_transient(src) + if src.heasarc.filter(polymorphic_ctype__in=[check_transient.polctypes[cat] for cat in ["sxps2", "3xmm", "csc2"]]).exists(): + src.transient = True + check_transient.check_csc_transients(src) + check_transient.check_sxps_transients(src) + check_transient.check_xmm3_transients(src) + if src.transient or src.rosat_transient: + srctoupdate.append(src) + EroTransSource.objects.bulk_update(srctoupdate, ["transient", "rosat_transient"]) + +def check_catsources_transient(cat): + check_sources_transient(cat.erotranssource_set.all()) + cat.ntransients = cat.erotranssource_set.filter(Q(transient=True) | Q(rosat_transient=True)).count() + cat.save() + + +def find_counterparts(srcquery, counterpatscat, counterpartname, neighsearch=1, verbose=False, **kwargs): + """ + for all sources in srcquery, which is assumed to have many-to-many relationtip to counterpatscat catalog, + performs search of sources in counterpart catalog within max(minrad[=30], 3*sqrt(src.poserr**2 + counterpart.poserr**2)) + + + Parameters: + ---------- + srcquery: django.srcquery - iterable providing access to catalog under consideration + + counterpartscat: catalog to search counterparts in + counterpartname: name of the many-to-many relationtip field in srcquery + **kwargs: additional arguments provided to makeposerrdist functions (see doc within) + """ + start_time = time.time() + + scat = read_frame(srcquery) + try: + scat["error_radius"] = scat.radec_error + except AttributeError: + scat["error_radius"] = scat.error_radius + + uhealpix = np.unique(scat.healpix.values) + for v in range(neighsearch): + uhealpix = np.unique(np.concatenate([uhealpix, neighbours(uhealpix, NSIDE_SOURCES, order=ORDER).ravel()])) + + if (verbose): + print("--- find healpix in {} seconds ---".format(time.time() - start_time)) + ts = time.time() + + ccat = read_frame(counterpatscat.filter(healpix__in=uhealpix)) + print(scat.index.size, ccat.index.size) + if 'error_radius' not in ccat.columns.values: + ccat['error_radius'] = np.zeros(ccat.index.size) + + if (verbose): + print("--- select healpix in {} seconds ---".format(time.time() - ts)) + ts = time.time() + + if scat.index.size == 0 or ccat.index.size == 0: + return 0 + ccat.error_radius.fillna(0., inplace=True) + dmat = correlate_utils.makeposerrdist(scat, ccat, **kwargs) + print("detected nonzero connections", dmat.nnz) + + if (verbose): + print("--- nonzero connections in {} seconds ---".format(time.time() - ts)) + ts = time.time() + + uidx, ucts = np.unique(dmat.row, return_counts=True) + print("unique detections", uidx.size) + cs = np.empty(ucts.size + 1, np.int) + cs[0] = 0 + cs[1:] = ucts.cumsum() + ce = np.lib.stride_tricks.as_strided(cs, (cs.size - 1, 2), cs.strides*2) + + for i, uid in enumerate(uidx): + src = srcquery.get(id=scat.iloc[uid].id) #[0] + getattr(src, counterpartname).add(*ccat.iloc[dmat.col[ce[i, 0]: ce[i, 1]]].id.values) + src.save() + + if (verbose): + print("--- main search in {} seconds ---".format(time.time() - ts)) + ts = time.time() + + if (verbose): + print("--- %s seconds ---" % (time.time() - start_time)) + +def find_erosita_gaia_counterparts(erosources): + """ + for provided erosita sources catalog makes connections to gaia catalog sources within 3 pos errors, but not less then 5'' + """ + find_counterparts(erosources, GAIADR2.objects, "gaia", minrad=3, maxdist=20.) + + +def find_heasarc_counterparts(cat): + """ + searches the heasarc counterparts for the source in provided catalogue + + + Parameters + ---------- + cat : an instance of django.models.Models which provides the catalogue, containing a set of sources (which can be accessed with erotranssource_set) + it is assumed that the sources, attached to the catalogue have a field of type manytomany, which can connect then to the heasarc sources load in the system + + Returns + ------- + status : int + + + :Author: + Andrey Semena + """ + + srcs = cat.source_set.all() + res = find_heasarc_counterparts_for_srcs(srcs) + cat.nmatched = cat.source_set.exclude(heasarc=None).count() + cat.save() + return res + +def find_heasarc_counterparts_for_srcs(srcs): + """ + searches the heasarc counterparts for the provided sources + + + Parameters + ---------- + srcs : an instance of django.models.Models which provides the sources of EroTransSource type + it is assumed that the sources, attached to the catalogue have a field of type manytomany, which can connect then to the heasarc sources load in the system + + Returns + ------- + status : int + + + :Author: + Andrey Semena + """ + + + for src in srcs: + src.heasarc.clear() + + ecat = read_frame(srcs) + ecat["error_radius"] = ecat.radec_error + uhealpix = np.unique(ecat.healpix_plate.values) + allhealpix = np.unique(np.concatenate([neighbours(uhealpix, NSIDE_PLATES, order=ORDER).ravel(), uhealpix])) + heasarccats = HeasarcBase.objects.filter(healpix_plate__in=allhealpix) + if not heasarccats.exists(): + print('Done') + return None + + ccat = read_frame(heasarccats) #ccat = crosss catalogues + print("preliminary healpix match", ccat.index.size) + + dmat = correlate_utils.makeposerrdist(ecat, ccat) + heasarccats = heasarccats.filter(id__in=np.unique(dmat.col)) + print("detected nonzero connections", dmat.nnz) + uidx, ucts = np.unique(dmat.row, return_counts=True) + print("unique erosita detections", uidx.size) + cs = np.empty(ucts.size + 1, np.int) + cs[0] = 0 + cs[1:] = ucts.cumsum() + ce = np.lib.stride_tricks.as_strided(cs, (cs.size - 1, 2), cs.strides*2) + + for i, uid in enumerate(uidx): + src = srcs.get(id=ecat.iloc[uid].id) #[0] + src.heasarc.add(*ccat.iloc[dmat.col[ce[i, 0]: ce[i, 1]]].id.values) #*list(heasarccats.filter(id__in=ccat.iloc[dmat.col[ce[i, 0]: ce[i, 1]]].id.values))) + print(i, "has matches", ce[i, 1] - ce[i, 0]) + src.save() + return 1 + + +def correlate_erotrans_with_green(srcs): + """ + for erotrans sources search SNR remnants nearby + """ + ecat = read_frame(srcs) + ecat["error_radius"] = ecat.radec_error + + green = read_frame(GreenSNR.objects.all()) + dmatsnr = coo_matrix(correlate_utils.makedistmat3d(ecat, green, 600)) + mask = dmatsnr.data < green.iloc[dmatsnr.col].MajDiam.values*60. + rows, cols = dmatsnr.row[mask], dmatsnr.col[mask] + urow, uii = np.unique(rows, return_index=True) + ucols = cols[uii] + + for i, esrcid in enumerate(ecat.iloc[urow].id.values): + src = srcs.get(id=esrcid) + src.green_snr_id = green.iloc[ucols[i]].id + src.save() + + +def load_erotrans_table(filename): + """ Loads catalog of eRosita sources to database. + + This procedure accepts filename in format 'catalog_OBSID.pkl', + where OBSID is 11-digit identificator. In case this OBSID is + already loaded to DB, the procedure exits. Otherwise, + catalog is loaded to DB. + + Parameters + ---------- + filename : str + Absolute path to the input catalog file + + Returns + ------- + status : int + Status code {0:'Already in DB', ID:'Object ID or Primary key in DB'} + + + :Author: + Andrey Semena + + """ + #metadata = json.load(open(filename.split(".")[0] + ".json")) + + obsid = re.findall("\d+", filename)[0] + + logging.debug("Loading %s, ObsID: %s" % (filename,obsid,)) + + check_repeat = EroTrans.objects.filter(eroday_start=int(obsid[:5])).first() + if check_repeat and check_repeat.obsid >= obsid: + print("obsid exists") + logging.info("ObsID %s is already loaded, skip" % obsid) + return 0 + + logging.info("Loading ObsID %s" % obsid) + + hp = HEALPix(nside=NSIDE_SOURCES, + order=ORDER, frame=FK5()) + + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + + + datafile=filename + + eroday_start = int(obsid[0:5]) + eroday_stop = int(obsid[5:10]) + ts_start = eroday_start*14400 + ts_stop = eroday_stop*14400 + start = mission2date(ts_start) + stop = mission2date(ts_stop) + dtime_start = Time(start , format='datetime') + dtime_stop = Time(stop , format='datetime') + + survey = EroTrans(obsid=obsid, + eroday_start=eroday_start, + eroday_stop=eroday_stop, + start=start, + stop=stop, + mjd_start=dtime_start.mjd, + mjd_stop=dtime_stop.mjd) + + + survey.save() + if check_repeat and check_repeat.obsid < obsid: + print("removing older version of the catalog") + logging.info("remove older version of the catalog: %s" % obsid) + check_repeat.delete() + + df=pd.read_pickle(datafile) + df.drop(columns=["tilenum", 'srcname', 'hpidx', "TSTART", "ID_SRC", + "TSTOP", "DIST_NN", "ID_CLUSTER", 'SRCDENS'], inplace=True, errors="ignore") + df.rename(columns={"RA":"ra", "DEC":"dec", "RADEC_ERR": "radec_error", + "srcID": "srcid", "ML_EXP_1":'APE_EXP_1'}, inplace=True) + df["flux"] = df.ML_FLUX_0.values*0.8911e-12 + sc = SkyCoord(df.ra.values, df.dec.values, unit="deg", frame="fk5") + df["lii"] = sc.galactic.l.value + df["bii"] = sc.galactic.b.value + print(df.index.size) + sc = sc[df.lii.values < 180.] + df = df[df.lii.values < 180.] + print(df.index.size) + df["healpix"] = hp.skycoord_to_healpix(sc) + df["name"] = [make_source_name("IKIe", row.ra, row.dec) for row in df[["ra", "dec"]].itertuples()] + df["healpix_plate"] = hp_plate.skycoord_to_healpix(sc) + + srclist = [EroTransSource(catalog=survey, **src) for idx, src in df.iterrows()] + EroTransSource.objects.bulk_create(srclist, ignore_conflicts=True) + + survey.nrows = df.index.size + survey.source_set = survey.erotranssource_set + find_counterparts(survey.erotranssource_set.all(), HeasarcBase.objects.all(), "heasarc", minrad=20) + find_counterparts(survey.erotranssource_set.all(), Flesch.objects.all(), "flesch", minrad=3, maxdist=20.) + find_counterparts(survey.erotranssource_set.all(), SDSSDR12Spec.objects.all(), "sdss_spec", minrad=3, maxdist=20.) + find_counterparts(survey.erotranssource_set.all(), GAIADR2.objects.all(), "gaia", minrad=3, maxdist=20.) + correlate_erotrans_with_green(survey.erotranssource_set.all()) + survey.nmatched = survey.erotranssource_set.exclude(heasarc=None).count() + check_catsources_transient(survey) + survey.ntransients = survey.erotranssource_set.filter(Q(transient=True) | Q(rosat_transient=True)).count() + survey.save() + load_simbad_sources(survey.erotranssource_set.all()) + return survey.pk + + + +def load_erotrans_table_test(filename): + """ Test + + Parameters + ---------- + filename : str + Absolute path to the input catalog file + + + :Author: + Roman Krivonos + + """ + obsid = re.findall("\d+", filename)[0] + + #logging.warning("Loading %s, ObsID: %s" % (filename,obsid,)) + + if EroTrans.objects.filter(obsid=obsid).exists(): + logging.warning("ObsID %s is already loaded" % obsid) + return None + logging.warning("ObsID %s loading to DB" % obsid) + + + + +def quat_to_pol_and_roll(qfin): + """ + it is assumed that quaternion is acting on the sattelite coordinate system + in order to orient in in icrs coordinates + opaxis - define dirrection of the main axis (x axis [1, 0, 0] coinside with optical axis) + we assume that north is oriented along z coordinate, we also name this coordinate + north for detectors + """ + opaxis=[1, 0, 0] + north=[0, 0, 1] + opticaxis = qfin.apply(opaxis) + print(opticaxis) + dec = np.arctan(opticaxis[2]/np.sqrt(opticaxis[1]**2 + opticaxis[0]**2)) + ra = np.arctan2(opticaxis[1], opticaxis[0])%(2.*pi) + print(ra/pi*180,dec/pi*180) + return + yzprojection = np.cross(opticaxis, north) + vort = np.cross(north, opaxis) + + rollangle = np.arctan2(np.sum(yzprojection*qfin.apply(vort), axis=1), + np.sum(yzprojection*qfin.apply(north), axis=1)) + print(ra, dec, rollangle) + return ra, dec, rollangle + +def slerp(v0, v1, t_array): + """Spherical linear interpolation.""" + # >>> slerp([1,0,0,0], [0,0,0,1], np.arange(0, 1, 0.001)) + t_array = np.array(t_array) + v0 = np.array(v0) + v1 = np.array(v1) + dot = np.sum(v0 * v1) + + if dot < 0.0: + v1 = -v1 + dot = -dot + + DOT_THRESHOLD = 0.9995 + if dot > DOT_THRESHOLD: + result = v0[np.newaxis,:] + t_array[:,np.newaxis] * (v1 - v0)[np.newaxis,:] + return (result.T / np.linalg.norm(result, axis=1)).T + + theta_0 = np.arccos(dot) + sin_theta_0 = np.sin(theta_0) + + theta = theta_0 * t_array + sin_theta = np.sin(theta) + + s0 = np.cos(theta) - dot * sin_theta / sin_theta_0 + s1 = sin_theta / sin_theta_0 + return (s0[:,np.newaxis] * v0[np.newaxis,:]) + (s1[:,np.newaxis] * v1[np.newaxis,:]) + +def load_srg_data_dumps(dir): + """ Loads data dumps stored in plates + + The format of the data damp directoery is srg_20191228_143827_000 + + + Parameters + ---------- + dir : str + Absolute path to the input directory + + + :Author: + Roman Krivonos + """ + logging.info("Loading data dumps from %s" % dir) + format_string='srg_%Y%m%d_%H%M%S' + for path in glob.glob("%s/??????" % (dir)): + plate = os.path.basename(path) + + try: + skymap = SkyMaps.objects.get(SMAPNR=int(plate)) + except: + logging.error("SkyMap %s is not found" % plate) + pass + for l3 in glob.glob("%s/L3/srg_????????_??????_000" % path): + name = os.path.basename(l3) + try: + dump = SrgDataDump.objects.get(name=name) + #for s in dump.skymap.all(): + #print("{} --> {}".format(dump,s)) + queryset = skymap.srgdatadump_set.all() + if not queryset.filter(pk=dump.pk).exists(): + print("Adding {} to {}".format(skymap,dump)) + dump.skymap.add(skymap) + dump.save() + continue + except SrgDataDump.DoesNotExist: + pass + + logging.info("Creating {} in SkyMap {}".format(name, skymap)) + + dtime = datetime.strptime(name[:-4], format_string) + tm = Time(dtime, format='datetime', scale='utc') + dt = tm.to_datetime(timezone=TZ_MSK) + + dump = SrgDataDump(name=name, date=dt, version=int(name[-3:])) + dump.catalog_loaded=False + dump.save() + dump.skymap.add(skymap) + dump.save() + + +def load_skymap_sources_file(filename): + """ Loads sources stored in plates + + The format of the data damp directory is srg_20191228_143827_000 + + + Parameters + ---------- + filename : str + Absolute path to the file + + + :Author: + Roman Krivonos + """ + + if not os.path.isfile(filename): + print("File {} does not exist".format(filename)) + return + + logging.debug("Loading sources from {}".format(filename)) + + name = os.path.basename(filename) + fits_name = os.path.splitext(name)[0] + """ without .gz """ + dump_name = os.path.splitext(fits_name)[0] + + """ Check whether this data dump exists """ + try: + dump = SrgDataDump.objects.get(name=dump_name) + if(dump.catalog_loaded): + logging.debug("Source catalog for {} is loaded, skip".format(dump)) + print("Source catalog for {} is loaded, skip".format(dump)) + return + else: + logging.debug("Clear source catalog for {}".format(dump)) + srcs = dump.skymapsource_set.all() + srcs.delete() + except SrgDataDump.DoesNotExist: + logging.debug("Create new data dump {}".format(dump_name)) + format_string='srg_%Y%m%d_%H%M%S' + dtime = datetime.strptime(dump_name[:-4], format_string) + tm = Time(dtime, format='datetime', scale='utc') + dt = tm.to_datetime(timezone=TZ_MSK) + dump = SrgDataDump(name=dump_name, date=dt) + dump.save() + + logging.debug("Read source list from {} and attach it to {}".format(filename,dump)) + print("Read source list from {} and attach it to {}".format(filename,dump)) + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + + + ext=1 + hdul = fits.open(filename) # open a FITS file + hdr = hdul[ext].header + data = hdul[ext].data + + for item in data: + try: + plate=int(item['FIELD']) + skymap = SkyMaps.objects.get(SMAPNR=plate) + except: + logging.info("SkyMap {} is not found".format(plate)) + pass + ra=float(item['RA']) + dec=float(item['DEC']) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + healpix_plate = hp_plate.skycoord_to_healpix(crd) + src=SkyMapSource(skymap=skymap,dump=dump,healpix=healpix, + healpix_plate=healpix_plate, + name=make_source_name('SRGA',ra,dec), + name_orig=item['NAME'], + lii=crd.galactic.l.value, + bii=crd.galactic.b.value, + ra=ra,dec=dec, + x = item['x'], + y = item['y'], + cnts = item['cnts'], + cnts_err = item['cnts_err'], + cnts_bg = item['cnts_bg'], + exptime = item['exptime'], + rate = item['rate'], + rate_err = item['rate_err'], + flux = item['flux'], + flux_err = item['flux_err'], + sig = item['sig'], + nfalse = item['nfalse'],) + src.save() + + dump.catalog_loaded=True + dump.save() + + srcs = dump.skymapsource_set.all()#.order_by('-rate') + + #clean_skymap_sources(srcs) + find_counterparts(srcs, HeasarcBase.objects.all(),'heasarc',minrad=40) + load_simbad_for_skymap_sources(srcs,minrad=40,maxdist=120) + #find_counterparts(srcs, GAIADR2.objects.all(), "gaia", maxdist=60., minrad=30.) + #mark_new_skymap_sources_in_latest() + + + + + + +def load_skymap_sources_for_artsurvey(filename,name,force=False): + """ Loads sources stored in plates + + The format of the data damp directory is srg_20191228_143827_000 + + + Parameters + ---------- + filename : str + Absolute path to the file + + + :Author: + Roman Krivonos + """ + + if not os.path.isfile(filename): + print("File {} does not exist".format(filename)) + return + + + logging.debug("Loading sources from {}".format(filename)) + print("Loading sources from {}".format(filename)) + + + #name = os.path.basename(filename) + #fits_name = os.path.splitext(name)[0] + #""" without .gz """ + #dump_name = os.path.splitext(fits_name)[0] + + print(name) + + + """ Check whether this data dump exists """ + try: + dump = SrgDataDump.objects.get(name=name) + if(force): + dump.catalog_loaded=False + dump.save() + if(dump.catalog_loaded): + print("Source catalog for {} is loaded, skip".format(dump)) + return + else: + print("Clear source catalog for {}".format(dump)) + srcs = dump.skymapsource_set.all() + srcs.delete() + except SrgDataDump.DoesNotExist: + print("Create new data dump {}".format(name)) + dt=localtime() + dump = SrgDataDump(name=name, date=dt) + dump.save() + + + print("Read source list from {} and attach it to {}".format(filename,dump)) + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + + + ext=1 + hdul = fits.open(filename) # open a FITS file + hdr = hdul[ext].header + data = hdul[ext].data + + + for item in hdul[ext].columns: + print(item) + + + + for item in data: + try: + plate=int(item['FIELD']) + skymap = SkyMaps.objects.get(SMAPNR=plate) + except: + logging.info("SkyMap plate is not found") + plate=0 + skymap=None + pass + + ra=float(item['RA']) + dec=float(item['DEC']) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + healpix_plate = hp_plate.skycoord_to_healpix(crd) + + """ Find nearest SkyMap if FIELD is not present """ + if not (skymap): + try: + my_hp_plate = SurveyHealpixPlate.objects.get(healpix=healpix_plate) + skymaps = my_hp_plate.skymaps_set.all() + sep_min=10000.0 + for smap in skymaps: + smap_crd = SkyCoord(smap.ra, smap.dec, frame=FK5(), unit="deg") + sep=crd.separation(smap_crd).arcmin + if(sep < sep_min): + sep_min = sep + skymap = smap + except: + pass + + print("Add {}".format(make_source_name('SRGA',ra,dec))) + src=SkyMapSource(skymap=skymap,dump=dump,healpix=healpix, + healpix_plate=healpix_plate, + name=make_source_name('SRGA',ra,dec), + name_orig=item['NAME']) + src.lii=crd.galactic.l.value + src.bii=crd.galactic.b.value + src.ra=ra + src.dec=dec + src.x = item['x'] + src.y = item['y'] + + src.cnts = item['ml_cnts'] + + try: + src.cnts_err = item['ml_cnts_err'] + except KeyError: + src.cnts_err = None + try: + src.cnts_bg = item['cnts_bg'] + except KeyError: + src.cnts_bg = None + try: + src.cnts_tot = item['cnts_tot'] + except KeyError: + src.cnts_tot = None + src.exptime = item['exptime'] + try: + src.ext = int(item['ext']) + except KeyError: + src.ext = None + try: + src.ext_id=item['ext_id'] + except KeyError: + src.ext_id = None + try: + src.rate = item['rate'] + except KeyError: + src.rate = None + try: + src.rate_err = item['rateerr_up'] + except KeyError: + src.rate_err = None + src.flux = item['flux'] + try: + src.flux_err = item['fluxerr_up']# obsolete + except KeyError: + src.flux_err = None + try: + src.fluxerr_lo = item['fluxerr_lo'] + except KeyError: + src.fluxerr_lo = None + try: + src.fluxerr_up = item['fluxerr_up'] + except KeyError: + src.fluxerr_up = None + try: + src.flux_uplim = item['flux_uplim'] + except KeyError: + src.flux_uplim = None + + try: + src.cnts0 = item['cnts0'] + except KeyError: + src.cnts0 =None + try: + src.cnts_er0 = item['cntserr0_up']# obsolete + except KeyError: + src.cnts_er0 =None + try: + src.cnts_bg0 = item['cnts_bg0'] + except KeyError: + src.cnts_bg0 = None + try: + src.cnts_t0 = item['cnts_t0'] + except KeyError: + src.cnts_t0 = None + try: + src.exptime0 = item['exptime0'] + except KeyError: + src.exptime0 =None + try: + src.flux0 = item['flux0'] + except KeyError: + src.flux0 = None + try: + src.flux_er0 = item['fluxerr0_up'] + except KeyError: + src.flux_er0 =None + + try: + src.cnts1 = item['cnts1'] + except KeyError: + src.cnts1 = None + try: + src.cnts_er1 = item['cntserr1_up']# obsolete + except KeyError: + src.cnts_er1 =None + try: + src.cnts_bg1 = item['cnts_bg1'] + except KeyError: + src.cnts_bg1 =None + try: + src.cnts_t1 = item['cnts_t1'] + except KeyError: + src.cnts_t1 =None + try: + src.exptime1 = item['exptime1'] + except KeyError: + src.exptime1 =None + try: + src.flux1 = item['flux1'] + except KeyError: + src.flux1 = None + try: + src.flux_er1 = item['fluxerr1_up'] + except KeyError: + src.flux_er1 = None + + try: + src.cnts2 = item['cnts2'] + except KeyError: + src.cnts2 = None + try: + src.cnts_er2 = item['cntserr2_up'] + except KeyError: + src.cnts_er2 = None + try: + src.cnts_bg2 = item['cnts_bg2'] + except KeyError: + src.cnts_bg2 =None + try: + src.cnts_t2 = item['cnts_t2'] + except KeyError: + src.cnts_t2 =None + try: + src.exptime2 = item['exptime2'] + except KeyError: + src.exptime2 =None + try: + src.flux2 = item['flux2'] + except KeyError: + src.flux2 =None + try: + src.flux_er2 = item['fluxerr2_up'] + except KeyError: + src.flux_er2 = None + + try: + src.cnts3 = item['cnts3'] + except KeyError: + src.cnts3 =None + try: + src.cnts_er3 = item['cntserr3_up'] + except KeyError: + src.cnts_er3 =None + try: + src.cnts_bg3 = item['cnts_bg3'] + except KeyError: + src.cnts_bg3 =None + try: + src.cnts_t3 = item['cnts_t3'] + except KeyError: + src.cnts_t3 =None + try: + src.exptime3 = item['exptime3'] + except KeyError: + src.exptime3 =None + try: + src.flux3 = item['flux3'] + except KeyError: + src.flux3 =None + try: + src.flux_er3 = item['fluxerr3_up'] + except KeyError: + src.flux_er3 = None + + try: + src.cnts4 = item['cnts4'] + except KeyError: + src.cnts4 =None + try: + src.cnts_er4 = item['cntserr4_up'] + except KeyError: + src.cnts_er4 =None + try: + src.cnts_bg4 = item['cnts_bg4'] + except KeyError: + src.cnts_bg4 =None + try: + src.cnts_t4 = item['cnts_t4'] + except KeyError: + src.cnts_t4 =None + try: + src.exptime4 = item['exptime4'] + except KeyError: + src.exptime4 =None + try: + src.flux4 = item['flux4'] + except KeyError: + src.flux4 = None + try: + src.flux_er4 = item['fluxerr4_up'] + except KeyError: + src.flux_er4 = None + + try: + src.sig = item['sig'] + except KeyError: + src.sig =None + try: + src.detlike = float(item['detlike'])#/2.30259 Andy's catalog fix + except KeyError: + src.detlike =None + try: + src.nfalse = item['nfalse'] + except KeyError: + src.nfalse = None + + try: + src.ml_sig = item['ml_sig'] + except KeyError: + src.ml_sig = None + try: + src.ml_detlike = item['ml_detlike'] + except KeyError: + src.ml_detlike = None + try: + src.ml_nfalse = item['ml_nfalse'] + except KeyError: + src.ml_nfalse = None + try: + src.ml_ra = item['ml_ra'] + except KeyError: + src.ml_ra = None + try: + src.ml_dec = item['ml_dec'] + except KeyError: + src.ml_dec = None + try: + src.ml_radec_err_90 = item['radec_err_90'] + except KeyError: + src.ml_radec_err_90 = None + try: + src.ml_radec_err_98 = item['radec_err_98'] + except KeyError: + src.ml_radec_err_98 = None + try: + src.ml_flux = item['ml_flux'] + except KeyError: + src.ml_flux = None + try: + src.ml_flux_err = item['ml_flux_err'] + except KeyError: + src.ml_flux_err = None + try: + src.ml_exp = item['ml_exp'] + except KeyError: + src.ml_exp = None + + src.save() + + dump.catalog_loaded=True + dump.save() + + srcs = dump.skymapsource_set.all()#.order_by('-rate') + print(srcs.count()) + + + +def update_skymap_sources_for_artsurvey(filename,name): + """ Update sources + + The format of the data damp directory is srg_20191228_143827_000 + + + Parameters + ---------- + filename : str + Absolute path to the file + + + :Author: + Roman Krivonos + """ + + if not os.path.isfile(filename): + print("File {} does not exist".format(filename)) + return + + + logging.debug("Loading sources from {}".format(filename)) + print("Loading sources from {}".format(filename)) + + + #name = os.path.basename(filename) + #fits_name = os.path.splitext(name)[0] + #""" without .gz """ + #dump_name = os.path.splitext(fits_name)[0] + + print(name) + + + """ Check whether this data dump exists """ + try: + dump = SrgDataDump.objects.get(name=name) + print("Use {}".format(dump)) + + except SrgDataDump.DoesNotExist: + print("Not found {}".format(name)) + + print("Read source list from {} and attach it to {}".format(filename,dump)) + + ext=1 + hdul = fits.open(filename) # open a FITS file + hdr = hdul[ext].header + data = hdul[ext].data + + """ + for item in hdul[ext].columns: + print(item) + """ + + for item in data: + try: + plate=int(item['FIELD']) + skymap = SkyMaps.objects.get(SMAPNR=plate) + except: + logging.info("SkyMap plate is not found") + plate=0 + skymap=None + pass + + """ find source by name_orig """ + name_orig=item['NAME'] + try: + src = SkyMapSource.objects.get(name_orig__exact=name_orig) + except: + continue + ext_id=item['EXT_ID'] + if not ext_id == '': + src.ext_id=ext_id + src.save() + print("{} --> {}".format(src, ext_id)) + + + + + + +def load_skymap_sources_erosita(filename): + """ Loads sources cross-matched with eRosita + + Parameters + ---------- + filename : str + Absolute path to the file + + + :Author: + Roman Krivonos + """ + logger = logging.getLogger('django') + + if not os.path.isfile(filename): + logger.error("File {} does not exist".format(filename)) + return + + logger.info("Loading sources from {}".format(filename)) + + + + name = os.path.basename(filename) + catalog_name = os.path.splitext(name)[0] + logger.info("Filename {} catalog {}".format(name, catalog_name)) + + + """ Check whether this data dump exists """ + try: + catalog = ArtCat.objects.get(name=catalog_name) + logger.info("Catalog {} is found".format(catalog)) + catalog.delete() + except ArtCat.DoesNotExist: + pass + + logger.info("Create catalog {}".format(catalog_name)) + + format_string='srg_%Y%m%d_%H%M%S' + dtime = datetime.strptime(catalog_name[:-4], format_string) + tm = Time(dtime, format='datetime', scale='utc') + dt = tm.to_datetime(timezone=TZ_MSK) + + catalog = ArtCat(name=catalog_name) + catalog.save() + + data = astropy.table.Table.read(filename,format='ascii.csv') + print(data.info) + + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + hp_plate = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, + frame=FK5()) + + for item in data: + + try: + plate=int(item['FIELD']) + skymap = SkyMaps.objects.get(SMAPNR=plate) + except: + logging.info("SkyMap {} is not found".format(plate)) + pass + ra=float(item['RA']) + dec=float(item['DEC']) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + healpix_plate = hp_plate.skycoord_to_healpix(crd) + src=SkyMapSource(skymap=skymap,catalog=catalog,healpix=healpix, + healpix_plate=healpix_plate, + name=make_source_name('SRGA',ra,dec), + name_orig=item['NAME'], + lii=crd.galactic.l.value, + bii=crd.galactic.b.value, + ra=ra,dec=dec, + x = item['X'], + y = item['Y'], + cnts = item['CNTS'], + cnts_err = item['CNTS_ERR'], + cnts_bg = item['CNTS_BG'], + exptime = item['EXPTIME'], + rate = item['RATE'], + rate_err = item['RATE_ERR'], + flux = item['FLUX'], + flux_err = item['FLUX_ERR'], + sig = item['SIG'], + nfalse = item['NFALSE'],) + src.save() + + if not (item['s_match'] == 'True'): + continue + + ero = eRositaMatch(source=src, + exp = item['exp'], + sep = item['s_sep'], + ra = item['s_ra'], + dec = item['s_dec'], + lkh = item['s_lkh'], + cts = item['s_cts'], + flux = item['s_fx']) + ero.save() + + srcs = catalog.skymapsource_set.all() + ##clean_skymap_sources(srcs) + find_counterparts(srcs, HeasarcBase.objects.all(),'heasarc',minrad=30) + load_simbad_sources(srcs,minrad=30,maxdist=120) + find_counterparts(srcs, GAIADR2.objects.all(), "gaia", maxdist=60., minrad=30.) + ##mark_new_skymap_sources_in_latest() + +def clean_skymap_sources(srcs, radius=36, threshold=10): + """ Marks sources as bad + + + Actually not used. TODO: preselection by healpix_plate + + Parameters + ---------- + srcs : + Sorted list of sources + + + :Author: + Roman Krivonos + """ + logging.info("Cleaning skymap sources") + + """ + for s in srcs: + s.bad=False + s.good = False + s.save() + return + """ + + for s in srcs: + + if(s.bad == True): + continue + crd = SkyCoord(s.ra, s.dec, frame="fk5", unit="deg") + count=0 + for s2 in srcs: + if(s == s2 or s2.bad == True): + continue + + crd2 = SkyCoord(s2.ra, s2.dec, frame="fk5", unit="deg") + s2.sep=crd.separation(crd2).arcmin + if(s2.sep <= radius): + count=count+1 + if(count >= threshold): + for s2 in srcs: + if(s == s2): + continue + if(s2.sep <= radius and s2.good != True): + s2.bad=True + s2.save() + s.good=True + s.save() + + logging.debug("{}: Rate {} SkyMap: {} Dump: {} {}".format(s,s.rate,s.skymap,s.dump,count)) + +def load_skymap_sources_dir(dir): + """ Loads sources stored in plates + + The format of the data dump directoery is srg_20191228_143827_000 + + + Parameters + ---------- + dir : str + Absolute path to the input directory + + + :Author: + Roman Krivonos + """ + logging.info("Loading sources from %s" % dir) + files = glob.glob("{}/srg_????????_??????_000.fits.gz".format(dir)) + files.sort(key=os.path.getmtime) + for filename in files: + print(filename) + load_skymap_sources_file(filename) + +def update_all_skymap_sources_counterparts(): + skymaps = SkyMaps.objects.all() + for cat in skymaps: + ntot = cat.skymapsource_set.count() + if (ntot): + find_counterparts(cat.skymapsource_set.all(), HeasarcBase.objects.all(),'heasarc',minrad=30) + load_simbad_sources(cat.skymapsource_set.all()) + find_counterparts(cat.skymapsource_set.all(), GAIADR2.objects.all(), "gaia", maxdist=20., minrad=3.) + +def update_allsky_missed(): + """ Select sources missed in published all-sky surveys """ + logging.info("Update All-sky missed") + missed = SelectAllskyMissed.objects.all() + missed.delete() + + #dump = SrgDataDump.objects.all().order_by('-date')[0] + dump = SrgDataDump.objects.filter(catalog_loaded=True).latest('date') + srcs = dump.skymapsource_set.all() + for src in srcs: + if (src.heasarc.filter(heasarcswiftbat105m__isnull=False).count()): + continue + if (src.heasarc.filter(heasarcxteasscat__isnull=False).count()): + continue + if (src.heasarc.filter(heasarcintrefcat__isnull=False).count()): + continue + if (src.heasarc.filter(heasarcrass2rxs__isnull=False).count()): + continue + if (src.heasarc.filter(heasarcmaxigschgl__isnull=False).count()): + continue + if (src.heasarc.filter(heasarcxmmsl2__isnull=False).count()): + continue + if (src.heasarc.filter(heasarc3maxi__isnull=False).count()): + continue + obj = SelectAllskyMissed(dump=dump, source=src) + obj.save() + +def mark_new_skymap_sources_in_latest(tolerance=10): + + if(SrgDataDump.objects.all().count() < 10): + print('mark_new_skymap_sources_in_latest: Too few dumps, skip') + return + + dumps = SrgDataDump.objects.all().order_by('-date')[:10] + latest=dumps[0] + for i in range(1,len(dumps)): + if(dumps[i].skymapsource_set.count() > 0): + second = dumps[i] + break + + try: + second + except NameError: + second = None + + if second is None: + return + + skymaps_latest = latest.skymap.all() + sources_latest = latest.skymapsource_set.filter(skymap__in=skymaps_latest) + logging.debug("Latest {} nsrc={} in SkyMap {}".format(latest,latest.skymapsource_set.count(),sources_latest.count())) + + sources_second = second.skymapsource_set.filter(skymap__in=skymaps_latest) + logging.debug("Second {} nsrc={} in SkyMap {}".format(second,second.skymapsource_set.count(),sources_second.count())) + + for src1 in sources_latest: + crd1 = SkyCoord(src1.ra, src1.dec, frame="fk5", unit="deg") + matched = False + for src2 in sources_second: + sep=crd1.separation(SkyCoord(src2.ra, src2.dec, frame="fk5", unit="deg")) + if(sep.arcsecond < tolerance): + matched = True + break + if matched: + continue + logging.debug("{} new".format(src1)) + src1.new=True + src1.save() + """ print some statistics """ + logging.debug("Latest {}: {} marked as new within {} arcseconds".format(latest,sources_latest.filter(new=True).count(),tolerance)) + +def transfer_data(child,clean_ads=True,clean_oname=True): + + if(child.survey.archived): + print("ERROR: Survey {} is archived ({})".format(child.survey,child)) + return + + parent=child.parent + + print("--> Transfer data from parent {} to {}".format(parent, child)) + + """ Clean ADS and OtherName in child """ + if(clean_ads): + child.ads.clear() + + if(clean_oname): + for o in child.othername_set.all(): + print("Delete {} in {}".format(o,child)) + o.delete() + child.save() + + + """ fill child with data """ + child.owner=parent.owner + child.notes=parent.notes + child.notes_paper=parent.notes_paper + child.follow_up=parent.follow_up + child.redshift=parent.redshift + child.class_id=parent.class_id + child.object_class=parent.object_class + child.cname=parent.cname + child.refid=parent.refid + child.erosita_data=parent.erosita_data + child.class_tentative=parent.class_tentative + child.category=parent.category + child.turkish=parent.turkish + child.turkish_date=parent.turkish_date + + for upload in parent.uploads.all(): + upload.artsources.add(child) + + for a in parent.ads.all(): + child.ads.add(a) + + """ clone OtherName """ + for o in parent.othername_set.all(): + o.source=child + o.pk=None + o.save() + + if(parent.allwise_primary): + child.allwise_primary=parent.allwise_primary + + if(parent.gaia3_primary): + child.gaia3_primary=parent.gaia3_primary + + if(parent.nvss_primary): + child.nvss_primary=parent.nvss_primary + + if(parent.first_primary): + child.first_primary=parent.first_primary + + child.save() + +def transfer(srcs1,srcs2,maxdist=120.0,minrad=40.0): + match_count=0 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + for s1 in srcs1: + crd1 = SkyCoord(s1.ra, s1.dec, frame="fk5", unit="deg") + + if(s1.ext): + if(s1.ext > minrad): + radec_error=float(s1.ext) + else: + radec_error=minrad + else: + radec_error=minrad + + heal = hp.cone_search_skycoord(crd1, radius=maxdist*u.arcsecond) + try: + sel2=srcs2.filter(healpix__in=heal) + except: + continue + + tot=0 + child=None + childs=[] + childs_sep=[] + for s2 in sel2: + crd2 = SkyCoord(s2.ra, s2.dec, frame="fk5", unit="deg") + sep=crd1.separation(crd2).arcsecond + if(sep <= radec_error): + #child=s2 + childs.append(s2) + childs_sep.append(sep) + tot=tot+1 + if(tot == 0): + continue + if(tot == 1): + child=childs[0] + if(tot > 1): + print("More than one child found for {}, please select:".format(s1)) + + for idx, child in enumerate(childs): + print("{} {} at {:.2f} arcsec".format(idx, childs[idx], childs_sep[idx])) + val = input("Enter your value: ") + print("You entered {}".format(int(val))) + child=childs[int(val)] + + child.parent=s1 + child.save() + print("Transfer data from {} to {}".format(s1,child)) + transfer_data(child) + + match_count=match_count+1 + + return(match_count) + + +def crossmatch_count(srcs1,srcs2,maxdist=120.0,minrad=40.0): + match_count=0 + match1_count=0 + match2_count=0 + match3_count=0 + match4_count=0 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + for s1 in srcs1: + crd1 = SkyCoord(s1.ra, s1.dec, frame="fk5", unit="deg") + heal = hp.cone_search_skycoord(crd1, radius=maxdist*u.arcsecond) + try: + sel2=srcs2.filter(healpix__in=heal) + except: + continue + + tot=0 + for s2 in sel2: + crd2 = SkyCoord(s2.ra, s2.dec, frame="fk5", unit="deg") + sep=crd1.separation(crd2).arcsecond + #print("{:.1f}".format(sep)) + if(sep <= minrad): + match_count=match_count+1 + tot=tot+1 + if(tot == 1): + match1_count=match1_count+1 + if(tot == 2): + match2_count=match2_count+1 + if(tot == 3): + match3_count=match3_count+1 + if(tot == 4): + match4_count=match4_count+1 + logging.debug("Confusion 1:{} 2:{} 3:{} 4:{}".format(match1_count,match2_count,match3_count,match4_count)) + return(match_count) + +def crossmatch(srcs1,srcs2,maxdist=120.0,minrad=40.0,ratio_min=0.001, ratio_max=1000.0): + match_count=0 + match1_count=0 + match2_count=0 + match3_count=0 + match4_count=0 + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + match_list = [] + + missed_count1=0 + missed_list1 = [] + for s1 in srcs1: + crd1 = SkyCoord(s1.ra, s1.dec, frame="fk5", unit="deg") + heal = hp.cone_search_skycoord(crd1, radius=maxdist*u.arcsecond) + try: + sel2=srcs2.filter(healpix__in=heal) + except: + continue + + tot=0 + for s2 in sel2: + crd2 = SkyCoord(s2.ra, s2.dec, frame="fk5", unit="deg") + sep=crd1.separation(crd2).arcsecond + #print("{:.1f}".format(sep)) + if(sep <= minrad): + if not (np.isnan(s1.flux) or s1.flux<=0.0): + flux_ratio = s2.flux/s1.flux + else: + flux_ratio = -100 + if(flux_ratio > ratio_min and flux_ratio < ratio_max): + match_list.append({ + 'dir':"2 in 1", + 'src1':s1, + 'src2':s2, + 'sep':sep, + 'f1f2_ratio':s2.flux/s1.flux, + }) + if(tot==0): + match_count=match_count+1 + tot=tot+1 + if(tot == 0): + missed_list1.append({'src1':s1,}) + missed_count1=missed_count1+1 + + if(tot == 1): + match1_count=match1_count+1 + if(tot == 2): + match2_count=match2_count+1 + if(tot == 3): + match3_count=match3_count+1 + if(tot == 4): + match4_count=match4_count+1 + + match_count2=0 + match1_count2=0 + match2_count2=0 + match3_count2=0 + match4_count2=0 + match_list2 = [] + missed_list2 = [] + missed_count2 = 0 + for s2 in srcs2: + crd2 = SkyCoord(s2.ra, s2.dec, frame="fk5", unit="deg") + heal = hp.cone_search_skycoord(crd2, radius=maxdist*u.arcsecond) + try: + sel1=srcs1.filter(healpix__in=heal) + except: + continue + + tot=0 + for s1 in sel1: + crd1 = SkyCoord(s1.ra, s1.dec, frame="fk5", unit="deg") + sep=crd2.separation(crd1).arcsecond + + if(sep <= minrad): + if not (np.isnan(s1.flux) or s1.flux<=0.0): + flux_ratio = s2.flux/s1.flux + else: + flux_ratio = -100 + + if(flux_ratio > ratio_min and flux_ratio < ratio_max): + match_list.append({ + 'dir':"1 in 2", + 'src1':s1, + 'src2':s2, + 'sep':sep, + 'f1f2_ratio':s2.flux/s1.flux, + }) + #if(tot==0): + match_count2=match_count2+1 + tot=tot+1 + if(tot == 0): + missed_list2.append({'src2':s2,}) + missed_count2=missed_count2+1 + + if(tot == 1): + match1_count2=match1_count2+1 + if(tot == 2): + match2_count2=match2_count2+1 + if(tot == 3): + match3_count2=match3_count2+1 + if(tot == 4): + match4_count2=match4_count2+1 + + logging.debug("Confusion 1:{} 2:{} 3:{} 4:{}".format(match1_count,match2_count,match3_count,match4_count)) + return match_count, match1_count,match2_count,match3_count,match4_count, match_list, missed_count1, missed_list1, missed_count2, missed_list2, match_count2, match1_count2, match2_count2,match3_count2,match4_count2 + + +def normalize(array): + """ + Normalize a 4 element array/list/numpy.array for use as a quaternion + + :param quat_array: 4 element list/array + :returns: normalized array + :rtype: numpy array + """ + quat = np.array(array) + return quat / np.sqrt(np.dot(quat, quat)) + + +def load_gyro_file(filename): + """ Loads sources stored in plates + + The format of the data damp directory is srg_20191228_143827_000 + + + Parameters + ---------- + filename : str + Absolute path to the file + + + :Author: + Roman Krivonos + """ + + if not os.path.isfile(filename): + print("File {} does not exist".format(filename)) + return + + logging.debug("Loading gyro from {}".format(filename)) + print("Loading {}".format(filename)) + + + + name = os.path.basename(filename) + fits_name = os.path.splitext(name)[0] + """ without .gz """ + dump_name = os.path.splitext(fits_name)[0] + dump_name=dump_name[:19] + + """ Check whether this data dump exists """ + try: + dump = SrgDataDump.objects.get(name=dump_name) + if(dump.gyro_loaded): + logging.debug("Gyro for {} is loaded, skip".format(dump)) + print("Gyro for {} is loaded, skip".format(dump)) + return + else: + logging.debug("Clear gyro for {}".format(dump)) + gyros = dump.gyro_set.all() + gyros.delete() + except SrgDataDump.DoesNotExist: + logging.debug("Create new data dump {}".format(dump_name)) + format_string='srg_%Y%m%d_%H%M%S' + dtime = datetime.strptime(dump_name, format_string) + tm = Time(dtime, format='datetime', scale='utc') + dt = tm.to_datetime(timezone=TZ_MSK) + dump = SrgDataDump(name=dump_name, date=dt) + dump.save() + + logging.debug("Read gyro from {} and attach it to {}".format(filename,dump)) + print("Read gyro from {} and attach it to {}".format(filename,dump)) + hp = HEALPix(nside=NSIDE_SOURCES, order=ORDER, frame=FK5()) + + ext=1 + hdul = fits.open(filename) # open a FITS file + hdr = hdul[ext].header + data = hdul[ext].data + + for item in data: + q1=float(item['QORT_0']) + q2=float(item['QORT_1']) + q3=float(item['QORT_2']) + q4=float(item['QORT_3']) + obt=float(item['TIME']) + #print(q1,q2,q3,q4) + quat=Quat(attitude=normalize([q2,q3,q4,q1])) + + ra=quat.ra + dec=quat.dec + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + gyro=Gyro(dump=dump,healpix=healpix,ra=ra,dec=dec,obt=obt,mjd=Time(mission2date(obt), format='datetime').mjd) + gyro.save() + + dump.gyro_loaded=True + dump.save() + + + diff --git a/srglib/views.py b/srglib/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/srglib/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/srglib/ztf.py b/srglib/ztf.py new file mode 100644 index 0000000..0a41f63 --- /dev/null +++ b/srglib/ztf.py @@ -0,0 +1,291 @@ +from astropy.coordinates import SkyCoord # High-level coordinates +from astropy.coordinates import ICRS, Galactic, FK4, FK5 # Low-level frames +from astropy.coordinates import Angle, Latitude, Longitude # Angles +import astropy.units as u +import re +import json +import pandas as pd +from astropy_healpix import HEALPix +from erotrans.models import EroTransSource, EroTrans +from erosurvey.models import NSIDE_PLATES, ORDER_PLATES +from heasarc.models import NSIDE_SOURCES, ORDER, HeasarcBase + +from astrobasis.models import GAIADR2 +from astrobasis.models import ZTFAlert +from astrobasis.models import ALeRCE +from astrobasis.models import alerce_early_class +from monthplan.models import Survey +#import datetime + +from datetime import datetime +from django.utils.timezone import localdate, localtime, now + +import astropy.units as u +from astropy.time.formats import erfa +from astropy.time import Time, TimeDelta, TimezoneInfo, TimeFromEpoch + +from astropy_healpix import HEALPix, neighbours +from django_pandas.io import read_frame +from srglib import correlate_utils, check_transient +import numpy as np + +from math import pi, cos, sin +from Quaternion import Quat +from scipy.spatial.transform import Rotation, Slerp + + +import logging +from pprint import pprint, pformat + +import requests +import json + +from srglib.utils import TZ_MSK + + +def find_ztf_in_survey(ztf): + """ Find ZTF alert in ALL surveys by HEALPIX plate""" + surveys = Survey.objects.all() + for survey in surveys: + #total = survey.surveyhealpixplate_set.count() + #print(survey.experiment, ztf, total) + if survey.surveyhealpixplate_set.filter(healpix=ztf.healpix_plate).exists(): + #print('add survey') + ztf.survey.add(survey) + ztf.save() + + + """ + if ztf.survey.exists(): + return True + else: + return False + """ +def load_ztf_alerts(): + """ Loads ZTF alerts + + https://zwickytransientfacility.github.io/ztf-avro-alert/filtering.html + + Based on tests done at IPAC (F. Masci, priv. comm), the following filter delivers a relatively pure sample: + + rb >= 0.65 and + nbad = 0 and + fwhm <= 5 and + elong <= 1.2 and + abs(magdiff) <= 0.1 + + + Parameters + ---------- + time : int + Request alerts since this time in seconds + + + :Author: + Roman Krivonos + + """ + + format_string='%a, %d %b %Y %H:%M:%S %Z' + + #date_string='Tue, 28 Jan 2020 09:30:30 GMT' + #dt = datetime.strptime(date_string, format_string) + + hp = HEALPix(nside=NSIDE_SOURCES, + order=ORDER, frame=FK5()) + + hp_plates = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, frame=FK5()) + + URL = "https://mars.lco.global/" + PARAMS={'format':'json', 'rb__gte':0.8,'nbad__lte':0,'fwhm__lte':5,'elong__lte':1.2} + r = requests.get(url = URL, params = PARAMS) + data = r.json() + #print(data['total']) + #pprint.pprint(data) + + for item in data['results']: + #print(item['lco_id'],item['objectId'],item['candidate']['wall_time']) + try: + ztf = ZTFAlert.objects.get(lco_id=item['lco_id']) + logging.info("ZTF alert ID %d is already loaded, skip" % item['lco_id']) + continue + except ZTFAlert.DoesNotExist: + logging.info("ZTF alert ID %d is not loaded" % item['lco_id']) + pass + date_string = item['candidate']['wall_time'] + dt = datetime.strptime(date_string, format_string) + ra = item['candidate']['ra'] + dec = item['candidate']['dec'] + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + healpix_plate = hp_plates.skycoord_to_healpix(crd) + ztf = ZTFAlert( + objectId = item['objectId'], + fid = item['candidate']['fid'], + lco_id = item['lco_id'], + healpix = healpix, healpix_plate = healpix_plate, + ra = ra, dec = dec, + programid = item['candidate']['programid'], + magpsf = item['candidate']['magpsf'], + sigmapsf = item['candidate']['sigmapsf'], + magap = item['candidate']['magap'], + magdiff = item['candidate']['magdiff'], + nbad = item['candidate']['nbad'], + fwhm = item['candidate']['fwhm'], + elong = item['candidate']['elong'], + sigmagap = item['candidate']['sigmagap'], + wall_time = datetime.strptime(date_string, format_string), + diffmaglim = item['candidate']['diffmaglim'], + deltamagref = item['candidate']['deltamagref'], + deltamaglatest = item['candidate']['deltamaglatest'], + rb = item['candidate']['rb']) + ztf.save() + + find_ztf_in_survey(ztf) + + +def load_ztf_alerce_to_django(item): + """ Loads ALeRCE ZTF alert to Django + + classearly=None, VS, Asteroids and Bogus objects are not loaded. + """ + + nobs = int(item['nobs']) + + classearly = item['classearly'] + + if (classearly == None): + print("ALeRCE ZTF alert %s: skip classearly=None" % item['oid']) + return + + """ + if (classearly == 20 or classearly == 21 or classearly == 22): + logging.debug("ALeRCE ZTF alert '{0}': skip '{1}'".format(item['oid'],alerce_early_class(classearly))) + return + """ + + try: + ztf = ALeRCE.objects.get(oid=item['oid']) + if(nobs > ztf.nobs): + """ if nobs is changed, reload this alert, else skip """ + logging.info("ALeRCE ZTF alert '{0}': reload, new nobs '{1}' > '{2}'".format(item['oid'],ztf.nobs,nobs)) + ztf.delete() + pass + else: + logging.debug("ALeRCE ZTF alert %s is found, skip" % item['oid']) + return + except ALeRCE.DoesNotExist: + logging.info("ALeRCE ZTF alert %s is not found, loading" % item['oid']) + pass + + hp = HEALPix(nside=NSIDE_SOURCES, + order=ORDER, frame=FK5()) + + hp_plates = HEALPix(nside=NSIDE_PLATES, + order=ORDER_PLATES, frame=FK5()) + + ra = float(item['meanra']) + dec = float(item['meandec']) + crd = SkyCoord(ra, dec, frame=FK5(), unit="deg") + healpix = hp.skycoord_to_healpix(crd) + healpix_plate = hp_plates.skycoord_to_healpix(crd) + + first_tm = Time(item['firstmjd'], format='mjd', scale='utc') + last_tm = Time(item['lastmjd'], format='mjd', scale='utc') + first_dt = first_tm.to_datetime(timezone=TZ_MSK) + last_dt = last_tm.to_datetime(timezone=TZ_MSK) + + ztf = ALeRCE( + healpix = healpix, + healpix_plate = healpix_plate, + classearly = item['classearly'], + classrf = item['classrf'], + oid = item['oid'], + firstdate = first_dt, + lastdate = last_dt, + firstmjd = item['firstmjd'], + lastmjd = item['lastmjd'], + mean_magap_g = item['mean_magap_g'], + mean_magap_r = item['mean_magap_r'], + mean_magpsf_g = item['mean_magpsf_g'], + mean_magpsf_r = item['mean_magpsf_r'], + dec = dec, + ra = ra, + nobs = item['nobs'], + pclassearly = item['pclassearly'], + pclassrf = item['pclassrf'], + sigma_magap_g = item['sigma_magap_g'], + sigma_magap_r = item['sigma_magap_r'], + sigma_magpsf_g = item['sigma_magpsf_g'], + sigma_magpsf_r = item['sigma_magpsf_r'], + sigmadec = item['sigmadec'], + sigmara = item['sigmara']) + + ztf.save() + find_ztf_in_survey(ztf) + +def load_ztf_alerce(days, extra=None): + classrf = "sn ia" + pclassrf = 0.1 + pclassearly = 0.1 + sortBy = "firstmjd" + nobsmin = 2 + nobsmax = 40 + url = "https://ztf.alerce.online/query" + + current_utc = now() + current_mjd = Time(current_utc).mjd + firstmjd_min = current_mjd - days + + post={ + "sortBy": sortBy, + "records_per_pages": 100, + "query_parameters":{ + "filters": { + "nobs": { + "min": nobsmin, + "max": nobsmax + }, + #"classrf": classrf, + #"pclassrf": pclassrf, + 'pclassearly':pclassearly, + }, + "dates":{ + "firstmjd":{ + "min": firstmjd_min + } + } + } + } + + if(extra): + post['query_parameters']['filters'].update(extra) + + #pprint.pprint(post) + + r = requests.post(url = url, json = post) + data = r.json() + for item in data['result']: + """ loads first page """ + packet=data['result'][item] + #pprint.pprint(packet['oid']) + load_ztf_alerce_to_django(packet) + + total=int(data['total']) + num_pages=int(data['num_pages']) + page=int(data['page']) + + logging.info("ALeRCE ZTF alerts total '{0}', num_pages '{1}'".format(int(total),int(num_pages))) + #return + pages = list(range(2,num_pages+1)) + for page in pages: + """ loads all other pages, staring from 2 """ + post.update( {'page':page,} ) + logging.debug(pformat(post)) + r = requests.post(url = url, json = post) + data = r.json() + for item in data['result']: + packet=data['result'][item] + #pprint.pprint(packet['oid']) + load_ztf_alerce_to_django(packet)