Many changes

scripts/03_init_obs.py

@@ -43,6 +43,8 @@ import sys, os, os.path, time, subprocess
 from pathlib import Path
 import numpy as np
 import glob
+from multiprocessing import Pool
+
 from os.path import dirname
 import inspect
 import uds
@@ -59,24 +61,29 @@ infile_dir=root_path+'/data/processed'
 outfile_dir=root_path+'/products'
 create_folder(outfile_dir)
 
-do_init         = True
-do_ermask       = True
+run_Pool=False
 
-do_erbox1       = True # local mode
-do_erbackmap1   = True # 
-do_erbox2       = True # map mode, with background map
-do_erbackmap2   = True # 
-do_erbox3       = True # map mode, with background map
-do_erbackmap3   = True # 
+do_init         = False
+do_ermask       = False
 
-do_ermldet      = True
-do_catprep      = True
+do_erbox1       = False # local mode
+do_erbackmap1   = False # 
+do_erbox2       = False # map mode, with background map
+do_erbackmap2   = False # 
+do_erbox3       = False # map mode, with background map
+do_erbackmap3   = False # 
+
+do_ermldet      = False
+do_catprep      = False
 do_cross_match  = False
 
-do_wcs_match    = True # Chandra task
-do_wcs_update   = True # Chandra task
+do_astro_corr   = False # search optimal shift
+do_astro_update = True 
 
-eband_selected=[0,1,2,3,4]
+do_wcs_match    = False # Chandra task -- DEPRECATED
+do_wcs_update   = False # Chandra task -- DEPRECATED
+
+eband_selected=[5]
 
 vign=True
 vignetting = 'vign' if (vign==True) else 'novign'
@@ -95,6 +102,11 @@ def runme(datakey):
     cheesemask=[]
     bkgimage=[]
     srcmaps=[]
+
+    print(datakey)
+    print('module name:', __name__)
+    print('parent process:', os.getppid())
+    print('process id:', os.getpid())
     
     for ii in range(len(eband_selected)):
         index=eband_selected[ii]
@@ -114,10 +126,10 @@ def runme(datakey):
         expmaps.append(outfile_expmap)
         events.append(outfile_evtool)
 
-    """
-    After astrometry-corrected files (*.attcorr.fits) are obtained, one can take them as original, in order to check the full chain:
-    events.append(outfile_evtool.replace(".fits", ".attcorr.fits"))
-    """
+        
+        # After astrometry-corrected files (*.attcorr.fits) are obtained, one can take them as original, in order to check the full chain:
+        #events.append(outfile_evtool.replace(".fits", ".attcorr.fits"))
+        
 
     """ Detmask """
     
@@ -339,12 +351,13 @@ def runme(datakey):
          ]
 
         if(do_ermldet==True):
+            test_exe('ermldet')
             remove_file(mllist)
             remove_file(srcmap)
 
             os.system((" ").join(cmd))
             print((" ").join(cmd))
-            #save_ermldet_ds9reg(mllist,scale=60*60)
+            save_ermldet_ds9reg(mllist,scale=60*60)
 
         catprep="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,datakey), eband[index], outfile_post)
         catprep_en0="{}_SourceCatalog_en{}{}".format(os.path.join(outfile_dir,datakey), eband[0], outfile_post)
@@ -359,12 +372,24 @@ def runme(datakey):
 
         if(do_cross_match==True):
             crossmatch_shu2019(catprep,dlmin=10,refimage=events[ii],crval=wcslist[datakey],
-                               catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_UDS.fits.catalog")
+                               catalog=root_path+"/data/Gaia_unWISE/Gaia_unWISE_UDS.fits.catalog",errlim=5.0)
 
-        if(do_wcs_match==True and eband[index]==0):
+        if(do_astro_corr==True and eband[index]=='0'):
+            """ run astro_corr for 0.3-2.3 keV only """
+            wcs_astro_corr(catprep)
+            #wcs_match_ciao(catprep, method='rst',radius=12,residlim=0,residtype=0,residfac=1)
+
+        if(do_astro_update==True):
+            """ run astro_corr for 0.3-2.3 keV only """
+            attcorr=wcs_update_shift(events[ii],flog=catprep_en0.replace(".fits", ".shift.log"))
+            do_evtool_esass(evfile=attcorr,outfile=attcorr,rmlock=False, do_center=True, ra_cen=ra_cen, de_cen=de_cen)
+
+
+        if(do_wcs_match==True and eband[index]=='0'):
             """ run wcs_match for 0.3-2.3 keV only """
-            wcs_match_ciao(catprep, method='rst',radius=12,residlim=5)
-        
+            wcs_match_ciao(catprep, method='trans',radius=12,residlim=5)
+            #wcs_match_ciao(catprep, method='rst',radius=12,residlim=0,residtype=0,residfac=1)
+
         if(do_wcs_update==True):
             """ use 0.3-2.3 keV transform matrix for all other bands """ 
             attcorr=wcs_update_ciao(events[ii],crval=wcslist[datakey],transformfile=catprep_en0.replace(".fits", ".xfm"),clean=False)
@@ -372,13 +397,28 @@ def runme(datakey):
 
 
 """
-testing
+# individual run, testing
 runme("tm7_obs_1")
 runme("tm5_obs_1")
+runme("tm6_scan_1")
 """
 
-for tmkey in keylist_tm.keys():
-    print("TM{} in work... init events".format(tmkey))
-    for datakey in keylist_tm[tmkey]:
-        print("--> {}".format(datakey))
-        runme(datakey)     
+
+
+if(run_Pool==True):
+    # parallel run
+    items=[]
+    for tmkey in keylist_tm.keys():
+        for datakey in keylist_tm[tmkey]:
+            items.append(datakey)     
+        
+    with Pool() as pool:
+        pool.map(runme, items)
+
+else:
+    # conventional run
+    for tmkey in keylist_tm.keys():
+        for datakey in keylist_tm[tmkey]:
+            print("--> {}".format(datakey))
+            runme(datakey)
+            #sys.exit()