In [1]:
import pandas as pd
import numpy as np
import os,sys
import random
import re
import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.colors import BoundaryNorm
import collections

from keras import models
from keras import layers
from keras.optimizers import SGD, Adadelta
from keras.callbacks import EarlyStopping, ModelCheckpoint
import datetime
import h5py

from sklearn.metrics import roc_curve, auc
from mpl_toolkits.axes_grid.inset_locator import inset_axes, InsetPosition

%matplotlib inline
/home/tomoesn/anaconda2/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Using TensorFlow backend.
/home/tomoesn/anaconda2/lib/python2.7/site-packages/matplotlib/cbook/deprecation.py:106: MatplotlibDeprecationWarning: The mpl_toolkits.axes_grid module was deprecated in version 2.1. Use mpl_toolkits.axes_grid1 and mpl_toolkits.axisartist provies the same functionality instead.
  warnings.warn(message, mplDeprecation, stacklevel=1)
In [2]:
def show_images(indice, ref_images, new_images, sub_images, title=None, Npanel=None, labels=None, classes=None, labelAsValue=True, labelAsFloat=False, save_name=None):
    if Npanel == None:
        Npanel = (len(indice)-1)/10 + 1
    for ii in range(Npanel):
        fig = plt.figure(figsize=(14, 14))
        if title != None:
            fig.suptitle(title+' (%d)' % (ii+1), fontsize=16)
        n = 0
        for i in indice[ii*10:]:
            ref_ax = fig.add_subplot(5, 6, 3*n+0+1)
            ref_ax.set_xticks([])
            ref_ax.set_yticks([])
            ref_ax.imshow(ref_images[i].reshape(29,29),
                          cmap=plt.cm.gray,
                          #norm=BoundaryNorm(np.linspace(ref_images[i].min(), ref_images[i].min()+0.2, 256), ncolors=256),
                          interpolation='nearest')

            new_ax = fig.add_subplot(5, 6, 3*n+1+1)
            new_ax.set_xticks([])
            new_ax.set_yticks([])
            new_ax.imshow(new_images[i].reshape(29,29),
                          cmap=plt.cm.gray,
                          #norm=BoundaryNorm(np.linspace(ref_images[i].min(), ref_images[i].mean()+500, 256), ncolors=256),
                          interpolation='nearest')

            sub_ax = fig.add_subplot(5, 6, 3*n+2+1)
            sub_ax.set_xticks([])
            sub_ax.set_yticks([])
            sub_ax.imshow(sub_images[i].reshape(29,29),
                          cmap=plt.cm.gray,
                          #norm=BoundaryNorm(np.linspace(ref_images[i].min(), ref_images[i].min()+2.0, 256), ncolors=256),
                          interpolation='nearest')

            if labels is not None and classes is not None:
                ref_ax.set_title('ref:{:1.0f}/{:.2f} ({})'.format(labels[i], classes[i], i))
                new_ax.set_title('new:{:1.0f}/{:.2f} ({})'.format(labels[i], classes[i], i))
                sub_ax.set_title('sub:{:1.0f}/{:.2f} ({})'.format(labels[i], classes[i], i))
                #ax.set_title('%d' % (i))
            elif labels is not None:
                if labelAsValue:
                    if labelAsFloat:
                        ref_ax.set_title('ref:%f' % (labels[i]))
                        new_ax.set_title('new:%f' % (labels[i]))
                        sub_ax.set_title('sub:%f' % (labels[i]))
                    else:
                        ref_ax.set_title('ref:{}'.format(labels[i]))
                        new_ax.set_title('new:{}'.format(labels[i]))
                        sub_ax.set_title('sub:{}'.format(labels[i]))
                else:
                    ref_ax.set_title('ref:%d' % (i))
                    new_ax.set_title('new:%d' % (i))
                    sub_ax.set_title('sub:%d' % (i))
            else:
                ref_ax.set_title('ref:%d' % (i))
                new_ax.set_title('new:%d' % (i))
                sub_ax.set_title('sub:%d' % (i))
            n += 1
            if n == 10:
                break
        if save_name is not None: plt.savefig(save_name+'_{}.png'.format(ii))
        plt.show()
        plt.close()
In [3]:
def norm_image(ref_image_nd, new_image_nd, sub_image_nd):
    ref_image_nd = ref_image_nd.reshape(ref_image_nd.shape[0],ref_image_nd.shape[1])
    new_image_nd = new_image_nd.reshape(new_image_nd.shape[0],new_image_nd.shape[1])
    sub_image_nd = sub_image_nd.reshape(sub_image_nd.shape[0],sub_image_nd.shape[1])

    ref_image_nd = (ref_image_nd-ref_image_nd.min())/(ref_image_nd.max()-ref_image_nd.min())
    new_image_nd = (new_image_nd-new_image_nd.min())/(new_image_nd.max()-new_image_nd.min())
    sub_image_nd = (sub_image_nd-sub_image_nd.min())/(sub_image_nd.max()-sub_image_nd.min())
    return [ref_image_nd, new_image_nd, sub_image_nd]

Use Model ver.5 (New)

In [4]:
root_directory = '/home/tomoesn/script/CNN/cnn_model_v5'

model_dist = {}

det_q_list   = [1,2,3,4]
det_num_list = [11,12,13,14,15,16,
                21,22,23,24,25,26,
                31,32,33,34,35,
                41,42,43,44]

load_det_num = [q*100+num for q in det_q_list for num in det_num_list]

for det_num in load_det_num:
    print 'Load {} model.'.format(det_num)
    model_file_path = os.path.join(root_directory, str(det_num), 'model{}.hdf5'.format(det_num))
    model_dist[det_num] = models.load_model(model_file_path)
Load 111 model.
/home/tomoesn/anaconda2/lib/python2.7/site-packages/Keras-2.1.6-py2.7.egg/keras/engine/sequential.py:109: UserWarning: `Sequential.model` is deprecated. `Sequential` is a subclass of `Model`, you can just use your `Sequential` instance directly.
Load 112 model.
Load 113 model.
Load 114 model.
Load 115 model.
Load 116 model.
Load 121 model.
Load 122 model.
Load 123 model.
Load 124 model.
Load 125 model.
Load 126 model.
Load 131 model.
Load 132 model.
Load 133 model.
Load 134 model.
Load 135 model.
Load 141 model.
Load 142 model.
Load 143 model.
Load 144 model.
Load 211 model.
Load 212 model.
Load 213 model.
Load 214 model.
Load 215 model.
Load 216 model.
Load 221 model.
Load 222 model.
Load 223 model.
Load 224 model.
Load 225 model.
Load 226 model.
Load 231 model.
Load 232 model.
Load 233 model.
Load 234 model.
Load 235 model.
Load 241 model.
Load 242 model.
Load 243 model.
Load 244 model.
Load 311 model.
Load 312 model.
Load 313 model.
Load 314 model.
Load 315 model.
Load 316 model.
Load 321 model.
Load 322 model.
Load 323 model.
Load 324 model.
Load 325 model.
Load 326 model.
Load 331 model.
Load 332 model.
Load 333 model.
Load 334 model.
Load 335 model.
Load 341 model.
Load 342 model.
Load 343 model.
Load 344 model.
Load 411 model.
Load 412 model.
Load 413 model.
Load 414 model.
Load 415 model.
Load 416 model.
Load 421 model.
Load 422 model.
Load 423 model.
Load 424 model.
Load 425 model.
Load 426 model.
Load 431 model.
Load 432 model.
Load 433 model.
Load 434 model.
Load 435 model.
Load 441 model.
Load 442 model.
Load 443 model.
Load 444 model.
In [5]:
csv_path = '/home/hamasaki/hamasaki-2/TestTransient/release/data/real_transient_2019-09-01_2019-11-01.csv'
npy_path = '/home/hamasaki/hamasaki-2/TestTransient/release/data/real_transient_2019-09-01_2019-11-01.npy'

real_params = pd.read_csv(csv_path)
real_images = np.load(npy_path)

print real_params.shape
print real_images.shape
(455, 74)
(455, 29, 29, 3)
In [6]:
pd.set_option('display.max_columns',80)
In [7]:
real_params[:6]
Out[7]:
tns_name tns_ra tns_dec tns_alias_name tns_disc_date tns_disc_mag tns_disc_filter tns_id tns_mjd ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist tom_raw_id tom_det_id tom_source_id tom_x tom_y tom_ra tom_dec tom_mag_auto tom_fwhm tom_cnn_cand tom_param_cand tom_variable_id tom_transient_id tom_cnn_prob tom_mjd tom_fits_path tom_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_mask sub_edge sub_saturation
0 SN 2019pff 251.457708 45.086331 ZTF19abvanim 2019-09-01 03:23:47 20.1197 r-ZTF 44358 58727.141516 False star 0 0 0.00 0.000000 5066877 111 632290078 716.0168 213.0173 251.45741 45.08638 18.2258 3.65 1 1 4594489 0 0.910008 58751.463169 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000216 561 715.9628 212.8853 251.340858 45.113945 14.7199 0.1148 2.298 1.372 30.92 1.675 7.26 1294.305 136.8022 45.3838 0.053 False star 162132513484553879 503635968 13.8702 0.006105 117 716.0168 213.0173 251.340883 45.113901 18.2258 0.1259 0.627 0.613 -0.98 1.023 3.65 5.124788 0.594095 0.334125 0.347 False False False False
1 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5063885 111 631818673 979.4688 596.1651 276.58267 7.82604 16.0397 4.07 0 0 0 0 0.001247 58751.522783 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000581 5928 978.9316 597.5719 276.395668 7.815827 10.9192 0.0049 3.445 2.542 -16.30 1.355 2.72 42887.980 192.0601 3309.6410 0.517 False extended 117382763949181683 512024576 18.1042 0.001992 341 979.4688 596.1651 276.395850 7.816291 16.0397 0.0295 1.393 1.287 41.47 1.082 4.07 38.382220 1.041897 9.940680 0.858 False False False False
2 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5099418 111 650369957 901.6376 611.2249 276.58280 7.82596 17.2657 6.65 0 0 0 0 0.001732 58757.451270 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000528 6517 900.6829 612.4434 276.582476 7.825561 10.6235 0.0041 3.982 2.939 -29.32 1.355 3.01 56311.410 214.4745 3383.3200 0.038 True star 117392765824611097 235200512 18.9900 0.000041 283 901.6376 611.2249 276.582796 7.825962 17.2657 0.0515 1.256 0.989 -52.85 1.270 6.65 12.408150 0.588507 3.692538 0.839 False False False False
3 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5235141 111 677264687 781.0756 609.5555 276.58259 7.82580 16.6846 2.36 0 0 0 0 0.001373 58765.459075 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000341 5595 780.7081 610.2126 276.582465 7.825586 10.5712 0.0050 2.865 2.029 28.87 1.412 2.51 59089.200 269.3798 5068.5100 0.772 True star 117392765824611097 235200512 18.9900 0.000053 174 781.0756 609.5555 276.582589 7.825802 16.6846 0.0438 1.051 0.835 -15.75 1.259 2.36 21.190820 0.855045 6.780733 0.716 False False False False
4 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5327681 111 695519476 826.5943 610.2678 276.58297 7.82574 16.1638 2.51 0 0 0 0 0.121597 58762.446115 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000441 933 825.3774 610.6802 276.582560 7.825610 12.0217 0.0211 1.855 1.674 39.36 1.108 2.81 15535.820 301.4909 1287.9090 0.191 True star 117392765824611097 235200512 18.9900 0.000137 79 826.5943 610.2678 276.582966 7.825744 16.1638 0.1219 0.574 0.328 39.15 1.750 2.51 34.234480 3.841384 5.085593 0.350 False False False False
5 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5381481 111 707667234 765.6056 606.3022 276.58287 7.82599 16.8779 3.48 0 0 0 0 0.001068 58766.465386 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000582 2391 764.5898 607.5141 276.582530 7.825589 10.5555 0.0056 3.146 2.302 18.02 1.367 3.16 59951.270 310.7070 3437.5030 0.032 True star 117392765824611097 235200512 18.9900 0.000101 190 765.6056 606.3022 276.582871 7.825987 16.8779 0.0653 1.154 1.093 53.46 1.056 3.48 17.735300 1.066942 6.175778 0.581 False False False False
In [8]:
index = [0,1,2,3,4,5]
images_ch_list = np.split(real_images, 3, axis=3)

print index
show_images(index,*images_ch_list, title='TNS Transient')
[0, 1, 2, 3, 4, 5]
In [9]:
index = np.array(real_params[real_params['ps_match']==False].index)
images_ch_list = np.split(real_images, 3, axis=3)

print index.shape
print index
show_images(index[10:20],*images_ch_list, title='TNS Transient without star')
(259,)
[  0   9  10  11  12  13  14  15  16  17  24  25  26  27  29  30  31  32
  33  34  35  51  52  53  54  55  56  57  58  59  60  61  62  63  68  69
  70  71  72  73  74  82  83  84  85  86  87  88  89  90  98 100 103 104
 118 119 120 121 122 123 124 125 126 127 128 129 135 138 163 164 165 166
 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
 203 204 205 206 207 209 212 213 214 215 223 224 225 226 227 228 229 230
 231 232 233 234 235 236 237 238 239 243 252 253 254 255 256 257 258 259
 264 265 266 267 268 269 276 277 280 283 289 290 291 293 299 300 306 307
 308 309 310 311 312 313 314 315 319 320 321 322 328 329 330 331 332 333
 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
 388 389 390 391 392 393 394 395 396 407 408 409 410 432 433 434 435 436
 437 438 439 440 441 442 454]

Let's look at REAL (TNS) Transient.

In [10]:
real_norm_images = np.array([np.stack(norm_image(*np.split(images, 3, axis=2)), axis=-1) for images in real_images])
In [11]:
real_prob  = []
real_class = []

for det_num, images in zip(real_params['tom_det_id'].values.astype('int64'), real_norm_images):
    images = images.reshape(1, images.shape[0], images.shape[1], images.shape[2])    
    predict_prob  = model_dist[det_num].predict_proba(images, verbose=False)
    predict_class = model_dist[det_num].predict_classes(images, verbose=False)
    real_prob.append(predict_prob[0,0])
    real_class.append(predict_class[0,0])

real_prob  = np.array(real_prob)
real_class = np.array(real_class)
In [12]:
real_params['ana_cnn_cand'] = real_class
real_params['ana_cnn_prob'] = real_prob
In [13]:
real_params[:5]
Out[13]:
tns_name tns_ra tns_dec tns_alias_name tns_disc_date tns_disc_mag tns_disc_filter tns_id tns_mjd ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist tom_raw_id tom_det_id tom_source_id tom_x tom_y tom_ra tom_dec tom_mag_auto tom_fwhm tom_cnn_cand tom_param_cand tom_variable_id tom_transient_id tom_cnn_prob tom_mjd tom_fits_path tom_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_mask sub_edge sub_saturation ana_cnn_cand ana_cnn_prob
0 SN 2019pff 251.457708 45.086331 ZTF19abvanim 2019-09-01 03:23:47 20.1197 r-ZTF 44358 58727.141516 False star 0 0 0.00 0.000000 5066877 111 632290078 716.0168 213.0173 251.45741 45.08638 18.2258 3.65 1 1 4594489 0 0.910008 58751.463169 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000216 561 715.9628 212.8853 251.340858 45.113945 14.7199 0.1148 2.298 1.372 30.92 1.675 7.26 1294.305 136.8022 45.3838 0.053 False star 162132513484553879 503635968 13.8702 0.006105 117 716.0168 213.0173 251.340883 45.113901 18.2258 0.1259 0.627 0.613 -0.98 1.023 3.65 5.124788 0.594095 0.334125 0.347 False False False False 1 0.979453
1 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5063885 111 631818673 979.4688 596.1651 276.58267 7.82604 16.0397 4.07 0 0 0 0 0.001247 58751.522783 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000581 5928 978.9316 597.5719 276.395668 7.815827 10.9192 0.0049 3.445 2.542 -16.30 1.355 2.72 42887.980 192.0601 3309.6410 0.517 False extended 117382763949181683 512024576 18.1042 0.001992 341 979.4688 596.1651 276.395850 7.816291 16.0397 0.0295 1.393 1.287 41.47 1.082 4.07 38.382220 1.041897 9.940680 0.858 False False False False 0 0.122927
2 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5099418 111 650369957 901.6376 611.2249 276.58280 7.82596 17.2657 6.65 0 0 0 0 0.001732 58757.451270 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000528 6517 900.6829 612.4434 276.582476 7.825561 10.6235 0.0041 3.982 2.939 -29.32 1.355 3.01 56311.410 214.4745 3383.3200 0.038 True star 117392765824611097 235200512 18.9900 0.000041 283 901.6376 611.2249 276.582796 7.825962 17.2657 0.0515 1.256 0.989 -52.85 1.270 6.65 12.408150 0.588507 3.692538 0.839 False False False False 0 0.048086
3 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5235141 111 677264687 781.0756 609.5555 276.58259 7.82580 16.6846 2.36 0 0 0 0 0.001373 58765.459075 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000341 5595 780.7081 610.2126 276.582465 7.825586 10.5712 0.0050 2.865 2.029 28.87 1.412 2.51 59089.200 269.3798 5068.5100 0.772 True star 117392765824611097 235200512 18.9900 0.000053 174 781.0756 609.5555 276.582589 7.825802 16.6846 0.0438 1.051 0.835 -15.75 1.259 2.36 21.190820 0.855045 6.780733 0.716 False False False False 0 0.298057
4 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5327681 111 695519476 826.5943 610.2678 276.58297 7.82574 16.1638 2.51 0 0 0 0 0.121597 58762.446115 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000441 933 825.3774 610.6802 276.582560 7.825610 12.0217 0.0211 1.855 1.674 39.36 1.108 2.81 15535.820 301.4909 1287.9090 0.191 True star 117392765824611097 235200512 18.9900 0.000137 79 826.5943 610.2678 276.582966 7.825744 16.1638 0.1219 0.574 0.328 39.15 1.750 2.51 34.234480 3.841384 5.085593 0.350 False False False False 0 0.067467
In [14]:
detected_real_params = real_params[(real_params['ana_cnn_cand']==1)&(real_params['ps_match']==False)&
                                   (real_params['sub_value_zero']==False)&(real_params['sub_mask']==False)&
                                   (real_params['sub_edge']==False)&(real_params['sub_saturation']==False)]
In [15]:
print real_params[real_params['ps_match']==False].shape
print detected_real_params.shape
(259, 76)
(243, 76)
In [16]:
not_detected_real_params = real_params[(real_params['ana_cnn_cand']==0)&(real_params['ps_match']==False)&
                                       (real_params['sub_value_zero']==False)&(real_params['sub_mask']==False)&
                                       (real_params['sub_edge']==False)&(real_params['sub_saturation']==False)]
In [17]:
index = np.array(detected_real_params.index)

images_ch_list = np.split(real_images, 3, axis=3)

print index
show_images(index[:10],*images_ch_list, title='Detected')
[  0   9  10  11  12  13  14  16  17  26  27  29  30  31  32  33  34  35
  51  52  53  54  55  56  57  58  59  60  61  62  63  68  69  70  71  72
  73  74  82  83  84  85  86  87  88  89  90 100 104 118 119 120 121 122
 123 124 125 127 128 129 163 164 165 166 167 168 169 170 171 172 173 174
 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 192 193 194
 195 196 197 198 199 200 201 202 203 204 205 206 207 212 214 215 223 224
 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 243 252 253
 254 255 256 257 258 259 264 265 266 267 268 269 276 277 280 283 290 291
 293 299 300 306 307 308 309 310 311 312 313 314 315 319 320 321 322 328
 329 330 331 332 333 334 335 336 339 340 341 342 343 344 345 346 347 348
 349 350 351 352 353 354 355 356 357 358 359 360 362 363 364 365 366 367
 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
 386 387 388 389 390 391 392 393 394 395 396 407 408 409 410 432 433 434
 435 436 437 438 439 440 441 442 454]
In [18]:
index = np.array(not_detected_real_params.index)

images_ch_list = np.split(real_images, 3, axis=3)

print index
show_images(index,*images_ch_list, title='Not Detected')
[ 15  24  25  98 103 126 135 138 190 191 209 213 289 337 338 361]
In [19]:
real_params[real_params['ps_type']=='star']
Out[19]:
tns_name tns_ra tns_dec tns_alias_name tns_disc_date tns_disc_mag tns_disc_filter tns_id tns_mjd ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist tom_raw_id tom_det_id tom_source_id tom_x tom_y tom_ra tom_dec tom_mag_auto tom_fwhm tom_cnn_cand tom_param_cand tom_variable_id tom_transient_id tom_cnn_prob tom_mjd tom_fits_path tom_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_mask sub_edge sub_saturation ana_cnn_cand ana_cnn_prob
0 SN 2019pff 251.457708 45.086331 ZTF19abvanim 2019-09-01 03:23:47 20.1197 r-ZTF 44358 58727.141516 False star 0 0 0.000000 0.000000 5066877 111 632290078 716.0168 213.0173 251.45741 45.08638 18.2258 3.65 1 1 4594489 0 9.100084e-01 58751.463169 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000216 561 715.9628 212.8853 251.340858 45.113945 14.7199 0.1148 2.298 1.372 30.92 1.675 7.26 1294.3050 136.80220 45.38380 0.053 False star 162132513484553879 503635968 13.870200 0.006105 117 716.0168 213.0173 251.340883 45.113901 18.2258 0.1259 0.627 0.613 -0.98 1.023 3.65 5.124788 0.594095 0.334125 0.347 False False False False 1 0.979453
1 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.990000 0.000196 5063885 111 631818673 979.4688 596.1651 276.58267 7.82604 16.0397 4.07 0 0 0 0 1.246867e-03 58751.522783 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000581 5928 978.9316 597.5719 276.395668 7.815827 10.9192 0.0049 3.445 2.542 -16.30 1.355 2.72 42887.9800 192.06010 3309.64100 0.517 False extended 117382763949181683 512024576 18.104200 0.001992 341 979.4688 596.1651 276.395850 7.816291 16.0397 0.0295 1.393 1.287 41.47 1.082 4.07 38.382220 1.041897 9.940680 0.858 False False False False 0 0.122927
2 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.990000 0.000196 5099418 111 650369957 901.6376 611.2249 276.58280 7.82596 17.2657 6.65 0 0 0 0 1.731714e-03 58757.451270 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000528 6517 900.6829 612.4434 276.582476 7.825561 10.6235 0.0041 3.982 2.939 -29.32 1.355 3.01 56311.4100 214.47450 3383.32000 0.038 True star 117392765824611097 235200512 18.990000 0.000041 283 901.6376 611.2249 276.582796 7.825962 17.2657 0.0515 1.256 0.989 -52.85 1.270 6.65 12.408150 0.588507 3.692538 0.839 False False False False 0 0.048086
3 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.990000 0.000196 5235141 111 677264687 781.0756 609.5555 276.58259 7.82580 16.6846 2.36 0 0 0 0 1.372755e-03 58765.459075 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000341 5595 780.7081 610.2126 276.582465 7.825586 10.5712 0.0050 2.865 2.029 28.87 1.412 2.51 59089.2000 269.37980 5068.51000 0.772 True star 117392765824611097 235200512 18.990000 0.000053 174 781.0756 609.5555 276.582589 7.825802 16.6846 0.0438 1.051 0.835 -15.75 1.259 2.36 21.190820 0.855045 6.780733 0.716 False False False False 0 0.298057
4 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.990000 0.000196 5327681 111 695519476 826.5943 610.2678 276.58297 7.82574 16.1638 2.51 0 0 0 0 1.215975e-01 58762.446115 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000441 933 825.3774 610.6802 276.582560 7.825610 12.0217 0.0211 1.855 1.674 39.36 1.108 2.81 15535.8200 301.49090 1287.90900 0.191 True star 117392765824611097 235200512 18.990000 0.000137 79 826.5943 610.2678 276.582966 7.825744 16.1638 0.1219 0.574 0.328 39.15 1.750 2.51 34.234480 3.841384 5.085593 0.350 False False False False 0 0.067467
5 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.990000 0.000196 5381481 111 707667234 765.6056 606.3022 276.58287 7.82599 16.8779 3.48 0 0 0 0 1.068264e-03 58766.465386 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000582 2391 764.5898 607.5141 276.582530 7.825589 10.5555 0.0056 3.146 2.302 18.02 1.367 3.16 59951.2700 310.70700 3437.50300 0.032 True star 117392765824611097 235200512 18.990000 0.000101 190 765.6056 606.3022 276.582871 7.825987 16.8779 0.0653 1.154 1.093 53.46 1.056 3.48 17.735300 1.066942 6.175778 0.581 False False False False 0 0.045617
6 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.990000 0.000196 5445710 111 723626344 722.5328 614.4581 276.58265 7.82602 17.2652 3.54 0 1 0 0 2.536714e-03 58771.413454 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000559 1910 722.1875 615.7745 276.582528 7.825582 10.5519 0.0053 2.908 2.649 25.77 1.098 3.84 60151.6300 292.99980 2677.76500 0.030 True star 117392765824611097 235200512 18.990000 0.000096 122 722.5328 614.4581 276.582646 7.826016 17.2652 0.0641 1.028 0.892 45.25 1.152 3.54 12.413800 0.733119 2.228744 0.523 False False False False 0 0.058606
9 AT 2019qfq 267.786496 24.803406 ZTF19abzfmak 2019-09-17 03:29:55 20.0300 g-ZTF 45055 58743.145775 False star 137772677879991784 503635968 16.665701 0.006103 5077237 113 642343874 520.3138 248.3151 267.78660 24.80348 18.5825 2.53 0 0 0 0 1.614271e-01 58752.480640 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000120 1514 520.2018 248.4171 267.786564 24.803450 15.3587 0.1446 1.476 1.002 63.78 1.473 7.27 718.6683 95.67133 44.25230 0.403 False star 137772677879991784 503635968 16.665701 0.006045 219 520.3138 248.3151 267.786605 24.803484 18.5825 0.1561 0.561 0.342 -45.07 1.642 2.53 3.689763 0.530253 0.598783 0.355 False False False False 1 0.987935
10 AT 2019qfq 267.786496 24.803406 ZTF19abzfmak 2019-09-17 03:29:55 20.0300 g-ZTF 45055 58743.145775 False star 137772677879991784 503635968 16.665701 0.006103 5118910 113 653533318 452.1517 243.8248 267.78655 24.80350 18.3196 0.21 0 0 0 0 3.192340e-01 58757.467502 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000106 1573 450.4736 245.5911 267.785945 24.802915 14.7349 0.0907 2.441 1.394 -38.39 1.751 11.05 1276.5240 106.58400 49.31821 0.023 False star 137762677810018917 503635968 16.936800 0.006056 253 452.1517 243.8248 267.786555 24.803499 18.3196 0.1222 0.752 0.569 -44.40 1.321 0.21 4.700558 0.528990 0.474208 0.355 False False False False 1 0.993935
11 AT 2019qaj 54.971217 24.957256 ZTF19abypphv 2019-09-12 11:32:59 19.1000 g-ZTF 44916 58738.481238 False star 137950549636646591 503635968 18.478600 0.009172 5193254 114 668796600 1457.2041 1009.0206 54.97125 24.95727 18.1636 1.95 0 1 0 0 1.691529e-01 58760.795553 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000033 972 1457.0322 1008.9995 54.971184 24.957278 15.0807 0.0833 1.370 1.126 -1.85 1.217 2.26 928.3583 71.19772 101.52470 0.483 False star 137950549636646591 503635968 18.478600 0.009135 186 1457.2041 1009.0206 54.971246 24.957271 18.1636 0.0866 0.872 0.745 0.79 1.170 1.95 5.426839 0.432625 0.783445 0.358 False False False False 1 0.973151
12 AT 2019qaj 54.971217 24.957256 ZTF19abypphv 2019-09-12 11:32:59 19.1000 g-ZTF 44916 58738.481238 False star 137950549636646591 503635968 18.478600 0.009172 5258896 114 682520630 1403.4861 997.0280 54.97124 24.95749 18.2264 0.00 1 0 0 0 9.342082e-01 58764.810296 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000235 675 1402.9115 996.8088 54.971030 24.957560 14.8990 0.1025 1.662 1.294 -24.84 1.284 8.78 1097.4600 103.55920 43.29323 0.065 False star 137950549636646591 503635968 18.478600 0.008844 96 1403.4861 997.0280 54.971240 24.957488 18.2264 0.0946 0.791 0.315 -57.69 2.507 0.17 5.122225 0.445981 0.352560 0.355 False False False False 1 0.977012
13 AT 2019qaj 54.971217 24.957256 ZTF19abypphv 2019-09-12 11:32:59 19.1000 g-ZTF 44916 58738.481238 False star 137950549636646591 503635968 18.478600 0.009172 5667494 114 767400003 1386.0000 991.5096 54.97130 24.95740 17.7483 0.00 1 0 0 0 9.995435e-01 58788.793620 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000163 612 1385.6843 991.3608 54.971186 24.957445 15.6967 0.1348 1.262 1.191 1.79 1.059 0.23 526.4257 65.36543 39.70267 0.408 False star 137950549636646591 503635968 18.478600 0.009026 97 1386.0000 991.5096 54.971302 24.957396 17.7483 0.1103 0.577 0.289 90.00 2.000 0.00 7.955543 0.807685 0.691456 0.347 False False False False 1 0.972763
14 AT 2019qaj 54.971217 24.957256 ZTF19abypphv 2019-09-12 11:32:59 19.1000 g-ZTF 44916 58738.481238 False star 137950549636646591 503635968 18.478600 0.009172 5718107 114 774967352 1388.0199 1010.7286 54.97125 24.95719 18.1717 4.86 1 1 717933 134923 9.947125e-01 58789.746298 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000072 919 1387.5812 1010.5961 54.971090 24.957237 15.0377 0.0811 1.388 1.208 35.16 1.149 2.75 965.8326 72.12723 95.73692 0.431 False star 137950549636646591 503635968 18.478600 0.009099 143 1388.0199 1010.7286 54.971250 24.957194 18.1717 0.0973 0.902 0.741 87.68 1.217 4.86 5.386670 0.482559 0.638645 0.437 False False False False 1 0.901932
15 AT 2019qaj 54.971217 24.957256 ZTF19abypphv 2019-09-12 11:32:59 19.1000 g-ZTF 44916 58738.481238 False star 137950549636646591 503635968 18.478600 0.009172 5728684 114 776856084 1396.5214 1033.6072 54.97125 24.95734 18.0675 1.05 0 0 0 0 3.894538e-02 58789.683688 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000090 1027 1396.2664 1033.6631 54.971159 24.957324 15.2582 0.0618 1.156 1.026 12.36 1.127 1.81 788.3357 44.84984 131.55620 0.538 False star 137950549636646591 503635968 18.478600 0.009088 26 1396.5214 1033.6072 54.971252 24.957343 18.0675 0.0806 0.857 0.842 -49.19 1.017 1.05 5.929348 0.440124 1.146774 0.504 False False False False 0 0.197139
16 AT 2019qaj 54.971217 24.957256 ZTF19abypphv 2019-09-12 11:32:59 19.1000 g-ZTF 44916 58738.481238 False star 137950549636646591 503635968 18.478600 0.009172 5735825 114 777838098 1360.5496 980.5219 54.97114 24.95723 18.0946 1.07 1 0 0 0 9.999604e-01 58790.822934 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000074 731 1360.1898 980.6349 54.971005 24.957194 15.2927 0.1150 1.359 1.099 -34.36 1.237 7.30 763.7139 80.88033 52.25178 0.027 False star 137950549636646591 503635968 18.478600 0.009071 124 1360.5496 980.5219 54.971136 24.957232 18.0946 0.1141 0.501 0.496 -55.50 1.011 1.07 5.783004 0.607581 0.530603 0.463 False False False False 1 0.931785
17 AT 2019qaj 54.971217 24.957256 ZTF19abypphv 2019-09-12 11:32:59 19.1000 g-ZTF 44916 58738.481238 False star 137950549636646591 503635968 18.478600 0.009172 5747358 114 779515576 1381.4449 1002.4793 54.97123 24.95725 18.2813 0.00 1 0 0 0 7.617022e-01 58790.749493 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000013 755 1381.1674 1002.3845 54.971127 24.957279 15.8127 0.1124 0.967 0.924 74.43 1.047 1.10 473.0529 48.95393 59.56844 0.359 False star 137950549636646591 503635968 18.478600 0.009095 18 1381.4449 1002.4793 54.971228 24.957248 18.2813 0.1301 0.501 0.495 58.04 1.012 2.12 4.869436 0.583204 0.706303 0.355 False False False False 1 0.809819
18 AT 2019sso 304.850292 33.006731 Gaia19epm 2019-10-09 05:16:48 14.0100 G-Gaia 46754 58765.220000 True star 147603048503468577 503635968 14.112900 0.000043 5600544 114 743610433 1589.6584 365.8270 304.84997 33.00687 16.5776 2.58 0 0 0 0 1.081726e-01 58778.384462 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000304 1674 1590.3616 366.2483 304.850250 33.006730 11.3570 0.0033 1.828 1.719 7.27 1.064 2.56 28654.5200 87.61550 2757.52100 0.926 True star 147603048503468577 503635968 14.112900 0.000076 572 1589.6584 365.8270 304.849972 33.006869 16.5776 0.0212 1.215 1.090 -24.11 1.115 2.58 23.385600 0.456741 4.411873 0.956 False False False False 0 0.403898
19 AT 2019sso 304.850292 33.006731 Gaia19epm 2019-10-09 05:16:48 14.0100 G-Gaia 46754 58765.220000 True star 147603048503468577 503635968 14.112900 0.000043 5633411 114 761450806 1485.9545 319.7631 304.85017 33.00714 16.9009 5.12 1 1 5148439 1795447 9.922403e-01 58788.476187 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000422 2275 1486.2745 320.9264 304.850301 33.006754 11.4348 0.0051 2.474 2.348 76.73 1.053 4.26 26673.3100 125.40520 1137.58100 0.028 True star 147603048503468577 503635968 14.112900 0.000054 424 1485.9545 319.7631 304.850174 33.007138 16.9009 0.0449 1.576 1.182 89.45 1.333 5.12 17.363900 0.717102 1.193365 0.279 False False False False 0 0.284716
24 SN 2019pzp 17.734900 71.433758 ZTF19abxiqve 2019-09-01 10:37:55 20.6300 g-ZTF 44895 58727.442998 False star 193710177362769287 503635968 16.895901 0.001716 5996466 116 819219923 414.6558 79.2907 17.73524 71.43324 14.6261 3.03 0 0 0 0 0.000000e+00 58797.441349 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000530 -1 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0 none 0 0 NaN NaN 1024 414.6558 79.2907 17.735235 71.433237 14.6261 0.0646 0.825 0.673 36.97 1.227 3.03 141.108900 8.392019 22.270260 0.479 False False False False 0 0.001693
25 AT 2019qcf 320.052154 38.007944 ATLAS19vbk 2019-09-14 10:32:09 16.3770 orange-ATLAS 44965 58740.438993 False star 153603200538239127 503635968 15.653000 0.001565 5532471 121 757966391 67.4709 295.6761 320.05221 38.00819 6.7299 0.00 0 0 0 0 3.706050e-02 58771.629259 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000249 -1 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0 none 0 0 NaN NaN 2300 67.4709 295.6761 320.052213 38.008186 6.7299 0.1486 2.088 0.600 3.95 3.482 0.00 203252.100000 27808.810000 43450.950000 0.292 False False False False 0 0.227651
28 AT 2019pud 318.229204 16.635297 ZTF19abxgmzr 2019-09-06 07:11:30 18.9100 g-ZTF 44753 58732.299653 True star 127963182284662716 503635968 16.021700 0.000718 5116738 122 654319786 1306.0000 79.4935 318.22835 16.63523 18.4482 2.35 0 0 0 0 5.161362e-02 58757.528538 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000821 102 1306.2395 79.5570 318.228430 16.635211 14.0298 0.0361 1.509 1.483 -68.51 1.018 2.64 2443.9780 81.16079 211.95260 0.799 True star 127963182284662716 503635968 16.021700 0.000038 26 1306.0000 79.4935 318.228348 16.635231 18.4482 0.1953 0.577 0.289 90.00 2.000 2.35 4.175464 0.750787 0.728571 0.355 False False False False 1 0.795695
29 AT 2019rql 298.921354 46.002325 ZTF19acbjagc 2019-09-28 03:24:28 17.3600 r-ZTF 46016 58754.141991 False star 163202989232511199 503635968 16.491899 0.002281 5085266 122 647732120 909.6196 1098.2565 298.92127 46.00235 17.2781 3.13 0 0 0 0 3.296874e-01 58757.397839 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000064 3084 909.6351 1098.2876 298.921277 46.002338 14.1652 0.0682 1.533 1.126 71.18 1.361 3.15 2157.3060 135.51190 130.26290 0.128 False star 163202989207186123 436527104 18.537500 0.002285 374 909.6196 1098.2565 298.921269 46.002348 17.2781 0.0664 1.102 0.917 62.05 1.202 3.13 12.267260 0.749751 1.103924 0.520 False False False False 1 0.999952
30 AT 2019rql 298.921354 46.002325 ZTF19acbjagc 2019-09-28 03:24:28 17.3600 r-ZTF 46016 58754.141991 False star 163202989232511199 503635968 16.491899 0.002281 5120866 122 654872974 895.0969 1048.3857 298.92139 46.00236 17.2906 5.23 0 1 0 0 2.178476e-01 58757.509226 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000043 3213 895.1024 1048.4016 298.921393 46.002354 14.8253 0.1045 1.523 1.132 71.63 1.346 7.85 1174.6260 113.07420 56.72371 0.008 False star 163202989207186123 436527104 18.537500 0.002285 347 895.0969 1048.3857 298.921390 46.002359 17.2906 0.0874 1.025 0.753 76.16 1.360 5.23 12.126910 0.976065 0.846220 0.048 False False False False 1 0.999995
31 AT 2019rql 298.921354 46.002325 ZTF19acbjagc 2019-09-28 03:24:28 17.3600 r-ZTF 46016 58754.141991 False star 163202989232511199 503635968 16.491899 0.002281 5147268 122 661790987 835.8397 1021.8313 298.92105 46.00243 17.6421 0.00 1 0 0 0 9.914740e-01 58760.565013 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000236 -1 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0 none 0 0 NaN NaN 287 835.8397 1021.8313 298.921054 46.002427 17.6421 0.0713 0.766 0.575 45.87 1.331 0.00 8.773608 0.576260 0.269013 0.347 False False False False 1 0.999896
32 AT 2019pgo 358.250150 25.121269 NaN 2019-09-01 14:24:00 18.4000 Clear- 44393 58727.600000 False star 0 0 0.000000 0.000000 4872537 124 600592965 1833.5186 534.5237 358.25017 25.12132 18.5361 0.47 1 0 0 0 9.215673e-01 58750.576583 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000054 410 1834.1960 534.3476 358.250420 25.121383 15.3312 0.0973 1.705 1.417 -22.43 1.204 3.13 737.1193 66.01575 43.17171 0.440 False star 0 0 0.000000 0.000000 77 1833.5186 534.5237 358.250173 25.121323 18.5361 0.1281 0.504 0.495 44.35 1.019 0.47 3.850935 0.454400 0.364636 0.355 False False False False 1 0.975047
33 AT 2019pgo 358.250150 25.121269 NaN 2019-09-01 14:24:00 18.4000 Clear- 44393 58727.600000 False star 0 0 0.000000 0.000000 4923893 124 604879317 1820.3304 531.6723 358.25023 25.12128 18.6849 0.00 0 0 0 0 4.655540e-01 58751.592802 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000073 371 1821.0146 531.0051 358.250476 25.121500 15.2867 0.1032 1.645 1.187 -46.30 1.385 8.36 767.9107 72.95628 31.83941 0.411 False star 0 0 0.000000 0.000000 69 1820.3304 531.6723 358.250228 25.121278 18.6849 0.1444 0.574 0.335 44.88 1.710 0.00 3.357782 0.446385 0.300414 0.347 False False False False 1 0.984514
34 AT 2019pgo 358.250150 25.121269 NaN 2019-09-01 14:24:00 18.4000 Clear- 44393 58727.600000 False star 0 0 0.000000 0.000000 4981289 124 611679142 1807.0000 524.5283 358.25008 25.12127 18.4584 2.35 0 1 0 0 5.415927e-02 58752.615259 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000063 381 1808.4854 524.2631 358.250621 25.121359 15.4091 0.1282 2.147 0.856 -20.34 2.509 8.11 686.0825 80.97824 41.05885 0.355 False star 0 0 0.000000 0.000000 101 1807.0000 524.5283 358.250080 25.121267 18.4584 0.1350 0.577 0.289 90.00 1.998 2.35 4.136609 0.514238 0.609378 0.355 False False False False 1 0.952180
35 AT 2019pof 18.157804 33.034936 ZTF19abxtcio 2019-09-07 09:49:20 19.2400 r-ZTF 44599 58733.409259 False star 147630181681908943 503635968 16.648300 0.009198 4988276 124 612582681 1022.7573 287.6220 18.15779 33.03497 18.7434 3.37 0 1 0 0 2.413593e-01 58752.555766 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000036 691 1022.8206 287.9348 18.157817 33.034867 15.9092 0.1556 1.141 0.772 36.90 1.478 3.22 432.8448 62.03650 45.82967 0.037 False star 147630181681908943 503635968 16.648300 0.009166 195 1022.7573 287.6220 18.157790 33.034970 18.7434 0.1281 0.725 0.454 17.63 1.599 3.37 3.181543 0.375165 0.552349 0.343 False False False False 1 0.967193
36 AT 2019sgc 68.652458 18.045761 Gaia19emm 2019-10-06 19:01:55 16.0100 G-Gaia 46426 58762.792998 True star 129650686525465493 503636160 17.824600 0.000067 5979718 124 814675702 1365.4590 643.9973 68.65250 18.04581 17.6526 4.65 1 1 852365 159774 9.910151e-01 58790.769190 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000063 408 1365.6105 644.4066 68.652559 18.045675 14.4258 0.0491 1.998 1.509 -79.37 1.325 4.13 1696.9760 76.66911 100.47150 0.010 True star 129650686525465493 503636160 17.824600 0.000136 62 1365.4590 643.9973 68.652505 18.045810 17.6526 0.0888 1.048 0.782 1.99 1.340 4.65 8.689065 0.710665 0.605591 0.436 False False False False 1 0.988137
37 AT 2019sso 304.850292 33.006731 Gaia19epm 2019-10-09 05:16:48 14.0100 G-Gaia 46754 58765.220000 True star 147603048503468577 503635968 14.112900 0.000043 5271649 124 681765465 1653.9460 344.2800 304.85005 33.00690 16.6008 6.25 0 1 0 0 6.705476e-02 58765.434114 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000264 2278 1654.2441 344.7835 304.850172 33.006734 11.4386 0.0065 2.013 1.991 38.07 1.011 3.24 26580.2200 159.88030 1810.95700 0.038 True star 147603048503468577 503635968 14.112900 0.000141 365 1653.9460 344.2800 304.850052 33.006899 16.6008 0.0572 1.564 1.014 65.37 1.542 6.25 22.890970 1.206166 1.749672 0.392 False False False False 1 0.552403
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
375 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5405798 425 713312250 954.5491 665.5178 317.35632 48.18112 14.7298 4.09 1 1 4549365 1399624 8.712495e-01 58761.650053 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000037 742 954.3010 665.3410 317.356198 48.181171 15.3483 0.1027 1.647 1.072 43.00 1.536 7.09 725.5569 68.62942 60.03645 0.055 False star 165823173568803200 503635968 16.065399 0.004296 92 954.5491 665.5178 317.356323 48.181115 14.7298 0.0895 1.097 0.847 44.77 1.294 4.09 128.252300 10.566620 9.664809 0.406 False False False False 1 0.999863
376 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5418755 425 716962854 972.4725 723.0164 317.35642 48.18116 14.8941 2.22 0 0 0 0 1.844065e-02 58766.429171 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000105 5241 972.4907 723.0460 317.356428 48.181153 12.6491 0.0133 1.520 1.394 13.97 1.090 2.24 8716.5060 106.52670 1135.03200 0.975 False star 165823173568803200 503635968 16.065399 0.004300 213 972.4725 723.0164 317.356418 48.181163 14.8941 0.0118 1.414 1.275 8.07 1.109 2.22 110.244400 1.202256 14.218690 0.974 False False False False 1 0.999966
377 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5449144 425 724336855 927.5474 704.8244 317.35635 48.18116 14.7858 3.72 1 1 4549365 1399624 9.966655e-01 58771.439777 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000081 1360 927.5476 704.7692 317.356352 48.181178 12.5938 0.0187 2.055 1.857 -36.06 1.107 3.72 9172.7360 158.10840 518.38420 0.012 False star 165823173568803200 503635968 16.065399 0.004279 130 927.5474 704.8244 317.356353 48.181160 14.7858 0.0169 1.840 1.707 -42.30 1.078 3.72 121.811600 1.900753 6.645267 0.449 False False False False 1 0.999996
378 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5498249 425 731349283 911.5875 692.6573 317.35631 48.18116 14.6887 3.44 0 1 0 0 4.653425e-01 58772.488427 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000077 1704 911.5370 692.5986 317.356281 48.181180 12.4608 0.0149 1.999 1.766 -50.05 1.132 3.45 10367.7600 142.11540 690.49440 0.091 False star 165823173568803200 503635968 16.065399 0.004281 191 911.5875 692.6573 317.356307 48.181161 14.6887 0.0136 1.884 1.590 -43.04 1.185 3.44 133.207600 1.666376 8.645084 0.873 False False False False 1 0.999997
379 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5485347 425 735335930 919.6202 739.5018 317.35640 48.18116 14.6907 3.04 1 1 4549365 1399624 9.873093e-01 58772.389308 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000096 4830 919.6303 739.5052 317.356409 48.181159 12.4446 0.0095 1.832 1.772 6.80 1.034 3.03 10523.5300 92.20958 759.37430 0.053 False star 165823173568803200 503635968 16.065399 0.004295 216 919.6202 739.5018 317.356404 48.181160 14.6907 0.0078 1.764 1.700 -1.27 1.037 3.04 132.958000 0.959575 9.492590 0.971 False False False False 1 0.999993
380 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5588669 425 740569013 361.9070 661.2247 317.35619 48.18117 14.4421 4.54 1 1 4549365 1399624 9.999969e-01 58781.566067 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000119 1782 361.9202 661.1827 317.356195 48.181187 12.2790 0.0101 2.478 2.278 42.46 1.088 4.50 12257.1300 114.50100 469.09010 0.018 False star 165823173568803200 503635968 16.065399 0.004280 213 361.9070 661.2247 317.356189 48.181173 14.4421 0.0082 2.327 2.152 47.70 1.082 4.54 167.171100 1.261240 6.214706 0.028 False False False False 1 0.999986
381 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5601599 425 744337927 429.9051 724.5355 317.35639 48.18116 14.5462 2.92 1 0 0 0 9.993872e-01 58778.377827 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000092 5251 429.8810 724.5261 317.356377 48.181165 12.3208 0.0088 1.732 1.630 1.34 1.062 2.92 11794.2800 95.78627 967.30340 0.972 False star 165823173568803200 503635968 16.065399 0.004290 293 429.9051 724.5355 317.356389 48.181162 14.5462 0.0076 1.645 1.522 -4.58 1.081 2.92 151.887800 1.058582 12.408020 0.999 False False False False 1 0.999998
382 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5648949 425 764743789 347.8698 676.0959 317.35630 48.18122 13.9377 5.17 1 0 0 0 9.999998e-01 58788.468478 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000137 1567 347.8868 676.0699 317.356312 48.181225 11.7369 0.0070 2.703 2.499 -45.23 1.082 5.16 20194.5300 129.76700 611.17080 0.022 False star 165823173568803200 503635968 16.065399 0.004234 175 347.8698 676.0959 317.356304 48.181216 13.9377 0.0053 2.575 2.384 -34.47 1.080 5.17 266.017100 1.306786 7.957870 0.026 False False False False 1 0.999986
383 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5697606 425 771671101 307.7822 651.7112 317.35606 48.18121 13.8752 3.58 1 0 0 0 9.999876e-01 58788.594853 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000211 1976 307.7697 651.6702 317.356057 48.181227 11.8004 0.0068 2.471 2.213 67.32 1.117 3.58 19047.2000 119.74600 1053.87400 0.031 False star 165823173568803200 503635968 16.065399 0.004251 374 307.7822 651.7112 317.356064 48.181213 13.8752 0.0055 2.231 2.059 65.47 1.084 3.58 281.790900 1.429983 15.697340 0.810 False False False False 1 0.999996
384 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5770724 425 782585474 356.3367 712.6931 317.35634 48.18119 13.7417 2.73 0 0 0 0 3.337860e-06 58789.389981 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000108 5067 356.3399 712.6845 317.356341 48.181189 12.1229 0.0065 1.620 1.461 -11.09 1.108 2.73 14152.6300 85.23186 1631.87400 0.983 False star 165823173568803200 503635968 16.065399 0.004269 205 356.3367 712.6931 317.356340 48.181186 13.7417 0.0059 1.553 1.409 -6.51 1.102 2.73 318.650700 1.743070 36.606720 0.981 False False False False 1 0.999995
385 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5884175 425 796843396 280.8891 644.4558 317.35606 48.18112 13.7810 4.16 0 0 0 0 3.215081e-01 58793.594623 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000172 1799 280.8440 644.4327 317.356036 48.181125 11.6942 0.0088 2.388 2.222 51.31 1.075 4.18 21005.4300 169.40110 966.74620 0.025 False star 165823173568803200 503635968 16.065399 0.004354 207 280.8891 644.4558 317.356059 48.181118 13.7810 0.0075 2.225 2.066 43.16 1.077 4.16 307.333400 2.117374 13.857520 0.071 False False False False 1 0.999963
386 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5903786 425 799949094 317.7873 668.0505 317.35621 48.18113 13.8829 3.35 0 0 0 0 1.845801e-02 58793.495163 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000083 5289 317.7562 668.0247 317.356196 48.181135 11.6333 0.0063 2.054 1.870 -38.82 1.098 3.39 22216.2500 128.26290 2027.26500 0.638 False star 165823173568803200 503635968 16.065399 0.004331 229 317.7873 668.0505 317.356212 48.181127 13.8829 0.0053 1.923 1.720 -33.75 1.118 3.35 279.796300 1.353167 25.604260 0.981 False False False False 1 0.999971
387 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5927853 425 805449794 339.9012 710.7394 317.35638 48.18115 13.8696 2.30 1 0 0 0 1.000000e+00 58793.383059 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000080 5685 339.8971 710.7275 317.356374 48.181152 11.6188 0.0046 1.639 1.475 -13.88 1.112 2.29 22514.5600 96.09938 3058.46000 0.983 False star 165823173568803200 503635968 16.065399 0.004304 313 339.9012 710.7394 317.356376 48.181148 13.8696 0.0041 1.567 1.371 -11.65 1.143 2.30 283.234600 1.064369 38.381730 0.982 False False False False 1 0.999911
388 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5947770 425 809210879 340.5807 696.0742 317.35628 48.18116 13.8908 2.55 0 0 0 0 4.207096e-02 58792.425668 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000080 5446 340.5841 696.0823 317.356286 48.181153 11.6327 0.0053 1.879 1.807 -29.84 1.040 2.55 22229.4400 108.04550 2207.34000 0.713 False star 165823173568803200 503635968 16.065399 0.004308 238 340.5807 696.0742 317.356284 48.181156 13.8908 0.0044 1.802 1.680 -21.03 1.073 2.55 277.774000 1.126538 27.692590 0.983 False False False False 1 0.999855
389 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 5957607 425 810641036 343.1271 690.4103 317.35629 48.18110 13.7061 4.15 0 0 0 0 3.496386e-01 58791.426722 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000022 1669 343.1189 690.3661 317.356281 48.181115 11.4687 0.0058 2.506 2.353 -29.97 1.065 4.14 25854.2600 138.14190 1062.59100 0.028 False star 165823173568803200 503635968 16.065399 0.004346 171 343.1271 690.4103 317.356286 48.181100 13.7061 0.0047 2.376 2.197 -34.57 1.082 4.15 329.289900 1.435054 13.451650 0.046 False False False False 1 0.999991
390 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 6146311 425 834627986 283.0901 661.8560 317.35632 48.18111 13.1066 5.32 1 0 0 0 6.494246e-01 58801.505747 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000027 1197 283.1457 661.2932 317.356336 48.181293 10.8530 0.0070 3.946 2.841 -84.63 1.389 5.78 45583.0400 292.50280 1151.64700 0.022 False star 165823173568803200 503635968 16.065399 0.004166 150 283.0901 661.8560 317.356318 48.181106 13.1066 0.0056 2.828 2.681 66.56 1.055 5.32 571.959000 2.927489 14.870740 0.025 False False False False 1 0.999977
391 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 6176544 425 837759638 313.6999 702.3613 317.35633 48.18114 13.1135 4.61 0 0 0 0 9.266290e-02 58801.394734 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000058 1345 313.6958 702.2867 317.356331 48.181162 10.8263 0.0044 2.871 2.794 -30.94 1.028 4.57 46718.7300 187.79870 1531.41100 0.028 False star 165823173568803200 503635968 16.065399 0.004296 115 313.6999 702.3613 317.356335 48.181138 13.1135 0.0035 2.725 2.587 -18.33 1.053 4.61 568.326600 1.841649 18.641460 0.029 False False False False 1 0.999948
392 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 6183286 425 839240108 267.6210 695.1888 317.35637 48.18113 13.9510 6.94 1 1 4549365 1399624 9.980237e-01 58800.390966 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000060 332 267.7672 694.9792 317.356441 48.181205 13.6309 0.0836 1.800 1.579 -24.74 1.140 9.90 3528.7850 271.49660 120.97280 0.032 False star 165823173568803200 503635968 16.065399 0.004248 47 267.6210 695.1888 317.356372 48.181134 13.9510 0.0749 1.289 1.049 -32.12 1.229 6.94 262.793700 18.115800 8.660407 0.028 False False False False 1 0.999587
393 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 6210900 425 844477067 284.9963 657.9671 317.35626 48.18115 13.7911 4.54 1 1 4549365 1399624 8.665041e-01 58799.515507 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000075 828 284.9156 657.9354 317.356221 48.181163 12.3064 0.0304 1.965 1.850 -50.76 1.062 4.48 11952.2200 334.39360 552.36850 0.002 False star 165823173568803200 503635968 16.065399 0.004302 82 284.9963 657.9671 317.356261 48.181154 13.7911 0.0269 1.696 1.588 -10.07 1.068 4.54 304.486100 7.547145 13.808540 0.149 False False False False 1 0.999996
394 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 6252928 425 848804984 325.4738 693.0833 317.35630 48.18116 13.7194 1.94 0 0 0 0 2.980232e-08 58796.421218 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000077 5469 325.4690 693.0569 317.356295 48.181168 11.4836 0.0053 1.598 1.470 -45.41 1.087 1.94 25501.2700 123.36320 4184.68100 0.983 False star 165823173568803200 503635968 16.065399 0.004293 251 325.4738 693.0833 317.356298 48.181159 13.7194 0.0046 1.531 1.364 -36.14 1.123 1.94 325.253300 1.372876 53.762130 0.980 False False False False 1 0.997868
395 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 6338783 425 861851350 292.2107 682.1893 317.35627 48.18104 13.7639 4.15 1 0 0 0 5.541216e-01 58807.405427 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000052 1862 292.2086 682.1683 317.356272 48.181045 11.5166 0.0051 2.538 2.298 -25.99 1.105 4.14 24738.4100 116.57280 1112.09900 0.029 False star 165823173568803200 503635968 16.065399 0.004416 254 292.2107 682.1893 317.356273 48.181038 13.7639 0.0040 2.454 2.170 -22.37 1.131 4.15 312.206100 1.153136 14.017130 0.150 False False False False 1 0.999983
396 AT 2019qwf 317.356313 48.181083 PGIR19brv 2019-09-24 06:14:24 11.5310 J-Bessel 45488 58750.260000 False star 165823173568803200 503635968 16.065399 0.004376 6415483 425 874061678 282.9724 674.7055 317.35625 48.18118 13.2460 8.50 1 0 0 0 9.996408e-01 58806.446620 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000105 529 282.8499 674.5845 317.356186 48.181222 12.7811 0.0227 3.364 3.171 -38.89 1.061 9.63 7719.0340 161.60960 107.50540 0.000 False star 165823173568803200 503635968 16.065399 0.004246 83 282.9724 674.7055 317.356248 48.181184 13.2460 0.0213 2.731 2.580 -32.39 1.058 8.50 503.026200 9.861201 6.970793 0.000 False False False False 1 0.999576
410 SN 2019sgo 334.293408 11.927533 ZTF19acbkyma 2019-09-28 06:34:33 19.1700 g-ZTF 46439 58754.273993 False star 0 0 0.000000 0.000000 5423544 434 718489314 1170.3217 494.3127 334.29337 11.92747 18.4869 2.53 1 1 5104883 0 8.303596e-01 58761.572521 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000074 429 1170.1034 494.2697 334.293295 11.927483 15.4011 0.1203 1.439 1.156 72.36 1.245 2.67 691.1597 76.56682 36.72660 0.438 False star 0 0 0.000000 0.000000 93 1170.3217 494.3127 334.293369 11.927470 18.4869 0.1338 0.563 0.341 -44.53 1.654 2.53 4.029294 0.496418 0.417643 0.355 False False False False 1 0.995217
432 AT 2019pzq 313.622971 26.511739 ZTF19abxzumq 2019-09-09 06:02:52 15.5500 r-ZTF 44896 58735.251991 False star 139813136182057343 503635968 17.564301 0.004815 4812741 435 593420846 1959.0400 31.0111 313.62302 26.51182 17.0284 3.75 1 1 4224652 1356738 9.988719e-01 58745.620316 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000092 104 1959.1458 31.0451 313.623063 26.511814 15.0618 0.1154 1.217 1.183 75.18 1.028 3.48 944.7009 100.36190 64.93517 0.386 False star 139813136182057343 503635968 17.564301 0.004854 14 1959.0400 31.0111 313.623024 26.511824 17.0284 0.0991 0.786 0.785 87.10 1.002 3.75 15.439330 1.409365 1.166584 0.350 False False False False 1 0.999965
433 AT 2019pzq 313.622971 26.511739 ZTF19abxzumq 2019-09-09 06:02:52 15.5500 r-ZTF 44896 58735.251991 False star 139813136182057343 503635968 17.564301 0.004815 5006048 435 617037308 1910.1714 103.4832 313.62299 26.51175 17.2917 4.39 1 1 4224652 1356738 9.374294e-01 58750.444074 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000020 333 1910.0342 103.3500 313.622939 26.511796 15.4727 0.0911 1.401 1.172 -17.36 1.195 5.45 646.9991 54.26406 54.91365 0.086 False star 139813136182057343 503635968 17.564301 0.004764 45 1910.1714 103.4832 313.622991 26.511753 17.2917 0.0848 1.218 0.890 -19.20 1.369 4.39 12.114750 0.945434 0.957419 0.419 False False False False 1 0.999666
434 AT 2019pzq 313.622971 26.511739 ZTF19abxzumq 2019-09-09 06:02:52 15.5500 r-ZTF 44896 58735.251991 False star 139813136182057343 503635968 17.564301 0.004815 5014460 435 619416898 1924.0310 76.9922 313.62296 26.51179 17.3616 3.17 1 1 4224652 1356738 9.823452e-01 58749.524471 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000052 267 1923.9932 77.0820 313.622950 26.511763 15.3904 0.0887 1.277 1.172 54.73 1.089 7.31 697.9672 56.97507 55.02839 0.332 False star 139813136182057343 503635968 17.564301 0.004788 29 1924.0310 76.9922 313.622964 26.511793 17.3616 0.0829 0.959 0.937 -10.62 1.023 3.17 11.359070 0.866671 0.949626 0.438 False False False False 1 0.999940
435 AT 2019pzq 313.622971 26.511739 ZTF19abxzumq 2019-09-09 06:02:52 15.5500 r-ZTF 44896 58735.251991 False star 139813136182057343 503635968 17.564301 0.004815 5045632 435 623347531 1873.3456 113.5647 313.62290 26.51175 17.3093 4.48 1 1 4547926 1392433 9.957677e-01 58753.422987 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000064 408 1873.3741 113.5427 313.622910 26.511756 15.5844 0.0839 1.266 1.170 -34.14 1.081 4.83 583.7902 45.11431 50.43771 0.086 False star 139813136182057343 503635968 17.564301 0.004759 56 1873.3456 113.5647 313.622900 26.511749 17.3093 0.0812 1.008 0.866 -46.02 1.164 4.48 11.920110 0.891563 0.922640 0.475 False False False False 1 0.999986
436 AT 2019pzq 313.622971 26.511739 ZTF19abxzumq 2019-09-09 06:02:52 15.5500 r-ZTF 44896 58735.251991 False star 139813136182057343 503635968 17.564301 0.004815 5033614 435 624877528 1880.6523 68.2380 313.62301 26.51180 17.2359 2.67 1 1 4547926 1392433 9.956660e-01 58752.527978 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000070 226 1880.5433 68.0492 313.622967 26.511866 15.4253 0.0945 1.506 1.077 38.59 1.399 4.16 675.9160 58.78621 55.58857 0.492 False star 139813136182057343 503635968 17.564301 0.004754 36 1880.6523 68.2380 313.623008 26.511805 17.2359 0.0787 1.006 0.890 52.30 1.130 2.67 12.754340 0.924314 0.943112 0.428 False False False False 1 0.999767
437 AT 2019pzq 313.622971 26.511739 ZTF19abxzumq 2019-09-09 06:02:52 15.5500 r-ZTF 44896 58735.251991 False star 139813136182057343 503635968 17.564301 0.004815 5042386 435 629706387 1886.2224 121.6769 313.62286 26.51180 17.3648 2.73 1 0 0 0 9.988061e-01 58752.411086 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000116 400 1886.2312 121.5869 313.622859 26.511825 15.7199 0.1042 1.163 1.055 43.03 1.102 1.80 515.2541 49.42215 57.63485 0.480 False star 139813136182057343 503635968 17.564301 0.004686 366 1886.2224 121.6769 313.622856 26.511795 17.3648 0.0890 0.950 0.868 -80.53 1.095 2.73 11.325580 0.927653 0.998713 0.369 False False False False 1 0.999980
454 AT 2019rle 302.467875 72.488731 Gaia19ejc 2019-09-28 23:24:00 17.1900 G-Gaia 45879 58754.975000 False star 194983024514291034 436527104 17.838699 0.007171 5095535 442 650688968 1541.4515 886.5005 302.46818 72.48865 18.2738 -1.61 0 0 0 0 3.087137e-01 58757.402224 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000122 213 1541.3307 886.4145 302.468029 72.488667 15.9604 0.1311 1.031 0.929 -49.05 1.110 1.82 412.8776 49.85655 52.10862 0.051 False star 194983024514291034 436527104 17.838699 0.007158 34 1541.4515 886.5005 302.468177 72.488647 18.2738 0.1567 0.504 0.494 -51.44 1.021 -1.61 4.903219 0.707467 0.671612 0.349 False False False False 1 0.897267

323 rows × 76 columns

In [20]:
real_params[real_params['ps_id']==0]
Out[20]:
tns_name tns_ra tns_dec tns_alias_name tns_disc_date tns_disc_mag tns_disc_filter tns_id tns_mjd ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist tom_raw_id tom_det_id tom_source_id tom_x tom_y tom_ra tom_dec tom_mag_auto tom_fwhm tom_cnn_cand tom_param_cand tom_variable_id tom_transient_id tom_cnn_prob tom_mjd tom_fits_path tom_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_mask sub_edge sub_saturation ana_cnn_cand ana_cnn_prob
0 SN 2019pff 251.457708 45.086331 ZTF19abvanim 2019-09-01 03:23:47 20.1197 r-ZTF 44358 58727.141516 False star 0 0 0.0 0.0 5066877 111 632290078 716.0168 213.0173 251.45741 45.08638 18.2258 3.65 1 1 4594489 0 0.910008 58751.463169 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000216 561 715.9628 212.8853 251.340858 45.113945 14.7199 0.1148 2.298 1.372 30.92 1.675 7.26 1294.3050 136.80220 45.38380 0.053 False star 162132513484553879 503635968 13.8702 0.006105 117 716.0168 213.0173 251.340883 45.113901 18.2258 0.1259 0.627 0.613 -0.98 1.023 3.65 5.124788 0.594095 0.334125 0.347 False False False False 1 0.979453
32 AT 2019pgo 358.250150 25.121269 NaN 2019-09-01 14:24:00 18.4000 Clear- 44393 58727.600000 False star 0 0 0.0 0.0 4872537 124 600592965 1833.5186 534.5237 358.25017 25.12132 18.5361 0.47 1 0 0 0 0.921567 58750.576583 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000054 410 1834.1960 534.3476 358.250420 25.121383 15.3312 0.0973 1.705 1.417 -22.43 1.204 3.13 737.1193 66.01575 43.17171 0.440 False star 0 0 0.0000 0.000000 77 1833.5186 534.5237 358.250173 25.121323 18.5361 0.1281 0.504 0.495 44.35 1.019 0.47 3.850935 0.454400 0.364636 0.355 False False False False 1 0.975047
33 AT 2019pgo 358.250150 25.121269 NaN 2019-09-01 14:24:00 18.4000 Clear- 44393 58727.600000 False star 0 0 0.0 0.0 4923893 124 604879317 1820.3304 531.6723 358.25023 25.12128 18.6849 0.00 0 0 0 0 0.465554 58751.592802 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000073 371 1821.0146 531.0051 358.250476 25.121500 15.2867 0.1032 1.645 1.187 -46.30 1.385 8.36 767.9107 72.95628 31.83941 0.411 False star 0 0 0.0000 0.000000 69 1820.3304 531.6723 358.250228 25.121278 18.6849 0.1444 0.574 0.335 44.88 1.710 0.00 3.357782 0.446385 0.300414 0.347 False False False False 1 0.984514
34 AT 2019pgo 358.250150 25.121269 NaN 2019-09-01 14:24:00 18.4000 Clear- 44393 58727.600000 False star 0 0 0.0 0.0 4981289 124 611679142 1807.0000 524.5283 358.25008 25.12127 18.4584 2.35 0 1 0 0 0.054159 58752.615259 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000063 381 1808.4854 524.2631 358.250621 25.121359 15.4091 0.1282 2.147 0.856 -20.34 2.509 8.11 686.0825 80.97824 41.05885 0.355 False star 0 0 0.0000 0.000000 101 1807.0000 524.5283 358.250080 25.121267 18.4584 0.1350 0.577 0.289 90.00 1.998 2.35 4.136609 0.514238 0.609378 0.355 False False False False 1 0.952180
100 AT 2019qvt 47.256413 24.043875 ZTF19abztknu 2019-09-21 10:35:04 19.6600 r-ZTF 45476 58747.441019 False star 0 0 0.0 0.0 5193769 212 668947704 573.5099 140.5061 47.25657 24.04400 18.5977 0.36 1 0 0 0 0.850264 58760.793080 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000191 613 573.4439 139.9865 47.256551 24.044169 14.9767 0.0703 1.546 1.258 -73.37 1.230 5.64 1021.7010 66.09620 61.85295 0.003 False star 0 0 0.0000 0.000000 131 573.5099 140.5061 47.256573 24.043997 18.5977 0.1120 0.500 0.500 -60.53 1.000 0.36 3.638625 0.375390 0.350176 0.368 False False False False 1 0.864460
135 AT 2019tji 11.753779 26.433714 ZTF19achtzyx 2019-10-04 08:35:31 19.9600 g-ZTF 47199 58760.357998 False star 0 0 0.0 0.0 5465472 225 727632466 1408.5116 189.0000 11.75400 26.43294 5.4805 0.00 0 0 0 0 0.000000 58771.515466 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000799 -1 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0 none 0 0 NaN NaN 5816 1408.5116 189.0000 11.753998 26.432944 5.4805 0.1999 0.577 0.289 0.00 2.000 0.00 642397.200000 118245.900000 93849.380000 0.347 False False False False 0 0.000008
410 SN 2019sgo 334.293408 11.927533 ZTF19acbkyma 2019-09-28 06:34:33 19.1700 g-ZTF 46439 58754.273993 False star 0 0 0.0 0.0 5423544 434 718489314 1170.3217 494.3127 334.29337 11.92747 18.4869 2.53 1 1 5104883 0 0.830360 58761.572521 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000074 429 1170.1034 494.2697 334.293295 11.927483 15.4011 0.1203 1.439 1.156 72.36 1.245 2.67 691.1597 76.56682 36.72660 0.438 False star 0 0 0.0000 0.000000 93 1170.3217 494.3127 334.293369 11.927470 18.4869 0.1338 0.563 0.341 -44.53 1.654 2.53 4.029294 0.496418 0.417643 0.355 False False False False 1 0.995217
In [21]:
print real_params[real_params['tns_name'].str.contains('SN')].shape
print real_params.shape
(74, 76)
(455, 76)
In [22]:
star_params = real_params[(real_params['ps_type']=='star')&(real_params['ps_id']!=0)]
ext_params = real_params[(real_params['ps_type']=='extended')&(real_params['ps_id']!=0)]
In [23]:
print star_params.shape
print ext_params.shape
(316, 76)
(132, 76)
In [24]:
fig = plt.figure(figsize=(15,7))
fig.suptitle('PS_dist vs PS_rmag  2019/09/01-2019/11/01',fontsize=18)

ax = fig.add_subplot(121)
ax.plot(star_params['ps_dist'].values*3600, star_params['ps_rmag'].values,
        '.', label='TNS (AT)', color='tab:blue')
ax.plot(star_params[star_params['tns_name'].str.contains('SN')]['ps_dist'].values*3600,
        star_params[star_params['tns_name'].str.contains('SN')]['ps_rmag'].values,
        'x', label='TNS (SN)', color='tab:red')

ax.vlines([3],20,16,'gray',linestyles='dashed',label='threshold')
ax.vlines([10],10,16,'gray',linestyles='dashed')
ax.hlines([16],3,10,'gray',linestyles='dashed')
ax.text(1,11,'PSstar==1', size=20, color='green')

ax.set_xlim(0, 60)
ax.set_ylim(20, 10)
#ax.invert_yaxis()
ax.set_xlabel('distance (arcsec)',fontsize=15)
ax.set_ylabel('r_mag',fontsize=15)
ax.tick_params(labelsize=12)

plt.legend(fontsize=12)



ax = fig.add_subplot(122)
ax.plot(ext_params['ps_dist'].values*3600, ext_params['ps_rmag'].values,
        '.', label='TNS (AT)', color='tab:blue')

ax.plot(ext_params[ext_params['tns_name'].str.contains('SN')]['ps_dist'].values*3600,
        ext_params[ext_params['tns_name'].str.contains('SN')]['ps_rmag'].values,
        'x', label='TNS (SN)', color='tab:red')

ax.vlines([3],20,16,'gray',linestyles='dashed',label='threshold')
ax.vlines([10],10,16,'gray',linestyles='dashed')
ax.hlines([16],3,10,'gray',linestyles='dashed')
ax.set_xlim(0, 60)
ax.set_ylim(20, 10)
#ax.invert_yaxis()
ax.set_xlabel('distance (arcsec)',fontsize=15)
ax.set_ylabel('r_mag',fontsize=15)
ax.tick_params(labelsize=12)
ax.text(1,11,'EXT==1', size=20, color='green')
plt.legend(fontsize=12)



plt.savefig('ps_dist_vs_ps_rmag.png',dpi=150)
plt.show()
plt.close()
In [25]:
index = np.array(real_params[real_params['ps_id']==0].index)
print index
show_images(index,*np.split(real_images, 3, axis=3), title='PS Over 60 arcsec')
[  0  32  33  34 100 135 410]
In [26]:
index = star_params[(star_params['ps_dist']<(3.0/3600))&(star_params['tns_name'].str.contains('AT'))].index.values
param=star_params[(star_params['ps_dist']<(3.0/3600))&(star_params['tns_name'].str.contains('AT'))]

print index
show_images(index[:10],*np.split(real_images, 3, axis=3), labelAsFloat=True,
            labels=param['ana_cnn_prob'],
            #save_name='psstar_and_dist_less_than_3arcsec',
            title='AT\nPSstar==1 & Dist<3arcsec')
[  1   2   3   4   5   6  18  19  28  36  37  38  39  40  41  42  43  44
  45  46  47  48  49  50  66  67  75  76  78  79  80  81  91  92  93  94
  95  96  97  99 131 132 133 134 136 137 139 140 141 142 143 144 145 146
 147 148 149 211 216 217 218 219 220 221 222 244 245 246 247 248 249 250
 251 270 271 272 273 274 281 282 302 303 304 305 323 324 325 326 327]
In [27]:
print param.shape
print param['ana_cnn_cand'].sum()
(89, 76)
42
In [28]:
index = ext_params[(ext_params['ps_dist']<(3.0/3600))&(ext_params['tns_name'].str.contains('AT'))].index.values
param=ext_params[(ext_params['ps_dist']<(3.0/3600))&(ext_params['tns_name'].str.contains('AT'))]

print index
show_images(index[:10],*np.split(real_images, 3, axis=3), labelAsFloat=True,
            labels=param['ana_cnn_prob'],
            #save_name='psext_and_dist_less_than_3arcsec',
            title='AT\nEXT==1 & Dist<3arcsec')
[  7   8  20  21  22  23  64  65  77 102 116 117 130 150 151 152 153 154
 155 156 157 158 159 160 161 162 208 240 241 242 260 261 262 263 275 278
 279 284 285 286 287 288 292 294 295 296 297 298 301 316 317 318 397 398
 399 400 401 402 403 404 405 406]

Let's look at BOGUS Transient.

In [29]:
csv_path = '/home/hamasaki/hamasaki-2/TestTransient/release/data/bogus_transient_2020-02-01_2020-03-01.csv'
npy_path = '/home/hamasaki/hamasaki-2/TestTransient/release/data/bogus_transient_2020-02-01_2020-03-01.npy'

bogus_params = pd.read_csv(csv_path)
bogus_images = np.load(npy_path)

print bogus_params.shape
print bogus_images.shape
(6980, 69)
(6980, 29, 29, 3)
In [30]:
import psycopg2
In [31]:
def inquire_tns(connection, bogus_params, temp_table_name='bogus_transient_tns_match'):
    ra = bogus_params['sub_ra'].values
    dec = bogus_params['sub_dec'].values
    mjd = bogus_params['tom_mjd'].values
    ra_dec_mjd = np.stack([ra,dec,mjd], axis=-1)
    ra_dec_mjd = tuple(map(tuple,ra_dec_mjd))
    ra_dec_mjd = ','.join(map(str, ra_dec_mjd))
    with connection.cursor() as cur:
        query = 'CREATE TEMPORARY TABLE {} (ra DOUBLE PRECISION, dec DOUBLE PRECISION, mjd DOUBLE PRECISION)'.format(temp_table_name)
        cur.execute(query)

        query = 'INSERT INTO {} (ra,dec,mjd) VALUES {}'.format(temp_table_name, ra_dec_mjd)
        cur.execute(query)
    connection.commit()

    with connection.cursor('match_tns_table') as cur: # ---!!!CAUTION!!!--- Heavy Processing
        cur.itersize = 10000
        # Search tns
        min_date = '2019-11-01'
        max_date = '2020-02-01'
        #query = 'SELECT ss.* FROM {} AS t LEFT JOIN LATERAL (SELECT s."tnsId", s."ra", s."dec", s."discMag", s."name", s."discDate", q3c_dist(t.ra, t.dec, s.ra, s.dec) FROM tns AS s WHERE s."discDate" > \'{}\' AND s."discDate" < \'{}\' AND q3c_join(t.ra, t.dec, s.ra, s.dec, 1./60) ORDER BY q3c_dist(t.ra, t.dec, s.ra, s.dec) ASC LIMIT 1 ) AS ss ON true'.format(temp_table_name, min_date, max_date)
        query = 'SELECT ss.* FROM {} AS t LEFT JOIN LATERAL (SELECT s."tnsId", s."ra", s."dec", s."discMag", s."name", s."discDate", q3c_dist(t.ra, t.dec, s.ra, s.dec) FROM tns AS s WHERE q3c_join(t.ra, t.dec, s.ra, s.dec, 60.0/3600.0) ORDER BY q3c_dist(t.ra, t.dec, s.ra, s.dec) ASC LIMIT 1 ) AS ss ON true'.format(temp_table_name)
        cur.execute(query)
        responses = cur.fetchall()
        data = [[0 if num is None else num for num in ps_param] for ps_param in responses]

    tns_df = pd.DataFrame(data, columns=['tns_id','tns_ra','tns_dec','tns_disc_mag','tns_name','tns_disc_date','tns_dist'])
    return tns_df
In [32]:
psql_login = 'host=gwdata dbname=supernova user=tomoesn password='
with psycopg2.connect(psql_login) as connection:
    tns_df = inquire_tns(connection, bogus_params)
In [33]:
tns_df[tns_df['tns_id']!=0][:5]
Out[33]:
tns_id tns_ra tns_dec tns_disc_mag tns_name tns_disc_date tns_dist
In [34]:
bogus_params[:3]
Out[34]:
tom_a tom_b tom_bogus_tran_name tom_date_obs tom_dec tom_det_id tom_elongation tom_fits_name tom_fits_path tom_fwhm tom_mag_auto tom_mag_auto_err tom_magmp tom_mjd tom_project tom_ra tom_raw_id tom_source_id tom_theta tom_x tom_y ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_saturation sub_mask sub_edge
0 1.964 0.790 bogus_tran_1096 2020-02-10 18.36716 225 2.484 20200210/sTMQ2202002100025061725.fits /lustre/tomoesn/raw/20200210/sTMQ2202002100025... 7.45 14.3389 0.0270 99.99 58889.574343 Supernova Survey 90.43399 7976273 1064080336 16.57 1981.3441 850.1444 False star 130040904305974847 503635968 16.1259 0.004473 -1 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0 none 0 0 NaN NaN 76 1981.3441 850.1444 90.433992 18.367156 14.3389 0.0270 1.964 0.790 -17.12 2.484 7.45 183.83860 4.562855 1.452905 0.064 False False True True
1 0.763 0.289 bogus_tran_1096 2020-02-10 18.36661 225 2.644 20200210/sTMQ2202002100025031925.fits /lustre/tomoesn/raw/20200210/sTMQ2202002100025... 2.73 16.7943 0.0702 99.99 58889.491869 Supernova Survey 90.43372 7994571 1065946921 -45.33 1983.4780 896.4780 False star 130040904305974847 503635968 16.1259 0.004473 -1 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0 none 0 0 NaN NaN 56 1983.4780 896.4780 90.433719 18.366614 16.7943 0.0702 0.763 0.289 45.00 2.644 2.73 19.15548 1.237628 0.948217 0.355 False False True True
2 3.140 1.751 bogus_tran_2718 2020-02-11 27.31503 343 1.793 20200211/sTMQ3202002110025299443.fits /lustre/tomoesn/raw/20200211/sTMQ3202002110025... 14.06 12.2133 0.0130 99.99 58890.771675 Supernova Survey 142.45087 8162148 1077312696 62.90 1976.6915 1055.8273 False star 130040904305974847 503635968 16.1259 0.004473 327 1977.1572 1053.9994 142.451066 27.315628 14.682 0.1844 0.94 0.472 52.92 1.994 0.0 1340.347 227.5316 78.71558 0.347 0 star 140791424520534962 503635968 17.194099 0.013066 177 1976.6915 1055.8273 142.450867 27.315031 12.2133 0.0130 3.140 1.751 -64.44 1.793 14.06 1302.25800 15.636790 2.933305 0.045 False False True True
In [35]:
index = [0,1,2,3,4,5]
print index
show_images(index,*np.split(bogus_images, 3, axis=3), title='BOGUS Transient')
[0, 1, 2, 3, 4, 5]
In [36]:
fig = plt.figure(figsize=(7,7))
fig.suptitle('BOGUS Coordinate Distribution',fontsize=18)

ax = fig.add_subplot(111)
ax.plot(bogus_params['sub_ra'].values, bogus_params['sub_dec'].values,
        '.', label='BOGUS Transient', color='tab:blue')

ax.set_xlim(0, 360)
ax.set_ylim(-20, 90)
#ax.invert_yaxis()
ax.set_xlabel('ra (deg)',fontsize=15)
ax.set_ylabel('dec (deg)',fontsize=15)
ax.tick_params(labelsize=12)

plt.legend(fontsize=12)


#plt.savefig('ps_dist_vs_ps_rmag.png',dpi=150)
plt.show()
plt.close()
In [37]:
bogus_norm_images = np.array([np.stack(norm_image(*np.split(images, 3, axis=2)), axis=-1) for images in bogus_images])
In [38]:
bogus_prob  = []
bogus_class = []

for det_num, images in zip(bogus_params['tom_det_id'].values.astype('int64'), bogus_norm_images):
    images = images.reshape(1, images.shape[0], images.shape[1], images.shape[2])
    predict_prob  = model_dist[det_num].predict_proba(images, verbose=False)
    predict_class = model_dist[det_num].predict_classes(images, verbose=False)
    bogus_prob.append(predict_prob[0,0])
    bogus_class.append(predict_class[0,0])

bogus_prob  = np.array(bogus_prob)
bogus_class = np.array(bogus_class)
bogus_params['ana_cnn_cand'] = bogus_class
bogus_params['ana_cnn_prob'] = bogus_prob

Let's look at REAL&BOGUS Transient.

In [39]:
def check_param_cand(param):
    param_cand_flag = True
    if param['sub_value_zero'] == True : param_cand_flag = False
    if param['sub_saturation'] == True : param_cand_flag = False
    if param['sub_mask'] == True :       param_cand_flag = False
    if param['sub_edge'] == True :       param_cand_flag = False
    return param_cand_flag
In [40]:
real_params['ana_param_cand'] = real_params.apply(check_param_cand, axis=1)
bogus_params['ana_param_cand'] = bogus_params.apply(check_param_cand, axis=1)
In [41]:
def check_ps_type(param):
    if param['ps_id'] == 0: return 'none'
    return param['ps_type']
In [42]:
real_params['ps_type'] = real_params.apply(check_ps_type, axis=1)
bogus_params['ps_type'] = bogus_params.apply(check_ps_type, axis=1)
In [43]:
bogus_params[bogus_params['ana_param_cand']==True][:5]
Out[43]:
tom_a tom_b tom_bogus_tran_name tom_date_obs tom_dec tom_det_id tom_elongation tom_fits_name tom_fits_path tom_fwhm tom_mag_auto tom_mag_auto_err tom_magmp tom_mjd tom_project tom_ra tom_raw_id tom_source_id tom_theta tom_x tom_y ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_saturation sub_mask sub_edge ana_cnn_cand ana_cnn_prob ana_param_cand
4 0.795 0.498 bogus_tran_2193 2020-02-03 10.984520 215 1.595 20200203/sTMQ2202002030024246915.fits /lustre/tomoesn/raw/20200203/sTMQ2202002030024... 0.58 18.3699 0.1249 99.99 58882.715336 Supernova Survey 121.178270 7846935 1045796553 -86.75 597.5139 735.0272 False star 130040904305974847 503635968 16.1259 0.004473 351 597.0903 734.5014 121.178132 10.984692 12.0598 0.0073 2.594 2.409 57.53 1.077 4.71 14999.55 100.38190 493.3157 0.020 1 extended 121181211781312092 512024576 14.6682 0.000021 73 597.5139 735.0272 121.178274 10.984518 18.3699 0.1249 0.795 0.498 86.61 1.595 0.58 4.488015 0.516076 0.570892 0.356 False False False False 0 0.014455 True
6 0.804 0.500 bogus_tran_2193 2020-02-04 10.984610 215 1.608 20200204/sTMQ2202002040024396615.fits /lustre/tomoesn/raw/20200204/sTMQ2202002040024... 3.74 18.1057 0.1388 99.99 58883.589116 Supernova Survey 121.178040 8637138 1093561175 0.27 610.9867 788.5120 False star 130040904305974847 503635968 16.1259 0.004473 308 611.1990 788.2391 121.178113 10.984701 11.9809 0.0082 1.765 1.722 -12.47 1.025 2.65 16129.54 121.92480 1413.4870 0.550 1 extended 121181211781312092 512024576 14.6682 0.000030 47 610.9867 788.5120 121.178041 10.984611 18.1057 0.1388 0.804 0.500 -0.39 1.608 3.74 5.724029 0.731825 0.835505 0.456 False False False False 0 0.128445 True
8 0.898 0.724 bogus_tran_2193 2020-02-24 10.984291 215 1.240 20200224/sTMQ2202002240026670115.fits /lustre/tomoesn/raw/20200224/sTMQ2202002240026... 4.24 17.8442 0.0781 99.99 58903.606141 Supernova Survey 121.177534 8818261 1110150691 -84.35 42.1250 773.8854 False star 130040904305974847 503635968 16.1259 0.004473 296 43.6984 773.0558 121.178064 10.984565 12.5081 0.0089 2.242 2.069 -89.89 1.083 4.24 9925.43 81.33785 425.6024 0.019 1 extended 121181211781312092 512024576 14.6682 0.000123 47 42.1250 773.8854 121.177534 10.984291 17.8442 0.0781 0.898 0.724 84.19 1.240 4.24 7.283351 0.523461 1.382973 0.429 False False False False 0 0.004141 True
9 1.992 1.378 bogus_tran_1581 2020-02-03 29.573760 411 1.446 20200203/sTMQ4202002030024233511.fits /lustre/tomoesn/raw/20200203/sTMQ4202002030024... 5.78 15.8161 0.0293 99.99 58882.645894 Supernova Survey 97.481830 7852753 1048227189 59.63 650.2703 933.3160 False star 130040904305974847 503635968 16.1259 0.004473 241 651.4354 934.9062 97.482276 29.573244 9.7377 0.0028 3.638 3.534 -58.86 1.029 6.24 127329.10 329.37550 2417.7990 0.028 1 star 143480974822028379 604299264 13.1691 0.000038 29 650.2703 933.3160 97.481827 29.573765 15.8161 0.0293 1.992 1.378 -59.24 1.446 5.78 47.159250 1.272882 2.978953 0.046 False False False False 0 0.009778 True
10 1.612 1.450 bogus_tran_1581 2020-02-03 29.573290 411 1.112 20200203/sTMQ4202002030024208811.fits /lustre/tomoesn/raw/20200203/sTMQ4202002030024... 3.89 15.1509 0.0147 99.99 58882.575685 Supernova Survey 97.482070 7865409 1052304550 -70.76 680.5984 158.0054 False star 130040904305974847 503635968 16.1259 0.004473 194 680.9657 158.2560 97.482213 29.573208 9.7093 0.0023 3.089 2.882 -72.48 1.072 4.68 130696.70 276.21040 4251.8660 0.029 1 star 143480974822028379 604299264 13.1691 0.000059 158 680.5984 158.0054 97.482073 29.573290 15.1509 0.0147 1.612 1.450 71.24 1.112 3.89 87.027150 1.178186 6.703943 0.972 False False False False 0 0.065893 True
In [44]:
real_params[:5]
Out[44]:
tns_name tns_ra tns_dec tns_alias_name tns_disc_date tns_disc_mag tns_disc_filter tns_id tns_mjd ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist tom_raw_id tom_det_id tom_source_id tom_x tom_y tom_ra tom_dec tom_mag_auto tom_fwhm tom_cnn_cand tom_param_cand tom_variable_id tom_transient_id tom_cnn_prob tom_mjd tom_fits_path tom_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_mask sub_edge sub_saturation ana_cnn_cand ana_cnn_prob ana_param_cand
0 SN 2019pff 251.457708 45.086331 ZTF19abvanim 2019-09-01 03:23:47 20.1197 r-ZTF 44358 58727.141516 False none 0 0 0.00 0.000000 5066877 111 632290078 716.0168 213.0173 251.45741 45.08638 18.2258 3.65 1 1 4594489 0 0.910008 58751.463169 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000216 561 715.9628 212.8853 251.340858 45.113945 14.7199 0.1148 2.298 1.372 30.92 1.675 7.26 1294.305 136.8022 45.3838 0.053 False star 162132513484553879 503635968 13.8702 0.006105 117 716.0168 213.0173 251.340883 45.113901 18.2258 0.1259 0.627 0.613 -0.98 1.023 3.65 5.124788 0.594095 0.334125 0.347 False False False False 1 0.979453 True
1 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5063885 111 631818673 979.4688 596.1651 276.58267 7.82604 16.0397 4.07 0 0 0 0 0.001247 58751.522783 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000581 5928 978.9316 597.5719 276.395668 7.815827 10.9192 0.0049 3.445 2.542 -16.30 1.355 2.72 42887.980 192.0601 3309.6410 0.517 False extended 117382763949181683 512024576 18.1042 0.001992 341 979.4688 596.1651 276.395850 7.816291 16.0397 0.0295 1.393 1.287 41.47 1.082 4.07 38.382220 1.041897 9.940680 0.858 False False False False 0 0.122927 True
2 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5099418 111 650369957 901.6376 611.2249 276.58280 7.82596 17.2657 6.65 0 0 0 0 0.001732 58757.451270 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000528 6517 900.6829 612.4434 276.582476 7.825561 10.6235 0.0041 3.982 2.939 -29.32 1.355 3.01 56311.410 214.4745 3383.3200 0.038 True star 117392765824611097 235200512 18.9900 0.000041 283 901.6376 611.2249 276.582796 7.825962 17.2657 0.0515 1.256 0.989 -52.85 1.270 6.65 12.408150 0.588507 3.692538 0.839 False False False False 0 0.048086 True
3 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5235141 111 677264687 781.0756 609.5555 276.58259 7.82580 16.6846 2.36 0 0 0 0 0.001373 58765.459075 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000341 5595 780.7081 610.2126 276.582465 7.825586 10.5712 0.0050 2.865 2.029 28.87 1.412 2.51 59089.200 269.3798 5068.5100 0.772 True star 117392765824611097 235200512 18.9900 0.000053 174 781.0756 609.5555 276.582589 7.825802 16.6846 0.0438 1.051 0.835 -15.75 1.259 2.36 21.190820 0.855045 6.780733 0.716 False False False False 0 0.298057 True
4 AT 2019pwh 276.582625 7.825461 Gaia19dzz 2019-09-08 15:04:19 17.4800 G-Gaia 44809 58734.627998 True star 117392765824611097 235200512 18.99 0.000196 5327681 111 695519476 826.5943 610.2678 276.58297 7.82574 16.1638 2.51 0 0 0 0 0.121597 58762.446115 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000441 933 825.3774 610.6802 276.582560 7.825610 12.0217 0.0211 1.855 1.674 39.36 1.108 2.81 15535.820 301.4909 1287.9090 0.191 True star 117392765824611097 235200512 18.9900 0.000137 79 826.5943 610.2678 276.582966 7.825744 16.1638 0.1219 0.574 0.328 39.15 1.750 2.51 34.234480 3.841384 5.085593 0.350 False False False False 0 0.067467 True
In [45]:
print bogus_params[bogus_params['ana_param_cand']==True].shape
print real_params[real_params['ana_param_cand']==True].shape
(4885, 72)
(449, 77)
In [46]:
label = np.concatenate([np.ones(real_params[real_params['ana_param_cand']==True]['ana_cnn_prob'].shape[0]),
                        np.zeros(bogus_params[bogus_params['ana_param_cand']==True]['ana_cnn_prob'].shape[0])],
                       axis=0)

prob = np.concatenate([real_params[real_params['ana_param_cand']==True]['ana_cnn_prob'].values,
                       bogus_params[bogus_params['ana_param_cand']==True]['ana_cnn_prob'].values],
                      axis=0)

fpr, tpr, thresholds = roc_curve(label, prob)
val_auc = auc(fpr, tpr)
fig = plt.figure(figsize=(8,6))
fig.suptitle("ROC curve for Test Set (AUC=%6.4f)" % (val_auc), fontsize=16)
ax = fig.add_subplot(111)
ax.plot(fpr, tpr, color='tab:red', label='AUC={:>6.3f}'.format(val_auc))
ax.set_xlabel("False Positive Rate",fontsize=15)
ax.set_ylabel("True Positive Rate",fontsize=15)
ax.set_ylim(-0.05,1.05)
ax.set_xlim(-0.05,1.05)
ax.tick_params(labelsize=12)
#ax.set_xscale('log')
ax.grid()
ax.set_aspect('equal')
iaxes = inset_axes(ax, width="50%", height="50%", loc=4)
ip = InsetPosition(ax, [0.3,0.1,0.6,0.6])
iaxes.set_axes_locator(ip)
iaxes.plot(fpr, tpr, color='tab:red')
iaxes.set_xticks(np.arange(0.0, 0.2, 0.050))
iaxes.set_xticks(np.arange(0.0, 0.2, 0.010), minor=True)
iaxes.set_yticks(np.arange(0.70, 1.0, 0.020))
iaxes.set_yticks(np.arange(0.70, 1.0, 0.010), minor=True)
iaxes.set_xlim(0.00, 0.15)
iaxes.set_ylim(0.80, 0.98)
iaxes.tick_params(labelsize=12)
iaxes.grid()
iaxes.set_aspect('equal')
plt.savefig('new_model_roc_curve.png')
plt.show()
plt.close()

Exclude 'variable star'

In [47]:
real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&(real_params['ana_param_cand']==True)][:5]
Out[47]:
tns_name tns_ra tns_dec tns_alias_name tns_disc_date tns_disc_mag tns_disc_filter tns_id tns_mjd ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist tom_raw_id tom_det_id tom_source_id tom_x tom_y tom_ra tom_dec tom_mag_auto tom_fwhm tom_cnn_cand tom_param_cand tom_variable_id tom_transient_id tom_cnn_prob tom_mjd tom_fits_path tom_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_mask sub_edge sub_saturation ana_cnn_cand ana_cnn_prob ana_param_cand
0 SN 2019pff 251.457708 45.086331 ZTF19abvanim 2019-09-01 03:23:47 20.1197 r-ZTF 44358 58727.141516 False none 0 0 0.000000 0.000000 5066877 111 632290078 716.0168 213.0173 251.45741 45.08638 18.2258 3.65 1 1 4594489 0 0.910008 58751.463169 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000216 561 715.9628 212.8853 251.340858 45.113945 14.7199 0.1148 2.298 1.372 30.92 1.675 7.26 1294.3050 136.80220 45.38380 0.053 False star 162132513484553879 503635968 13.870200 0.006105 117 716.0168 213.0173 251.340883 45.113901 18.2258 0.1259 0.627 0.613 -0.98 1.023 3.65 5.124788 0.594095 0.334125 0.347 False False False False 1 0.979453 True
7 AT 2019tiq 57.583454 30.035031 ZTF19acgxbwo 2019-10-25 09:41:55 19.5194 g-ZTF 47181 58781.404109 True extended 144040575837162770 512024576 18.747499 0.000265 5655069 111 765426144 455.0000 812.4873 57.58332 30.03506 18.9776 2.35 1 0 0 0 0.993657 58788.780284 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000120 314 455.8197 811.1992 57.583633 30.035492 13.7605 0.0533 3.080 2.240 89.97 1.375 7.01 3131.9610 153.82800 86.09808 0.002 True extended 144040575837162770 512024576 18.747499 0.000330 39 455.0000 812.4873 57.583322 30.035065 18.9776 0.1491 0.577 0.289 90.00 2.000 1.54 2.564136 0.352048 0.405629 0.355 False False False False 1 0.991029 True
8 AT 2019tiq 57.583454 30.035031 ZTF19acgxbwo 2019-10-25 09:41:55 19.5194 g-ZTF 47181 58781.404109 True extended 144040575837162770 512024576 18.747499 0.000265 5690210 111 770486614 453.6156 858.7574 57.58341 30.03512 18.4628 3.41 1 1 6149126 0 0.968779 58789.683350 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000097 300 454.1608 857.8607 57.583614 30.035415 13.8167 0.0478 3.281 1.902 -85.43 1.726 12.45 2973.8400 130.82320 97.39457 0.000 True extended 144040575837162770 512024576 18.747499 0.000260 48 453.6156 858.7574 57.583408 30.035117 18.4628 0.1155 0.741 0.455 72.86 1.629 3.41 4.119688 0.437961 0.388668 0.184 False False False False 1 0.994041 True
9 AT 2019qfq 267.786496 24.803406 ZTF19abzfmak 2019-09-17 03:29:55 20.0300 g-ZTF 45055 58743.145775 False star 137772677879991784 503635968 16.665701 0.006103 5077237 113 642343874 520.3138 248.3151 267.78660 24.80348 18.5825 2.53 0 0 0 0 0.161427 58752.480640 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000120 1514 520.2018 248.4171 267.786564 24.803450 15.3587 0.1446 1.476 1.002 63.78 1.473 7.27 718.6683 95.67133 44.25230 0.403 False star 137772677879991784 503635968 16.665701 0.006045 219 520.3138 248.3151 267.786605 24.803484 18.5825 0.1561 0.561 0.342 -45.07 1.642 2.53 3.689763 0.530253 0.598783 0.355 False False False False 1 0.987935 True
10 AT 2019qfq 267.786496 24.803406 ZTF19abzfmak 2019-09-17 03:29:55 20.0300 g-ZTF 45055 58743.145775 False star 137772677879991784 503635968 16.665701 0.006103 5118910 113 653533318 452.1517 243.8248 267.78655 24.80350 18.3196 0.21 0 0 0 0 0.319234 58757.467502 /gwkiso/hamasaki/test_data/real_transient_old/... 0.000106 1573 450.4736 245.5911 267.785945 24.802915 14.7349 0.0907 2.441 1.394 -38.39 1.751 11.05 1276.5240 106.58400 49.31821 0.023 False star 137762677810018917 503635968 16.936800 0.006056 253 452.1517 243.8248 267.786555 24.803499 18.3196 0.1222 0.752 0.569 -44.40 1.321 0.21 4.700558 0.528990 0.474208 0.355 False False False False 1 0.993935 True
In [48]:
bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&(bogus_params['ana_param_cand']==True)][:5]
Out[48]:
tom_a tom_b tom_bogus_tran_name tom_date_obs tom_dec tom_det_id tom_elongation tom_fits_name tom_fits_path tom_fwhm tom_mag_auto tom_mag_auto_err tom_magmp tom_mjd tom_project tom_ra tom_raw_id tom_source_id tom_theta tom_x tom_y ps_match ps_type ps_id ps_obj_info_flag ps_rmag ps_dist new_number new_x new_y new_ra new_dec new_mag_auto new_magerr_auto new_a new_b new_theta new_elongation new_fwhm new_flux_auto new_fluxerr_auto new_flux_max new_class_star new_ps_match new_ps_type new_ps_id new_obj_info_flag new_ps_rmag new_ps_dist sub_number sub_x sub_y sub_ra sub_dec sub_mag_auto sub_magerr_auto sub_a sub_b sub_theta sub_elongation sub_fwhm sub_flux_auto sub_fluxerr_auto sub_flux_max sub_class_star sub_value_zero sub_saturation sub_mask sub_edge ana_cnn_cand ana_cnn_prob ana_param_cand
4 0.795 0.498 bogus_tran_2193 2020-02-03 10.984520 215 1.595 20200203/sTMQ2202002030024246915.fits /lustre/tomoesn/raw/20200203/sTMQ2202002030024... 0.58 18.3699 0.1249 99.99 58882.715336 Supernova Survey 121.178270 7846935 1045796553 -86.75 597.5139 735.0272 False star 130040904305974847 503635968 16.1259 0.004473 351 597.0903 734.5014 121.178132 10.984692 12.0598 0.0073 2.594 2.409 57.53 1.077 4.71 14999.55 100.38190 493.3157 0.020 1 extended 121181211781312092 512024576 14.6682 0.000021 73 597.5139 735.0272 121.178274 10.984518 18.3699 0.1249 0.795 0.498 86.61 1.595 0.58 4.488015 0.516076 0.570892 0.356 False False False False 0 0.014455 True
6 0.804 0.500 bogus_tran_2193 2020-02-04 10.984610 215 1.608 20200204/sTMQ2202002040024396615.fits /lustre/tomoesn/raw/20200204/sTMQ2202002040024... 3.74 18.1057 0.1388 99.99 58883.589116 Supernova Survey 121.178040 8637138 1093561175 0.27 610.9867 788.5120 False star 130040904305974847 503635968 16.1259 0.004473 308 611.1990 788.2391 121.178113 10.984701 11.9809 0.0082 1.765 1.722 -12.47 1.025 2.65 16129.54 121.92480 1413.4870 0.550 1 extended 121181211781312092 512024576 14.6682 0.000030 47 610.9867 788.5120 121.178041 10.984611 18.1057 0.1388 0.804 0.500 -0.39 1.608 3.74 5.724029 0.731825 0.835505 0.456 False False False False 0 0.128445 True
8 0.898 0.724 bogus_tran_2193 2020-02-24 10.984291 215 1.240 20200224/sTMQ2202002240026670115.fits /lustre/tomoesn/raw/20200224/sTMQ2202002240026... 4.24 17.8442 0.0781 99.99 58903.606141 Supernova Survey 121.177534 8818261 1110150691 -84.35 42.1250 773.8854 False star 130040904305974847 503635968 16.1259 0.004473 296 43.6984 773.0558 121.178064 10.984565 12.5081 0.0089 2.242 2.069 -89.89 1.083 4.24 9925.43 81.33785 425.6024 0.019 1 extended 121181211781312092 512024576 14.6682 0.000123 47 42.1250 773.8854 121.177534 10.984291 17.8442 0.0781 0.898 0.724 84.19 1.240 4.24 7.283351 0.523461 1.382973 0.429 False False False False 0 0.004141 True
9 1.992 1.378 bogus_tran_1581 2020-02-03 29.573760 411 1.446 20200203/sTMQ4202002030024233511.fits /lustre/tomoesn/raw/20200203/sTMQ4202002030024... 5.78 15.8161 0.0293 99.99 58882.645894 Supernova Survey 97.481830 7852753 1048227189 59.63 650.2703 933.3160 False star 130040904305974847 503635968 16.1259 0.004473 241 651.4354 934.9062 97.482276 29.573244 9.7377 0.0028 3.638 3.534 -58.86 1.029 6.24 127329.10 329.37550 2417.7990 0.028 1 star 143480974822028379 604299264 13.1691 0.000038 29 650.2703 933.3160 97.481827 29.573765 15.8161 0.0293 1.992 1.378 -59.24 1.446 5.78 47.159250 1.272882 2.978953 0.046 False False False False 0 0.009778 True
10 1.612 1.450 bogus_tran_1581 2020-02-03 29.573290 411 1.112 20200203/sTMQ4202002030024208811.fits /lustre/tomoesn/raw/20200203/sTMQ4202002030024... 3.89 15.1509 0.0147 99.99 58882.575685 Supernova Survey 97.482070 7865409 1052304550 -70.76 680.5984 158.0054 False star 130040904305974847 503635968 16.1259 0.004473 194 680.9657 158.2560 97.482213 29.573208 9.7093 0.0023 3.089 2.882 -72.48 1.072 4.68 130696.70 276.21040 4251.8660 0.029 1 star 143480974822028379 604299264 13.1691 0.000059 158 680.5984 158.0054 97.482073 29.573290 15.1509 0.0147 1.612 1.450 71.24 1.112 3.89 87.027150 1.178186 6.703943 0.972 False False False False 0 0.065893 True
In [49]:
label = np.concatenate([np.ones(real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&(real_params['ana_param_cand']==True)]['ana_cnn_prob'].shape[0]),
                        np.zeros(bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&(bogus_params['ana_param_cand']==True)]['ana_cnn_prob'].shape[0])],
                       axis=0)

prob = np.concatenate([real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&(real_params['ana_param_cand']==True)]['ana_cnn_prob'].values,
                       bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&(bogus_params['ana_param_cand']==True)]['ana_cnn_prob'].values],
                      axis=0)

fpr, tpr, thresholds = roc_curve(label, prob)
val_auc = auc(fpr, tpr)
fig = plt.figure(figsize=(8,6))
fig.suptitle("ROC curve for Test Set (AUC=%6.4f)" % (val_auc), fontsize=16)
ax = fig.add_subplot(111)
ax.plot(fpr, tpr, color='tab:red', label='AUC={:>6.3f}'.format(val_auc))
ax.set_xlabel("False Positive Rate",fontsize=15)
ax.set_ylabel("True Positive Rate",fontsize=15)
ax.set_ylim(-0.05,1.05)
ax.set_xlim(-0.05,1.05)
ax.tick_params(labelsize=12)
#ax.set_xscale('log')
ax.grid()
ax.set_aspect('equal')
iaxes = inset_axes(ax, width="50%", height="50%", loc=4)
ip = InsetPosition(ax, [0.3,0.1,0.6,0.6])
iaxes.set_axes_locator(ip)
iaxes.plot(fpr, tpr, color='tab:red')
iaxes.set_xticks(np.arange(0.0, 0.2, 0.050))
iaxes.set_xticks(np.arange(0.0, 0.2, 0.010), minor=True)
iaxes.set_yticks(np.arange(0.70, 1.0, 0.020))
iaxes.set_yticks(np.arange(0.70, 1.0, 0.010), minor=True)
iaxes.set_xlim(0.00, 0.15)
iaxes.set_ylim(0.80, 0.98)
iaxes.tick_params(labelsize=12)
iaxes.grid()
iaxes.set_aspect('equal')
plt.savefig('new_model_roc_curve.png')
plt.show()
plt.close()
In [50]:
real = real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&
                   (real_params['ana_param_cand']==True)]

bogus = bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&
                     (bogus_params['ana_param_cand']==True)]

tp = real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&
                 (real_params['ana_param_cand']==True)&
                 (real_params['ana_cnn_prob']>=0.5)]

tn = bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&
                  (bogus_params['ana_param_cand']==True)&
                  (bogus_params['ana_cnn_prob']<0.5)]

fp = bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&
                  (bogus_params['ana_param_cand']==True)&
                  (bogus_params['ana_cnn_prob']>=0.5)]

print 'True Positive  : {:>5}/{:>5} = {:>5.3f}'.format(tp.shape[0], real.shape[0], float(tp.shape[0])/real.shape[0])
print 'False Positive : {:>5}/{:>5} = {:>5.3f}'.format(fp.shape[0], bogus.shape[0], float(fp.shape[0])/bogus.shape[0])
True Positive  :   344/  366 = 0.940
False Positive :   199/ 4885 = 0.041
In [53]:
fig = plt.figure(figsize=(12,6))
fig.suptitle("TPR, FPR vs. Threshold", fontsize=18)
ax = fig.add_subplot(121)
ax.plot(thresholds, fpr, color='tab:red')
ax.set_xlabel("Threshold", fontsize=15)
ax.set_ylabel("False Positive Rate", fontsize=15)
ax.set_ylim(-0.05,1.05)
ax.set_xlim(-0.05,1.05)
ax.tick_params(labelsize=12)
ax.set_aspect('equal')
iaxes = inset_axes(ax, width="50%", height="50%", loc=4)
ip = InsetPosition(ax, [0.20,0.65,0.7,0.3])
iaxes.set_axes_locator(ip)
iaxes.tick_params(labelsize=12)
iaxes.plot(thresholds, fpr, color='tab:red')
iaxes.set_xticks(np.arange(0.0, 1.1, 0.100))
iaxes.set_xticks(np.arange(0.0, 1.1, 0.050), minor=True)
iaxes.set_yticks(np.arange(0.0, 1.1, 0.010))
iaxes.set_yticks(np.arange(0.0, 1.1, 0.005), minor=True)
iaxes.set_xlim(0.50, 1.00)
iaxes.set_ylim(0.00, 0.05)
iaxes.grid()
#iaxes.set_aspect('equal')

ax = fig.add_subplot(122)
ax.plot(thresholds, tpr, color='tab:red')
ax.set_xlabel("Threshold", fontsize=15)
ax.set_ylabel("True Positive Rate", fontsize=15)
ax.set_ylim(-0.05,1.05)
ax.set_xlim(-0.05,1.05)
ax.tick_params(labelsize=12)
ax.set_aspect('equal')
iaxes = inset_axes(ax, width="50%", height="50%", loc=4)
ip = InsetPosition(ax, [0.15,0.10,0.7,0.3])
iaxes.set_axes_locator(ip)
iaxes.tick_params(labelsize=12)
iaxes.plot(thresholds, tpr, color='tab:red')
iaxes.set_xticks(np.arange(0.0, 1.1, 0.100))
iaxes.set_xticks(np.arange(0.0, 1.1, 0.050), minor=True)
iaxes.set_yticks(np.arange(0.0, 1.1, 0.040))
iaxes.set_yticks(np.arange(0.0, 1.1, 0.010), minor=True)
iaxes.set_xlim(0.50, 1.00)
iaxes.set_ylim(0.76, 0.96)
iaxes.grid()
#iaxes.set_aspect('equal')
#plt.savefig('TPR_FPR_vs_Threshold.png',dpi=100)
plt.show()
plt.close()
In [51]:
index = fp.index.values

print index
show_images(index[:10],*np.split(bogus_images, 3, axis=3), #labelAsFloat=True,
            labels=bogus_params.index.values,
            classes=bogus_params['ana_cnn_prob'],
            #save_name='psstar_and_dist_less_than_3arcsec',
            title='False Positive')
[  16   37   58   61   62  100  261  455  460  461  462  491  492  497
  515  527  557  576  577  632  668  669  707  725  749  750  789  794
  796  845  874  907  909  919  920  990 1025 1147 1187 1196 1202 1211
 1212 1317 1374 1375 1418 1419 1480 1522 1523 1538 1539 1649 1650 1716
 1717 1718 1722 1723 1769 1770 1777 1881 1882 1883 1944 1981 1982 2087
 2088 2197 2198 2216 2246 2247 2248 2289 2290 2681 2682 2684 2771 2805
 3056 3057 3141 3142 3196 3197 3219 3292 3306 3316 3410 3411 3412 3470
 3512 3513 3595 3628 3701 3702 3703 3708 3709 3710 3711 3712 3857 3858
 3859 3860 3861 3925 4129 4130 4132 4144 4145 4375 4376 4377 4378 4446
 4447 4464 4487 4488 4561 4562 4578 4669 4697 4727 4744 4745 4751 4795
 4850 4851 4852 4859 4861 4887 4889 4946 4948 5072 5170 5171 5172 5173
 5189 5190 5191 5270 5284 5308 5309 5506 5507 5622 5646 5647 5648 5653
 5761 5782 5785 5819 5820 5833 5870 5871 5872 5881 5977 5988 5989 5990
 6029 6030 6036 6159 6180 6264 6420 6436 6499 6500 6618 6636 6706 6707
 6744 6760 6979]
In [58]:
real_params['sub_snr']  = real_params['sub_flux_auto']  / real_params['sub_fluxerr_auto']
bogus_params['sub_snr'] = bogus_params['sub_flux_auto'] / bogus_params['sub_fluxerr_auto']
In [61]:
label = np.concatenate([np.ones(real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&
                                            (real_params['ana_param_cand']==True)]['ana_cnn_prob'].shape[0]),
                        np.zeros(bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&
                                              (bogus_params['ana_param_cand']==True)]['ana_cnn_prob'].shape[0])],
                       axis=0)

prob = np.concatenate([real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&
                                   (real_params['ana_param_cand']==True)]['ana_cnn_prob'].values,
                       bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&
                                    (bogus_params['ana_param_cand']==True)]['ana_cnn_prob'].values],
                      axis=0)

snr = np.concatenate([real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&
                                   (real_params['ana_param_cand']==True)]['sub_snr'].values,
                       bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&
                                    (bogus_params['ana_param_cand']==True)]['sub_snr'].values],
                      axis=0)
In [71]:
fig = plt.figure(figsize=(12,6))
fig.suptitle('Frequency Distribution of SNR', fontsize=18)
ax = fig.add_subplot(121)
ax.hist([bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&(bogus_params['ana_param_cand']==True)]['sub_snr']],
        label=['BOGUS Transient'],
        color=['tab:red'],
        range=(0,50),
        bins=10, density=True)
#ax.set_xlim(0,50)
#ax.set_ylim(0,200)
ax.set_xlabel('SNR', fontsize=15)
#ax.set_yscale('log')
ax.set_ylabel('Frequency', fontsize=15)
ax.tick_params(labelsize=12)
plt.legend(fontsize=14)

ax = fig.add_subplot(122)
ax.hist([bogus_params[~((bogus_params['ps_type']=='star')&(bogus_params['ps_match']==True))&(bogus_params['ana_param_cand']==True)]['sub_snr']],
        label=['BOGUS Transient'],
        color=['tab:red'],
        range=(0,50),
        bins=10, density=True)
ax.hist([real_params[~((real_params['ps_type']=='star')&(real_params['ps_match']==True))&(real_params['ana_param_cand']==True)]['sub_snr']],
        label=['REAL Transient'],
        color=['tab:blue'],
        range=(0,50),
        bins=10, density=True)
#ax.set_xlim(0,50)
#ax.set_ylim(0,200)
ax.set_xlabel('SNR', fontsize=15)
#ax.set_yscale('log')
#ax.set_ylabel('Frequency')
ax.tick_params(labelsize=12)

plt.legend(fontsize=14)
#plt.savefig('/home/hamasaki/master_reseach/dist_snr.png')
plt.show()
plt.close()
In [72]:
def snr_roc(sn_thresh=20):
    high_sn_index = np.where(snr >= sn_thresh)[0]
    low_sn_index  = np.where(snr < sn_thresh)[0]

    fpr_high, tpr_high, thresholds_high = roc_curve(label[high_sn_index], prob[high_sn_index])
    auc_high = auc(fpr_high, tpr_high)

    fpr_low, tpr_low, thresholds_low = roc_curve(label[low_sn_index], prob[low_sn_index])
    auc_low = auc(fpr_low, tpr_low)
    return auc_high, auc_low

sn_thresh_list = range(8, 101, 1)
auc_high_list = []
auc_low_list = []
for sn_thresh in sn_thresh_list:
    auc_high, auc_low = snr_roc(sn_thresh)
    auc_high_list.append(auc_high)
    auc_low_list.append(auc_low)

#auc_ratio = (np.array(auc_high_list) - base_auc_value)/(np.array(auc_low_list) - base_auc_value)

#print auc_high_list
#print auc_low_list
#print auc_ratio



fig = plt.figure(figsize=(15,7))
fig.suptitle('SNR vs. AUC',fontsize=18)

ax = fig.add_subplot(121)
ax.plot(np.array(sn_thresh_list), np.array(auc_high_list), '-', label='AUC (SNR > SNR threshold)', color='tab:orange')
ax.plot(np.array(sn_thresh_list), np.array(auc_low_list), '-', label='AUC (SNR < SNR threshold)', color='tab:cyan')
#ax.hlines([base_auc],8,101,'gray',linestyles='dashed')
#ax.set_xlim(0, 100)
#ax.set_ylim(0, 1500)
ax.tick_params(labelsize=12)
ax.set_xlabel('SNR threshold',fontsize=15)
ax.set_ylabel('AUC',fontsize=15)
plt.legend(fontsize=15)
#plt.show()
#plt.close()

def snr_true_num(sn_thresh=20):
    high_sn_index = np.where( (snr >= sn_thresh)&(label==1) )[0]
    low_sn_index  = np.where( (snr < sn_thresh)&(label==1) )[0]
    return [high_sn_index.shape[0], low_sn_index.shape[0]]
sn_high_low_num = [ snr_true_num(sn_thresh=thresh) for thresh in sn_thresh_list]
sn_high_num = np.array([num[0] for num in sn_high_low_num])
sn_low_num  = np.array([num[1] for num in sn_high_low_num])
true_num_ratio = sn_high_num.astype('float64') / (sn_low_num + sn_high_num)

def snr_false_num(sn_thresh=20):
    high_sn_index = np.where( (snr >= sn_thresh)&(label==0) )[0]
    low_sn_index  = np.where( (snr < sn_thresh)&(label==0) )[0]
    return [high_sn_index.shape[0], low_sn_index.shape[0]]
sn_high_low_num = [ snr_false_num(sn_thresh=thresh) for thresh in sn_thresh_list]
sn_high_num = np.array([num[0] for num in sn_high_low_num])
sn_low_num  = np.array([num[1] for num in sn_high_low_num])
false_num_ratio = sn_high_num.astype('float64') / (sn_low_num + sn_high_num)

#fig = plt.figure(figsize=(8,6))
#fig.suptitle('Rate of Number of High SNR VS SNR (All=1)', fontsize=20)
ax = fig.add_subplot(122)
ax.plot(np.array(sn_thresh_list), true_num_ratio, '-', label='REAL Transient', color='tab:blue')
ax.plot(np.array(sn_thresh_list), false_num_ratio, '-', label='BOGUS Transient', color='tab:red')
#ax.set_xlim(0, 100)
#ax.set_ylim(0, 1)
ax.tick_params(labelsize=12)
plt.legend(fontsize=15)
ax.set_xlabel('SNR threshold', fontsize=15)
ax.set_ylabel('Ratio of Number (SNR > SNR threshold) ', fontsize=15)
#plt.savefig('/home/hamasaki/master_reseach/snr_vs_auc.png')
plt.show()
plt.close()
In [73]:
sn_thresh = 20
high_thresh_index = np.where(snr >= sn_thresh)[0]
low_thresh_index  = np.where(snr < sn_thresh)[0]

#prob_test = model0.predict_proba(image_test, verbose=False)

fpr_high, tpr_high, thresholds_high = roc_curve(label[high_thresh_index], prob[high_thresh_index])
val_auc_high = auc(fpr_high, tpr_high)

fpr_low, tpr_low, thresholds_low = roc_curve(label[low_thresh_index], prob[low_thresh_index])
val_auc_low = auc(fpr_low, tpr_low)

fig = plt.figure(figsize=(7,7))
fig.suptitle("ROC curve (AUC = %6.4f / %6.4f)" % (val_auc_high, val_auc_low), fontsize=18)
ax = fig.add_subplot(111)
ax.plot(fpr_low, tpr_low, 'k:', label='SNR < %3.1f' % (sn_thresh))
ax.plot(fpr_high, tpr_high, 'r-', label='SNR > %3.1f' % (sn_thresh))
ax.set_xlabel("False Positive Rate", fontsize=15)
ax.set_ylabel("True Positive Rate", fontsize=15)
ax.set_ylim(-0.05,1.05)
ax.set_xlim(-0.05,1.05)
#ax.tick_params(labelsize=15)
#ax.grid()
ax.set_aspect('equal')
plt.legend(bbox_to_anchor=(0.95,0.9), fontsize=14)
iaxes = inset_axes(ax, width="50%", height="50%", loc=4)
ip = InsetPosition(ax, [0.4,0.1,0.5,0.5])
iaxes.set_axes_locator(ip)
iaxes.plot(fpr_low, tpr_low, 'k:')
iaxes.plot(fpr_high, tpr_high, 'r-')
iaxes.set_xticks(np.arange(0.0, 0.2, 0.050))
iaxes.set_xticks(np.arange(0.0, 0.2, 0.010), minor=True)
iaxes.set_yticks(np.arange(0.80, 1.0, 0.020))
iaxes.set_yticks(np.arange(0.80, 1.0, 0.010), minor=True)
iaxes.set_xlim(0.00, 0.15)
iaxes.set_ylim(0.80, 0.98)
#iaxes.tick_params(labelsize=14)
iaxes.grid()
iaxes.set_aspect('equal')
plt.show()
plt.close()