Changed timing output

zhangkaiitugithub · Nov 15, 2021 · b4aecc4 · b4aecc4
1 parent 84fb0df
commit b4aecc4
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 14 deletions.
diff --git a/opt/autotune.py b/opt/autotune.py
@@ -58,19 +58,14 @@ def run_exp(env, eval_mode:bool, enable_render:bool, train_dir, data_dir, config
         print('! RUN opt.py -t', train_dir)
         opt_cmd = ' '.join(opt_base_cmd + flags + common_flags)
         print(opt_cmd)
-        start_time = datetime.now()
         try:
             opt_ret = subprocess.check_output(opt_cmd, shell=True, env=env).decode(
                     sys.stdout.encoding)
         except subprocess.CalledProcessError:
             print('Error occurred while running OPT for exp', train_dir, 'on', env["CUDA_VISIBLE_DEVICES"])
             return
-        end_time = datetime.now()
         with open(log_file_path, 'w') as f:
             f.write(opt_ret)
-        secs = (end_time - start_time).total_seconds()
-        timings_file = open(os.path.join(train_dir, 'time_mins.txt'), 'a')
-        timings_file.write(f"{secs / 60}\n")
 
     if eval_mode:
         eval_base_cmd = [

diff --git a/opt/opt.py b/opt/opt.py
@@ -23,6 +23,7 @@
 from util import config_util
 
 from warnings import warn
+from datetime import datetime
 from torch.utils.tensorboard import SummaryWriter
 
 from tqdm import tqdm
@@ -249,7 +250,6 @@
 np.random.seed(20200823)
 
 factor = 1
-
 dset = datasets[args.dataset_type](
                args.data_dir,
                split="train",
@@ -264,6 +264,8 @@
 dset_test = datasets[args.dataset_type](
         args.data_dir, split="test", **config_util.build_data_options(args))
 
+global_start_time = datetime.now()
+
 grid = svox2.SparseGrid(reso=reso_list[reso_id],
                         center=dset.scene_center,
                         radius=dset.scene_radius,
@@ -278,7 +280,6 @@
                         background_nlayers=args.background_nlayers,
                         background_reso=args.background_reso)
 
-
 # DC -> gray; mind the SH scaling!
 grid.sh_data.data[:] = 0.0
 grid.density_data.data[:] = args.init_sigma
@@ -362,8 +363,8 @@ def eval_step():
             stats_test = {'psnr' : 0.0, 'mse' : 0.0}
 
             # Standard set
-            N_IMGS_TO_SAVE = min(5, dset_test.n_images) if not args.tune_mode else 1
             N_IMGS_TO_EVAL = min(20 if epoch_id > 0 else 5, dset_test.n_images)
+            N_IMGS_TO_SAVE = N_IMGS_TO_EVAL if not args.tune_mode else 1
             img_eval_interval = dset_test.n_images // N_IMGS_TO_EVAL
             img_save_interval = (N_IMGS_TO_EVAL // N_IMGS_TO_SAVE)
             img_ids = range(0, dset_test.n_images, img_eval_interval)
@@ -373,7 +374,7 @@ def eval_step():
             #             44, 45, 47, 49, 56,
             #             80, 88, 99, 115, 120,
             #             154]
-            img_save_interval = 1
+            #  img_save_interval = 1
 
             n_images_gen = 0
             for i, img_id in tqdm(enumerate(img_ids), total=len(img_ids)):
@@ -439,7 +440,8 @@ def eval_step():
                         stats_test[stat_name], global_step=gstep_id_base)
             summary_writer.add_scalar('epoch_id', float(epoch_id), global_step=gstep_id_base)
             print('eval stats:', stats_test)
-    if epoch_id % max(factor, args.eval_every) == 0:
+    if epoch_id % max(factor, args.eval_every) == 0 and (epoch_id > 0 or not args.tune_mode):
+        # NOTE: we do an eval sanity check, if not in tune_mode
         eval_step()
         gc.collect()
 
@@ -609,6 +611,10 @@ def train_step():
     if gstep_id_base >= args.n_iters:
         print('* Final eval and save')
         eval_step()
+        global_stop_time = datetime.now()
+        secs = (global_stop_time - global_start_time).total_seconds()
+        timings_file = open(os.path.join(args.train_dir, 'time_mins.txt'), 'a')
+        timings_file.write(f"{secs / 60}\n")
         if not args.tune_nosave:
             grid.save(ckpt_path)
         break
diff --git a/opt/tasks/eval_tnt.json b/opt/tasks/eval_tnt.json
@@ -1,7 +1,7 @@
 {
     "eval": true,
     "data_root": "/home/sxyu/data/TanksAndTempleBG",
-    "train_root": "/home/sxyu/proj/svox2/opt/ckpt_auto/tnt_equirectlin_fasttv_autoscale",
+    "train_root": "/home/sxyu/proj/svox2/opt/ckpt_auto/tnt_equirectlin_fasttv_autoscale_time",
     "variables": {
         "scene": ["Train", "M60", "Truck", "Playground"]
     },

diff --git a/opt/util/nsvf_dataset.py b/opt/util/nsvf_dataset.py
@@ -146,15 +146,14 @@ def look_for_dir(cands, required=True):
                 warn('normalize_by_bbox=True but bbox.txt was not available')
         elif normalize_by_camera:
             norm_pose_files = sorted(os.listdir(path.join(root, pose_dir_name)), key=sort_key)
-            norm_pose_files = [x for x in norm_pose_files if not x.startswith('2_')]
             norm_poses = np.stack([np.loadtxt(path.join(root, pose_dir_name, x)).reshape(-1, 4)
                                     for x in norm_pose_files], axis=0)
 
             # Select subset of files
-            center = np.median(norm_poses[:, :3, 3], axis=0)
+            center = np.mean(norm_poses[:, :3, 3], axis=0)
             radius = np.median(np.linalg.norm(norm_poses[:, :3, 3] - center, axis=-1))
             self.c2w_f64[:, :3, 3] -= center
-            scene_scale = 1.0 / radius
+            scene_scale = 0.95 / radius
 
         print('scene_scale', scene_scale)
         self.c2w_f64[:, :3, 3] *= scene_scale