easyvolcap: add support for mipnerf360 dataset & add nvitop

configs/datasets/mipnerf360/bicycle.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/bicycle
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/bonsai.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/bonzai
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/counter.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/counter
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/flowers.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/flowers
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/garden.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/garden
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/kitchen.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/kitchen
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/mipnerf360.yaml

@@ -0,0 +1,7 @@
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        images_dir: images_easyvolcap
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/room.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/garden
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/stump.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/stump
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/datasets/mipnerf360/treehill.yaml

@@ -0,0 +1,9 @@
+configs: configs/datasets/mipnerf360/mipnerf360.yaml
+
+dataloader_cfg:
+    dataset_cfg: &dataset_cfg
+        data_root: data/mipnerf360/treehill
+
+val_dataloader_cfg:
+    dataset_cfg:
+        <<: *dataset_cfg

configs/exps/l3mhet/l3mhet_kitchen.yaml

@@ -0,0 +1,16 @@
+configs:
+    - configs/base.yaml # default arguments for the whole codebase
+    - configs/models/l3mhet.yaml # network model configuration
+    - configs/datasets/mipnerf360/kitchen.yaml
+    - configs/specs/blurry.yaml # empty temporal embedder
+    - configs/specs/optcam.yaml # optimize camera parameters
+    - configs/specs/transient.yaml # add per-image coding
+
+runner_cfg:
+    epochs: 10
+    save_ep: 1
+    save_latest_ep: 1
+    eval_ep: 1
+
+# prettier-ignore
+exp_name: {{fileBasenameNoExtension}}

requirements-devel.txt

@@ -60,6 +60,7 @@ ruamel.yaml
 commentjson
 
 # to support visualization, we need:
+nvitop
 gpustat @ git+https://github.com/wookayin/gpustat
 # for winding_number_remesh
 nvdiffrast @ git+https://github.com/NVlabs/nvdiffrast

scripts/preprocess/mipnerf360_to_easyvolcap.py

@@ -0,0 +1,77 @@
+"""
+Convert the mipnerf360 dataset to easyvolcap format
+Read camera pose from poses_bounds.npy
+Symlink image files into separated folders per camera
+"""
+
+from easyvolcap.utils.console_utils import *
+from easyvolcap.utils.easy_utils import write_camera
+from easyvolcap.utils.data_utils import as_numpy_func
+from easyvolcap.utils.math_utils import affine_inverse
+
+
+@catch_throw
+def main():
+    args = dotdict(
+        mipnerf360_root='data/mipnerf360',
+        easyvolcap_root='data/mipnerf360',
+        raw_images_dir='images',
+        out_images_dir='images_easyvolcap',
+        image_name_no_ext='000000',
+        camera_pose='poses_bounds.npy',
+        scenes=['bonsai', 'bicycle', 'counter', 'flowers', 'garden', 'kitchen', 'room', 'stump', 'treehill'],
+    )
+    args = dotdict(vars(build_parser(args, description=__doc__).parse_args()))
+
+    scenes = args.scenes
+    for scene in tqdm(scenes):
+        keys = []
+        # for scene in scenes:
+        for i, img in enumerate(sorted(os.listdir(join(args.mipnerf360_root, scene, args.raw_images_dir)))):
+            # shutil.copy(join(args.mipnerf360_root, scene, args.raw_images_dir, img), join(args.easyvolcap_root, scene, args.out_images_dir, img))
+            key = f'{i:06d}'
+            src = join(args.mipnerf360_root, scene, args.raw_images_dir, img)
+            tar = join(args.easyvolcap_root, scene, args.out_images_dir, key, args.image_name_no_ext + splitext(img)[-1]).lower()
+
+            if exists(tar): os.remove(tar)
+            os.makedirs(dirname(tar), exist_ok=True)
+            os.symlink(relpath(src, dirname(tar)), tar)
+            keys.append(key)
+
+        # https://github.com/kwea123/nerf_pl/blob/52aeb387da64a9ad9a0f914ea9b049ffc598b20c/datasets/llff.py#L177
+        raw = np.load(join(args.mipnerf360_root, scene, args.camera_pose), allow_pickle=True)  # 21, 17
+        poses = raw[:, :15].reshape(-1, 3, 5)  # N, 3, 5
+        bounds = raw[:, -2:]  # N, 2
+        # Step 1: rescale focal length according to training resolution
+        H, W, F = poses[0, :, -1]  # original intrinsics, same for all images
+
+        # Step 2: correct poses
+        # Original poses has rotation in form "down right back", change to "right down front"
+        # See https://github.com/bmild/nerf/issues/34
+        poses = np.concatenate([poses[..., 1:2], poses[..., :1], -poses[..., 2:3], poses[..., 3:4]], -1)  # (N_images, 3, 4) exclude H, W, focal
+        cameras = dotdict()
+
+        for i in range(len(poses)):
+            key = keys[i]
+
+            c2w = poses[i]
+            w2c = as_numpy_func(affine_inverse)(c2w)
+
+            cameras[key] = dotdict()
+            cameras[key].R = w2c[:3, :3]
+            cameras[key].T = w2c[:3, 3:]
+            cameras[key].K = np.zeros_like(cameras[key].R)
+            cameras[key].K[0, 0] = F
+            cameras[key].K[1, 1] = F
+            cameras[key].K[0, 2] = W / 2
+            cameras[key].K[1, 2] = H / 2
+            cameras[key].K[2, 2] = 1.0
+            cameras[key].n = bounds[i, 0]  # camera has near and far
+            cameras[key].f = bounds[i, 1]  # camera has near and far
+
+        write_camera(cameras, join(args.easyvolcap_root, scene))
+        log(yellow(f'Converted cameras saved to {blue(join(args.easyvolcap_root, scene, "{intri.yml,extri.yml}"))}'))
+
+
+if __name__ == '__main__':
+    main()

scripts/preprocess/neural3dv_to_easyvolcap.py

@@ -1,7 +1,16 @@
-# Convert neural3dv dataset to easyvolcap format
-# Assume we've got the images processed by haotong
-# Need some validation since we're kind of blindly converting from pose_bounds.npy
-# Need a way to store the bound info in the camera parameters file
+"""
+Convert neural3dv dataset to easyvolcap format
+Assume we've got the images processed by haotong
+Need some validation since we're kind of blindly converting from pose_bounds.npy
+Need a way to store the bound info in the camera parameters file
+
+Assume ./data/neural3dv/XXX
+in easyvolcap:
+python3 scripts/preprocess/neural3dv_to_easyvolcap.py --only XXX
+python3 scripts/colmap/easymocap_to_colmap.py --data_root data/neural3dv/XXX --image_dir images --output_dir colmap
+in spg_colmap:
+python3 sfm_renbody.py --root_dir ./data/neural3dv/XXX/colmap --colmap_path $PATHTOCOLMAP
+"""
 
 import argparse
 import subprocess
@@ -18,15 +27,7 @@
 
 
 def main():
-    """
-    Assume ./data/neural3dv/XXX
-    in easyvolcap:
-    python3 scripts/preprocess/neural3dv_to_easyvolcap.py --only XXX
-    python3 scripts/colmap/easymocap_to_colmap.py --data_root data/neural3dv/XXX --image_dir images --output_dir colmap
-    in spg_colmap:
-    python3 sfm_renbody.py --root_dir ./data/neural3dv/XXX/colmap --colmap_path $PATHTOCOLMAP
-    """
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--neural3dv_root', type=str, default='data/neural3dv')
     parser.add_argument('--easyvolcap_root', type=str, default='data/neural3dv')
     parser.add_argument('--camera_pose', type=str, default='poses_bounds.npy')