From a8cfd0e8fdd6cdc8f6a8b5b3c18cc52a0fb60c0f Mon Sep 17 00:00:00 2001 From: Alex Yu Date: Sat, 14 Jan 2023 21:22:16 -0800 Subject: [PATCH] WIP --- README.md | 15 +++++++++++++-- opt/scripts/run_colmap.py | 5 ++++- opt/util/nsvf_dataset.py | 20 ++++++++++++++------ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 508bf855..35735f0b 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,11 @@ normal perspective camera. For custom datasets we adopt a data format similar to that in NSVF +**Update:** I found that the original release which uses colmap's undistorter made custom scene results much worse. There might be some issue here, but the current version does not use the undistorter. +Supporting a full OPENCV camera model should be fairly easy, as done in nerfstudio and multi-nerf +https://github.com/google-research/multinerf/blob/1c8b1c552133cdb2de1c1f3c871b2813f6662265/internal/camera_utils.py#L477 + + You should be able to use this dataset directly afterwards. The format will be auto-detected. To view the data use: @@ -151,17 +156,23 @@ Now follow the "Voxel Optimization (aka Training)" section to train: `./launch.sh -c configs/custom.json` -You can also try `configs/custom_alt.json` which has some minor differences. +custom.json was used for the real lego bulldozer scene. +You can also try `configs/custom_alt.json` which has some minor differences **especially that near_clip is eliminated**. If the scene's central object is totally messed up, this might be due to the aggressive near clip, and the alt config fixes it. You may need to tune the TV for best results. + To render a video, please see the "rendering a spiral" section. To convert to a svox1-compatible PlenOctree (not perfect quality since interpolation is not implemented) you can try `to_svox1.py ` +### Recent updates + +- Removed COLMAP undistorter since it is very unhelpful (using SIMPLE_PINHOLE for simplicity, could easily use OPENCV camera instead) +- Updated the camera normalization + ## Random tip: how to make pip install faster for native extensions You may notice that this CUDA extension takes forever to install. A suggestion is using ninja. On Ubuntu, install it with `sudo apt install ninja-build`. -Then set the environment variable `MAX_JOBS` to the number of CPUS to use in parallel (e.g. 12) in your shell startup script. This will enable parallel compilation and significantly improve iteration speed. diff --git a/opt/scripts/run_colmap.py b/opt/scripts/run_colmap.py index 13739765..aad070f6 100644 --- a/opt/scripts/run_colmap.py +++ b/opt/scripts/run_colmap.py @@ -259,6 +259,9 @@ def run_colmap(vid_root, args, factor, overwrite=False): os.system(mapper_cmd) if not args.noradial: + print("Warning: I've found the undistorter to work very poorly, substantially reducing quality.") + print("A potential (fairly easy) improvement is to support OPENCV camera model in the codebase, " + "and without doing undistorting.") undist_dir = os.path.join(vid_root, args.undistorted_output) if not os.path.exists(undist_dir) or overwrite: os.makedirs(undist_dir, exist_ok=True) @@ -387,7 +390,7 @@ def preprocess(vid_root, args): parser.add_argument('--known-intrin', action='store_true', default=False, help='use intrinsics in /intrinsics.txt if available') parser.add_argument('--fix-intrin', action='store_true', default=False, help='fix intrinsics in bundle adjustment, only used if --known-intrin is given and intrinsics.txt exists') parser.add_argument('--debug', action='store_true', default=False, help='render debug video') - parser.add_argument('--noradial', action='store_true', default=False, help='do not use radial distortion') + parser.add_argument('--noradial', action='store_true', default=True, help='do not use radial distortion') parser.add_argument('--use-masks', action='store_true', default=False, help='use automatic masks') parser.add_argument( '--images-resized', default='images_resized', help='location for resized/renamed images') diff --git a/opt/util/nsvf_dataset.py b/opt/util/nsvf_dataset.py index 1e3c2f8a..d18c260f 100644 --- a/opt/util/nsvf_dataset.py +++ b/opt/util/nsvf_dataset.py @@ -82,20 +82,28 @@ def look_for_dir(cands, required=True): img_dir_name = look_for_dir(["images", "image", "rgb"]) pose_dir_name = look_for_dir(["poses", "pose"]) # intrin_dir_name = look_for_dir(["intrin"], required=False) - img_files = sorted(os.listdir(path.join(root, img_dir_name)), key=sort_key) + orig_img_files = sorted(os.listdir(path.join(root, img_dir_name)), key=sort_key) # Select subset of files if self.split == "train" or self.split == "test_train": - img_files = [x for x in img_files if x.startswith("0_")] + img_files = [x for x in orig_img_files if x.startswith("0_")] elif self.split == "val": - img_files = [x for x in img_files if x.startswith("1_")] + img_files = [x for x in orig_img_files if x.startswith("1_")] elif self.split == "test": - test_img_files = [x for x in img_files if x.startswith("2_")] + test_img_files = [x for x in orig_img_files if x.startswith("2_")] if len(test_img_files) == 0: - test_img_files = [x for x in img_files if x.startswith("1_")] + test_img_files = [x for x in orig_img_files if x.startswith("1_")] img_files = test_img_files + else: + img_files = orig_img_files + + if len(img_files) == 0: + if self.split == "train": + img_files = [x for i, x in enumerate(orig_img_files) if i % 16 != 0] + else: + img_files = orig_img_files[::16] - assert len(img_files) > 0, "No matching images in directory: " + path.join(data_dir, img_dir_name) + assert len(img_files) > 0, "No matching images in directory: " + path.join(root, img_dir_name) self.img_files = img_files dynamic_resize = scale < 1