diff --git a/CMakeLists.txt b/CMakeLists.txt
index 469d619fafd06993c33620c34a1429a437088648..4b3a4ba1cbe1b77ffa2b604408aff87dbd2bce3e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -205,6 +205,7 @@ endif()
list(APPEND NGP_SOURCES
${GUI_SOURCES}
src/camera_path.cu
+ src/common.cu
src/common_device.cu
src/marching_cubes.cu
src/nerf_loader.cu
diff --git a/README.md b/README.md
index 2f916509bc5f15672371420bae6800dcbfeb8863..54fcd050c65fb82e3ce4154f33ff7ba7a20871f2 100644
--- a/README.md
+++ b/README.md
@@ -16,6 +16,7 @@ To get started with NVIDIA Instant NeRF, check out the [blog post](https://devel
For business inquiries, please submit the [NVIDIA research licensing form](https://www.nvidia.com/en-us/research/inquiries/).
+
## Windows binary release
If you have Windows and if you do not need Python bindings, you can download one of the following binary releases and then jump directly to the [usage instructions](https://github.com/NVlabs/instant-ngp#interactive-training-and-rendering). These releases are automatically regenerated whenever the code gets updated, so you can be sure that they have the latest features.
@@ -26,6 +27,7 @@ If you have Windows and if you do not need Python bindings, you can download one
If you use Linux, or want the Python bindings, or if your GPU is not listed above (e.g. Hopper, Volta, or Maxwell generations), use the following step-by-step instructions to compile __instant-ngp__ yourself.
+
## Requirements
- An __NVIDIA GPU__; tensor cores increase performance when available. All shown results come from an RTX 3090.
@@ -37,7 +39,7 @@ If you use Linux, or want the Python bindings, or if your GPU is not listed abov
- __Linux:__ CUDA 10.2 or higher
- __[CMake](https://cmake.org/) v3.21 or higher__.
- __(optional) [Python](https://www.python.org/) 3.7 or higher__ for interactive bindings. Also, run `pip install -r requirements.txt`.
-- __(optional) [OptiX](https://developer.nvidia.com/optix) 7.3 or higher__ for faster mesh SDF training.
+- __(optional) [OptiX](https://developer.nvidia.com/optix) 7.6 or higher__ for faster mesh SDF training.
- __(optional) [Vulkan SDK](https://vulkan.lunarg.com/)__ for DLSS support.
@@ -49,7 +51,7 @@ sudo apt-get install build-essential git python3-dev python3-pip libopenexr-dev
Alternatively, if you are using Arch or Arch derivatives, install the following packages
```sh
-sudo pacman -S base-devel cmake openexr libxi glfw openmp libxinerama libxcursor
+sudo pacman -S cuda base-devel cmake openexr libxi glfw openmp libxinerama libxcursor
```
We also recommend installing [CUDA](https://developer.nvidia.com/cuda-toolkit) and [OptiX](https://developer.nvidia.com/optix) in `/usr/local/` and adding the CUDA installation to your PATH.
@@ -60,11 +62,6 @@ export PATH="/usr/local/cuda-11.4/bin:$PATH"
export LD_LIBRARY_PATH="/usr/local/cuda-11.4/lib64:$LD_LIBRARY_PATH"
```
-For Arch and derivatives,
-```sh
-sudo pacman -S cuda
-```
-
## Compilation (Windows & Linux)
@@ -112,13 +109,13 @@ Let's start using __instant-ngp__; more information about the GUI and other scri
One test scene is provided in this repository, using a small number of frames from a casually captured phone video:
```sh
-instant-ngp$ ./build/instant-ngp --scene data/nerf/fox
+instant-ngp$ ./build/instant-ngp data/nerf/fox
```
On Windows you need to reverse the slashes here (and below), i.e.:
```sh
-instant-ngp> .\build\instant-ngp --scene data\nerf\fox
+instant-ngp> .\build\instant-ngp data\nerf\fox
```
<img src="docs/assets_readme/fox.png"/>
@@ -127,7 +124,7 @@ Alternatively, download any NeRF-compatible scene (e.g. from the [NeRF authors'
Now you can run:
```sh
-instant-ngp$ ./build/instant-ngp --scene data/nerf_synthetic/lego/transforms_train.json
+instant-ngp$ ./build/instant-ngp data/nerf_synthetic/lego/transforms_train.json
```
**[To prepare your own dataset for use with our NeRF implementation, click here.](docs/nerf_dataset_tips.md)** See also [this video](https://www.youtube.com/watch?v=8GbENSmdVeE) for a guided walkthrough.
@@ -135,7 +132,7 @@ instant-ngp$ ./build/instant-ngp --scene data/nerf_synthetic/lego/transforms_tra
### SDF armadillo
```sh
-instant-ngp$ ./build/instant-ngp --scene data/sdf/armadillo.obj
+instant-ngp$ ./build/instant-ngp data/sdf/armadillo.obj
```
<img src="docs/assets_readme/armadillo.png"/>
@@ -143,7 +140,7 @@ instant-ngp$ ./build/instant-ngp --scene data/sdf/armadillo.obj
### Image of Einstein
```sh
-instant-ngp$ ./build/instant-ngp --scene data/image/albert.exr
+instant-ngp$ ./build/instant-ngp data/image/albert.exr
```
<img src="docs/assets_readme/albert.png"/>
@@ -151,7 +148,7 @@ instant-ngp$ ./build/instant-ngp --scene data/image/albert.exr
To reproduce the gigapixel results, download, for example, [the Tokyo image](https://www.flickr.com/photos/trevor_dobson_inefekt69/29314390837) and convert it to `.bin` using the `scripts/convert_image.py` script. This custom format improves compatibility and loading speed when resolution is high. Now you can run:
```sh
-instant-ngp$ ./build/instant-ngp --scene data/image/tokyo.bin
+instant-ngp$ ./build/instant-ngp data/image/tokyo.bin
```
@@ -160,7 +157,7 @@ instant-ngp$ ./build/instant-ngp --scene data/image/tokyo.bin
Download the [nanovdb volume for the Disney cloud](https://drive.google.com/drive/folders/1SuycSAOSG64k2KLV7oWgyNWyCvZAkafK?usp=sharing), which is derived [from here](https://disneyanimation.com/data-sets/?drawer=/resources/clouds/) ([CC BY-SA 3.0](https://media.disneyanimation.com/uploads/production/data_set_asset/6/asset/License_Cloud.pdf)).
```sh
-instant-ngp$ ./build/instant-ngp --mode volume --scene data/volume/wdas_cloud_quarter.nvdb
+instant-ngp$ ./build/instant-ngp data/volume/wdas_cloud_quarter.nvdb
```
<img src="docs/assets_readme/cloud.png"/>
@@ -204,7 +201,7 @@ For an example of how the `./build/instant-ngp` application can be implemented a
Here is a typical command line using `scripts/run.py` to generate a 5-second flythrough of the fox dataset to the (default) file `video.mp4`, after using the GUI to save a (default) NeRF snapshot `base.msgpack` and a set of camera key frames: (see [this video](https://www.youtube.com/watch?v=8GbENSmdVeE) for a guided walkthrough)
```sh
-instant-ngp$ python scripts/run.py --mode nerf --scene data/nerf/fox --load_snapshot data/nerf/fox/base.msgpack --video_camera_path data/nerf/fox/base_cam.json --video_n_seconds 5 --video_fps 60 --width 1920 --height 1080
+instant-ngp$ python scripts/run.py data/nerf/fox/base.msgpack --video_camera_path data/nerf/fox/base_cam.json --video_n_seconds 5 --video_fps 60 --width 1920 --height 1080
```
If you'd rather build new models from the hash encoding and fast neural networks, consider the [__tiny-cuda-nn__'s PyTorch extension](https://github.com/nvlabs/tiny-cuda-nn#pytorch-extension).
diff --git a/docs/nerf_dataset_tips.md b/docs/nerf_dataset_tips.md
index 47de07acc1b8e0ed0d733034e9ca7aacc5730272..7952082976e0510f0f0a57013c2e4a4399097a33 100644
--- a/docs/nerf_dataset_tips.md
+++ b/docs/nerf_dataset_tips.md
@@ -84,7 +84,7 @@ The `aabb_scale` parameter is the most important `instant-ngp` specific paramete
Assuming success, you can now train your NeRF model as follows, starting in the `instant-ngp` folder:
```sh
-instant-ngp$ ./build/instant-ngp --mode nerf --scene [path to training data folder containing transforms.json]
+instant-ngp$ ./build/instant-ngp [path to training data folder containing transforms.json]
```
### Record3D
@@ -102,7 +102,7 @@ With an >=iPhone 12 Pro, one can use [Record3D](https://record3d.app/) to collec
5. Launch Instant-NGP training:
```
- ./build/instant-ngp --scene path/to/data
+ ./build/instant-ngp path/to/data
```
## Tips for NeRF training data
diff --git a/include/neural-graphics-primitives/common.h b/include/neural-graphics-primitives/common.h
index 9b5c0cf3337d0b175826afa4e0d0352c9e3cf8a8..30c59cd9478dbdcb6339d07ce43a572e5e471298 100644
--- a/include/neural-graphics-primitives/common.h
+++ b/include/neural-graphics-primitives/common.h
@@ -53,8 +53,17 @@
#include <chrono>
#include <functional>
+#if defined(__NVCC__) || (defined(__clang__) && defined(__CUDA__))
+#define NGP_HOST_DEVICE __host__ __device__
+#else
+#define NGP_HOST_DEVICE
+#endif
+
NGP_NAMESPACE_BEGIN
+bool ends_with(const std::string& str, const std::string& ending);
+bool ends_with_case_insensitive(const std::string& str, const std::string& ending);
+
using Vector2i32 = Eigen::Matrix<uint32_t, 2, 1>;
using Vector3i16 = Eigen::Matrix<uint16_t, 3, 1>;
using Vector4i16 = Eigen::Matrix<uint16_t, 4, 1>;
@@ -155,8 +164,13 @@ enum class ETestbedMode : int {
Sdf,
Image,
Volume,
+ None,
};
+ETestbedMode mode_from_scene(const std::string& scene);
+ETestbedMode mode_from_string(const std::string& str);
+std::string to_string(ETestbedMode);
+
enum class ESDFGroundTruthMode : int {
RaytracedMesh,
SpheretracedMesh,
@@ -185,12 +199,6 @@ struct Lens {
float params[7] = {};
};
-#if defined(__NVCC__) || (defined(__clang__) && defined(__CUDA__))
-#define NGP_HOST_DEVICE __host__ __device__
-#else
-#define NGP_HOST_DEVICE
-#endif
-
inline NGP_HOST_DEVICE float sign(float x) {
return copysignf(1.0, x);
}
diff --git a/include/neural-graphics-primitives/testbed.h b/include/neural-graphics-primitives/testbed.h
index 8e55eeb08c90198e5b45320549e89a0b789dbe82..1efa2b27040b34160fe207f48ce235d7e1726195 100644
--- a/include/neural-graphics-primitives/testbed.h
+++ b/include/neural-graphics-primitives/testbed.h
@@ -61,7 +61,7 @@ class GLTexture;
class Testbed {
public:
EIGEN_MAKE_ALIGNED_OPERATOR_NEW
- Testbed(ETestbedMode mode);
+ Testbed(ETestbedMode mode = ETestbedMode::None);
~Testbed();
Testbed(ETestbedMode mode, const std::string& data_path) : Testbed(mode) { load_training_data(data_path); }
Testbed(ETestbedMode mode, const std::string& data_path, const std::string& network_config_path) : Testbed(mode, data_path) { reload_network_from_file(network_config_path); }
@@ -69,6 +69,8 @@ public:
void load_training_data(const std::string& data_path);
void clear_training_data();
+ void set_mode(ETestbedMode mode);
+
using distance_fun_t = std::function<void(uint32_t, const Eigen::Vector3f*, float*, cudaStream_t)>;
using normals_fun_t = std::function<void(uint32_t, const Eigen::Vector3f*, Eigen::Vector3f*, cudaStream_t)>;
@@ -284,8 +286,9 @@ public:
void render_image(CudaRenderBuffer& render_buffer, cudaStream_t stream);
void render_frame(const Eigen::Matrix<float, 3, 4>& camera_matrix0, const Eigen::Matrix<float, 3, 4>& camera_matrix1, const Eigen::Vector4f& nerf_rolling_shutter, CudaRenderBuffer& render_buffer, bool to_srgb = true) ;
void visualize_nerf_cameras(ImDrawList* list, const Eigen::Matrix<float, 4, 4>& world2proj);
+ filesystem::path find_network_config(const filesystem::path& network_config_path);
nlohmann::json load_network_config(const filesystem::path& network_config_path);
- void reload_network_from_file(const std::string& network_config_path);
+ void reload_network_from_file(const std::string& network_config_path = "");
void reload_network_from_json(const nlohmann::json& json, const std::string& config_base_path=""); // config_base_path is needed so that if the passed in json uses the 'parent' feature, we know where to look... be sure to use a filename, or if a directory, end with a trailing slash
void reset_accumulation(bool due_to_camera_movement = false, bool immediate_redraw = true);
void redraw_next_frame() {
@@ -306,7 +309,7 @@ public:
void translate_camera(const Eigen::Vector3f& rel);
void mouse_drag(const Eigen::Vector2f& rel, int button);
void mouse_wheel(Eigen::Vector2f m, float delta);
- void handle_file(const std::string& file);
+ void load_file(const std::string& file);
void set_nerf_camera_matrix(const Eigen::Matrix<float, 3, 4>& cam);
Eigen::Vector3f look_at() const;
void set_look_at(const Eigen::Vector3f& pos);
@@ -463,7 +466,7 @@ public:
bool m_training_data_available = false;
bool m_render = true;
int m_max_spp = 0;
- ETestbedMode m_testbed_mode = ETestbedMode::Sdf;
+ ETestbedMode m_testbed_mode = ETestbedMode::None;
bool m_max_level_rand_training = false;
// Rendering stuff
@@ -844,7 +847,7 @@ public:
bool m_train_network = true;
filesystem::path m_data_path;
- filesystem::path m_network_config_path;
+ filesystem::path m_network_config_path = "base.json";
nlohmann::json m_network_config;
diff --git a/notebooks/instant_ngp.ipynb b/notebooks/instant_ngp.ipynb
index 55590117cf7e4be1bc7582c8a6cb00e59ba54ad1..4d2cda757d7d326b57df3cdeabc4197d611034db 100644
--- a/notebooks/instant_ngp.ipynb
+++ b/notebooks/instant_ngp.ipynb
@@ -5847,7 +5847,7 @@
"source": [
"train_steps = 2000 #@param {type:\"integer\"}\n",
"snapshot_path = os.path.join(scene_path, f\"{train_steps}.msgpack\")\n",
- "!python ./scripts/run.py --scene {scene_path} --mode nerf --n_steps {train_steps} --save_snapshot {snapshot_path}"
+ "!python ./scripts/run.py {scene_path} --n_steps {train_steps} --save_snapshot {snapshot_path}"
]
},
{
@@ -5863,7 +5863,7 @@
"\n",
"Example command:\n",
"```\n",
- "./build/instant-ngp --scene data/nerf/fox --no-train --snapshot /data/nerf/fox/2000.msgpack\n",
+ "./build/instant-ngp /data/nerf/fox/2000.msgpack\n",
"```\n",
"\n",
"After you're done, **upload `base_cam.json` to the root folder of your scene.**"
@@ -5999,7 +5999,7 @@
"height = 720 #@param {type:\"integer\"}\n",
"output_video_path = os.path.join(scene_path, \"output_video.mp4\")\n",
"\n",
- "!python scripts/run.py --mode nerf --scene {scene_path} --load_snapshot {snapshot_path} --video_camera_path {video_camera_path} --video_n_seconds 2 --video_fps 25 --width 720 --height 720 --video_output {output_video_path}\n",
+ "!python scripts/run.py {snapshot_path} --video_camera_path {video_camera_path} --video_n_seconds 2 --video_fps 25 --width 720 --height 720 --video_output {output_video_path}\n",
"print(f\"Generated video saved to:\\n{output_video_path}\")"
]
}
@@ -6018,11 +6018,11 @@
},
"language_info": {
"name": "python",
- "version": "3.10.5 (tags/v3.10.5:f377153, Jun 6 2022, 16:14:13) [MSC v.1929 64 bit (AMD64)]"
+ "version": "3.9.7 (tags/v3.9.7:1016ef3, Aug 30 2021, 20:19:38) [MSC v.1929 64 bit (AMD64)]"
},
"vscode": {
"interpreter": {
- "hash": "80f0ca567e8a8332be8d0227e77114b80c729e82298f4777b19db59a6217bb0d"
+ "hash": "76392a4a51364e66be8d9e8c24551b51109eb4003b2256c2b9df7166893b45c1"
}
}
},
diff --git a/scripts/run.py b/scripts/run.py
index 56636dbc435df7521e373aaf3ae4893a4a7c3e06..6b4929a90dd00e7d5dd6a39f070dca76cf4e3aa8 100644
--- a/scripts/run.py
+++ b/scripts/run.py
@@ -27,8 +27,10 @@ import pyngp as ngp # noqa
def parse_args():
parser = argparse.ArgumentParser(description="Run instant neural graphics primitives with additional configuration & output options")
- parser.add_argument("--scene", "--training_data", default="", help="The scene to load. Can be the scene's name or a full path to the training data.")
- parser.add_argument("--mode", default="", const="nerf", nargs="?", choices=["nerf", "sdf", "image", "volume"], help="Mode can be 'nerf', 'sdf', 'image' or 'volume'. Inferred from the scene if unspecified.")
+ parser.add_argument("files", nargs="*", help="Files to be loaded. Can be a scene, network config, snapshot, camera path, or a combination of those.")
+
+ parser.add_argument("--scene", "--training_data", default="", help="The scene to load. Can be the scene's name or a full path to the training data. Can be NeRF dataset, a *.obj/*.ply mesh for training a SDF, an image, or a *.nvdb volume.")
+ parser.add_argument("--mode", default="", type=str, help=argparse.SUPPRESS) # deprecated
parser.add_argument("--network", default="", help="Path to the network config. Uses the scene's default if unspecified.")
parser.add_argument("--load_snapshot", default="", help="Load this snapshot before training. recommended extension: .msgpack")
@@ -68,70 +70,53 @@ def parse_args():
return parser.parse_args()
+def get_scene(scene):
+ for scenes in [scenes_sdf, scenes_nerf, scenes_image, scenes_volume]:
+ if scene in scenes:
+ return scenes[scene]
+ return None
+
if __name__ == "__main__":
args = parse_args()
- args.mode = args.mode or mode_from_scene(args.scene) or mode_from_scene(args.load_snapshot)
- if not args.mode:
- raise ValueError("Must specify either a valid '--mode' or '--scene' argument.")
-
- if args.mode == "sdf":
- mode = ngp.TestbedMode.Sdf
- configs_dir = os.path.join(ROOT_DIR, "configs", "sdf")
- scenes = scenes_sdf
- elif args.mode == "nerf":
- mode = ngp.TestbedMode.Nerf
- configs_dir = os.path.join(ROOT_DIR, "configs", "nerf")
- scenes = scenes_nerf
- elif args.mode == "image":
- mode = ngp.TestbedMode.Image
- configs_dir = os.path.join(ROOT_DIR, "configs", "image")
- scenes = scenes_image
- elif args.mode == "volume":
- mode = ngp.TestbedMode.Volume
- configs_dir = os.path.join(ROOT_DIR, "configs", "volume")
- scenes = scenes_volume
- else:
- raise ValueError("Must specify either a valid '--mode' or '--scene' argument.")
-
- base_network = os.path.join(configs_dir, "base.json")
- if args.scene in scenes:
- network = scenes[args.scene]["network"] if "network" in scenes[args.scene] else "base"
- base_network = os.path.join(configs_dir, network+".json")
- network = args.network if args.network else base_network
- if not os.path.isabs(network):
- network = os.path.join(configs_dir, network)
-
- testbed = ngp.Testbed(mode)
- testbed.nerf.sharpen = float(args.sharpen)
- testbed.exposure = args.exposure
- if mode == ngp.TestbedMode.Sdf:
- testbed.tonemap_curve = ngp.TonemapCurve.ACES
+ if args.mode:
+ print("Warning: the '--mode' argument is no longer in use. It has no effect. The mode is automatically chosen based on the scene.")
+
+ testbed = ngp.Testbed()
+
+ for file in args.files:
+ scene_info = get_scene(file)
+ if scene_info:
+ file = os.path.join(scene_info["data_dir"], scene_info["dataset"])
+ testbed.load_file(file)
if args.scene:
- scene = args.scene
- if not os.path.exists(args.scene) and args.scene in scenes:
- scene = os.path.join(scenes[args.scene]["data_dir"], scenes[args.scene]["dataset"])
- testbed.load_training_data(scene)
+ scene_info = get_scene(args.scene)
+ if scene_info is not None:
+ args.scene = os.path.join(scene_info["data_dir"], scene_info["dataset"])
+ if not args.network and "network" in scene_info:
+ args.network = scene_info["network"]
+
+ testbed.load_training_data(args.scene)
if args.gui:
# Pick a sensible GUI resolution depending on arguments.
sw = args.width or 1920
sh = args.height or 1080
- while sw*sh > 1920*1080*4:
+ while sw * sh > 1920 * 1080 * 4:
sw = int(sw / 2)
sh = int(sh / 2)
testbed.init_window(sw, sh, second_window = args.second_window or False)
if args.load_snapshot:
- snapshot = args.load_snapshot
- if not os.path.exists(snapshot) and snapshot in scenes:
- snapshot = default_snapshot_filename(scenes[snapshot])
- print("Loading snapshot ", snapshot)
- testbed.load_snapshot(snapshot)
- else:
- testbed.reload_network_from_file(network)
+ scene_info = get_scene(args.load_snapshot)
+ if scene_info is not None:
+ args.load_snapshot = default_snapshot_filename(scene_info)
+ print("Loading snapshot ", args.load_snapshot)
+ testbed.load_snapshot(args.load_snapshot)
+ elif args.network:
+ testbed.reload_network_from_file(args.network)
ref_transforms = {}
if args.screenshot_transforms: # try to load the given file straight away
@@ -139,13 +124,18 @@ if __name__ == "__main__":
with open(args.screenshot_transforms) as f:
ref_transforms = json.load(f)
+ if testbed.mode == ngp.TestbedMode.Sdf:
+ testbed.tonemap_curve = ngp.TonemapCurve.ACES
+
+ testbed.nerf.sharpen = float(args.sharpen)
+ testbed.exposure = args.exposure
testbed.shall_train = args.train if args.gui else True
testbed.nerf.render_with_lens_distortion = True
- network_stem = os.path.splitext(os.path.basename(network))[0]
- if args.mode == "sdf":
+ network_stem = os.path.splitext(os.path.basename(args.network))[0] if args.network else "base"
+ if testbed.mode == ngp.TestbedMode.Sdf:
setup_colored_sdf(testbed, args.scene)
if args.near_distance >= 0.0:
diff --git a/scripts/scenes.py b/scripts/scenes.py
index a96f60b9038e277c325789719326ba7551114eb5..ca9e30c39ca87b5f99819b3a7cbfecf4ee554eda 100644
--- a/scripts/scenes.py
+++ b/scripts/scenes.py
@@ -12,7 +12,6 @@ import os
from common import *
-
def ours_real_converted(path, frameidx):
return {
"data_dir" : os.path.join(NERF_DATA_FOLDER, path),
@@ -74,7 +73,6 @@ scenes_nerf = {
"stump" : mipnerf_360("stump", frameidx=0),
}
-
def ours_mesh(name, up = [0,1,0], infolder=True):
return {
"data_dir" : os.path.join(SDF_DATA_FOLDER, f"{name}") if infolder else SDF_DATA_FOLDER,
@@ -86,7 +84,6 @@ scenes_sdf = {
"armadillo" : ours_mesh("armadillo", infolder=False),
}
-
def ours_image(name, infolder=True):
data_dir = os.path.join(IMAGE_DATA_FOLDER, f"{name}") if infolder else IMAGE_DATA_FOLDER
dataset = f"{name}.bin"
@@ -107,7 +104,6 @@ scenes_image = {
}
-
def ours_volume(name, ds):
return {
"data_dir" : os.path.join(VOLUME_DATA_FOLDER, f"{name}"),
@@ -225,20 +221,8 @@ def setup_colored_sdf(testbed, scene, softshadow=True):
testbed.sdf.shadow_sharpness = 16 if softshadow else 2048
testbed.scale = testbed.scale * 1.13
-def default_snapshot_filename(scene):
+def default_snapshot_filename(scene_info):
filename = "base.msgpack"
- if scene["dataset"]:
- filename = f"{os.path.splitext(scene['dataset'])[0]}_{filename}"
- return os.path.join(scene["data_dir"], filename)
-
-def mode_from_scene(scene):
- if scene in scenes_sdf:
- return "sdf"
- elif scene in scenes_nerf:
- return "nerf"
- elif scene in scenes_image:
- return "image"
- elif scene in scenes_volume:
- return "volume"
- else:
- return ""
+ if scene_info["dataset"]:
+ filename = f"{os.path.splitext(scene_info['dataset'])[0]}_{filename}"
+ return os.path.join(scene_info["data_dir"], filename)
diff --git a/src/common.cu b/src/common.cu
new file mode 100644
index 0000000000000000000000000000000000000000..e9538ce2c7659d90385ea6f15304d25bceef638a
--- /dev/null
+++ b/src/common.cu
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * NVIDIA CORPORATION and its licensors retain all intellectual property
+ * and proprietary rights in and to this software, related documentation
+ * and any modifications thereto. Any use, reproduction, disclosure or
+ * distribution of this software and related documentation without an express
+ * license agreement from NVIDIA CORPORATION is strictly prohibited.
+ */
+
+/** @file common_device.cu
+ * @author Thomas Müller, NVIDIA
+ */
+
+#include <neural-graphics-primitives/common.h>
+
+#include <tiny-cuda-nn/common.h>
+
+#include <filesystem/path.h>
+
+using namespace tcnn;
+namespace fs = filesystem;
+
+NGP_NAMESPACE_BEGIN
+
+bool ends_with(const std::string& str, const std::string& ending) {
+ if (ending.length() > str.length()) {
+ return false;
+ }
+ return std::equal(std::rbegin(ending), std::rend(ending), std::rbegin(str));
+}
+
+bool ends_with_case_insensitive(const std::string& str, const std::string& ending) {
+ return ends_with(to_lower(str), to_lower(ending));
+}
+
+ETestbedMode mode_from_scene(const std::string& scene) {
+ fs::path scene_path = scene;
+ if (!scene_path.exists()) {
+ return ETestbedMode::None;
+ }
+
+ if (scene_path.is_directory() || equals_case_insensitive(scene_path.extension(), "json")) {
+ return ETestbedMode::Nerf;
+ } else if (equals_case_insensitive(scene_path.extension(), "obj") || equals_case_insensitive(scene_path.extension(), "stl")) {
+ return ETestbedMode::Sdf;
+ } else if (equals_case_insensitive(scene_path.extension(), "nvdb")) {
+ return ETestbedMode::Volume;
+ } else { // probably an image. Too bothersome to list all supported ones: exr, bin, jpg, png, tga, hdr, ...
+ return ETestbedMode::Image;
+ }
+}
+
+ETestbedMode mode_from_string(const std::string& str) {
+ if (equals_case_insensitive(str, "nerf")) {
+ return ETestbedMode::Nerf;
+ } else if (equals_case_insensitive(str, "sdf")) {
+ return ETestbedMode::Sdf;
+ } else if (equals_case_insensitive(str, "image")) {
+ return ETestbedMode::Image;
+ } else if (equals_case_insensitive(str, "volume")) {
+ return ETestbedMode::Volume;
+ } else {
+ return ETestbedMode::None;
+ }
+}
+
+std::string to_string(ETestbedMode mode) {
+ switch (mode) {
+ case ETestbedMode::Nerf: return "nerf";
+ case ETestbedMode::Sdf: return "sdf";
+ case ETestbedMode::Image: return "image";
+ case ETestbedMode::Volume: return "volume";
+ case ETestbedMode::None: return "none";
+ default: throw std::runtime_error{fmt::format("Can not convert mode {} to string.", (int)mode)};
+ }
+}
+
+NGP_NAMESPACE_END
diff --git a/src/main.cu b/src/main.cu
index 480d3afec0040b5b1261a6a24fefec1d47eea94c..ee39a5645abdbd2428d5348b4728400898833c6b 100644
--- a/src/main.cu
+++ b/src/main.cu
@@ -43,7 +43,7 @@ int main(int argc, char** argv) {
ValueFlag<string> mode_flag{
parser,
"MODE",
- "Mode can be 'nerf', 'sdf', or 'image' or 'volume'. Inferred from the scene if unspecified.",
+ "Deprecated. Do not use.",
{'m', "mode"},
};
@@ -71,7 +71,7 @@ int main(int argc, char** argv) {
ValueFlag<string> scene_flag{
parser,
"SCENE",
- "The scene to load. Can be NeRF dataset, a *.obj mesh for training a SDF, an image, or a *.nvdb volume.",
+ "The scene to load. Can be NeRF dataset, a *.obj/*.ply mesh for training a SDF, an image, or a *.nvdb volume.",
{'s', "scene"},
};
@@ -103,6 +103,12 @@ int main(int argc, char** argv) {
{'v', "version"},
};
+ PositionalList<string> files{
+ parser,
+ "files",
+ "Files to be loaded. Can be a scene, network config, snapshot, camera path, or a combination of those.",
+ };
+
// Parse command line arguments and react to parsing
// errors using exceptions.
try {
@@ -126,99 +132,32 @@ int main(int argc, char** argv) {
}
try {
- ETestbedMode mode;
- if (!mode_flag) {
- if (!scene_flag) {
- tlog::error() << "Must specify either a mode or a scene";
- return 1;
- }
-
- fs::path scene_path = get(scene_flag);
- if (!scene_path.exists()) {
- tlog::error() << "Scene path " << scene_path << " does not exist.";
- return 1;
- }
-
- if (scene_path.is_directory() || equals_case_insensitive(scene_path.extension(), "json")) {
- mode = ETestbedMode::Nerf;
- } else if (equals_case_insensitive(scene_path.extension(), "obj") || equals_case_insensitive(scene_path.extension(), "stl")) {
- mode = ETestbedMode::Sdf;
- } else if (equals_case_insensitive(scene_path.extension(), "nvdb")) {
- mode = ETestbedMode::Volume;
- } else {
- mode = ETestbedMode::Image;
- }
- } else {
- auto mode_str = get(mode_flag);
- if (equals_case_insensitive(mode_str, "nerf")) {
- mode = ETestbedMode::Nerf;
- } else if (equals_case_insensitive(mode_str, "sdf")) {
- mode = ETestbedMode::Sdf;
- } else if (equals_case_insensitive(mode_str, "image")) {
- mode = ETestbedMode::Image;
- } else if (equals_case_insensitive(mode_str, "volume")) {
- mode = ETestbedMode::Volume;
- } else {
- tlog::error() << "Mode must be one of 'nerf', 'sdf', 'image', and 'volume'.";
- return 1;
- }
+ if (mode_flag) {
+ tlog::warning() << "The '--mode' argument is no longer in use. It has no effect. The mode is automatically chosen based on the scene.";
}
- Testbed testbed{mode};
+ Testbed testbed;
- if (scene_flag) {
- fs::path scene_path = get(scene_flag);
- if (!scene_path.exists()) {
- tlog::error() << "Scene path " << scene_path << " does not exist.";
- return 1;
- }
- testbed.load_training_data(scene_path.str());
+ for (auto file : get(files)) {
+ testbed.load_file(file);
}
- std::string mode_str;
- switch (mode) {
- case ETestbedMode::Nerf: mode_str = "nerf"; break;
- case ETestbedMode::Sdf: mode_str = "sdf"; break;
- case ETestbedMode::Image: mode_str = "image"; break;
- case ETestbedMode::Volume: mode_str = "volume"; break;
+ if (scene_flag) {
+ testbed.load_training_data(get(scene_flag));
}
if (snapshot_flag) {
- // Load network from a snapshot if one is provided
- fs::path snapshot_path = get(snapshot_flag);
- if (!snapshot_path.exists()) {
- tlog::error() << "Snapshot path " << snapshot_path << " does not exist.";
- return 1;
- }
-
- testbed.load_snapshot(snapshot_path.str());
- testbed.m_train = false;
- } else {
- // Otherwise, load the network config and prepare for training
- fs::path network_config_path = fs::path{"configs"}/mode_str;
- if (network_config_flag) {
- auto network_config_str = get(network_config_flag);
- if ((network_config_path/network_config_str).exists()) {
- network_config_path = network_config_path/network_config_str;
- } else {
- network_config_path = network_config_str;
- }
- } else {
- network_config_path = network_config_path/"base.json";
- }
-
- if (!network_config_path.exists()) {
- tlog::error() << "Network config path " << network_config_path << " does not exist.";
- return 1;
- }
-
- testbed.reload_network_from_file(network_config_path.str());
- testbed.m_train = !no_train_flag;
+ testbed.load_snapshot(get(snapshot_flag));
+ } else if (network_config_flag) {
+ testbed.reload_network_from_file(get(network_config_flag));
}
+ testbed.m_train = !no_train_flag;
+
+#ifdef NGP_GUI
bool gui = !no_gui_flag;
-#ifndef NGP_GUI
- gui = false;
+#else
+ bool gui = false;
#endif
if (gui) {
diff --git a/src/nerf_loader.cu b/src/nerf_loader.cu
index eb0555bf75a26edebdd87b267842a0e8b17864c9..003c423fc3fc4f6f5c314f61fd316384aa8c2eda 100644
--- a/src/nerf_loader.cu
+++ b/src/nerf_loader.cu
@@ -168,10 +168,6 @@ __global__ void compute_sharpness(Eigen::Vector2i sharpness_resolution, Eigen::V
*sharpness_data = (variance_of_laplacian) ; // / max(0.00001f,tot_lum*tot_lum); // var / (tot+0.001f);
}
-bool ends_with(const std::string& str, const std::string& suffix) {
- return str.size() >= suffix.size() && 0 == str.compare(str.size()-suffix.size(), suffix.size(), suffix);
-}
-
NerfDataset create_empty_nerf_dataset(size_t n_images, int aabb_scale, bool is_hdr) {
NerfDataset result{};
result.n_images = n_images;
diff --git a/src/python_api.cu b/src/python_api.cu
index d6785ba4056036baf79f93d8306706ef8776303f..8d4b27a2705572e9f0869643611c7be12fac47cd 100644
--- a/src/python_api.cu
+++ b/src/python_api.cu
@@ -237,8 +237,12 @@ PYBIND11_MODULE(pyngp, m) {
.value("Sdf", ETestbedMode::Sdf)
.value("Image", ETestbedMode::Image)
.value("Volume", ETestbedMode::Volume)
+ .value("None", ETestbedMode::None)
.export_values();
+ m.def("mode_from_scene", &mode_from_scene);
+ m.def("mode_from_string", &mode_from_string);
+
py::enum_<EGroundTruthRenderMode>(m, "GroundTruthRenderMode")
.value("Shade", EGroundTruthRenderMode::Shade)
.value("Depth", EGroundTruthRenderMode::Depth)
@@ -336,9 +340,10 @@ PYBIND11_MODULE(pyngp, m) {
py::class_<Testbed> testbed(m, "Testbed");
testbed
- .def(py::init<ETestbedMode>())
+ .def(py::init<ETestbedMode>(), py::arg("mode") = ETestbedMode::None)
.def(py::init<ETestbedMode, const std::string&, const std::string&>())
.def(py::init<ETestbedMode, const std::string&, const json&>())
+ .def_readonly("mode", &Testbed::m_testbed_mode)
.def("create_empty_nerf_dataset", &Testbed::create_empty_nerf_dataset, "Allocate memory for a nerf dataset with a given size", py::arg("n_images"), py::arg("aabb_scale")=1, py::arg("is_hdr")=false)
.def("load_training_data", &Testbed::load_training_data, py::call_guard<py::gil_scoped_release>(), "Load training data from a given path.")
.def("clear_training_data", &Testbed::clear_training_data, "Clears training data to free up GPU memory.")
@@ -404,6 +409,7 @@ PYBIND11_MODULE(pyngp, m) {
.def("save_snapshot", &Testbed::save_snapshot, py::arg("path"), py::arg("include_optimizer_state")=false, "Save a snapshot of the currently trained model")
.def("load_snapshot", &Testbed::load_snapshot, py::arg("path"), "Load a previously saved snapshot")
.def("load_camera_path", &Testbed::load_camera_path, "Load a camera path", py::arg("path"))
+ .def("load_file", &Testbed::load_file, "Load a file and automatically determine how to handle it. Can be a snapshot, dataset, network config, or camera path.", py::arg("path"))
.def_property("loop_animation", &Testbed::loop_animation, &Testbed::set_loop_animation)
.def("compute_and_save_png_slices", &Testbed::compute_and_save_png_slices,
py::arg("filename"),
@@ -434,10 +440,7 @@ PYBIND11_MODULE(pyngp, m) {
"`thresh` is the density threshold; use 0 for SDF; 2.5 works well for NeRF. "
"If the aabb parameter specifies an inside-out (\"empty\") box (default), the current render_aabb bounding box is used."
)
- ;
-
- // Interesting members.
- testbed
+ // Interesting members.
.def_readwrite("dynamic_res", &Testbed::m_dynamic_res)
.def_readwrite("dynamic_res_target_fps", &Testbed::m_dynamic_res_target_fps)
.def_readwrite("fixed_res_factor", &Testbed::m_fixed_res_factor)
diff --git a/src/testbed.cu b/src/testbed.cu
index de8a921e52df0d04a73d1ad56d91beda6ec2beeb..e70f3c933272d7c7c66bf12b56ee5bf777e4b9cb 100644
--- a/src/testbed.cu
+++ b/src/testbed.cu
@@ -85,18 +85,24 @@ json merge_parent_network_config(const json &child, const fs::path &child_filena
return parent;
}
-static bool ends_with(const std::string& str, const std::string& ending) {
- if (ending.length() > str.length()) {
- return false;
+void Testbed::load_training_data(const std::string& data_path_str) {
+ fs::path data_path = data_path_str;
+ if (!data_path.exists()) {
+ throw std::runtime_error{fmt::format("Data path '{}' does not exist.", data_path.str())};
+ }
+
+ // Automatically determine the mode from the first scene that's loaded
+ ETestbedMode scene_mode = mode_from_scene(data_path.str());
+ if (scene_mode == ETestbedMode::None) {
+ throw std::runtime_error{fmt::format("Unknown scene format for path '{}'.", data_path.str())};
}
- return std::equal(std::rbegin(ending), std::rend(ending), std::rbegin(str));
-}
-void Testbed::load_training_data(const std::string& data_path) {
+ set_mode(scene_mode);
+
m_data_path = data_path;
if (!m_data_path.exists()) {
- throw std::runtime_error{fmt::format("Data path {} does not exist.", m_data_path.str())};
+ throw std::runtime_error{fmt::format("Data path '{}' does not exist.", m_data_path.str())};
}
switch (m_testbed_mode) {
@@ -115,17 +121,68 @@ void Testbed::clear_training_data() {
m_nerf.training.dataset.metadata.clear();
}
-json Testbed::load_network_config(const fs::path& network_config_path) {
- if (!network_config_path.empty()) {
- m_network_config_path = network_config_path;
+void Testbed::set_mode(ETestbedMode mode) {
+ if (mode == m_testbed_mode) {
+ return;
+ }
+
+ // Reset mode-specific members
+ m_image = {};
+ m_mesh = {};
+ m_nerf = {};
+ m_sdf = {};
+ m_volume = {};
+
+ // Kill training-related things
+ m_encoding = {};
+ m_loss = {};
+ m_network = {};
+ m_nerf_network = {};
+ m_optimizer = {};
+ m_trainer = {};
+ m_envmap = {};
+ m_distortion = {};
+ m_training_data_available = false;
+
+ // Reset paths that might be attached to the chosen mode
+ m_data_path = {};
+
+ m_testbed_mode = mode;
+
+ reset_camera();
+}
+
+fs::path Testbed::find_network_config(const fs::path& network_config_path) {
+ if (network_config_path.exists()) {
+ return network_config_path;
+ }
+
+ // The following resolution steps do not work if the path is absolute. Treat it as nonexistent.
+ if (network_config_path.is_absolute()) {
+ return network_config_path;
+ }
+
+ fs::path candidate = fs::path{"configs"}/to_string(m_testbed_mode)/network_config_path;
+ if (candidate.exists()) {
+ return candidate;
+ }
+
+ candidate = fs::path{"../"}/candidate;
+ if (candidate.exists()) {
+ return candidate;
}
- tlog::info() << "Loading network config from: " << network_config_path;
+ return network_config_path;
+}
+json Testbed::load_network_config(const fs::path& network_config_path) {
+ bool is_snapshot = equals_case_insensitive(network_config_path.extension(), "msgpack");
if (network_config_path.empty() || !network_config_path.exists()) {
- throw std::runtime_error{fmt::format("Network config {} does not exist.", network_config_path.str())};
+ throw std::runtime_error{fmt::format("Network {} '{}' does not exist.", is_snapshot ? "snapshot" : "config", network_config_path.str())};
}
+ tlog::info() << "Loading network " << (is_snapshot ? "snapshot" : "config") << " from: " << network_config_path;
+
json result;
if (equals_case_insensitive(network_config_path.extension(), "json")) {
std::ifstream f{network_config_path.str()};
@@ -140,13 +197,38 @@ json Testbed::load_network_config(const fs::path& network_config_path) {
return result;
}
-void Testbed::reload_network_from_file(const std::string& network_config_path) {
- if (!network_config_path.empty()) {
- m_network_config_path = network_config_path;
+void Testbed::reload_network_from_file(const std::string& network_config_path_string) {
+ if (!network_config_path_string.empty()) {
+ fs::path candidate = find_network_config(network_config_path_string);
+ if (candidate.exists()) {
+ // Store the path _argument_ in the member variable. E.g. for the base config,
+ // it'll store `base.json`, even though the loaded config will be
+ // config/<mode>/base.json. This has the benefit of switching to the
+ // appropriate config when switching modes.
+ m_network_config_path = network_config_path_string;
+ }
}
- m_network_config = load_network_config(m_network_config_path);
- reset_network();
+ // If the testbed mode hasn't been decided yet, don't load a network yet, but
+ // still keep track of the requested config (see above).
+ if (m_testbed_mode == ETestbedMode::None) {
+ return;
+ }
+
+ fs::path full_network_config_path = find_network_config(m_network_config_path);
+ bool is_snapshot = equals_case_insensitive(full_network_config_path.extension(), "msgpack");
+
+ if (!full_network_config_path.exists()) {
+ tlog::warning() << "Network " << (is_snapshot ? "snapshot" : "config") << " path '" << full_network_config_path << "' does not exist.";
+ } else {
+ m_network_config = load_network_config(full_network_config_path);
+ }
+
+ // Reset training if we haven't loaded a snapshot of an already trained model, in which case, presumably the network
+ // configuration changed and the user is interested in seeing how it trains from scratch.
+ if (!is_snapshot) {
+ reset_network();
+ }
}
void Testbed::reload_network_from_json(const json& json, const std::string& config_base_path) {
@@ -156,36 +238,55 @@ void Testbed::reload_network_from_json(const json& json, const std::string& conf
reset_network();
}
-void Testbed::handle_file(const std::string& file) {
- if (ends_with(file, ".msgpack")) {
- load_snapshot(file);
- }
- else if (ends_with(file, ".json")) {
- reload_network_from_file(file);
- } else if (ends_with(file, ".obj") || ends_with(file, ".stl")) {
- m_data_path = file;
- m_testbed_mode = ETestbedMode::Sdf;
- load_mesh();
- } else if (ends_with(file, ".exr") || ends_with(file, ".bin")) {
- m_data_path = file;
- m_testbed_mode = ETestbedMode::Image;
- try {
- load_image();
- } catch (std::runtime_error& e) {
- tlog::error() << "Failed to open image: " << e.what();
+void Testbed::load_file(const std::string& file_path) {
+ if (!fs::path{file_path}.exists()) {
+ // If the path doesn't exist, but a network config can be resolved, load that.
+ if (ends_with_case_insensitive(file_path, ".json") && find_network_config(file_path).exists()) {
+ reload_network_from_file(file_path);
+ return;
+ }
+
+ tlog::error() << "File '" << file_path << "' does not exist.";
+ return;
+ }
+
+ if (ends_with_case_insensitive(file_path, ".msgpack")) {
+ load_snapshot(file_path);
+ return;
+ }
+
+ // If we get a json file, we need to parse it to determine its purpose.
+ if (ends_with_case_insensitive(file_path, ".json")) {
+ json file;
+ {
+ std::ifstream f{file_path};
+ file = json::parse(f, nullptr, true, true);
+ }
+
+ // Snapshot in json format... inefficient, but technically supported.
+ if (file.contains("snapshot")) {
+ load_snapshot(file_path);
return;
}
- } else if (ends_with(file, ".nvdb")) {
- m_data_path = file;
- m_testbed_mode = ETestbedMode::Volume;
- try {
- load_volume();
- } catch (std::runtime_error& e) {
- tlog::error() << "Failed to open volume: " << e.what();
+
+ // Regular network config
+ if (file.contains("parent") || file.contains("network") || file.contains("encoding") || file.contains("loss") || file.contains("optimizer")) {
+ reload_network_from_file(file_path);
+ return;
+ }
+
+ // Camera path
+ if (file.contains("path")) {
+ load_camera_path(file_path);
return;
}
- } else {
- tlog::error() << "Tried to open unknown file type: " << file;
+ }
+
+ // If the dragged file isn't any of the above, assume that it's training data
+ try {
+ load_training_data(file_path);
+ } catch (std::runtime_error& e) {
+ tlog::error() << "Failed to load training data: " << e.what();
}
}
@@ -529,7 +630,7 @@ void Testbed::imgui() {
ImGui::Checkbox("Quaternion format", &export_extrinsics_in_quat_format);
}
if (imgui_colored_button("Reset training", 0.f)) {
- reload_network_from_file("");
+ reload_network_from_file();
}
ImGui::SameLine();
ImGui::DragInt("Seed", (int*)&m_seed, 1.0f, 0, std::numeric_limits<int>::max());
@@ -1306,23 +1407,33 @@ bool Testbed::keyboard_event() {
set_exposure(m_exposure + (shift ? -0.5f : 0.5f));
redraw_next_frame();
}
+
if (ImGui::IsKeyPressed('R')) {
if (shift) {
reset_camera();
} else {
- reload_network_from_file("");
+ reload_network_from_file();
}
}
- if (ImGui::IsKeyPressed('O')) {
- m_nerf.training.render_error_overlay = !m_nerf.training.render_error_overlay;
- }
- if (ImGui::IsKeyPressed('G')) {
- m_render_ground_truth = !m_render_ground_truth;
- reset_accumulation();
- if (m_render_ground_truth) {
- m_nerf.training.view = find_best_training_view(m_nerf.training.view);
+
+ if (m_training_data_available) {
+ if (ImGui::IsKeyPressed('O')) {
+ m_nerf.training.render_error_overlay = !m_nerf.training.render_error_overlay;
+ }
+
+ if (ImGui::IsKeyPressed('G')) {
+ m_render_ground_truth = !m_render_ground_truth;
+ reset_accumulation();
+ if (m_render_ground_truth) {
+ m_nerf.training.view = find_best_training_view(m_nerf.training.view);
+ }
+ }
+
+ if (ImGui::IsKeyPressed('T')) {
+ set_train(!m_train);
}
}
+
if (ImGui::IsKeyPressed('.')) {
if (m_single_view) {
if (m_visualized_dimension == m_network->width(m_visualized_layer)-1 && m_visualized_layer < m_network->num_forward_activations()-1) {
@@ -1335,6 +1446,7 @@ bool Testbed::keyboard_event() {
set_visualized_layer(std::max(0, std::min((int)m_network->num_forward_activations()-1, m_visualized_layer+1)));
}
}
+
if (ImGui::IsKeyPressed(',')) {
if (m_single_view) {
if (m_visualized_dimension == 0 && m_visualized_layer > 0) {
@@ -1347,14 +1459,14 @@ bool Testbed::keyboard_event() {
set_visualized_layer(std::max(0, std::min((int)m_network->num_forward_activations()-1, m_visualized_layer-1)));
}
}
+
if (ImGui::IsKeyPressed('M')) {
m_single_view = !m_single_view;
set_visualized_dim(-1);
reset_accumulation();
}
- if (ImGui::IsKeyPressed('T')) {
- set_train(!m_train);
- }
+
+
if (ImGui::IsKeyPressed('N')) {
m_sdf.analytic_normals = !m_sdf.analytic_normals;
reset_accumulation();
@@ -1389,29 +1501,37 @@ bool Testbed::keyboard_event() {
if (ImGui::IsKeyDown('W')) {
translate_vec.z() += 1.0f;
}
+
if (ImGui::IsKeyDown('A')) {
translate_vec.x() += -1.0f;
}
+
if (ImGui::IsKeyDown('S')) {
translate_vec.z() += -1.0f;
}
+
if (ImGui::IsKeyDown('D')) {
translate_vec.x() += 1.0f;
}
+
if (ImGui::IsKeyDown(' ')) {
translate_vec.y() += -1.0f;
}
+
if (ImGui::IsKeyDown('C')) {
translate_vec.y() += 1.0f;
}
+
translate_vec *= m_camera_velocity * m_frame_ms.val() / 1000.0f;
if (shift) {
translate_vec *= 5;
}
+
if (translate_vec != Vector3f::Zero()) {
m_fps_camera = true;
translate_camera(translate_vec);
}
+
return false;
}
@@ -1661,6 +1781,15 @@ void Testbed::train_and_render(bool skip_rendering) {
train(m_training_batch_size);
}
+ // If we don't have a trainer, as can happen when having loaded training data or changed modes without having
+ // explicitly loaded a new neural network.
+ if (m_testbed_mode != ETestbedMode::None && !m_network) {
+ reload_network_from_file();
+ if (!m_network) {
+ throw std::runtime_error{"Unable to reload neural network."};
+ }
+ }
+
if (m_mesh.optimize_mesh) {
optimise_mesh_step(1);
}
@@ -1905,14 +2034,7 @@ void Testbed::init_window(int resw, int resh, bool hidden, bool second_window) {
glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GLFW_TRUE);
glfwWindowHint(GLFW_VISIBLE, hidden ? GLFW_FALSE : GLFW_TRUE);
- std::string title = "Instant Neural Graphics Primitives v" NGP_VERSION " (";
- switch (m_testbed_mode) {
- case ETestbedMode::Image: title += "Image"; break;
- case ETestbedMode::Sdf: title += "SDF"; break;
- case ETestbedMode::Nerf: title += "NeRF"; break;
- case ETestbedMode::Volume: title += "Volume"; break;
- }
- title += ")";
+ std::string title = "Instant Neural Graphics Primitives";
m_glfw_window = glfwCreateWindow(m_window_res.x(), m_window_res.y(), title.c_str(), NULL, NULL);
if (m_glfw_window == NULL) {
throw std::runtime_error{"GLFW window could not be created."};
@@ -1939,7 +2061,7 @@ void Testbed::init_window(int resw, int resh, bool hidden, bool second_window) {
testbed->redraw_gui_next_frame();
for (int i = 0; i < count; i++) {
- testbed->handle_file(paths[i]);
+ testbed->load_file(paths[i]);
}
});
@@ -2117,8 +2239,8 @@ fs::path Testbed::training_data_path() const {
}
bool Testbed::want_repl() {
- bool b=m_want_repl;
- m_want_repl=false;
+ bool b = m_want_repl;
+ m_want_repl = false;
return b;
}
@@ -2275,6 +2397,11 @@ void Testbed::reset_network(bool clear_density_grid) {
m_nerf.training.reset_camera_extrinsics();
+ if (clear_density_grid) {
+ m_nerf.density_grid.memset(0);
+ m_nerf.density_grid_bitfield.memset(0);
+ }
+
m_loss_graph_samples = 0;
// Default config
@@ -2425,17 +2552,17 @@ void Testbed::reset_network(bool clear_density_grid) {
tcnn::string_to_interpolation_type(encoding_config.value("interpolation", "linear"))
));
- m_network = std::make_shared<NetworkWithInputEncoding<precision_t>>(m_encoding, dims.n_output, network_config);
m_sdf.uses_takikawa_encoding = true;
} else {
m_encoding.reset(create_encoding<precision_t>(dims.n_input, encoding_config));
- m_network = std::make_shared<NetworkWithInputEncoding<precision_t>>(m_encoding, dims.n_output, network_config);
+
m_sdf.uses_takikawa_encoding = false;
if (m_sdf.octree_depth_target == 0 && encoding_config.contains("n_levels")) {
m_sdf.octree_depth_target = encoding_config["n_levels"];
}
}
+ m_network = std::make_shared<NetworkWithInputEncoding<precision_t>>(m_encoding, dims.n_output, network_config);
n_encoding_params = m_encoding->n_params();
tlog::info()
@@ -2472,15 +2599,9 @@ void Testbed::reset_network(bool clear_density_grid) {
}
}
- if (clear_density_grid) {
- m_nerf.density_grid.memset(0);
- m_nerf.density_grid_bitfield.memset(0);
- }
}
-Testbed::Testbed(ETestbedMode mode)
-: m_testbed_mode(mode)
-{
+Testbed::Testbed(ETestbedMode mode) {
if (!(__CUDACC_VER_MAJOR__ > 10 || (__CUDACC_VER_MAJOR__ == 10 && __CUDACC_VER_MINOR__ >= 2))) {
throw std::runtime_error{"Testbed required CUDA 10.2 or later."};
}
@@ -2544,11 +2665,12 @@ Testbed::Testbed(ETestbedMode mode)
}},
};
- reset_camera();
-
+ set_mode(mode);
set_exposure(0);
set_min_level(0.f);
set_max_level(1.f);
+
+ reset_camera();
}
Testbed::~Testbed() {
@@ -2564,6 +2686,19 @@ void Testbed::train(uint32_t batch_size) {
return;
}
+ if (m_testbed_mode == ETestbedMode::None) {
+ throw std::runtime_error{"Cannot train without a mode."};
+ }
+
+ // If we don't have a trainer, as can happen when having loaded training data or changed modes without having
+ // explicitly loaded a new neural network.
+ if (!m_trainer) {
+ reload_network_from_file();
+ if (!m_trainer) {
+ throw std::runtime_error{"Unable to create a neural network trainer."};
+ }
+ }
+
if (!m_dlss) {
// No immediate redraw necessary
reset_accumulation(false, false);
@@ -2714,9 +2849,8 @@ void Testbed::render_frame(const Matrix<float, 3, 4>& camera_matrix0, const Matr
Vector2f focal_length = calc_focal_length(render_buffer.in_resolution(), m_fov_axis, m_zoom);
Vector2f screen_center = render_screen_center();
- if (m_quilting_dims != Vector2i::Ones() && m_quilting_dims != Vector2i{2, 1}) {
- // In the case of a holoplay lenticular screen, m_scale represents the inverse distance of the head above the display.
- m_parallax_shift.z() = 1.0f / m_scale;
+ if (!m_network) {
+ return;
}
switch (m_testbed_mode) {
@@ -3025,6 +3159,7 @@ void Testbed::save_snapshot(const std::string& filepath_string, bool include_opt
auto& snapshot = m_network_config["snapshot"];
snapshot["version"] = SNAPSHOT_FORMAT_VERSION;
+ snapshot["mode"] = to_string(m_testbed_mode);
if (m_testbed_mode == ETestbedMode::Nerf) {
snapshot["density_grid_size"] = NERF_GRIDSIZE();
@@ -3063,10 +3198,21 @@ void Testbed::load_snapshot(const std::string& filepath_string) {
throw std::runtime_error{fmt::format("File {} does not contain a snapshot.", filepath_string)};
}
+ m_network_config_path = filepath_string;
+
const auto& snapshot = config["snapshot"];
if (snapshot.value("version", 0) < SNAPSHOT_FORMAT_VERSION) {
- throw std::runtime_error{"Snapshot uses an old format."};
+ throw std::runtime_error{"Snapshot uses an old format and can not be loaded."};
+ }
+
+ if (snapshot.contains("mode")) {
+ set_mode(mode_from_string(snapshot["mode"]));
+ } else if (snapshot.contains("nerf")) {
+ // To be able to load old NeRF snapshots that don't specify their mode yet
+ set_mode(ETestbedMode::Nerf);
+ } else if (m_testbed_mode == ETestbedMode::None) {
+ throw std::runtime_error{"Unknown snapshot mode. Snapshot must be regenerated with a new version of instant-ngp."};
}
m_aabb = snapshot.value("aabb", m_aabb);
diff --git a/src/testbed_image.cu b/src/testbed_image.cu
index 21610716e6570dce31ed08b5ca1dab179da44281..742a3b1784c3d9c670b4d2cd43069dbcc96bb627 100644
--- a/src/testbed_image.cu
+++ b/src/testbed_image.cu
@@ -384,7 +384,7 @@ void Testbed::load_image() {
void Testbed::load_exr_image() {
if (!m_data_path.exists()) {
- throw std::runtime_error{m_data_path.str() + " does not exist."};
+ throw std::runtime_error{fmt::format("Image file '{}' does not exist.", m_data_path.str())};
}
tlog::info() << "Loading EXR image from " << m_data_path;
@@ -399,7 +399,7 @@ void Testbed::load_exr_image() {
void Testbed::load_stbi_image() {
if (!m_data_path.exists()) {
- throw std::runtime_error{m_data_path.str() + " does not exist."};
+ throw std::runtime_error{fmt::format("Image file '{}' does not exist.", m_data_path.str())};
}
tlog::info() << "Loading STBI image from " << m_data_path;
@@ -415,7 +415,7 @@ void Testbed::load_stbi_image() {
void Testbed::load_binary_image() {
if (!m_data_path.exists()) {
- throw std::runtime_error{m_data_path.str() + " does not exist."};
+ throw std::runtime_error{fmt::format("Image file '{}' does not exist.", m_data_path.str())};
}
tlog::info() << "Loading binary image from " << m_data_path;