From 664031a1cfeaabcce8ab4658f2104c3f1d8c5a10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20M=C3=BCller?= <tmueller@nvidia.com>
Date: Fri, 3 Feb 2023 15:36:21 +0100
Subject: [PATCH] VR: permit optional depth-based reprojection

---
 include/neural-graphics-primitives/openxr_hmd.h |  2 +-
 include/neural-graphics-primitives/testbed.h    |  2 ++
 src/openxr_hmd.cu                               | 15 +++++++++------
 src/testbed.cu                                  |  4 +++-
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/include/neural-graphics-primitives/openxr_hmd.h b/include/neural-graphics-primitives/openxr_hmd.h
index f31ceca..8d574cb 100644
--- a/include/neural-graphics-primitives/openxr_hmd.h
+++ b/include/neural-graphics-primitives/openxr_hmd.h
@@ -121,7 +121,7 @@ public:
 	// begin OpenXR frame, return views to render
 	FrameInfoPtr begin_frame();
 	// must be called for each begin_frame
-	void end_frame(FrameInfoPtr frame_info, float znear, float zfar);
+	void end_frame(FrameInfoPtr frame_info, float znear, float zfar, bool submit_depth);
 
 	void set_environment_blend_mode(EEnvironmentBlendMode mode) {
 		m_environment_blend_mode = mode;
diff --git a/include/neural-graphics-primitives/testbed.h b/include/neural-graphics-primitives/testbed.h
index add17a3..c8ade45 100644
--- a/include/neural-graphics-primitives/testbed.h
+++ b/include/neural-graphics-primitives/testbed.h
@@ -604,6 +604,8 @@ public:
 
 	std::unique_ptr<OpenXRHMD> m_hmd;
 	OpenXRHMD::FrameInfoPtr m_vr_frame_info;
+	bool m_vr_depth_reproject = false;
+
 	void init_vr();
 	void update_vr_performance_settings();
 	void set_n_views(size_t n_views);
diff --git a/src/openxr_hmd.cu b/src/openxr_hmd.cu
index 77b98cf..a60ad43 100644
--- a/src/openxr_hmd.cu
+++ b/src/openxr_hmd.cu
@@ -1208,7 +1208,7 @@ OpenXRHMD::FrameInfoPtr OpenXRHMD::begin_frame() {
 	return frame_info;
 }
 
-void OpenXRHMD::end_frame(FrameInfoPtr frame_info, float znear, float zfar) {
+void OpenXRHMD::end_frame(FrameInfoPtr frame_info, float znear, float zfar, bool submit_depth) {
 	std::vector<XrCompositionLayerProjectionView> layer_projection_views(frame_info->views.size());
 	for (size_t i = 0; i < layer_projection_views.size(); ++i) {
 		auto& v = frame_info->views[i];
@@ -1224,11 +1224,14 @@ void OpenXRHMD::end_frame(FrameInfoPtr frame_info, float znear, float zfar) {
 			XR_CHECK_THROW(xrReleaseSwapchainImage(v.depth_info.subImage.swapchain, &release_info));
 			v.depth_info.nearZ = znear;
 			v.depth_info.farZ = zfar;
-			// The following line being commented means that our provided depth buffer
-			// _isn't_ actually passed to the runtime for reprojection. So far,
-			// experimentation has shown that runtimes do a better job at reprojection
-			// without getting a depth buffer from us, so we leave it disabled for now.
-			// view.next = &v.depth_info;
+
+			// Submitting the depth buffer to the runtime for reprojection is optional,
+			// because, while depth-based reprojection can make the experience smoother,
+			// it also results in distortion around geometric edges. Many users prefer
+			// a more stuttery experience without this distortion.
+			if (submit_depth) {
+				view.next = &v.depth_info;
+			}
 		}
 	}
 
diff --git a/src/testbed.cu b/src/testbed.cu
index b39de31..f87d0c1 100644
--- a/src/testbed.cu
+++ b/src/testbed.cu
@@ -946,10 +946,12 @@ void Testbed::imgui() {
 					ImGui::Checkbox("Multi-GPU rendering (one per eye)", &m_use_aux_devices);
 				}
 
+				accum_reset |= ImGui::Checkbox("Depth-based reprojection", &m_vr_depth_reproject);
 				accum_reset |= ImGui::Checkbox("Foveated rendering", &m_foveated_rendering) && !m_dlss;
 				if (m_foveated_rendering) {
 					accum_reset |= ImGui::SliderFloat("Maximum foveation", &m_foveated_rendering_max_scaling, 1.0f, 16.0f, "%.01f", ImGuiSliderFlags_Logarithmic | ImGuiSliderFlags_NoRoundToFormat) && !m_dlss;
 				}
+
 				ImGui::TreePop();
 			}
 		}
@@ -3202,7 +3204,7 @@ bool Testbed::frame() {
 		// Far and near planes are intentionally reversed, because we map depth inversely
 		// to z. I.e. a window-space depth of 1 refers to the near plane and a depth of 0
 		// to the far plane. This results in much better numeric precision.
-		m_hmd->end_frame(m_vr_frame_info, m_ndc_zfar / m_scale, m_ndc_znear / m_scale);
+		m_hmd->end_frame(m_vr_frame_info, m_ndc_zfar / m_scale, m_ndc_znear / m_scale, m_vr_depth_reproject);
 	}
 #endif
 
-- 
GitLab