Fixed depth calculations (inverse depth is 1/d, not 1 - d). Absolute depth is now inferred from inverse depth by exposing a scale factor as a material property.

Author: julienkay

Packages/com.doji.genesis/Editor/DepthSkyboxPrefabUtility.cs

@@ -9,7 +9,7 @@ namespace Genesis.Editor {
     public static class DepthSkyboxPrefabUtility {
 
         /// <summary>
-        /// Imput given png fileb, generates depth and creates a prefab for a DepthSkybox.
+        /// Given an input png file, generates depth and creates a prefab for a DepthSkybox.
         /// </summary>
         public static void CreateSkyboxAsset(string pngFilePath) {
             string name = Path.GetFileNameWithoutExtension(pngFilePath);
@@ -18,8 +18,8 @@ public static void CreateSkyboxAsset(string pngFilePath) {
 
             string pngTargetPath = Path.Combine(assetPath, $"{name}_rgb.png");
             Texture2D skybox = ImportImageFile(pngFilePath, pngTargetPath);
-            GenerateDepth(assetPath, name, skybox);
-            CreateSkyboxPrefab(assetPath, name);
+            var range = GenerateDepth(assetPath, name, skybox);
+            CreateSkyboxPrefab(assetPath, name, range);
         }
 
         /// <summary>
@@ -31,27 +31,31 @@ public static async Task CreateSkyboxAsset(string id, string name) {
 
             string imageFile = Path.Combine(assetPath, $"{name}_rgb.png");
             Texture2D skybox = await DownloadSkyboxById(imageFile, id);
-            GenerateDepth(assetPath, name, skybox);
-            CreateSkyboxPrefab(assetPath, name);
+            var range = GenerateDepth(assetPath, name, skybox);
+            CreateSkyboxPrefab(assetPath, name, range);
         }
 
-        private static void GenerateDepth(string assetPath, string name, Texture2D skybox) {
+        private static Vector2 GenerateDepth(string assetPath, string name, Texture2D skybox) {
             int progressId = Progress.Start("Generating depth texture", "Generating a depth texture from skybox...");
 
             DepthEstimator depthEstimator = new DepthEstimator();
             depthEstimator.GenerateDepth(skybox);
-            var depthTexture = depthEstimator.GetNormalizedDepth();
+            var depthTexture = depthEstimator.PostProcessDepth();
 
             string depthTextureFile = Path.Combine(assetPath, $"{name}_depth.asset");
             CreateAsset(depthTexture, depthTextureFile);
-            AssetDatabase.SaveAssets();
-            AssetDatabase.Refresh();
 
-            // for some reason, saving out as exr or png and reimporting produces 
+            // for some reason, saving out as exr or png and reimporting produces what looks like precision artifacts
             //byte[] bytes = depthTexture.EncodeToEXR(Texture2D.EXRFlags.OutputAsFloat);
 
             depthEstimator.Dispose();
+
+            AssetDatabase.SaveAssets();
+            AssetDatabase.Refresh();
+
             Progress.Remove(progressId);
+
+            return new Vector2(depthEstimator.MinDepth, depthEstimator.MaxDepth);
         }
 
         /// <summary>
@@ -66,7 +70,9 @@ private static void CreateAsset(Object obj, string path) {
             AssetDatabase.CreateAsset(obj, path);
         }
 
-        public static void CreateSkyboxPrefab(string assetPath, string name) {
+        public static void CreateSkyboxPrefab(string assetPath, string name, Vector2 range) {
+            int progressId = Progress.Start("Creating skybox assets", "Your skybox assets are being created...");
+
             GameObject skyboxPrefab = (GameObject)Resources.Load("Prefabs/DepthSkybox");
 
             GameObject instance = (GameObject)PrefabUtility.InstantiatePrefab(skyboxPrefab);
@@ -82,6 +88,8 @@ public static void CreateSkyboxPrefab(string assetPath, string name) {
             renderer.sharedMaterial = m;
             m.SetTexture("_MainTex", skybox);
             m.SetTexture("_Depth", skyboxDepth);
+            m.SetFloat("_Min", range.x);
+            m.SetFloat("_Max", range.y);
 
             string materialPath = Path.Combine(assetPath, $"{name}_material.mat");
             CreateAsset(m, materialPath);
@@ -93,8 +101,12 @@ public static void CreateSkyboxPrefab(string assetPath, string name) {
             }
             GameObject variant = PrefabUtility.SaveAsPrefabAsset(instance, prefabPath);
             Object.DestroyImmediate(instance);
-
             PrefabUtility.InstantiatePrefab(variant);
+
+            AssetDatabase.SaveAssets();
+            AssetDatabase.Refresh();
+
+            Progress.Remove(progressId);
         }
 
         /// <summary>

Packages/com.doji.genesis/Runtime/Materials/EquirectDepth.mat

@@ -69,12 +69,12 @@ Material:
     - _GlossMapScale: 1
     - _Glossiness: 0.5
     - _GlossyReflections: 1
-    - _MaxDepth: 100
+    - _Max: 1000
     - _Metallic: 0
-    - _MinDepth: 2
     - _Mode: 0
     - _OcclusionStrength: 1
     - _Parallax: 0.02
+    - _Scale: 1
     - _SmoothnessTextureChannel: 0
     - _SpecularHighlights: 1
     - _SrcBlend: 1

Packages/com.doji.genesis/Runtime/Scripts/DepthEstimator.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Linq;
 using Unity.Barracuda;
+using Unity.Collections;
 using UnityEngine;
 
 namespace Genesis {
@@ -20,12 +21,8 @@ public static NNModel MiDaSv2 {
         public RenderTexture _input, _output;
         private int _width, _height;
 
-        private float MinDepth { get; set; }
-        private float MaxDepth { get; set; }
-
-        // Shader Properties
-        private static readonly int _minShaderProp = Shader.PropertyToID("_Min");
-        private static readonly int _maxShaderProp = Shader.PropertyToID("_Max");
+        public float MinDepth { get; private set; }
+        public float MaxDepth { get; private set; }
 
         public DepthEstimator() {
             InitializeNetwork();
@@ -49,20 +46,6 @@ public RenderTexture GenerateDepth(Texture2D inputTexture) {
             return _output;
         }
 
-        public void SetProperties(Material mat) {
-            // In Barracuda 1.0.4 the output of MiDaS can be passed  directly to a texture as it is shaped correctly.
-            // In later versions we have to swap X and Y axes, and also flip the X axis...
-            // We need to inform the shader of this change.
-#if BARRACUDA_1_0_5_OR_NEWER
-            mat.SetInt("_SwapChannels", 1);
-#else
-            mat.SetInt("_SwapChannels", 0);
-#endif
-
-            mat.SetFloat(_minShaderProp, MinDepth);
-            mat.SetFloat(_maxShaderProp, MaxDepth);
-        }
-
         /// <summary>
         /// Loads the <see cref="NNModel"/> asset in memory and creates a Barracuda <see cref="IWorker"/>
         /// </summary>
@@ -89,24 +72,39 @@ private void InitializeNetwork() {
 #endif
         }
 
-        public Texture2D GetNormalizedDepth() {
+        /// <summary>
+        /// Do some post processing of the depth texture to
+        /// - fix depth discontinuities / seams
+        /// - fix distortions at the poles
+        /// - normalize depth values
+        /// </summary>
+        public Texture2D PostProcessDepth() {
             RenderTexture depth = _output;
-            Texture2D depthTexture = new Texture2D(depth.width, depth.height, TextureFormat.RFloat, false);
+            Texture2D depthTexture = new Texture2D(depth.width, depth.height, TextureFormat.RFloat, mipChain: false, linear: true);
+            depthTexture.wrapModeU = TextureWrapMode.Clamp;
+            depthTexture.wrapModeV = TextureWrapMode.Repeat;
             RenderTexture.active = depth;
             depthTexture.ReadPixels(new Rect(0, 0, depth.width, depth.height), 0, 0);
             RenderTexture.active = null;
 
-            //noramlize to range [0, 1] and convert from MiDaS' reversed depth to depth
             var depthData = depthTexture.GetPixelData<float>(0);
-            float min = MinDepth;
-            float max = MaxDepth;
+
+            NormalizeDepth(depthData);
+
+            depthTexture.Apply(false);
+            return depthTexture;
+        }
+
+        /// <summary>
+        /// Makes sure all values of the depth texture are in the [0, 1] range
+        /// and converts from MiDaS' reversed depth to depth.
+        /// </summary>
+        private void NormalizeDepth(NativeArray<float> depthData) {
             for (int i = 0; i < depthData.Length; i++) {
-                float depthValue = depthData[i];
-                float normalizedDepth = 1.0f - Mathf.Clamp01((depthValue - min) / (max - min));
+                float depth = depthData[i];
+                float normalizedDepth = Mathf.Clamp01((depth - MinDepth) / (MaxDepth - MinDepth));
                 depthData[i] = normalizedDepth;
             }
-            depthTexture.Apply(false);
-            return depthTexture;
         }
 
         /// <summary>
@@ -124,7 +122,7 @@ private void AllocateObjects() {
 
             // Declare texture resources
             _input = RenderTexture.GetTemporary(_width, _height, 0, RenderTextureFormat.ARGB32);
-            _output = RenderTexture.GetTemporary(_width, _height, 0, RenderTextureFormat.RFloat);
+            _output = RenderTexture.GetTemporary(_width, _height, 0, RenderTextureFormat.RFloat, RenderTextureReadWrite.Linear);
         }
 
         /// <summary>

Packages/com.doji.genesis/Runtime/Shaders/EquirectDepth.shader

@@ -2,8 +2,8 @@ Shader "Genesis/EquirectDepth" {
     Properties {
         _MainTex ("Texture", 2D) = "white" {}
         _Depth("Depth", 2D) = "white" {}
-        _MinDepth("Min Depth", Range(0.0, 100.0)) = 2.0
-        _MaxDepth("Max Depth", Range(10.0, 10000.0)) = 100.0
+        _Scale("Depth Multiplier", float) = 1
+        _Max("Max Depth Cutoff", float) = 1000
     }
     SubShader {
         Tags { "RenderType"="Opaque" }
@@ -20,8 +20,9 @@ Shader "Genesis/EquirectDepth" {
             float4 _MainTex_ST;
             sampler2D _Depth;
 
-            float _MinDepth;
-            float _MaxDepth;
+            float _Min;
+            float _Max;
+            float _Scale;
 
             struct appdata {
                 float4 vertex : POSITION;
@@ -46,9 +47,8 @@ Shader "Genesis/EquirectDepth" {
                 uv.x = 1 - uv.x;     // invert x because we use an outward-facing sphere with 'Cull Front'
 
                 float depth = tex2Dlod(_Depth, float4(uv, 0, 0));
-
-                // assume normalized depth, map to a given min, max range
-                depth = (depth * _MaxDepth) + _MinDepth;
+                depth = _Scale / (depth);
+                depth = clamp(depth, 0, _Max * _Scale);
 
                 // Vertex displacement (assumes rendering on a unit sphere with radius 1)
                 o.vertex = UnityObjectToClipPos(v.vertex * depth);