Custom/extended splat encoding ranges, Premultiplied alpha for hyper-saturated colors (#134)

* Externalized SparkRenderer.maxPixelRadius and .minAlpha settings. Added to documentation. Added to examples/editor under Debug folder, moved sort32 there.

* Added SplatEncoding concept, applied individually to SparkRenderer (rgbMin/Max, lnScaleMinMax), SplatMesh (additionally sh1Min/Max, sh2, sh3). Updated examples/editor to control settings via gui. Updated all related code to pipe along settings where needed.
Added SparkRenderer.premultipliedAlpha to control blend mode and enable hyper-saturated color contributions by default.

* Fix SplatLoader not passing along encoding. Complete renaming to splat encoding.

Author: asundqui

examples/editor/index.html

@@ -69,7 +69,7 @@
     import * as THREE from "three";
     import { OrbitControls } from "three/addons/controls/OrbitControls.js";
     import { GUI } from "lil-gui";
-    import { constructGrid, SparkControls, SparkRenderer, SplatMesh, textSplats, dyno, transcodeSpz, isMobile, isPcSogs } from "@sparkjsdev/spark";
+    import { constructGrid, SparkControls, SparkRenderer, SplatMesh, textSplats, dyno, transcodeSpz, isMobile, isPcSogs, LN_SCALE_MIN, LN_SCALE_MAX } from "@sparkjsdev/spark";
     import { getAssetFileURL } from "/examples/js/get-asset-url.js";
 
     const scene = new THREE.Scene();
@@ -230,15 +230,21 @@
     applyCameraFromQuery();
 
     const cameraFolder = gui.addFolder("Camera");
-    cameraFolder.add(camera.position, "x", -10, 10, 0.01).name("X").listen();
-    cameraFolder.add(camera.position, "y", -10, 10, 0.01).name("Y").listen();
-    cameraFolder.add(camera.position, "z", -10, 10, 0.01).name("Z").listen();
-    const rotX = cameraFolder.add(camera.rotation, "x", -Math.PI, Math.PI, 0.01).name("RotateX").listen();
-    const rotY = cameraFolder.add(camera.rotation, "y", -Math.PI, Math.PI, 0.01).name("RotateY").listen();
-    const rotZ = cameraFolder.add(camera.rotation, "z", -Math.PI, Math.PI, 0.01).name("RotateZ").listen();
-    cameraFolder.add(camera, "fov", 1, 179, 1).name("Fov Y degrees").listen().onChange((value) => {
+    const cameraPose = cameraFolder.addFolder("Camera Pose");
+    cameraPose.add(camera.position, "x", -10, 10, 0.01).name("X").listen();
+    cameraPose.add(camera.position, "y", -10, 10, 0.01).name("Y").listen();
+    cameraPose.add(camera.position, "z", -10, 10, 0.01).name("Z").listen();
+    const rotX = cameraPose.add(camera.rotation, "x", -Math.PI, Math.PI, 0.01).name("RotateX").listen();
+    const rotY = cameraPose.add(camera.rotation, "y", -Math.PI, Math.PI, 0.01).name("RotateY").listen();
+    const rotZ = cameraPose.add(camera.rotation, "z", -Math.PI, Math.PI, 0.01).name("RotateZ").listen();
+    cameraPose.add(camera, "fov", 1, 179, 1).name("Fov Y degrees").listen().onChange((value) => {
       camera.updateProjectionMatrix();
     });
+    cameraPose.close();
+
+    function touch() {
+      spark.needsUpdate = true;
+    }
 
     // Progress bar functions
     const progressBar = document.getElementById('progress-bar');
@@ -383,6 +389,7 @@
           }
 
           const init = url ? { url } : { fileBytes: fileBytes.slice(), fileName };
+          init.splatEncoding = { ...splatEncoding };
           const splatMesh = new SplatMesh(init);
           const translate = guiOptions.loadOffset * index
           splatMesh.position.set(translate, 0.5 * translate, 0.1 * translate);
@@ -480,8 +487,6 @@
       stats.dom.style.display = value ? "block" : "none";
     });
     gui.add(spark.defaultView, "sortRadial").name("Radial sort").listen();
-    spark.defaultView.sort32 = true;
-    gui.add(spark.defaultView, "sort32").name("Float32 sort").listen();
     gui.add(grid, "opacity", 0, 1, 0.01).name("Grid opacity").listen();
     gui.add({
       logFocalDistance: 0.0,
@@ -520,6 +525,37 @@
     debugFolder.add(spark, "maxPixelRadius", 1, 1024, 1).name("Max pixel radius").listen();
     debugFolder.add(spark, "minAlpha", 0, 1, 0.001).name("Min alpha").listen();
 
+    debugFolder.add(spark, "premultipliedAlpha").name("Premultiplied alpha").listen();
+    const accumFolder = debugFolder.addFolder("Accumulator encoding").close();;
+    accumFolder.add(spark.splatEncoding, "rgbMin", -1, 1, 0.1).name("RGB min").onChange(touch);
+    accumFolder.add(spark.splatEncoding, "rgbMax", 0, 4, 0.1).name("RGB max").onChange(touch);
+    accumFolder.add(spark.splatEncoding, "lnScaleMin", -14, -2.5, 0.1).name("Ln scale min").onChange(touch);
+    accumFolder.add(spark.splatEncoding, "lnScaleMax", -14, 14, 0.1).name("Ln scale max").onChange(touch);
+
+    const splatEncoding = {
+      rgbMin: 0.0,
+      rgbMax: 1.0,
+      lnScaleMin: LN_SCALE_MIN,
+      lnScaleMax: LN_SCALE_MAX,
+      sh1Min: -1,
+      sh1Max: 1,
+      sh2Min: -1,
+      sh2Max: 1,
+      sh3Min: -1,
+      sh3Max: 1,
+    };
+    const splatFolder = debugFolder.addFolder("SplatMesh encoding").close();
+    splatFolder.add(splatEncoding, "rgbMin", -1, 1, 0.1).name("RGB min").onChange(touch);
+    splatFolder.add(splatEncoding, "rgbMax", 0, 4, 0.1).name("RGB max").onChange(touch);
+    splatFolder.add(splatEncoding, "lnScaleMin", -14, -2.5, 0.1).name("Ln scale min").onChange(touch);
+    splatFolder.add(splatEncoding, "lnScaleMax", -14, 14, 0.1).name("Ln scale max").onChange(touch);
+    splatFolder.add(splatEncoding, "sh1Min", -6, 6, 0.1).name("SH1 min").onChange(touch);
+    splatFolder.add(splatEncoding, "sh1Max", -6, 6, 0.1).name("SH1 max").onChange(touch);
+    splatFolder.add(splatEncoding, "sh2Min", -6, 6, 0.1).name("SH2 min").onChange(touch);
+    splatFolder.add(splatEncoding, "sh2Max", -6, 6, 0.1).name("SH2 max").onChange(touch);
+    splatFolder.add(splatEncoding, "sh3Min", -6, 6, 0.1).name("SH3 min").onChange(touch);
+    splatFolder.add(splatEncoding, "sh3Max", -6, 6, 0.1).name("SH3 max").onChange(touch);
+
     const splatsFolder = secondGui.addFolder("Files");
 
     const clipFolder = gui.addFolder("Clip Splats").close();

rust/spark-internal-rs/src/lib.rs

@@ -82,6 +82,7 @@ pub fn raycast_splats(
     near: f32, far: f32,
     num_splats: u32, packed_splats: Uint32Array,
     raycast_ellipsoid: bool,
+    ln_scale_min: f32, ln_scale_max: f32,
 ) -> Float32Array {
     let mut distances = Vec::<f32>::new();
 
@@ -94,9 +95,9 @@ pub fn raycast_splats(
             subarray.copy_to(subbuffer);
 
             if raycast_ellipsoid {
-                raycast_ellipsoids(subbuffer, &mut distances, [origin_x, origin_y, origin_z], [dir_x, dir_y, dir_z], near, far);
+                raycast_ellipsoids(subbuffer, &mut distances, [origin_x, origin_y, origin_z], [dir_x, dir_y, dir_z], near, far, ln_scale_min, ln_scale_max);
             } else {
-                raycast_spheres(subbuffer, &mut distances, [origin_x, origin_y, origin_z], [dir_x, dir_y, dir_z], near, far);
+                raycast_spheres(subbuffer, &mut distances, [origin_x, origin_y, origin_z], [dir_x, dir_y, dir_z], near, far, ln_scale_min, ln_scale_max);
             }
 
             base += chunk_size;

rust/spark-internal-rs/src/raycast.rs

@@ -2,30 +2,19 @@ use half::f16;
 
 const MIN_OPACITY: f32 = 0.1;
 
-pub const LN_SCALE_MIN: f32 = -12.0;
-pub const LN_SCALE_MAX: f32 = 9.0;
-pub const LN_RESCALE: f32 = (LN_SCALE_MAX - LN_SCALE_MIN) / 254.0; // 1..=255
-
-// pub fn encode_scale(scale: f32) -> u8 {
-//     if scale == 0.0 {
-//         0
-//     } else {
-//         // Allow scales below LN_SCALE_MIN to be encoded as 0, which signifies a 2DGS
-//         ((scale.ln() - LN_SCALE_MIN) / LN_RESCALE + 1.0).clamp(0.0, 255.0).round() as u8
-//     }
-// }
-
-pub fn decode_scale(scale: u8) -> f32 {
+pub fn decode_scale(scale: u8, ln_scale_min: f32, ln_scale_max: f32) -> f32 {
     if scale == 0 {
         0.0
     } else {
-        (LN_SCALE_MIN + (scale - 1) as f32 * LN_RESCALE).exp()
+        let ln_scale_scale = (ln_scale_max - ln_scale_min) / 254.0;
+        (ln_scale_min + (scale - 1) as f32 * ln_scale_scale).exp()
     }
 }
 
 pub fn raycast_spheres(
     buffer: &[u32], distances: &mut Vec<f32>, 
     origin: [f32; 3], dir: [f32; 3], near: f32, far: f32,
+    ln_scale_min: f32, ln_scale_max: f32,
 ) {
     let quad_a = vec3_dot(dir, dir);
 
@@ -36,7 +25,7 @@ pub fn raycast_spheres(
         }
 
         let origin = vec3_sub(origin, extract_center(packed));
-        let scale = extract_scale(packed);
+        let scale = extract_scale(packed, ln_scale_min, ln_scale_max);
 
         // Model the Gsplat as a sphere for faster approximate raycasting
         let radius = (scale[0] + scale[1] + scale[2]) / 3.0;
@@ -58,6 +47,7 @@ pub fn raycast_spheres(
 pub fn raycast_ellipsoids(
     buffer: &[u32], distances: &mut Vec<f32>, 
     origin: [f32; 3], dir: [f32; 3], near: f32, far: f32,
+    ln_scale_min: f32, ln_scale_max: f32,
 ) {
     for packed in buffer.chunks(4) {
         let opacity = ((packed[0] >> 24) as u8) as f32 / 255.0;
@@ -66,7 +56,7 @@ pub fn raycast_ellipsoids(
         }
 
         let origin = vec3_sub(origin, extract_center(packed));
-        let scale = extract_scale(packed);
+        let scale = extract_scale(packed, ln_scale_min, ln_scale_max);
         let quat = extract_quat(packed);
         let inv_quat = [-quat[0], -quat[1], -quat[2], quat[3]];
 
@@ -139,10 +129,10 @@ fn extract_center(packed: &[u32]) -> [f32; 3] {
     [x, y, z]
 }
 
-fn extract_scale(packed: &[u32]) -> [f32; 3] {
-    let scale_x = decode_scale(packed[3] as u8);
-    let scale_y = decode_scale((packed[3] >> 8) as u8);
-    let scale_z = decode_scale((packed[3] >> 16) as u8);
+fn extract_scale(packed: &[u32], ln_scale_min: f32, ln_scale_max: f32) -> [f32; 3] {
+    let scale_x = decode_scale(packed[3] as u8, ln_scale_min, ln_scale_max);
+    let scale_y = decode_scale((packed[3] >> 8) as u8, ln_scale_min, ln_scale_max);
+    let scale_z = decode_scale((packed[3] >> 16) as u8, ln_scale_min, ln_scale_max);
     [scale_x, scale_y, scale_z]
 }