From 604715c3ca2436802768384bf4a7018527990547 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Tue, 16 Jul 2024 08:42:52 -0700 Subject: [PATCH 01/15] Add 2D scene mode option to demo page --- demo/index.html | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/demo/index.html b/demo/index.html index d2ec1aef..29916fb9 100644 --- a/demo/index.html +++ b/demo/index.html @@ -285,6 +285,7 @@ let currentCameraPositionArray; let currentCameraLookAtArray; let currentAntialiased; + let current2DScene; let currentSphericalHarmonicsDegree; + + + + + + + + + + \ No newline at end of file diff --git a/demo/dynamic_scenes.html b/demo/dynamic_scenes.html index 43197d94..4441395a 100644 --- a/demo/dynamic_scenes.html +++ b/demo/dynamic_scenes.html @@ -37,7 +37,8 @@ 'initialCameraLookAt': [1.52976, 2.27776, 1.65898], 'dynamicScene': true }); - const lp = viewer.addSplatScenes([ + + viewer.addSplatScenes([ { 'path': 'assets/data/garden/garden.ksplat', 'splatAlphaRemovalThreshold': 20, @@ -69,6 +70,8 @@ const material = new THREE.MeshBasicMaterial({color: 0xff0000}); const meshA = new THREE.Mesh(sphereGeometry, material); const meshB = new THREE.Mesh(sphereGeometry, material); + + // add splat mesh parent objects to the scene viewer.splatMesh.add(meshA); viewer.splatMesh.add(meshB); From 7c57f3a8152d6ce1d1e0a404505e763c2a588854 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Fri, 23 Aug 2024 10:57:18 -0700 Subject: [PATCH 12/15] Commenting spherical harmonics shader code --- src/splatmesh/SplatMaterial.js | 84 +++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/src/splatmesh/SplatMaterial.js b/src/splatmesh/SplatMaterial.js index c01eed0b..fde5c4a5 100644 --- a/src/splatmesh/SplatMaterial.js +++ b/src/splatmesh/SplatMaterial.js @@ -169,6 +169,7 @@ export class SplatMaterial { vColor = uintToRGBAVec(sampledCenterColor.r); `; + // Proceed to sampling and rendering 1st degree spherical harmonics if (maxSphericalHarmonicsDegree >= 1) { vertexShaderSource += ` @@ -193,31 +194,20 @@ export class SplatMaterial { if (maxSphericalHarmonicsDegree >= 2) { vertexShaderSource += ` - vec4 sampledSH0123; - vec4 sampledSH4567; - vec4 sampledSH891011; - - vec4 sampledSH0123R; - vec4 sampledSH0123G; - vec4 sampledSH0123B; - - if (sphericalHarmonicsMultiTextureMode == 0) { - sampledSH0123 = texture(sphericalHarmonicsTexture, getDataUV(6, 0, sphericalHarmonicsTextureSize)); - sampledSH4567 = texture(sphericalHarmonicsTexture, getDataUV(6, 1, sphericalHarmonicsTextureSize)); - sampledSH891011 = texture(sphericalHarmonicsTexture, getDataUV(6, 2, sphericalHarmonicsTextureSize)); - sh1 = sampledSH0123.rgb; - sh2 = vec3(sampledSH0123.a, sampledSH4567.rg); - sh3 = vec3(sampledSH4567.ba, sampledSH891011.r); - } else { - sampledSH0123R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sampledSH0123G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sampledSH0123B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sh1 = vec3(sampledSH0123R.rgb); - sh2 = vec3(sampledSH0123G.rgb); - sh3 = vec3(sampledSH0123B.rgb); - } + vec3 sh4; + vec3 sh5; + vec3 sh6; + vec3 sh7; + vec3 sh8; `; - } else { + } + + // Determining how to sample spherical harmonics textures to get the coefficients for calculations for a given degree + // depends on how many total degrees (maxSphericalHarmonicsDegree) are present in the textures. This is because that + // number affects how they are packed in the textures, and therefore the offset & stride required to access them. + + // Sample spherical harmonics textures with 1 degree worth of data for 1st degree calculations, and store in sh1, sh2, and sh3 + if (maxSphericalHarmonicsDegree === 1) { vertexShaderSource += ` if (sphericalHarmonicsMultiTextureMode == 0) { vec2 shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset, sphericalHarmonicsTextureSize); @@ -241,8 +231,36 @@ export class SplatMaterial { sh3 = vec3(sampledSH01B.rg, sampledSH23B.r); } `; + // Sample spherical harmonics textures with 2 degrees worth of data for 1st degree calculations, and store in sh1, sh2, and sh3 + } else if (maxSphericalHarmonicsDegree === 2) { + vertexShaderSource += ` + vec4 sampledSH0123; + vec4 sampledSH4567; + vec4 sampledSH891011; + + vec4 sampledSH0123R; + vec4 sampledSH0123G; + vec4 sampledSH0123B; + + if (sphericalHarmonicsMultiTextureMode == 0) { + sampledSH0123 = texture(sphericalHarmonicsTexture, getDataUV(6, 0, sphericalHarmonicsTextureSize)); + sampledSH4567 = texture(sphericalHarmonicsTexture, getDataUV(6, 1, sphericalHarmonicsTextureSize)); + sampledSH891011 = texture(sphericalHarmonicsTexture, getDataUV(6, 2, sphericalHarmonicsTextureSize)); + sh1 = sampledSH0123.rgb; + sh2 = vec3(sampledSH0123.a, sampledSH4567.rg); + sh3 = vec3(sampledSH4567.ba, sampledSH891011.r); + } else { + sampledSH0123R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sampledSH0123G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sampledSH0123B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sh1 = vec3(sampledSH0123R.rgb); + sh2 = vec3(sampledSH0123G.rgb); + sh3 = vec3(sampledSH0123B.rgb); + } + `; } + // Perform 1st degree spherical harmonics calculations vertexShaderSource += ` if (sphericalHarmonics8BitMode == 1) { sh1 = sh1 * sh8BitCompressionRangeForScene + vec8BitSHShift; @@ -255,6 +273,7 @@ export class SplatMaterial { vColor.rgb += SH_C1 * (-sh1 * y + sh2 * z - sh3 * x); `; + // Proceed to sampling and rendering 2nd degree spherical harmonics if (maxSphericalHarmonicsDegree >= 2) { vertexShaderSource += ` @@ -265,13 +284,12 @@ export class SplatMaterial { float xy = x * y; float yz = y * z; float xz = x * z; + `; - vec3 sh4; - vec3 sh5; - vec3 sh6; - vec3 sh7; - vec3 sh8; - + // Sample spherical harmonics textures with 2 degrees worth of data for 2nd degree calculations, + // and store in sh4, sh5, sh6, sh7, and sh8 + if (maxSphericalHarmonicsDegree === 2) { + vertexShaderSource += ` if (sphericalHarmonicsMultiTextureMode == 0) { vec4 sampledSH12131415 = texture(sphericalHarmonicsTexture, getDataUV(6, 3, sphericalHarmonicsTextureSize)); vec4 sampledSH16171819 = texture(sphericalHarmonicsTexture, getDataUV(6, 4, sphericalHarmonicsTextureSize)); @@ -291,7 +309,11 @@ export class SplatMaterial { sh7 = vec3(sampledSH4567G.a, sampledSH0123B.a, sampledSH4567B.r); sh8 = vec3(sampledSH4567B.gba); } + `; + } + // Perform 2nd degree spherical harmonics calculations + vertexShaderSource += ` if (sphericalHarmonics8BitMode == 1) { sh4 = sh4 * sh8BitCompressionRangeForScene + vec8BitSHShift; sh5 = sh5 * sh8BitCompressionRangeForScene + vec8BitSHShift; @@ -311,7 +333,7 @@ export class SplatMaterial { } vertexShaderSource += ` - + vColor.rgb = clamp(vColor.rgb, vec3(0.), vec3(1.)); } From 7ff42ebc2a12d6c80d5b8f76f29735c7b326e7eb Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 24 Aug 2024 09:47:40 -0700 Subject: [PATCH 13/15] Update README for create-ksplat.js usage --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dc17746a..3a407040 100644 --- a/README.md +++ b/README.md @@ -354,7 +354,13 @@ Both of the above methods will prompt your browser to automatically start downlo The third option is to use the included nodejs script: ``` -node util/create-ksplat.js [path to .PLY or .SPLAT] [output file] [compression level = 0] [alpha removal threshold = 1] +node util/create-ksplat.js [path to .PLY or .SPLAT] [output file] [compression level = 0] [alpha removal threshold = 1] [scene center = "0,0,0"] [block size = 5.0] [bucket size = 256] [spherical harmonics level = 0] +``` + +For the nodejs script, it may be necessary to increase the heap size for larger scenes: + +``` +node util/create-ksplat.js --max-old-space-size=8192 ... ``` Currently supported values for `compressionLevel` are `0`, `1`, or `2`. `0` means no compression and `1` means compression of scale, rotation, position, and spherical harmonics coefficient values from 32-bit to 16-bit. `2` is similar to `1` except spherical harmonics coefficients are compressed to 8-bit. From e2d2b6ec3df3c079bbd4ec056c9c008a537742e8 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 24 Aug 2024 17:19:51 -0700 Subject: [PATCH 14/15] Cleanup --- README.md | 4 ++-- demo/dynamic_dropin.html | 6 +++--- demo/dynamic_scenes.html | 6 +++--- src/Viewer.js | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3a407040..59290a8a 100644 --- a/README.md +++ b/README.md @@ -357,10 +357,10 @@ The third option is to use the included nodejs script: node util/create-ksplat.js [path to .PLY or .SPLAT] [output file] [compression level = 0] [alpha removal threshold = 1] [scene center = "0,0,0"] [block size = 5.0] [bucket size = 256] [spherical harmonics level = 0] ``` -For the nodejs script, it may be necessary to increase the heap size for larger scenes: +For the nodejs script, it may be necessary to increase the heap size for larger scenes. Use the parameter `--max-old-space-size=[heap size in MB]` to do so: ``` -node util/create-ksplat.js --max-old-space-size=8192 ... +node util/create-ksplat.js --max-old-space-size=8192 [... remaining arguments] ``` Currently supported values for `compressionLevel` are `0`, `1`, or `2`. `0` means no compression and `1` means compression of scale, rotation, position, and spherical harmonics coefficient values from 32-bit to 16-bit. `2` is similar to `1` except spherical harmonics coefficients are compressed to 8-bit. diff --git a/demo/dynamic_dropin.html b/demo/dynamic_dropin.html index 2e9cccd0..1724a662 100644 --- a/demo/dynamic_dropin.html +++ b/demo/dynamic_dropin.html @@ -134,9 +134,9 @@ // You can modify the transform components (position, quaternion, scale) of a SplatScene // directly like any three.js object OR you can just attach them to another three.js object - // and they will be transformed accordingly. Below we are going with the latetr approach. - // The splat scenes at index 1 & 2 are children of viewer.splatMesh, so we re-parent them - // to meshA and meshB respectively. + // and they will be transformed accordingly. Below we are going with the latter approach. + // The splat scenes at index 1 & 2 are (by default) children of viewer.splatMesh, so we + // re-parent them to meshA and meshB respectively. meshA.add(viewer.getSplatScene(1)); meshB.add(viewer.getSplatScene(2)); diff --git a/demo/dynamic_scenes.html b/demo/dynamic_scenes.html index 4441395a..ecaa682c 100644 --- a/demo/dynamic_scenes.html +++ b/demo/dynamic_scenes.html @@ -77,9 +77,9 @@ // You can modify the transform components (position, quaternion, scale) of a SplatScene // directly like any three.js object OR you can just attach them to another three.js object - // and they will be transformed accordingly. Below we are going with the latetr approach. - // The splat scenes at index 1 & 2 are children of viewer.splatMesh, so we re-parent them - // to meshA and meshB respectively. + // and they will be transformed accordingly. Below we are going with the latter approach. + // The splat scenes at index 1 & 2 are (by default) children of viewer.splatMesh, so we + // re-parent them to meshA and meshB respectively. meshA.add(viewer.getSplatScene(1)); meshB.add(viewer.getSplatScene(2)); diff --git a/src/Viewer.js b/src/Viewer.js index b5f73731..244cd914 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -1079,7 +1079,7 @@ export class Viewer { }); } - this.updateSplatSort(true).then((sortRunning) => { + this.runSplatSort(true).then((sortRunning) => { if (!this.sortWorker || !sortRunning) { this.splatRenderReady = true; removeSplatProcessingTask(); @@ -1354,7 +1354,7 @@ export class Viewer { this.splatMesh.updateTransforms(); this.splatRenderReady = false; - this.updateSplatSort(true) + this.runSplatSort(true) .then(() => { if (checkForEarlyExit()) { this.splatRenderReady = true; @@ -1570,7 +1570,7 @@ export class Viewer { Viewer.setCameraPositionFromZoom(this.camera, this.camera, this.controls); } } - this.updateSplatSort(); + this.runSplatSort(); this.updateForRendererSizeChanges(); this.updateSplatMesh(); this.updateMeshCursor(); @@ -1767,7 +1767,7 @@ export class Viewer { } } - updateSplatSort = function() { + runSplatSort = function() { const mvpMatrix = new THREE.Matrix4(); const cameraPositionArray = []; From b3ec449e045c28c7358476a8d2706c79fcbb02b0 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sun, 25 Aug 2024 16:50:27 -0700 Subject: [PATCH 15/15] Update version to 0.4.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7f8c763d..7b829a9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.3", + "version": "0.4.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.3", + "version": "0.4.4", "license": "MIT", "devDependencies": { "@babel/core": "7.22.0", diff --git a/package.json b/package.json index d4e59048..d899299e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "git", "url": "https://github.com/mkkellogg/GaussianSplats3D" }, - "version": "0.4.3", + "version": "0.4.4", "description": "Three.js-based 3D Gaussian splat viewer", "module": "build/gaussian-splats-3d.module.js", "main": "build/gaussian-splats-3d.umd.cjs",