Support for Trickplay thumbnails in Playlet backend

.vscode/launch.json

@@ -34,7 +34,8 @@
             "preLaunchTask": "build-dev",
             "injectRdbOnDeviceComponent": true,
             "injectRaleTrackerTask": true,
-            "raleTrackerTaskFileLocation": "${workspaceFolder}/tools/RALE/TrackerTask.xml"
+            "raleTrackerTaskFileLocation": "${workspaceFolder}/tools/RALE/TrackerTask.xml",
+            "enableDebugProtocol": true
         },
         {
             "name": "Playlet (prod)",

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- Support for Trickplay thumbnails in Playlet backend. Done by parsing storyboards and injecting them into HLS manifest.
+
 ## [0.30.0] - 2024-11-22
 
 ### Added

playlet-lib/src/components/Services/Innertube/InnertubeService.bs

@@ -260,16 +260,17 @@ namespace InnertubeService
             }
         end if
 
+        lengthSeconds = videoDetails["lengthSeconds"].ToInt()
         videoInfo = {
             "type": "video"
             "title": videoDetails["title"]
             "videoId": videoDetails["videoId"]
             "videoThumbnails": videoDetails["thumbnail"]["thumbnails"]
-            "storyboards": []
+            "storyboards": ParseStoryboards(payload, lengthSeconds)
             "viewCount": videoDetails["viewCount"].ToInt()
             "author": videoDetails["author"]
             "authorId": videoDetails["channelId"]
-            "lengthSeconds": videoDetails["lengthSeconds"].ToInt()
+            "lengthSeconds": lengthSeconds
             "liveNow": ValidBool(videoDetails["isLive"])
             "hlsUrl": streamingData["hlsManifestUrl"]
             "adaptiveFormats": []
@@ -284,6 +285,88 @@ namespace InnertubeService
         }
     end function
 
+    function ParseStoryboards(payload as object, lengthSeconds as integer) as object
+        storyboard = ObjectUtils.Dig(payload, ["storyboards", "playerLiveStoryboardSpecRenderer", "spec"])
+        if IsString(storyboard)
+            storyboard = storyboard.Split("#")
+            if storyboard.Count() <> 5
+                return []
+            end if
+            return [{
+                "templateUrl": storyboard[0]
+                "width": storyboard[1].ToInt()
+                "height": storyboard[2].ToInt()
+                "count": -1
+                "interval": 5000
+                "storyboardHeight": storyboard[3].ToInt()
+                "storyboardWidth": storyboard[4].ToInt()
+            }]
+        end if
+
+        storyboards = ObjectUtils.Dig(payload, ["storyboards", "playerStoryboardSpecRenderer", "spec"])
+        if not IsString(storyboards)
+            return []
+        end if
+
+        storyboardsData = storyboards.Split("|")
+        if storyboardsData.Count() < 2
+            return []
+        end if
+
+        baseUrl = storyboardsData.Shift()
+        storyboards = []
+
+        index = 0
+        for each sb in storyboardsData
+            sbData = sb.Split("#")
+            if sbData.Count() <> 8
+                index += 1
+                continue for
+            end if
+
+            width = sbData[0].ToInt()
+            height = sbData[1].ToInt()
+            _count = sbData[2].ToInt()
+            columns = sbData[3].ToInt()
+            rows = sbData[4].ToInt()
+            interval = sbData[5].ToInt()
+            name = sbData[6]
+            sigh = sbData[7]
+
+            url = baseUrl
+            url = url.Replace("$L", `${index}`)
+            url = url.Replace("$N", name)
+
+            queryComponents = UrlUtils.ParseQueryComponents(url)
+            queryComponents["sigh"] = sigh.DecodeUriComponent()
+            url = UrlUtils.SetQueryParams(url, queryComponents)
+
+            thumbnailsPerImage = columns * rows
+            imagesCount = _count \ thumbnailsPerImage
+            if _count mod thumbnailsPerImage > 0
+                imagesCount += 1
+            end if
+
+            if interval = 0 and _count > 0
+                interval = Cint(Cdbl(lengthSeconds) * 1000.0 / Cdbl(_count))
+            end if
+
+            storyboards.Push({
+                "templateUrl": url
+                "width": width
+                "height": height
+                "count": _count
+                "interval": interval
+                "storyboardWidth": columns
+                "storyboardHeight": rows
+                "storyboardCount": imagesCount
+            })
+            index += 1
+        end for
+
+        return storyboards
+    end function
+
     function ParseCaptions(payload as object) as object
         captions = payload["captions"]
         if not IsAssociativeArray(captions)

playlet-lib/src/components/VideoPlayer/VideoContentTask.bs

@@ -60,17 +60,17 @@ function VideoContentTask(input as object) as object
     end if
 
     streamUrls = CreateStreamUrls(metadata, service, preferencesNode, playletServerPort)
-    contentNode.url = streamUrls[0]
-
-    SetCaptions(metadata, service, contentNode)
-
     contentNode.addFields({
         metadata: metadata
         ' StreamUrls is taken
         playletStreamUrls: streamUrls
         playletStreamUrlIndex: 0
     })
 
+    contentNode.url = streamUrls[0]
+
+    SetCaptions(metadata, service, contentNode)
+
     if metadata.liveNow = true
         contentNode.Live = true
     end if
@@ -87,7 +87,7 @@ function CreateStreamUrls(metadata as object, service as Invidious.InvidiousServ
     streamUrls = []
 
     if metadata.hlsUrl <> invalid
-        AddHlsUrls(streamUrls, metadata, proxyVideos)
+        AddHlsUrls(streamUrls, metadata, proxyVideos, playletServerPort)
     else
         preferredQuality = preferences["playback.preferred_quality"]
 
@@ -105,10 +105,17 @@ function CreateStreamUrls(metadata as object, service as Invidious.InvidiousServ
     return streamUrls
 end function
 
-function AddHlsUrls(streamUrls as object, metadata as object, proxyVideos as string)
+function AddHlsUrls(streamUrls as object, metadata as object, proxyVideos as string, playletServerPort as integer)
     hlsUrl = metadata.hlsUrl
+    if hlsUrl.StartsWith("/api/manifest")
+        hlsUrl = "https://manifest.googlevideo.com" + hlsUrl
+    end if
     hlsUrlLocal = AddLocalFlag(hlsUrl)
 
+    ' Redirect to Playet's local web server, where we will be
+    ' dynamically adding storyboards
+    hlsUrl = `http://127.0.0.1:${playletServerPort}/api/hls?url=${hlsUrl.EncodeUriComponent()}`
+
     AddUrls(streamUrls, hlsUrl, hlsUrlLocal, proxyVideos)
 end function
 

playlet-lib/src/components/Web/PlayletWebServer/Middleware/HlsRouter.bs

@@ -0,0 +1,195 @@
+import "pkg:/source/services/HttpClient.bs"
+
+namespace Http
+
+    class HlsRouter extends HttpRouter
+        function new(server as object)
+            super()
+
+            m.videoQueue = server.task.videoQueue
+        end function
+
+        @get("/api/hls")
+        function GenerateHlsManifest(context as object) as boolean
+            request = context.request
+            response = context.response
+
+            url = request.query.url
+
+            if StringUtils.IsNullOrEmpty(url)
+                response.Default(400, "Missing url")
+                return true
+            end if
+
+            hlsRequest = HttpClient.Get(url)
+            headers = request.headers
+            headers.Delete("Host")
+            hlsRequest.Headers(headers)
+
+            hlsResponse = hlsRequest.Await()
+
+            if not hlsResponse.IsSuccess()
+                LogError("Failed to fetch HLS manifest:", hlsResponse.ErrorMessage())
+            end if
+
+            hlsManifest = hlsResponse.Text()
+            hlsManifest = m.AppendStoryboardPlaylists(hlsManifest)
+
+            response.http_code = hlsResponse.StatusCode()
+            response.headers = hlsResponse.Headers()
+            response.SetBodyDataString(ValidString(hlsManifest))
+
+            return true
+        end function
+
+        @get("/api/hls/storyboards")
+        function GetStoryboardsHls(context as object) as boolean
+            request = context.request
+            response = context.response
+
+            metadata = m.GetPlayerMetadata()
+            if metadata = invalid
+                response.Default(500, "Player metadata is invalid")
+                return true
+            end if
+
+            storyboards = metadata.storyboards
+            if not IsArray(storyboards)
+                response.Default(500, "Player metadata storyboards is invalid")
+                return true
+            end if
+
+            lengthSeconds = metadata.lengthSeconds
+            if not IsInt(lengthSeconds)
+                response.Default(500, "Player metadata lengthSeconds is invalid")
+                return true
+            end if
+
+            index = request.query.index
+            if StringUtils.IsNullOrEmpty(index)
+                response.Default(400, "Missing index")
+                return true
+            end if
+
+            index = index.ToInt()
+            if index < 0 or index >= storyboards.Count()
+                response.Default(400, "Invalid index")
+                return true
+            end if
+
+            storyboard = storyboards[index]
+            hlsManifest = m.GenerateStoryboardManifest(storyboard, lengthSeconds)
+
+            response.http_code = 200
+            response.SetBodyDataString(hlsManifest)
+            response.ContentType("application/vnd.apple.mpegurl")
+
+            return true
+        end function
+
+        function AppendStoryboardPlaylists(hlsManifest as dynamic) as string
+            if StringUtils.IsNullOrEmpty(hlsManifest) or not hlsManifest.StartsWith("#EXTM3U")
+                return hlsManifest
+            end if
+
+            metadata = m.GetPlayerMetadata()
+            if metadata = invalid
+                return hlsManifest
+            end if
+
+            ' TODO:P2: handle live videos
+            if ValidBool(metadata.liveNow)
+                return hlsManifest
+            end if
+
+            storyboards = metadata.storyboards
+            if not IsArray(storyboards)
+                return hlsManifest
+            end if
+
+            ' Perhaps this is due to the bandwidth calculation being off, but the
+            ' Video node ends up using the first storyboard, which is the lowest
+            ' quality. That's why we reverse the order here.
+            for i = storyboards.Count() - 1 to 0 step -1
+                storyboard = storyboards[i]
+                ' estimate bandwidth based on storyboard size
+                bandwidth = Cint((storyboard.width * storyboard.height * storyboard.storyboardWidth * storyboard.storyboardHeight * storyboard.storyboardCount / (Cdbl(storyboard.interval) / 1000.0)) * 0.01)
+                hlsManifest += `\n#EXT-X-IMAGE-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${storyboard.width}x${storyboard.height},CODECS="jpeg",URI="/api/hls/storyboards?index=${i}"`
+            end for
+
+            hlsManifest += `\n`
+
+            return hlsManifest
+        end function
+
+        function GenerateStoryboardManifest(storyboard as object, lengthSeconds as float) as string
+            tileCount = storyboard.storyboardWidth * storyboard.storyboardHeight
+            intervalSeconds = Cdbl(storyboard.interval) / 1000.0
+            targetDuration = tileCount * intervalSeconds
+
+            hlsManifest = `#EXTM3U
+#EXT-X-TARGETDURATION:${targetDuration}
+#EXT-X-VERSION:7
+#EXT-X-MEDIA-SEQUENCE:1
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-IMAGES-ONLY
+
+`
+            for i = 0 to storyboard.storyboardCount - 1
+                currentTileCount = tileCount
+                currentStoryboardWidth = storyboard.storyboardWidth
+                currentStoryboardHeight = storyboard.storyboardHeight
+                if i = storyboard.storyboardCount - 1
+                    currentTileCount = storyboard.count mod tileCount
+                    if currentTileCount = 0
+                        currentTileCount = tileCount
+                    end if
+
+                    if currentTileCount < currentStoryboardWidth
+                        currentStoryboardWidth = currentTileCount
+                    end if
+
+                    currentStoryboardHeight = Cint(Cdbl(currentTileCount) / Cdbl(currentStoryboardWidth))
+                    if currentStoryboardHeight * currentStoryboardWidth < currentTileCount
+                        currentStoryboardHeight += 1
+                    end if
+                end if
+
+                extinf = currentTileCount * intervalSeconds
+
+                ' TODO:P2: although we took care of the last storyboard to get the
+                ' correct tile count and layout, and still looks wrong, similar to DASH.
+                hlsManifest += `#EXTINF:${extinf},
+#EXT-X-TILES:RESOLUTION=${storyboard.width}x${storyboard.height},LAYOUT=${currentStoryboardWidth}x${currentStoryboardHeight},DURATION=${intervalSeconds}
+${storyboard.templateUrl.replace("$M", `${i}`)}
+
+`
+            end for
+
+            hlsManifest += `#EXT-X-ENDLIST`
+
+            return hlsManifest
+        end function
+
+        function GetPlayerMetadata() as object
+            player = m.videoQueue.player
+
+            if player = invalid
+                return invalid
+            end if
+
+            content = player.content
+            if content = invalid
+                return invalid
+            end if
+
+            metadata = content.metadata
+            if not IsAssociativeArray(metadata)
+                return invalid
+            end if
+
+            return metadata
+        end function
+    end class
+
+end namespace

playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs

@@ -3,6 +3,7 @@ import "pkg:/components/Web/PlayletWebServer/Middleware/BookmarksRouter.bs"
 import "pkg:/components/Web/PlayletWebServer/Middleware/CacheRouter.bs"
 import "pkg:/components/Web/PlayletWebServer/Middleware/DashRouter.bs"
 import "pkg:/components/Web/PlayletWebServer/Middleware/DialRouter.bs"
+import "pkg:/components/Web/PlayletWebServer/Middleware/HlsRouter.bs"
 import "pkg:/components/Web/PlayletWebServer/Middleware/HomeLayoutRouter.bs"
 import "pkg:/components/Web/PlayletWebServer/Middleware/HomeRouter.bs"
 import "pkg:/components/Web/PlayletWebServer/Middleware/InvidiousRouter.bs"
@@ -42,6 +43,7 @@ function SetupRoutes(server as object)
     server.UseRouter(new Http.HomeRouter())
     server.UseRouter(new Http.StateApiRouter(server))
     server.UseRouter(new Http.DashRouter(server))
+    server.UseRouter(new Http.HlsRouter(server))
     server.UseRouter(new Http.PreferencesRouter(server))
     server.UseRouter(new Http.InvidiousRouter(server))
     server.UseRouter(new Http.ProfilesRouter(server))