From 3e5280c46e8ab0ffc894b359b36fe22208f4ec49 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Sat, 23 Nov 2024 16:11:40 -0500 Subject: [PATCH 1/3] Support for Trickplay thumbnails in Playlet backend --- .vscode/launch.json | 3 +- CHANGELOG.md | 6 + .../Services/Innertube/InnertubeService.bs | 87 +++++++- .../VideoPlayer/VideoContentTask.bs | 19 +- .../PlayletWebServer/Middleware/HlsRouter.bs | 195 ++++++++++++++++++ .../Web/PlayletWebServer/PlayletWebServer.bs | 2 + playlet-lib/src/source/services/HttpClient.bs | 5 +- playlet-lib/src/source/utils/UrlUtils.bs | 8 +- 8 files changed, 310 insertions(+), 15 deletions(-) create mode 100644 playlet-lib/src/components/Web/PlayletWebServer/Middleware/HlsRouter.bs diff --git a/.vscode/launch.json b/.vscode/launch.json index 604436a9..dd61863e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -34,7 +34,8 @@ "preLaunchTask": "build-dev", "injectRdbOnDeviceComponent": true, "injectRaleTrackerTask": true, - "raleTrackerTaskFileLocation": "${workspaceFolder}/tools/RALE/TrackerTask.xml" + "raleTrackerTaskFileLocation": "${workspaceFolder}/tools/RALE/TrackerTask.xml", + "enableDebugProtocol": true }, { "name": "Playlet (prod)", diff --git a/CHANGELOG.md b/CHANGELOG.md index ba2159df..50df8b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Support for Trickplay thumbnails in Playlet backend. Done by parsing storyboards and injecting them into HLS manifest. + ## [0.30.0] - 2024-11-22 ### Added diff --git a/playlet-lib/src/components/Services/Innertube/InnertubeService.bs b/playlet-lib/src/components/Services/Innertube/InnertubeService.bs index f3f7e787..9694a6b0 100644 --- a/playlet-lib/src/components/Services/Innertube/InnertubeService.bs +++ b/playlet-lib/src/components/Services/Innertube/InnertubeService.bs @@ -260,16 +260,17 @@ namespace InnertubeService } end if + lengthSeconds = videoDetails["lengthSeconds"].ToInt() videoInfo = { "type": "video" "title": videoDetails["title"] "videoId": videoDetails["videoId"] "videoThumbnails": videoDetails["thumbnail"]["thumbnails"] - "storyboards": [] + "storyboards": ParseStoryboards(payload, lengthSeconds) "viewCount": videoDetails["viewCount"].ToInt() "author": videoDetails["author"] "authorId": videoDetails["channelId"] - "lengthSeconds": videoDetails["lengthSeconds"].ToInt() + "lengthSeconds": lengthSeconds "liveNow": ValidBool(videoDetails["isLive"]) "hlsUrl": streamingData["hlsManifestUrl"] "adaptiveFormats": [] @@ -284,6 +285,88 @@ namespace InnertubeService } end function + function ParseStoryboards(payload as object, lengthSeconds as integer) as object + storyboard = ObjectUtils.Dig(payload, ["storyboards", "playerLiveStoryboardSpecRenderer", "spec"]) + if IsString(storyboard) + storyboard = storyboard.Split("#") + if storyboard.Count() <> 5 + return [] + end if + return [{ + "templateUrl": storyboard[0] + "width": storyboard[1].ToInt() + "height": storyboard[2].ToInt() + "count": -1 + "interval": 5000 + "storyboardHeight": storyboard[3].ToInt() + "storyboardWidth": storyboard[4].ToInt() + }] + end if + + storyboards = ObjectUtils.Dig(payload, ["storyboards", "playerStoryboardSpecRenderer", "spec"]) + if not IsString(storyboards) + return [] + end if + + storyboardsData = storyboards.Split("|") + if storyboardsData.Count() < 2 + return [] + end if + + baseUrl = storyboardsData.Shift() + storyboards = [] + + index = 0 + for each sb in storyboardsData + sbData = sb.Split("#") + if sbData.Count() <> 8 + index += 1 + continue for + end if + + width = sbData[0].ToInt() + height = sbData[1].ToInt() + _count = sbData[2].ToInt() + columns = sbData[3].ToInt() + rows = sbData[4].ToInt() + interval = sbData[5].ToInt() + name = sbData[6] + sigh = sbData[7] + + url = baseUrl + url = url.Replace("$L", `${index}`) + url = url.Replace("$N", name) + + queryComponents = UrlUtils.ParseQueryComponents(url) + queryComponents["sigh"] = sigh.DecodeUriComponent() + url = UrlUtils.SetQueryParams(url, queryComponents) + + thumbnailsPerImage = columns * rows + imagesCount = _count \ thumbnailsPerImage + if _count mod thumbnailsPerImage > 0 + imagesCount += 1 + end if + + if interval = 0 and _count > 0 + interval = Cint(Cdbl(lengthSeconds) * 1000.0 / Cdbl(_count)) + end if + + storyboards.Push({ + "templateUrl": url + "width": width + "height": height + "count": _count + "interval": interval + "storyboardWidth": columns + "storyboardHeight": rows + "storyboardCount": imagesCount + }) + index += 1 + end for + + return storyboards + end function + function ParseCaptions(payload as object) as object captions = payload["captions"] if not IsAssociativeArray(captions) diff --git a/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs b/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs index 7bd54c03..8ef48593 100644 --- a/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs +++ b/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs @@ -60,10 +60,6 @@ function VideoContentTask(input as object) as object end if streamUrls = CreateStreamUrls(metadata, service, preferencesNode, playletServerPort) - contentNode.url = streamUrls[0] - - SetCaptions(metadata, service, contentNode) - contentNode.addFields({ metadata: metadata ' StreamUrls is taken @@ -71,6 +67,10 @@ function VideoContentTask(input as object) as object playletStreamUrlIndex: 0 }) + contentNode.url = streamUrls[0] + + SetCaptions(metadata, service, contentNode) + if metadata.liveNow = true contentNode.Live = true end if @@ -87,7 +87,7 @@ function CreateStreamUrls(metadata as object, service as Invidious.InvidiousServ streamUrls = [] if metadata.hlsUrl <> invalid - AddHlsUrls(streamUrls, metadata, proxyVideos) + AddHlsUrls(streamUrls, metadata, proxyVideos, playletServerPort) else preferredQuality = preferences["playback.preferred_quality"] @@ -105,10 +105,17 @@ function CreateStreamUrls(metadata as object, service as Invidious.InvidiousServ return streamUrls end function -function AddHlsUrls(streamUrls as object, metadata as object, proxyVideos as string) +function AddHlsUrls(streamUrls as object, metadata as object, proxyVideos as string, playletServerPort as integer) hlsUrl = metadata.hlsUrl + if hlsUrl.StartsWith("/api/manifest") + hlsUrl = "https://manifest.googlevideo.com" + hlsUrl + end if hlsUrlLocal = AddLocalFlag(hlsUrl) + ' Redirect to Playet's local web server, where will will be + ' dynamically adding storyboards + hlsUrl = `http://127.0.0.1:${playletServerPort}/api/hls?url=${hlsUrl.EncodeUriComponent()}` + AddUrls(streamUrls, hlsUrl, hlsUrlLocal, proxyVideos) end function diff --git a/playlet-lib/src/components/Web/PlayletWebServer/Middleware/HlsRouter.bs b/playlet-lib/src/components/Web/PlayletWebServer/Middleware/HlsRouter.bs new file mode 100644 index 00000000..0611abf5 --- /dev/null +++ b/playlet-lib/src/components/Web/PlayletWebServer/Middleware/HlsRouter.bs @@ -0,0 +1,195 @@ +import "pkg:/source/services/HttpClient.bs" + +namespace Http + + class HlsRouter extends HttpRouter + function new(server as object) + super() + + m.videoQueue = server.task.videoQueue + end function + + @get("/api/hls") + function GenerateHlsManifest(context as object) as boolean + request = context.request + response = context.response + + url = request.query.url + + if StringUtils.IsNullOrEmpty(url) + response.Default(400, "Missing url") + return true + end if + + hlsRequest = HttpClient.Get(url) + headers = request.headers + headers.Delete("Host") + hlsRequest.Headers(headers) + + hlsResponse = hlsRequest.Await() + + if not hlsResponse.IsSuccess() + LogError("Failed to fetch HLS manifest:", hlsResponse.ErrorMessage()) + end if + + hlsManifest = hlsResponse.Text() + hlsManifest = m.AppendStoryboardPlaylists(hlsManifest) + + response.http_code = hlsResponse.StatusCode() + response.headers = hlsResponse.Headers() + response.SetBodyDataString(ValidString(hlsManifest)) + + return true + end function + + @get("/api/hls/storyboards") + function GetStoryboardsHls(context as object) as boolean + request = context.request + response = context.response + + metadata = m.GetPlayerMetadata() + if metadata = invalid + response.Default(500, "Player metadata is invalid") + return true + end if + + storyboards = metadata.storyboards + if not IsArray(storyboards) + response.Default(500, "Player metadata storyboards is invalid") + return true + end if + + lengthSeconds = metadata.lengthSeconds + if not IsInt(lengthSeconds) + response.Default(500, "Player metadata lengthSeconds is invalid") + return true + end if + + index = request.query.index + if StringUtils.IsNullOrEmpty(index) + response.Default(400, "Missing index") + return true + end if + + index = index.ToInt() + if index < 0 or index >= storyboards.Count() + response.Default(400, "Invalid index") + return true + end if + + storyboard = storyboards[index] + hlsManifest = m.GenerateStoryboardManifest(storyboard, lengthSeconds) + + response.http_code = 200 + response.SetBodyDataString(hlsManifest) + response.ContentType("application/vnd.apple.mpegurl") + + return true + end function + + function AppendStoryboardPlaylists(hlsManifest as dynamic) as string + if StringUtils.IsNullOrEmpty(hlsManifest) or not hlsManifest.StartsWith("#EXTM3U") + return hlsManifest + end if + + metadata = m.GetPlayerMetadata() + if metadata = invalid + return hlsManifest + end if + + ' TODO:P2: handle live videos + if ValidBool(metadata.liveNow) + return hlsManifest + end if + + storyboards = metadata.storyboards + if not IsArray(storyboards) + return hlsManifest + end if + + ' Perhaps this is due to the bandwidth calculation being off, but the + ' Video node ends up using the first storyboard, which is the lowest + ' quality. That's why we reverse the order here. + for i = storyboards.Count() - 1 to 0 step -1 + storyboard = storyboards[i] + ' estimate bandwidth based on storyboard size + bandwidth = Cint((storyboard.width * storyboard.height * storyboard.storyboardWidth * storyboard.storyboardHeight * storyboard.storyboardCount / (Cdbl(storyboard.interval) / 1000.0)) * 0.01) + hlsManifest += `\n#EXT-X-IMAGE-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${storyboard.width}x${storyboard.height},CODECS="jpeg",URI="/api/hls/storyboards?index=${i}"` + end for + + hlsManifest += `\n` + + return hlsManifest + end function + + function GenerateStoryboardManifest(storyboard as object, lengthSeconds as float) as string + tileCount = storyboard.storyboardWidth * storyboard.storyboardHeight + intervalSeconds = Cdbl(storyboard.interval) / 1000.0 + targetDuration = tileCount * intervalSeconds + + hlsManifest = `#EXTM3U +#EXT-X-TARGETDURATION:${targetDuration} +#EXT-X-VERSION:7 +#EXT-X-MEDIA-SEQUENCE:1 +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-IMAGES-ONLY + +` + for i = 0 to storyboard.storyboardCount - 1 + currentTileCount = tileCount + currentStoryboardWidth = storyboard.storyboardWidth + currentStoryboardHeight = storyboard.storyboardHeight + if i = storyboard.storyboardCount - 1 + currentTileCount = storyboard.count mod tileCount + if currentTileCount = 0 + currentTileCount = tileCount + end if + + if currentTileCount < currentStoryboardWidth + currentStoryboardWidth = currentTileCount + end if + + currentStoryboardHeight = Cint(Cdbl(currentTileCount) / Cdbl(currentStoryboardWidth)) + if currentStoryboardHeight * currentStoryboardWidth < currentTileCount + currentStoryboardHeight += 1 + end if + end if + + extinf = currentTileCount * intervalSeconds + + ' TODO:P2: although we took care of the last storyboard to get the + ' correct tile count and layout, and still looks wrong, similar to DASH. + hlsManifest += `#EXTINF:${extinf}, +#EXT-X-TILES:RESOLUTION=${storyboard.width}x${storyboard.height},LAYOUT=${currentStoryboardWidth}x${currentStoryboardHeight},DURATION=${intervalSeconds} +${storyboard.templateUrl.replace("$M", `${i}`)} + +` + end for + + hlsManifest += `#EXT-X-ENDLIST` + + return hlsManifest + end function + + function GetPlayerMetadata() as object + player = m.videoQueue.player + + if player = invalid + return invalid + end if + + content = player.content + if content = invalid + return invalid + end if + + metadata = content.metadata + if not IsAssociativeArray(metadata) + return invalid + end if + + return metadata + end function + end class + +end namespace diff --git a/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs b/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs index e765121f..35448f68 100644 --- a/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs +++ b/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs @@ -19,6 +19,7 @@ import "pkg:/components/Web/WebServer/Middleware/CorsMiddleware.bs" import "pkg:/components/Web/WebServer/Middleware/EtagMiddleware.bs" import "pkg:/components/Web/WebServer/Middleware/HttpStaticFilesRouter.bs" import "pkg:/components/Web/WebServer/WebSockets/WebSocketMiddleware.bs" +import "pkg:/components/Web/PlayletWebServer/Middleware/HlsRouter.bs" function Init() m.top.packageName = "Playlet" @@ -42,6 +43,7 @@ function SetupRoutes(server as object) server.UseRouter(new Http.HomeRouter()) server.UseRouter(new Http.StateApiRouter(server)) server.UseRouter(new Http.DashRouter(server)) + server.UseRouter(new Http.HlsRouter(server)) server.UseRouter(new Http.PreferencesRouter(server)) server.UseRouter(new Http.InvidiousRouter(server)) server.UseRouter(new Http.ProfilesRouter(server)) diff --git a/playlet-lib/src/source/services/HttpClient.bs b/playlet-lib/src/source/services/HttpClient.bs index 10816626..70a05f62 100644 --- a/playlet-lib/src/source/services/HttpClient.bs +++ b/playlet-lib/src/source/services/HttpClient.bs @@ -651,10 +651,11 @@ namespace HttpClient return m._text end if - if not m.IsSuccess() + if type(m.event) <> "roUrlEvent" return invalid + else + m._text = m.event.GetString() end if - m._text = m.event.GetString() return m._text end function diff --git a/playlet-lib/src/source/utils/UrlUtils.bs b/playlet-lib/src/source/utils/UrlUtils.bs index d90b20ae..85fb4d6c 100644 --- a/playlet-lib/src/source/utils/UrlUtils.bs +++ b/playlet-lib/src/source/utils/UrlUtils.bs @@ -181,10 +181,10 @@ namespace UrlUtils key = component value = "" hasValue = false - if Instr(0, component, "=") - keyValue = component.split("=") - key = keyValue[0] - value = keyValue[1].DecodeUriComponent() + equalSignIndex = component.InStr("=") + if equalSignIndex <> -1 + key = component.Left(equalSignIndex) + value = component.Mid(equalSignIndex + 1).DecodeUriComponent() hasValue = true end if From 9e738e99ef5c69c35d6dde48dc8503f43492d605 Mon Sep 17 00:00:00 2001 From: github-action linter Date: Sat, 23 Nov 2024 21:12:50 +0000 Subject: [PATCH 2/3] Lint fix --- .../src/components/Web/PlayletWebServer/PlayletWebServer.bs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs b/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs index 35448f68..dc6a75a2 100644 --- a/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs +++ b/playlet-lib/src/components/Web/PlayletWebServer/PlayletWebServer.bs @@ -3,6 +3,7 @@ import "pkg:/components/Web/PlayletWebServer/Middleware/BookmarksRouter.bs" import "pkg:/components/Web/PlayletWebServer/Middleware/CacheRouter.bs" import "pkg:/components/Web/PlayletWebServer/Middleware/DashRouter.bs" import "pkg:/components/Web/PlayletWebServer/Middleware/DialRouter.bs" +import "pkg:/components/Web/PlayletWebServer/Middleware/HlsRouter.bs" import "pkg:/components/Web/PlayletWebServer/Middleware/HomeLayoutRouter.bs" import "pkg:/components/Web/PlayletWebServer/Middleware/HomeRouter.bs" import "pkg:/components/Web/PlayletWebServer/Middleware/InvidiousRouter.bs" @@ -19,7 +20,6 @@ import "pkg:/components/Web/WebServer/Middleware/CorsMiddleware.bs" import "pkg:/components/Web/WebServer/Middleware/EtagMiddleware.bs" import "pkg:/components/Web/WebServer/Middleware/HttpStaticFilesRouter.bs" import "pkg:/components/Web/WebServer/WebSockets/WebSocketMiddleware.bs" -import "pkg:/components/Web/PlayletWebServer/Middleware/HlsRouter.bs" function Init() m.top.packageName = "Playlet" From d05fba6193f8c0e8ab2e6eec78b41b41019b8daf Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Sat, 23 Nov 2024 16:13:50 -0500 Subject: [PATCH 3/3] typo --- playlet-lib/src/components/VideoPlayer/VideoContentTask.bs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs b/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs index 8ef48593..0be040b5 100644 --- a/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs +++ b/playlet-lib/src/components/VideoPlayer/VideoContentTask.bs @@ -112,7 +112,7 @@ function AddHlsUrls(streamUrls as object, metadata as object, proxyVideos as str end if hlsUrlLocal = AddLocalFlag(hlsUrl) - ' Redirect to Playet's local web server, where will will be + ' Redirect to Playet's local web server, where we will be ' dynamically adding storyboards hlsUrl = `http://127.0.0.1:${playletServerPort}/api/hls?url=${hlsUrl.EncodeUriComponent()}`