From c1219654fdfda8d0582bfc196e1f161bbca01928 Mon Sep 17 00:00:00 2001 From: mikhail Date: Mon, 20 May 2024 21:59:33 +0500 Subject: [PATCH 01/13] wip(niconico): service added --- src/modules/processing/match.js | 4 ++++ src/modules/processing/services/nicovideo.js | 4 ++++ src/modules/processing/servicesConfig.json | 6 ++++++ src/modules/processing/servicesPatternTesters.js | 4 ++++ 4 files changed, 18 insertions(+) create mode 100644 src/modules/processing/services/nicovideo.js diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 5dfd27650..97ac7a076 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -24,6 +24,7 @@ import streamable from "./services/streamable.js"; import twitch from "./services/twitch.js"; import rutube from "./services/rutube.js"; import dailymotion from "./services/dailymotion.js"; +import nicovideo from "./services/nicovideo.js"; let freebind; @@ -185,6 +186,9 @@ export default async function(host, patternMatch, lang, obj) { case "dailymotion": r = await dailymotion(patternMatch); break; + case "nicovideo": + r = await nicovideo({ id: patternMatch.id }); + break; default: return createResponse("error", { t: loc(lang, 'ErrorUnsupported') diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js new file mode 100644 index 000000000..db7375351 --- /dev/null +++ b/src/modules/processing/services/nicovideo.js @@ -0,0 +1,4 @@ +// TODO @synzr implement nicovideo +export default function nicovideo() { + return { error: "ErrorEmptyDownload" }; +} diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index f714b827c..65e7d8647 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -110,6 +110,12 @@ "alias": "dailymotion videos", "patterns": ["video/:id"], "enabled": true + }, + "nicovideo": { + "alias": "niconico videos", + "tld": "jp", + "patterns": ["watch/:id"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 25b8943a9..cde8b8206 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -54,4 +54,8 @@ export const testers = { "youtube": (patternMatch) => patternMatch.id?.length <= 11, + + "nicovideo": (patternMatch) => + // checking if this page is video and if identifier is number + patternMatch.id.startsWith("sm") && !isNaN(parseInt(patternMatch.id.substring(2), 10)) } From c06fac08c0b927f42b22184791556ff1e2c97fec Mon Sep 17 00:00:00 2001 From: mikhail Date: Mon, 20 May 2024 23:23:15 +0500 Subject: [PATCH 02/13] wip: untested implementation TODO @synzr setting up something like vpn and test this --- src/modules/processing/services/nicovideo.js | 130 ++++++++++++++++++- 1 file changed, 127 insertions(+), 3 deletions(-) diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index db7375351..554912ac7 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -1,4 +1,128 @@ -// TODO @synzr implement nicovideo -export default function nicovideo() { - return { error: "ErrorEmptyDownload" }; +import { genericUserAgent } from "../../config.js"; +import HLS from "hls-parser"; +import util from "node:util"; + +const NICOVIDEO_EMBED_URL = "https://embed.nicovideo.jp/watch/%s"; +const NICOVIDEO_GUEST_API_URL = + // frontend is embed player + "https://www.nicovideo.jp/api/watch/v3_guest/%s?_frontendId=70&_frontendVersion=0&actionTrackId=%s"; +const NICOVIDEO_HLS_API_URL = + "https://nvapi.nicovideo.jp/v1/watch/%s/access-rights/hls?actionTrackId=%s"; + +const ACTION_TRACK_ID_REGEXP = + /"actionTrackId":"[A-Za-z0-9]+_[0-9]+"/; + +// working +async function getActionTrackId(id) { + const page = await fetch(util.format(NICOVIDEO_EMBED_URL, id), { + headers: { "user-agent": genericUserAgent }, + }).then((response) => response.text()); + + if (!ACTION_TRACK_ID_REGEXP.test(page)) { + throw new Error(); // we can't fetch the embed page + } + + const actionTrackId = page + // getting the regexp results + .match(ACTION_TRACK_ID_REGEXP) + .shift() + // getting the actionTrackId field's value + .split(":"") + .pop() + // cleaning from double quotation mark + .replaceAll(""", ""); + + return actionTrackId; +} + +// not tested +async function fetchGuestData(id, actionTrackId) { + const data = await fetch( + util.format(NICOVIDEO_GUEST_API_URL, id, actionTrackId), + { + headers: { "user-agent": genericUserAgent }, + } + ).then((response) => response.json()); + + if (data?.meta?.status !== 200) { + console.debug("fetchGuestData():", data) + throw new Error(); + } + + const { videos, audios, accessRightKey } = data.data.media.domand; + + // getting the HQ audio + const { id: audioId } = audios + .filter((audio) => audio.isAvailable) + .sort((firstAudio, secondAudio) => firstAudio.bitrate - secondAudio.bitrate) + .pop(); + + return { + accessRightKey, + outputs: videos + .filter((video) => video.isAvailable) + .map((video) => [video.id, audioId]), + }; +} + +// not tested +async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { + const data = await fetch( + util.format(NICOVIDEO_HLS_API_URL, id, actionTrackId), + { + method: "POST", + headers: { + "user-agent": genericUserAgent, + "content-type": "application/json; charset=utf-8", + "x-access-right-key": accessRightKey, + }, + body: JSON.stringify({ outputs }), + } + ).then((response) => response.json()); + + if (data?.meta?.status !== 201) { + throw new Error(); + } + + return data.data.contentURL; +} + +// not tested +async function getHighestQualityHLS(contentURL) { + const hls = await fetch(contentURL) + .then((response) => response.text()) + .then((response) => HLS.parse(response)); + + const highestQualityHLS = hls.variants + .sort((firstVariant, secondVariant) => { + const firstVariantPixels = + firstVariant.resolution.width * firstVariant.resolution.height; + const secondVariantPixels = + secondVariant.resolution.width * secondVariant.resolution.height; + + return firstVariantPixels - secondVariantPixels; + }) + .pop(); + + return highestQualityHLS; +} + +export default async function nicovideo({ id }) { + try { + const actionTrackId = await getActionTrackId(id); + const highestQualityHLS = await fetchGuestData(actionTrackId) + .then(({ accessRightKey, outputs }) => + fetchContentURL(id, actionTrackId, accessRightKey, outputs) + ) + .then((contentURL) => getHighestQualityHLS(contentURL)); + + return { + urls: highestQualityHLS, + isM3U8: true, + // TODO @synzr get video information from embed page props + filenameAttributes: { service: "niconico", id }, + }; + } catch (error) { + return { error: "ErrorEmptyDownload" }; + } } From 449ee1dac3055d6a5603d2f5f7bfe9ee4b26a24d Mon Sep 17 00:00:00 2001 From: mikhail Date: Tue, 21 May 2024 18:06:23 +0500 Subject: [PATCH 03/13] wip: tested implementation but broken stream TODO @synzr ffmpeg doesn't work for no reason --- src/modules/processing/matchActionDecider.js | 1 + src/modules/processing/services/nicovideo.js | 37 +++++++++----------- src/modules/stream/manage.js | 2 +- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/modules/processing/matchActionDecider.js b/src/modules/processing/matchActionDecider.js index f54bb3f56..389f2f309 100644 --- a/src/modules/processing/matchActionDecider.js +++ b/src/modules/processing/matchActionDecider.js @@ -85,6 +85,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di case "video": switch (host) { case "bilibili": + case "nicovideo": params = { type: "render" }; break; case "youtube": diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index 554912ac7..d7179a983 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -2,9 +2,15 @@ import { genericUserAgent } from "../../config.js"; import HLS from "hls-parser"; import util from "node:util"; +const NICOVIDEO_EMBED_FRONTEND_HEADERS = { + "x-frontend-id": "70", + "x-frontend-version": "0", + "x-niconico-langauge": "ja-jp", + "x-request-with": "https://embed.nicovideo.jp", +}; + const NICOVIDEO_EMBED_URL = "https://embed.nicovideo.jp/watch/%s"; const NICOVIDEO_GUEST_API_URL = - // frontend is embed player "https://www.nicovideo.jp/api/watch/v3_guest/%s?_frontendId=70&_frontendVersion=0&actionTrackId=%s"; const NICOVIDEO_HLS_API_URL = "https://nvapi.nicovideo.jp/v1/watch/%s/access-rights/hls?actionTrackId=%s"; @@ -12,7 +18,6 @@ const NICOVIDEO_HLS_API_URL = const ACTION_TRACK_ID_REGEXP = /"actionTrackId":"[A-Za-z0-9]+_[0-9]+"/; -// working async function getActionTrackId(id) { const page = await fetch(util.format(NICOVIDEO_EMBED_URL, id), { headers: { "user-agent": genericUserAgent }, @@ -35,7 +40,6 @@ async function getActionTrackId(id) { return actionTrackId; } -// not tested async function fetchGuestData(id, actionTrackId) { const data = await fetch( util.format(NICOVIDEO_GUEST_API_URL, id, actionTrackId), @@ -45,7 +49,6 @@ async function fetchGuestData(id, actionTrackId) { ).then((response) => response.json()); if (data?.meta?.status !== 200) { - console.debug("fetchGuestData():", data) throw new Error(); } @@ -65,7 +68,6 @@ async function fetchGuestData(id, actionTrackId) { }; } -// not tested async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { const data = await fetch( util.format(NICOVIDEO_HLS_API_URL, id, actionTrackId), @@ -75,6 +77,7 @@ async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { "user-agent": genericUserAgent, "content-type": "application/json; charset=utf-8", "x-access-right-key": accessRightKey, + ...NICOVIDEO_EMBED_FRONTEND_HEADERS, }, body: JSON.stringify({ outputs }), } @@ -84,43 +87,37 @@ async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { throw new Error(); } - return data.data.contentURL; + return data.data.contentUrl; } -// not tested async function getHighestQualityHLS(contentURL) { const hls = await fetch(contentURL) .then((response) => response.text()) .then((response) => HLS.parse(response)); const highestQualityHLS = hls.variants - .sort((firstVariant, secondVariant) => { - const firstVariantPixels = - firstVariant.resolution.width * firstVariant.resolution.height; - const secondVariantPixels = - secondVariant.resolution.width * secondVariant.resolution.height; - - return firstVariantPixels - secondVariantPixels; - }) + .sort( + (firstVariant, secondVariant) => + firstVariant.bandwidth - secondVariant.bandwidth + ) .pop(); - return highestQualityHLS; + return [highestQualityHLS.uri, highestQualityHLS.audio.pop().uri]; } export default async function nicovideo({ id }) { try { const actionTrackId = await getActionTrackId(id); - const highestQualityHLS = await fetchGuestData(actionTrackId) + const [video, audio] = await fetchGuestData(id, actionTrackId) .then(({ accessRightKey, outputs }) => fetchContentURL(id, actionTrackId, accessRightKey, outputs) ) .then((contentURL) => getHighestQualityHLS(contentURL)); return { - urls: highestQualityHLS, - isM3U8: true, + urls: [video, audio], // TODO @synzr get video information from embed page props - filenameAttributes: { service: "niconico", id }, + filenameAttributes: { service: "nicovideo", id, extension: "mp4" }, }; } catch (error) { return { error: "ErrorEmptyDownload" }; diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 6c9e9d3c8..4702dbdac 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -9,7 +9,7 @@ import { strict as assert } from "assert"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); -const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube']; +const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube', 'nicovideo']; const streamCache = new NodeCache({ stdTTL: env.streamLifespan, From cadbeefd09732665ff0d734b79e3596189666ac6 Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 14:28:48 +0500 Subject: [PATCH 04/13] wip: two m3u services filter -> isM3UService function --- src/modules/stream/manage.js | 5 ++--- src/modules/stream/shared.js | 8 +++++++- src/modules/stream/types.js | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/modules/stream/manage.js b/src/modules/stream/manage.js index 4702dbdac..c051a9663 100644 --- a/src/modules/stream/manage.js +++ b/src/modules/stream/manage.js @@ -5,12 +5,11 @@ import { nanoid } from "nanoid"; import { decryptStream, encryptStream, generateHmac } from "../sub/crypto.js"; import { env } from "../config.js"; import { strict as assert } from "assert"; +import { isM3UService } from "./shared.js"; // optional dependency const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); -const M3U_SERVICES = ['dailymotion', 'vimeo', 'rutube', 'nicovideo']; - const streamCache = new NodeCache({ stdTTL: env.streamLifespan, checkperiod: 10, @@ -108,7 +107,7 @@ export function destroyInternalStream(url) { function wrapStream(streamInfo) { /* m3u8 links are currently not supported * for internal streams, skip them */ - if (M3U_SERVICES.includes(streamInfo.service)) { + if (isM3UService(streamInfo.service)) { return streamInfo; } diff --git a/src/modules/stream/shared.js b/src/modules/stream/shared.js index 42df2758c..cdd0edb1e 100644 --- a/src/modules/stream/shared.js +++ b/src/modules/stream/shared.js @@ -29,4 +29,10 @@ export function getHeaders(service) { // Converting all header values to strings return Object.entries({ ...defaultHeaders, ...serviceHeaders[service] }) .reduce((p, [key, val]) => ({ ...p, [key]: String(val) }), {}) -} \ No newline at end of file +} + +const M3U_SERVICES = ["dailymotion", "vimeo", "rutube", "nicovideo"]; + +export function isM3UService(service) { + return M3U_SERVICES.includes(service); +} diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a1db0c606..d5745c61f 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -6,7 +6,7 @@ import { create as contentDisposition } from "content-disposition-header"; import { metadataManager } from "../sub/utils.js"; import { destroyInternalStream } from "./manage.js"; import { env, ffmpegArgs } from "../config.js"; -import { getHeaders, closeResponse } from "./shared.js"; +import { getHeaders, closeResponse, isM3UService } from "./shared.js"; function toRawHeaders(headers) { return Object.entries(headers) @@ -215,7 +215,7 @@ export function streamVideoOnly(streamInfo, res) { args.push('-an') } - if (["vimeo", "rutube", "dailymotion"].includes(streamInfo.service)) { + if (isM3UService(streamInfo.service)) { args.push('-bsf:a', 'aac_adtstoasc') } From 60f9f995689825b9beda6d8cd2aefcefb384465e Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 21:47:32 +0500 Subject: [PATCH 05/13] feat: nicovideo guest implementation --- src/modules/processing/services/nicovideo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index d7179a983..343d7c870 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -58,7 +58,7 @@ async function fetchGuestData(id, actionTrackId) { const { id: audioId } = audios .filter((audio) => audio.isAvailable) .sort((firstAudio, secondAudio) => firstAudio.bitrate - secondAudio.bitrate) - .pop(); + .shift(); return { accessRightKey, From 6524d4d44caa9af70d35dc20e062bd6e2fd87005 Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 22:29:33 +0500 Subject: [PATCH 06/13] improvement: filenameAttributes and quality picker in nicovideo --- src/modules/processing/match.js | 5 +- src/modules/processing/services/nicovideo.js | 91 +++++++++++++------- src/modules/processing/servicesConfig.json | 1 + 3 files changed, 64 insertions(+), 33 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 97ac7a076..a02693fd8 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -187,7 +187,10 @@ export default async function(host, patternMatch, lang, obj) { r = await dailymotion(patternMatch); break; case "nicovideo": - r = await nicovideo({ id: patternMatch.id }); + r = await nicovideo({ + id: patternMatch.id, + quality: obj.vQuality + }); break; default: return createResponse("error", { diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index 343d7c870..52b1fa91e 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -1,4 +1,5 @@ import { genericUserAgent } from "../../config.js"; +import { cleanString } from "../../sub/utils.js"; import HLS from "hls-parser"; import util from "node:util"; @@ -10,34 +11,34 @@ const NICOVIDEO_EMBED_FRONTEND_HEADERS = { }; const NICOVIDEO_EMBED_URL = "https://embed.nicovideo.jp/watch/%s"; +const NICOVIDEO_AUTHOR_DATA_URL = "https://embed.nicovideo.jp/users/%d"; const NICOVIDEO_GUEST_API_URL = "https://www.nicovideo.jp/api/watch/v3_guest/%s?_frontendId=70&_frontendVersion=0&actionTrackId=%s"; const NICOVIDEO_HLS_API_URL = "https://nvapi.nicovideo.jp/v1/watch/%s/access-rights/hls?actionTrackId=%s"; -const ACTION_TRACK_ID_REGEXP = - /"actionTrackId":"[A-Za-z0-9]+_[0-9]+"/; - -async function getActionTrackId(id) { +async function getBasicVideoInformation(id) { const page = await fetch(util.format(NICOVIDEO_EMBED_URL, id), { headers: { "user-agent": genericUserAgent }, }).then((response) => response.text()); - if (!ACTION_TRACK_ID_REGEXP.test(page)) { - throw new Error(); // we can't fetch the embed page - } + const data = JSON.parse( + page + .split('data-props="') + .pop() + .split('" data-style-map="') + .shift() + .replaceAll(""", '"') + ); + + const author = await fetch( + util.format(NICOVIDEO_AUTHOR_DATA_URL, data.videoUploaderId), + { + headers: { "user-agent": genericUserAgent }, + } + ).then((response) => response.json()); - const actionTrackId = page - // getting the regexp results - .match(ACTION_TRACK_ID_REGEXP) - .shift() - // getting the actionTrackId field's value - .split(":"") - .pop() - // cleaning from double quotation mark - .replaceAll(""", ""); - - return actionTrackId; + return { ...data, author }; } async function fetchGuestData(id, actionTrackId) { @@ -90,34 +91,60 @@ async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { return data.data.contentUrl; } -async function getHighestQualityHLS(contentURL) { +async function getHLSContent(contentURL, quality) { const hls = await fetch(contentURL) .then((response) => response.text()) .then((response) => HLS.parse(response)); - const highestQualityHLS = hls.variants - .sort( - (firstVariant, secondVariant) => - firstVariant.bandwidth - secondVariant.bandwidth - ) - .pop(); + const height = quality === "max" ? 9000 : parseInt(quality, 10); + let hlsContent = hls.variants.find( + (variant) => variant.resolution.height === height + ); + + if (hlsContent === undefined) { + hlsContent = hls.variants + .sort( + (firstVariant, secondVariant) => + firstVariant.bandwidth - secondVariant.bandwidth + ) + .shift(); + } - return [highestQualityHLS.uri, highestQualityHLS.audio.pop().uri]; + return { + resolution: hlsContent.resolution, + urls: [hlsContent.uri, hlsContent.audio.pop().uri], + }; } -export default async function nicovideo({ id }) { +// TODO @synzr only audio support +// TODO @synzr better error handling +export default async function nicovideo({ id, quality }) { try { - const actionTrackId = await getActionTrackId(id); - const [video, audio] = await fetchGuestData(id, actionTrackId) + const { actionTrackId, title, author } = await getBasicVideoInformation(id); + const { + resolution, + urls: [video, audio], + } = await fetchGuestData(id, actionTrackId) .then(({ accessRightKey, outputs }) => fetchContentURL(id, actionTrackId, accessRightKey, outputs) ) - .then((contentURL) => getHighestQualityHLS(contentURL)); + .then((contentURL) => getHLSContent(contentURL, quality)); return { urls: [video, audio], - // TODO @synzr get video information from embed page props - filenameAttributes: { service: "nicovideo", id, extension: "mp4" }, + filenameAttributes: { + service: "nicovideo", + id, + title, + author: author.nickname, + resolution: `${resolution.width}x${resolution.height}`, + qualityLabel: `${resolution.height}p`, + extension: "mp4", + }, + fileMetadata: { + title: cleanString(title.trim()), + artist: cleanString(author.nickname.trim()), + }, }; } catch (error) { return { error: "ErrorEmptyDownload" }; diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 65e7d8647..7bc302a79 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -115,6 +115,7 @@ "alias": "niconico videos", "tld": "jp", "patterns": ["watch/:id"], + "subdomains": ["www", "sp", "embed"], "enabled": true } } From f956af06ed2a330746006094b0a7c5012bf89557 Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 22:47:18 +0500 Subject: [PATCH 07/13] improvement: isAudioOnly support in nicovideo --- src/modules/processing/match.js | 3 +- src/modules/processing/services/nicovideo.js | 47 +++++++++++--------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index a02693fd8..eab768e24 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -189,7 +189,8 @@ export default async function(host, patternMatch, lang, obj) { case "nicovideo": r = await nicovideo({ id: patternMatch.id, - quality: obj.vQuality + quality: obj.vQuality, + isAudioOnly: isAudioOnly, }); break; default: diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index 52b1fa91e..5196982fc 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -91,7 +91,7 @@ async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { return data.data.contentUrl; } -async function getHLSContent(contentURL, quality) { +async function getHLSContent(contentURL, quality, isAudioOnly) { const hls = await fetch(contentURL) .then((response) => response.text()) .then((response) => HLS.parse(response)); @@ -110,41 +110,48 @@ async function getHLSContent(contentURL, quality) { .shift(); } - return { - resolution: hlsContent.resolution, - urls: [hlsContent.uri, hlsContent.audio.pop().uri], - }; + const audioUrl = hlsContent.audio.pop().uri; + return isAudioOnly + ? { resolution: null, urls: audioUrl, type: "audio" } + : { + resolution: hlsContent.resolution, + urls: [hlsContent.uri, audioUrl], + type: "video", + }; } -// TODO @synzr only audio support // TODO @synzr better error handling -export default async function nicovideo({ id, quality }) { +export default async function nicovideo({ id, quality, isAudioOnly }) { try { const { actionTrackId, title, author } = await getBasicVideoInformation(id); - const { - resolution, - urls: [video, audio], - } = await fetchGuestData(id, actionTrackId) + const { resolution, urls, type } = await fetchGuestData(id, actionTrackId) .then(({ accessRightKey, outputs }) => fetchContentURL(id, actionTrackId, accessRightKey, outputs) ) - .then((contentURL) => getHLSContent(contentURL, quality)); + .then((contentURL) => getHLSContent(contentURL, quality, isAudioOnly)); return { - urls: [video, audio], + urls, + isAudioOnly: type === "audio", + fileMetadata: { + title: cleanString(title.trim()), + artist: cleanString(author.nickname.trim()), + }, + // bible accurate object concatenation filenameAttributes: { service: "nicovideo", id, title, author: author.nickname, - resolution: `${resolution.width}x${resolution.height}`, - qualityLabel: `${resolution.height}p`, - extension: "mp4", - }, - fileMetadata: { - title: cleanString(title.trim()), - artist: cleanString(author.nickname.trim()), + ...(type === "video" + ? { + extension: "mp4", + qualityLabel: `${resolution.height}p`, + resolution: `${resolution.width}x${resolution.height}`, + } + : {}), }, + ...(type === "audio" ? { isM3U8: true, bestAudio: "mp3" } : {}), }; } catch (error) { return { error: "ErrorEmptyDownload" }; From 2d8bb1d103bbde9e4ba7e0013b0dfd27e92920bb Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 22:54:01 +0500 Subject: [PATCH 08/13] improvement: better error handling in nicovideo --- src/modules/processing/services/nicovideo.js | 37 +++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index 5196982fc..1cd22f9c7 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -17,10 +17,20 @@ const NICOVIDEO_GUEST_API_URL = const NICOVIDEO_HLS_API_URL = "https://nvapi.nicovideo.jp/v1/watch/%s/access-rights/hls?actionTrackId=%s"; +class CobaltError extends Error { + constructor(localizatedMessage) { + this.localizatedMessage = localizatedMessage; + } +} + async function getBasicVideoInformation(id) { const page = await fetch(util.format(NICOVIDEO_EMBED_URL, id), { headers: { "user-agent": genericUserAgent }, - }).then((response) => response.text()); + }) + .then((response) => response.text()) + .catch(() => { + throw new CobaltError("ErrorCouldntFetch"); + }); const data = JSON.parse( page @@ -36,7 +46,11 @@ async function getBasicVideoInformation(id) { { headers: { "user-agent": genericUserAgent }, } - ).then((response) => response.json()); + ) + .then((response) => response.json()) + .catch(() => { + throw new CobaltError("ErrorCouldntFetch"); + }); return { ...data, author }; } @@ -47,10 +61,14 @@ async function fetchGuestData(id, actionTrackId) { { headers: { "user-agent": genericUserAgent }, } - ).then((response) => response.json()); + ) + .then((response) => response.json()) + .catch(() => { + throw new CobaltError("ErrorCouldntFetch"); + }); if (data?.meta?.status !== 200) { - throw new Error(); + throw new CobaltError("ErrorBadFetch"); } const { videos, audios, accessRightKey } = data.data.media.domand; @@ -82,10 +100,14 @@ async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { }, body: JSON.stringify({ outputs }), } - ).then((response) => response.json()); + ) + .then((response) => response.json()) + .catch(() => { + throw new CobaltError("ErrorCouldntFetch"); + }); if (data?.meta?.status !== 201) { - throw new Error(); + throw new CobaltError("ErrorBadFetch"); } return data.data.contentUrl; @@ -120,7 +142,6 @@ async function getHLSContent(contentURL, quality, isAudioOnly) { }; } -// TODO @synzr better error handling export default async function nicovideo({ id, quality, isAudioOnly }) { try { const { actionTrackId, title, author } = await getBasicVideoInformation(id); @@ -154,6 +175,6 @@ export default async function nicovideo({ id, quality, isAudioOnly }) { ...(type === "audio" ? { isM3U8: true, bestAudio: "mp3" } : {}), }; } catch (error) { - return { error: "ErrorEmptyDownload" }; + return { error: error.localizatedMessage ?? "ErrorSomethingWentWrong" }; } } From d39fd063877e807eeab55fc61b24bd5303a2b6bc Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 22:56:47 +0500 Subject: [PATCH 09/13] fix: video length check in nicovideo --- src/modules/processing/services/nicovideo.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index 1cd22f9c7..9548581d1 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -1,4 +1,4 @@ -import { genericUserAgent } from "../../config.js"; +import { env, genericUserAgent } from "../../config.js"; import { cleanString } from "../../sub/utils.js"; import HLS from "hls-parser"; import util from "node:util"; @@ -144,7 +144,12 @@ async function getHLSContent(contentURL, quality, isAudioOnly) { export default async function nicovideo({ id, quality, isAudioOnly }) { try { - const { actionTrackId, title, author } = await getBasicVideoInformation(id); + const { actionTrackId, title, author, lengthInSeconds } = await getBasicVideoInformation(id); + + if (lengthInSeconds > env.durationLimit) { + throw new CobaltError(['ErrorLengthLimit', env.durationLimit / 60]); + } + const { resolution, urls, type } = await fetchGuestData(id, actionTrackId) .then(({ accessRightKey, outputs }) => fetchContentURL(id, actionTrackId, accessRightKey, outputs) From 9a5ff35eb79d26017c2f23d545fd97b7f78d3e9b Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 23:27:57 +0500 Subject: [PATCH 10/13] improvement: isAudioMuted support in nicovideo --- src/modules/processing/match.js | 1 + src/modules/processing/services/nicovideo.js | 34 +++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index eab768e24..8ed29e3a1 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -191,6 +191,7 @@ export default async function(host, patternMatch, lang, obj) { id: patternMatch.id, quality: obj.vQuality, isAudioOnly: isAudioOnly, + isAudioMuted: obj.isAudioMuted, }); break; default: diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index 9548581d1..92f1dee24 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -18,8 +18,9 @@ const NICOVIDEO_HLS_API_URL = "https://nvapi.nicovideo.jp/v1/watch/%s/access-rights/hls?actionTrackId=%s"; class CobaltError extends Error { - constructor(localizatedMessage) { - this.localizatedMessage = localizatedMessage; + constructor(locMessage) { + super(); // gdsfkgjsoiredgjhredszdfpijgkertoindsuf + this.locMessage = locMessage; } } @@ -113,7 +114,7 @@ async function fetchContentURL(id, actionTrackId, accessRightKey, outputs) { return data.data.contentUrl; } -async function getHLSContent(contentURL, quality, isAudioOnly) { +async function getHLSContent(contentURL, quality, isAudioOnly, isAudioMuted) { const hls = await fetch(contentURL) .then((response) => response.text()) .then((response) => HLS.parse(response)); @@ -137,31 +138,41 @@ async function getHLSContent(contentURL, quality, isAudioOnly) { ? { resolution: null, urls: audioUrl, type: "audio" } : { resolution: hlsContent.resolution, - urls: [hlsContent.uri, audioUrl], + urls: isAudioMuted ? hlsContent.uri : [hlsContent.uri, audioUrl], type: "video", }; } -export default async function nicovideo({ id, quality, isAudioOnly }) { +export default async function nicovideo({ + id, + quality, + isAudioOnly, + isAudioMuted, +}) { try { - const { actionTrackId, title, author, lengthInSeconds } = await getBasicVideoInformation(id); + const { actionTrackId, title, author, lengthInSeconds } = + await getBasicVideoInformation(id); if (lengthInSeconds > env.durationLimit) { - throw new CobaltError(['ErrorLengthLimit', env.durationLimit / 60]); + throw new CobaltError(["ErrorLengthLimit", env.durationLimit / 60]); } const { resolution, urls, type } = await fetchGuestData(id, actionTrackId) .then(({ accessRightKey, outputs }) => fetchContentURL(id, actionTrackId, accessRightKey, outputs) ) - .then((contentURL) => getHLSContent(contentURL, quality, isAudioOnly)); + .then((contentURL) => + getHLSContent(contentURL, quality, isAudioOnly, isAudioMuted) + ); return { urls, isAudioOnly: type === "audio", fileMetadata: { title: cleanString(title.trim()), - artist: cleanString(author.nickname.trim()), + artist: author.nickname + ? cleanString(author.nickname.trim()) + : undefined, }, // bible accurate object concatenation filenameAttributes: { @@ -177,9 +188,10 @@ export default async function nicovideo({ id, quality, isAudioOnly }) { } : {}), }, - ...(type === "audio" ? { isM3U8: true, bestAudio: "mp3" } : {}), + ...(type === "audio" || typeof urls === "string" ? { isM3U8: true } : {}), + ...(type === "audio" ? { bestAudio: "mp3" } : {}), }; } catch (error) { - return { error: error.localizatedMessage ?? "ErrorSomethingWentWrong" }; + return { error: error.locMessage ?? "ErrorSomethingWentWrong" }; } } From c23e8649551bef74b3446c665037ce5970f7658f Mon Sep 17 00:00:00 2001 From: mikhail Date: Wed, 22 May 2024 23:28:18 +0500 Subject: [PATCH 11/13] test: nicovideo added in tests.json --- src/test/tests.json | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/test/tests.json b/src/test/tests.json index 565fa5a47..95542b1f1 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -531,6 +531,45 @@ "status": "error" } }], + "nicovideo": [{ + "name": "only 360p video", + "url": "https://www.nicovideo.jp/watch/sm13876659", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "only 360p video muted", + "url": "https://www.nicovideo.jp/watch/sm7055008", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "niconico video converted to audio", + "url": "https://www.nicovideo.jp/watch/sm6287843", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "max 720p video in 480p", + "url": "https://www.nicovideo.jp/watch/sm43808466", + "params": { + "vQuality": "480" + }, + "expected": { + "code": 200, + "status": "stream" + } + }], "bilibili": [{ "name": "1080p video", "url": "https://www.bilibili.com/video/BV18i4y1m7xV/", From 4124c503c46d25d6276076c36eb2e8d49e11f19d Mon Sep 17 00:00:00 2001 From: mikhail Date: Thu, 23 May 2024 00:18:03 +0500 Subject: [PATCH 12/13] improvement: sort().shift() -> reduce() in nicovideo --- src/modules/processing/services/nicovideo.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index 92f1dee24..dac568393 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -77,8 +77,9 @@ async function fetchGuestData(id, actionTrackId) { // getting the HQ audio const { id: audioId } = audios .filter((audio) => audio.isAvailable) - .sort((firstAudio, secondAudio) => firstAudio.bitrate - secondAudio.bitrate) - .shift(); + .reduce((firstAudio, secondAudio) => + firstAudio.bitRate > secondAudio.bitRate ? firstAudio : secondAudio + ); return { accessRightKey, @@ -125,12 +126,11 @@ async function getHLSContent(contentURL, quality, isAudioOnly, isAudioMuted) { ); if (hlsContent === undefined) { - hlsContent = hls.variants - .sort( - (firstVariant, secondVariant) => - firstVariant.bandwidth - secondVariant.bandwidth - ) - .shift(); + hlsContent = hls.variants.reduce((firstVariant, secondVariant) => + firstVariant.bandwidth > secondVariant.bandwidth + ? firstVariant + : secondVariant + ); } const audioUrl = hlsContent.audio.pop().uri; From f8dd35c62dcb65be08157fff136897389a25fe29 Mon Sep 17 00:00:00 2001 From: mikhail Date: Sun, 26 May 2024 23:48:06 +0500 Subject: [PATCH 13/13] improvement: better expression for isM3U8 in nicovideo --- src/modules/processing/services/nicovideo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/processing/services/nicovideo.js b/src/modules/processing/services/nicovideo.js index dac568393..9a28a4dca 100644 --- a/src/modules/processing/services/nicovideo.js +++ b/src/modules/processing/services/nicovideo.js @@ -188,7 +188,7 @@ export default async function nicovideo({ } : {}), }, - ...(type === "audio" || typeof urls === "string" ? { isM3U8: true } : {}), + ...(typeof urls === "string" ? { isM3U8: true } : {}), ...(type === "audio" ? { bestAudio: "mp3" } : {}), }; } catch (error) {