diff --git a/.github/workflows/test-services.yml b/.github/workflows/test-services.yml index ad6e25b7d..c722084e6 100644 --- a/.github/workflows/test-services.yml +++ b/.github/workflows/test-services.yml @@ -30,7 +30,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - - run: pnpm i --frozen-lockfile && node api/src/util/test run-tests-for ${{ matrix.service }} + - run: pnpm i --frozen-lockfile + - run: node api/src/util/test run-tests-for ${{ matrix.service }} env: - API_EXTERNAL_PROXY: ${{ secrets.API_EXTERNAL_PROXY }} + HTTP_PROXY: ${{ secrets.API_EXTERNAL_PROXY }} TEST_IGNORE_SERVICES: ${{ vars.TEST_IGNORE_SERVICES }} diff --git a/README.md b/README.md index 795eb7e30..1014b598e 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,20 @@ cobalt is a media downloader that doesn't piss you off. it's friendly, efficient paste the link, get the file, move on. that simple, just how it should be. +### sponsors +
+ special thanks to Warp for sponsoring the development of cobalt +
+ + Warp banner + + +### [Warp, built for coding with multiple AI agents](https://go.warp.dev/cobalt) +
+ +#### RoyaleHosting +cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), and a part of our infrastructure is hosted on their network. we really appreciate their kindness and support! + ### cobalt monorepo this monorepo includes source code for api, frontend, and related packages: - [api tree & readme](/api/) @@ -53,9 +67,6 @@ same content can be downloaded via dev tools of any modern web browser. ### contributing if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away. -### thank you -cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support! - ### licenses for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs. unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE). diff --git a/api/README.md b/api/README.md index 36c1dc89d..3bdfb5190 100644 --- a/api/README.md +++ b/api/README.md @@ -23,6 +23,7 @@ if the desired service isn't supported yet, feel free to create an appropriate i | instagram | ✅ | ✅ | ✅ | ➖ | ➖ | | facebook | ✅ | ❌ | ✅ | ➖ | ➖ | | loom | ✅ | ❌ | ✅ | ✅ | ➖ | +| newgrounds | ✅ | ✅ | ✅ | ✅ | ✅ | | ok.ru | ✅ | ❌ | ✅ | ✅ | ✅ | | pinterest | ✅ | ✅ | ✅ | ➖ | ➖ | | reddit | ✅ | ✅ | ✅ | ❌ | ❌ | diff --git a/api/package.json b/api/package.json index 1611247d2..664ffbd72 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "11.2", + "version": "11.5", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -37,9 +37,9 @@ "mime": "^4.0.4", "nanoid": "^5.0.9", "set-cookie-parser": "2.6.0", - "undici": "^5.19.1", + "undici": "^6.21.3", "url-pattern": "1.0.3", - "youtubei.js": "^14.0.0", + "youtubei.js": "15.1.1", "zod": "^3.23.8" }, "optionalDependencies": { diff --git a/api/src/config.js b/api/src/config.js index 8f1579b8b..2d8ac7a08 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -3,12 +3,12 @@ import { loadEnvs, validateEnvs } from "./core/env.js"; const version = await getVersion(); +const canonicalEnv = Object.freeze(structuredClone(process.env)); const env = loadEnvs(); -const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; +const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"; const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`; -export const canonicalEnv = Object.freeze(structuredClone(process.env)); export const setTunnelPort = (port) => env.tunnelPort = port; export const isCluster = env.instanceCount > 1; export const updateEnv = (newEnv) => { @@ -18,6 +18,10 @@ export const updateEnv = (newEnv) => { newEnv.tunnelPort = env.tunnelPort; for (const key in env) { + if (key === 'subscribe') { + continue; + } + if (String(env[key]) !== String(newEnv[key])) { changes.push(key); } @@ -31,6 +35,7 @@ await validateEnvs(env); export { env, + canonicalEnv, genericUserAgent, cobaltUserAgent, } diff --git a/api/src/core/api.js b/api/src/core/api.js index 248f93579..d1f6545ac 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -1,7 +1,7 @@ import cors from "cors"; import http from "node:http"; import rateLimit from "express-rate-limit"; -import { setGlobalDispatcher, ProxyAgent } from "undici"; +import { setGlobalDispatcher, EnvHttpProxyAgent } from "undici"; import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info"; import jwt from "../security/jwt.js"; @@ -337,9 +337,17 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { randomizeCiphers(); setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes - if (env.externalProxy) { - setGlobalDispatcher(new ProxyAgent(env.externalProxy)) - } + env.subscribe(['externalProxy', 'httpProxyValues'], () => { + // TODO: remove env.externalProxy in a future version + const options = {}; + if (env.externalProxy) { + options.httpProxy = env.externalProxy; + } + + setGlobalDispatcher( + new EnvHttpProxyAgent(options) + ); + }); http.createServer(app).listen({ port: env.apiPort, diff --git a/api/src/core/env.js b/api/src/core/env.js index f26b3986e..e504a8dce 100644 --- a/api/src/core/env.js +++ b/api/src/core/env.js @@ -10,6 +10,34 @@ import { Green, Yellow } from "../misc/console-text.js"; const forceLocalProcessingOptions = ["never", "session", "always"]; const youtubeHlsOptions = ["never", "key", "always"]; +const httpProxyVariables = ["NO_PROXY", "HTTP_PROXY", "HTTPS_PROXY"].flatMap( + k => [ k, k.toLowerCase() ] +); + +const changeCallbacks = {}; + +const onEnvChanged = (changes) => { + for (const key of changes) { + if (changeCallbacks[key]) { + changeCallbacks[key].map(fn => { + try { fn() } catch {} + }); + } + } +} + +const subscribe = (keys, fn) => { + keys = [keys].flat(); + + for (const key of keys) { + if (key in currentEnv && key !== 'subscribe') { + changeCallbacks[key] ??= []; + changeCallbacks[key].push(fn); + fn(); + } else throw `invalid env key ${key}`; + } +} + export const loadEnvs = (env = process.env) => { const allServices = new Set(Object.keys(services)); const disabledServices = env.DISABLED_SERVICES?.split(',') || []; @@ -19,6 +47,18 @@ export const loadEnvs = (env = process.env) => { } })); + // we need to copy the proxy envs (HTTP_PROXY, HTTPS_PROXY) + // back into process.env, so that EnvHttpProxyAgent can pick + // them up later + for (const key of httpProxyVariables) { + const value = env[key] ?? canonicalEnv[key]; + if (value !== undefined) { + process.env[key] = env[key]; + } else { + delete process.env[key]; + } + } + return { apiURL: env.API_URL || '', apiPort: env.API_PORT || 9000, @@ -55,6 +95,9 @@ export const loadEnvs = (env = process.env) => { externalProxy: env.API_EXTERNAL_PROXY, + // used only for comparing against old values when envs are being updated + httpProxyValues: httpProxyVariables.map(k => String(env[k])).join(''), + turnstileSitekey: env.TURNSTILE_SITEKEY, turnstileSecret: env.TURNSTILE_SECRET, jwtSecret: env.JWT_SECRET, @@ -87,9 +130,13 @@ export const loadEnvs = (env = process.env) => { envFile: env.API_ENV_FILE, envRemoteReloadInterval: 300, + + subscribe, }; } +let loggedProxyWarning = false; + export const validateEnvs = async (env) => { if (env.sessionEnabled && env.jwtSecret.length < 16) { throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)"); @@ -127,6 +174,15 @@ export const validateEnvs = async (env) => { throw new Error('freebind is not available when external proxy is enabled') } + if (env.externalProxy && !loggedProxyWarning) { + console.error('API_EXTERNAL_PROXY is deprecated and will be removed in a future release.'); + console.error('Use HTTP_PROXY or HTTPS_PROXY instead.'); + console.error('You can read more about the new proxy variables in docs/api-env-variables.md\n'); + + // prevent the warning from being printed on every env validation + loggedProxyWarning = true; + } + return env; } @@ -170,11 +226,14 @@ const wrapReload = (contents) => { return; } + onEnvChanged(changes); + console.log(`${Green('[✓]')} envs reloaded successfully!`); for (const key of changes) { const value = currentEnv[key]; const isSecret = key.toLowerCase().includes('apikey') - || key.toLowerCase().includes('secret'); + || key.toLowerCase().includes('secret') + || key === 'httpProxyValues'; if (!value) { console.log(` removed: ${key}`); diff --git a/api/src/misc/language-codes.js b/api/src/misc/language-codes.js index c18006b58..1f6926015 100644 --- a/api/src/misc/language-codes.js +++ b/api/src/misc/language-codes.js @@ -49,5 +49,6 @@ const maps = { } export const convertLanguageCode = (code) => { + code = code?.split("-")[0]?.split("_")[0] || ""; return maps[code.length]?.[code.toLowerCase()] || null; } diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 1cde3cdc7..82fe40a78 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,4 +1,4 @@ -import { request } from 'undici'; +import { request } from "undici"; const redirectStatuses = new Set([301, 302, 303, 307, 308]); export async function getRedirectingURL(url, dispatcher, headers) { @@ -8,18 +8,34 @@ export async function getRedirectingURL(url, dispatcher, headers) { headers, redirect: 'manual' }; + const getParams = { + ...params, + method: 'GET', + }; - let location = await request(url, params).then(r => { + const callback = (r) => { if (redirectStatuses.has(r.statusCode) && r.headers['location']) { return r.headers['location']; } - }).catch(() => null); + } - location ??= await fetch(url, params).then(r => { - if (redirectStatuses.has(r.status) && r.headers.has('location')) { - return r.headers.get('location'); - } - }).catch(() => null); + /* + try request() with HEAD & GET, + then do the same with fetch + (fetch is required for shortened reddit links) + */ + + let location = await request(url, params) + .then(callback).catch(() => null); + + location ??= await request(url, getParams) + .then(callback).catch(() => null); + + location ??= await fetch(url, params) + .then(callback).catch(() => null); + + location ??= await fetch(url, getParams) + .then(callback).catch(() => null); return location; } diff --git a/api/src/processing/cookie/manager.js b/api/src/processing/cookie/manager.js index 9e23374b9..cad11f77f 100644 --- a/api/src/processing/cookie/manager.js +++ b/api/src/processing/cookie/manager.js @@ -13,6 +13,7 @@ const VALID_SERVICES = new Set([ 'reddit', 'twitter', 'youtube', + 'vimeo_bearer', ]); const invalidCookies = {}; diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index 555705e86..5852b19d7 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -180,6 +180,7 @@ export default function({ case "ok": case "xiaohongshu": + case "newgrounds": params = { type: "proxy" }; break; @@ -263,8 +264,9 @@ export default function({ // extractors usually return ISO 639-1 language codes, // but video players expect ISO 639-2, so we convert them here - if (defaultParams.fileMetadata?.sublanguage?.length === 2) { - const code = convertLanguageCode(defaultParams.fileMetadata.sublanguage); + const sublanguage = defaultParams.fileMetadata?.sublanguage; + if (sublanguage && sublanguage.length !== 3) { + const code = convertLanguageCode(sublanguage); if (code) { defaultParams.fileMetadata.sublanguage = code; } else { diff --git a/api/src/processing/match.js b/api/src/processing/match.js index 360ed532a..1265297cd 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -29,6 +29,7 @@ import loom from "./services/loom.js"; import facebook from "./services/facebook.js"; import bluesky from "./services/bluesky.js"; import xiaohongshu from "./services/xiaohongshu.js"; +import newgrounds from "./services/newgrounds.js"; let freebind; @@ -268,6 +269,13 @@ export default async function({ host, patternMatch, params, authType }) { }); break; + case "newgrounds": + r = await newgrounds({ + ...patternMatch, + quality: params.videoQuality, + }); + break; + default: return createResponse("error", { code: "error.api.service.unsupported" @@ -314,7 +322,7 @@ export default async function({ host, patternMatch, params, authType }) { let localProcessing = params.localProcessing; const lpEnv = env.forceLocalProcessing; const shouldForceLocal = lpEnv === "always" || (lpEnv === "session" && authType === "session"); - const localDisabled = (!localProcessing || localProcessing === "none"); + const localDisabled = (!localProcessing || localProcessing === "disabled"); if (shouldForceLocal && localDisabled) { localProcessing = "preferred"; diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 3ffcf10a4..0a35838bd 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -7,6 +7,7 @@ export const services = { bilibili: { patterns: [ "video/:comId", + "video/:comId?p=:partId", "_shortLink/:comShortLink", "_tv/:lang/video/:tvId", "_tv/video/:tvId" @@ -74,6 +75,12 @@ export const services = { "url_shortener/:shortLink" ], }, + newgrounds: { + patterns: [ + "portal/view/:id", + "audio/listen/:audioId", + ] + }, reddit: { patterns: [ "comments/:id", @@ -159,6 +166,7 @@ export const services = { twitch: { patterns: [":channel/clip/:clip"], tld: "tv", + subdomains: ["clips", "www", "m"], }, twitter: { patterns: [ @@ -201,7 +209,7 @@ export const services = { patterns: [ "explore/:id?xsec_token=:token", "discovery/item/:id?xsec_token=:token", - "a/:shareId" + ":shareType/:shareId", ], altDomains: ["xhslink.com"], }, diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index fd6daef96..6dc3ccbd0 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -1,53 +1,72 @@ export const testers = { "bilibili": pattern => - pattern.comId?.length <= 12 || pattern.comShortLink?.length <= 16 - || pattern.tvId?.length <= 24, + (pattern.comId?.length <= 12 && pattern.partId?.length <= 3) || + (pattern.comId?.length <= 12 && !pattern.partId) || + pattern.comShortLink?.length <= 16 || + pattern.tvId?.length <= 24, + + "bsky": pattern => + pattern.user?.length <= 128 && pattern.post?.length <= 128, "dailymotion": pattern => pattern.id?.length <= 32, + "facebook": pattern => + pattern.shortLink?.length <= 11 || + pattern.username?.length <= 30 || + pattern.caption?.length <= 255 || + pattern.id?.length <= 20 && !pattern.shareType || + pattern.id?.length <= 20 && pattern.shareType?.length === 1, + "instagram": pattern => - pattern.postId?.length <= 48 - || pattern.shareId?.length <= 16 - || (pattern.username?.length <= 30 && pattern.storyId?.length <= 24), + pattern.postId?.length <= 48 || + pattern.shareId?.length <= 16 || + (pattern.username?.length <= 30 && pattern.storyId?.length <= 24), "loom": pattern => pattern.id?.length <= 32, + "newgrounds": pattern => + pattern.id?.length <= 12 || + pattern.audioId?.length <= 12, + "ok": pattern => pattern.id?.length <= 16, "pinterest": pattern => - pattern.id?.length <= 128 || pattern.shortLink?.length <= 32, + pattern.id?.length <= 128 || + pattern.shortLink?.length <= 32, "reddit": pattern => - pattern.id?.length <= 16 && !pattern.sub && !pattern.user - || (pattern.sub?.length <= 22 && pattern.id?.length <= 16) - || (pattern.user?.length <= 22 && pattern.id?.length <= 16) - || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16) - || (pattern.shortId?.length <= 16), + pattern.id?.length <= 16 && !pattern.sub && !pattern.user || + (pattern.sub?.length <= 22 && pattern.id?.length <= 16) || + (pattern.user?.length <= 22 && pattern.id?.length <= 16) || + (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16) || + (pattern.shortId?.length <= 16), "rutube": pattern => (pattern.id?.length === 32 && pattern.key?.length <= 32) || - pattern.id?.length === 32 || pattern.yappyId?.length === 32, - - "soundcloud": pattern => - (pattern.author?.length <= 255 && pattern.song?.length <= 255) - || pattern.shortLink?.length <= 32, + pattern.id?.length === 32 || + pattern.yappyId?.length === 32, "snapchat": pattern => - (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255)) - || pattern.spotlightId?.length <= 255 - || pattern.shortLink?.length <= 16, + (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255)) || + pattern.spotlightId?.length <= 255 || + pattern.shortLink?.length <= 16, + + "soundcloud": pattern => + (pattern.author?.length <= 255 && pattern.song?.length <= 255) || + pattern.shortLink?.length <= 32, "streamable": pattern => pattern.id?.length <= 6, "tiktok": pattern => - pattern.postId?.length <= 21 || pattern.shortLink?.length <= 21, + pattern.postId?.length <= 21 || + pattern.shortLink?.length <= 21, "tumblr": pattern => - pattern.id?.length < 21 - || (pattern.id?.length < 21 && pattern.user?.length <= 32), + pattern.id?.length < 21 || + (pattern.id?.length < 21 && pattern.user?.length <= 32), "twitch": pattern => pattern.channel && pattern.clip?.length <= 100, @@ -56,27 +75,16 @@ export const testers = { pattern.id?.length < 20, "vimeo": pattern => - pattern.id?.length <= 11 - && (!pattern.password || pattern.password.length < 16), + pattern.id?.length <= 11 && (!pattern.password || pattern.password.length < 16), "vk": pattern => (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) || (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18), + "xiaohongshu": pattern => + pattern.id?.length <= 24 && pattern.token?.length <= 64 || + pattern.shareId?.length <= 24 && pattern.shareType?.length === 1, + "youtube": pattern => pattern.id?.length <= 11, - - "facebook": pattern => - pattern.shortLink?.length <= 11 - || pattern.username?.length <= 30 - || pattern.caption?.length <= 255 - || pattern.id?.length <= 20 && !pattern.shareType - || pattern.id?.length <= 20 && pattern.shareType?.length === 1, - - "bsky": pattern => - pattern.user?.length <= 128 && pattern.post?.length <= 128, - - "xiaohongshu": pattern => - pattern.id?.length <= 24 && pattern.token?.length <= 64 - || pattern.shareId?.length <= 24, } diff --git a/api/src/processing/services/bilibili.js b/api/src/processing/services/bilibili.js index 8747a7812..297c5f9f2 100644 --- a/api/src/processing/services/bilibili.js +++ b/api/src/processing/services/bilibili.js @@ -17,8 +17,14 @@ function extractBestQuality(dashData) { return [ bestVideo, bestAudio ]; } -async function com_download(id) { - let html = await fetch(`https://bilibili.com/video/${id}`, { +async function com_download(id, partId) { + const url = new URL(`https://bilibili.com/video/${id}`); + + if (partId) { + url.searchParams.set('p', partId); + } + + const html = await fetch(url, { headers: { "user-agent": genericUserAgent } @@ -34,7 +40,10 @@ async function com_download(id) { return { error: "fetch.empty" }; } - let streamData = JSON.parse(html.split('')[0]); + const streamData = JSON.parse( + html.split('')[0] + ); + if (streamData.data.timelength > env.durationLimit * 1000) { return { error: "content.too_long" }; } @@ -44,11 +53,15 @@ async function com_download(id) { return { error: "fetch.empty" }; } + let filenameBase = `bilibili_${id}`; + if (partId) { + filenameBase += `_${partId}`; + } + return { urls: [video.baseUrl, audio.baseUrl], - audioFilename: `bilibili_${id}_audio`, - filename: `bilibili_${id}_${video.width}x${video.height}.mp4`, - isHLS: true + audioFilename: `${filenameBase}_audio`, + filename: `${filenameBase}_${video.width}x${video.height}.mp4`, }; } @@ -87,14 +100,14 @@ async function tv_download(id) { }; } -export default async function({ comId, tvId, comShortLink }) { +export default async function({ comId, tvId, comShortLink, partId }) { if (comShortLink) { const patternMatch = await resolveRedirectingURL(`https://b23.tv/${comShortLink}`); comId = patternMatch?.comId; } if (comId) { - return com_download(comId); + return com_download(comId, partId); } else if (tvId) { return tv_download(tvId); } diff --git a/api/src/processing/services/newgrounds.js b/api/src/processing/services/newgrounds.js new file mode 100644 index 000000000..7519a6cfe --- /dev/null +++ b/api/src/processing/services/newgrounds.js @@ -0,0 +1,103 @@ +import { genericUserAgent } from "../../config.js"; + +const getVideo = async ({ id, quality }) => { + const json = await fetch(`https://www.newgrounds.com/portal/video/${id}`, { + headers: { + "User-Agent": genericUserAgent, + "X-Requested-With": "XMLHttpRequest", // required to get the JSON response + } + }) + .then(r => r.json()) + .catch(() => {}); + + if (!json) return { error: "fetch.empty" }; + + const videoSources = json.sources; + const videoQualities = Object.keys(videoSources); + + if (videoQualities.length === 0) { + return { error: "fetch.empty" }; + } + + const bestVideo = videoSources[videoQualities[0]]?.[0], + userQuality = quality === "2160" ? "4k" : `${quality}p`, + preferredVideo = videoSources[userQuality]?.[0], + video = preferredVideo || bestVideo, + videoQuality = preferredVideo ? userQuality : videoQualities[0]; + + if (!bestVideo || !video.type.includes("mp4")) { + return { error: "fetch.empty" }; + } + + const fileMetadata = { + title: decodeURIComponent(json.title), + artist: decodeURIComponent(json.author), + } + + return { + urls: video.src, + filenameAttributes: { + service: "newgrounds", + id, + title: fileMetadata.title, + author: fileMetadata.artist, + extension: "mp4", + qualityLabel: videoQuality, + resolution: videoQuality, + }, + fileMetadata, + } +} + +const getMusic = async ({ id }) => { + const html = await fetch(`https://www.newgrounds.com/audio/listen/${id}`, { + headers: { + "User-Agent": genericUserAgent, + } + }) + .then(r => r.text()) + .catch(() => {}); + + if (!html) return { error: "fetch.fail" }; + + const params = JSON.parse( + `{${html.split(',"params":{')[1]?.split(',"images":')[0]}}` + ); + if (!params) return { error: "fetch.empty" }; + + if (!params.name || !params.artist || !params.filename || !params.icon) { + return { error: "fetch.empty" }; + } + + const fileMetadata = { + title: decodeURIComponent(params.name), + artist: decodeURIComponent(params.artist), + } + + return { + urls: params.filename, + filenameAttributes: { + service: "newgrounds", + id, + title: fileMetadata.title, + author: fileMetadata.artist, + }, + fileMetadata, + cover: + params.icon.includes(".png?") || params.icon.includes(".jpg?") + ? params.icon + : undefined, + isAudioOnly: true, + bestAudio: "mp3", + } +} + +export default function({ id, audioId, quality }) { + if (id) { + return getVideo({ id, quality }); + } else if (audioId) { + return getMusic({ id: audioId }); + } + + return { error: "fetch.empty" }; +} diff --git a/api/src/processing/services/soundcloud.js b/api/src/processing/services/soundcloud.js index 1c04a3663..a73e6474c 100644 --- a/api/src/processing/services/soundcloud.js +++ b/api/src/processing/services/soundcloud.js @@ -148,7 +148,14 @@ export default async function(obj) { let cover; if (json.artwork_url) { - cover = json.artwork_url.replace(/-large/, "-t1080x1080"); + const coverUrl = json.artwork_url.replace(/-large/, "-t1080x1080"); + const testCover = await fetch(coverUrl) + .then(r => r.status === 200) + .catch(() => {}); + + if (testCover) { + cover = coverUrl; + } } return { diff --git a/api/src/processing/services/tiktok.js b/api/src/processing/services/tiktok.js index 3f5ea1fc9..1e1605e01 100644 --- a/api/src/processing/services/tiktok.js +++ b/api/src/processing/services/tiktok.js @@ -168,4 +168,6 @@ export default async function(obj) { headers: { cookie } } } + + return { error: "fetch.empty" }; } diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index 8bbe0eeb3..572bc2f0a 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -3,12 +3,12 @@ import { genericUserAgent } from "../../config.js"; import { createStream } from "../../stream/manage.js"; import { getCookie, updateCookie } from "../cookie/manager.js"; -const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId'; +const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail'; const tokenURL = 'https://api.x.com/1.1/guest/activate.json'; -const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}); +const tweetFeatures = JSON.stringify({"rweb_video_screen_enabled":false,"payments_enabled":false,"rweb_xchat_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false}); -const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false}); +const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}); const commonHeaders = { "user-agent": genericUserAgent, @@ -100,10 +100,14 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { graphqlTweetURL.searchParams.set('variables', JSON.stringify({ - tweetId, - withCommunity: false, - includePromotedContent: false, - withVoice: false + focalTweetId: tweetId, + with_rux_injections: false, + rankingMode: "Relevance", + includePromotedContent: true, + withCommunity: true, + withQuickPromoteEligibilityTweetFields: true, + withBirdwatchNotes: true, + withVoice: true }) ); graphqlTweetURL.searchParams.set('features', tweetFeatures); @@ -129,24 +133,48 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { return result } -const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => { - let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; +const parseCard = (cardOuter) => { + const card = JSON.parse( + (cardOuter?.legacy?.binding_values[0].value + || cardOuter?.binding_values?.unified_card)?.string_value, + ); + + if (!["video_website", "image_website"].includes(card?.type) + || !card?.media_entities + || card?.component_objects?.media_1?.type !== "media") { + return; + } + + const mediaId = card.component_objects?.media_1?.data?.id; + return [card.media_entities[mediaId]]; +}; + +const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) => { + const addInsn = thread?.data?.threaded_conversation_with_injections_v2?.instructions?.find( + insn => insn.type === 'TimelineAddEntries' + ); + + const tweetResult = addInsn?.entries?.find( + entry => entry.entryId === `tweet-${id}` + )?.content?.itemContent?.tweet_results?.result; + + let tweetTypename = tweetResult?.__typename; if (!tweetTypename) { return { error: "fetch.empty" } } - if (tweetTypename === "TweetUnavailable") { - const reason = tweet?.data?.tweetResult?.result?.reason; - switch(reason) { - case "Protected": - return { error: "content.post.private" }; - case "NsfwLoggedOut": - if (cookie) { - tweet = await requestTweet(dispatcher, id, guestToken, cookie); - tweet = await tweet.json(); - tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - } else return { error: "content.post.age" }; + if (tweetTypename === "TweetUnavailable" || tweetTypename === "TweetTombstone") { + const reason = tweetResult?.result?.reason; + if (reason === 'Protected') { + return { error: "content.post.private" }; + } else if (reason === "NsfwLoggedOut" || tweetResult?.tombstone?.text?.text?.startsWith('Age-restricted')) { + if (!cookie) { + return { error: "content.post.age" }; + } + + const tweet = await requestTweet(dispatcher, id, guestToken, cookie).then(t => t.json()); + return extractGraphqlMedia(tweet, dispatcher, id, guestToken); } } @@ -154,8 +182,7 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => return { error: "content.post.unavailable" } } - let tweetResult = tweet.data.tweetResult.result, - baseTweet = tweetResult.legacy, + let baseTweet = tweetResult.legacy, repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities; if (tweetTypename === "TweetWithVisibilityResults") { @@ -164,81 +191,51 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => } if (tweetResult.card?.legacy?.binding_values?.length) { - const card = JSON.parse(tweetResult.card.legacy.binding_values[0].value?.string_value); - - if (!["video_website", "image_website"].includes(card?.type) || - !card?.media_entities || - card?.component_objects?.media_1?.type !== "media") { - return; - } - - const mediaId = card.component_objects?.media_1?.data?.id; - return [card.media_entities[mediaId]]; + return parseCard(tweetResult.card); } return (repostedTweet?.media || baseTweet?.extended_entities?.media); } -const testResponse = (result) => { - const contentLength = result.headers.get("content-length"); - - if (!contentLength || contentLength === '0') { - return false; - } - - if (!result.headers.get("content-type").startsWith("application/json")) { - return false; - } - - return true; -} - export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) { const cookie = await getCookie('twitter'); - let syndication = false; - let guestToken = await getGuestToken(dispatcher); if (!guestToken) return { error: "fetch.fail" }; - // for now we assume that graphql api will come back after some time, - // so we try it first - let tweet = await requestTweet(dispatcher, id, guestToken); - // get new token & retry if old one expired - if ([403, 429].includes(tweet.status)) { - guestToken = await getGuestToken(dispatcher, true); - if (cookie) { - tweet = await requestTweet(dispatcher, id, guestToken, cookie); - } else { - tweet = await requestTweet(dispatcher, id, guestToken); + if ([403, 404, 429].includes(tweet.status)) { + // get new token & retry if old one expired + if ([403, 429].includes(tweet.status)) { + guestToken = await getGuestToken(dispatcher, true); } + tweet = await requestTweet(dispatcher, id, guestToken, cookie); } - const testGraphql = testResponse(tweet); + let media; + try { + tweet = await tweet.json(); + media = await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie); + } catch {} // if graphql requests fail, then resort to tweet embed api - if (!testGraphql) { - syndication = true; - tweet = await requestSyndication(dispatcher, id); + if (!media || 'error' in media) { + try { + tweet = await requestSyndication(dispatcher, id); + tweet = await tweet.json(); - const testSyndication = testResponse(tweet); + if (tweet?.card) { + media = parseCard(tweet.card); + } + } catch {} - // if even syndication request failed, then cry out loud - if (!testSyndication) { - return { error: "fetch.fail" }; - } + media = tweet?.mediaDetails ?? media; } - tweet = await tweet.json(); - - let media = - syndication - ? tweet.mediaDetails - : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie); - - if (!media) return { error: "fetch.empty" }; + if (!media || 'error' in media) { + return { error: media?.error || "fetch.empty" }; + } // check if there's a video at given index (/video/) if (index >= 0 && index < media?.length) { diff --git a/api/src/processing/services/vimeo.js b/api/src/processing/services/vimeo.js index 7a65f17a1..c12269bc6 100644 --- a/api/src/processing/services/vimeo.js +++ b/api/src/processing/services/vimeo.js @@ -1,6 +1,7 @@ import HLS from "hls-parser"; import { env } from "../../config.js"; import { merge } from '../../misc/utils.js'; +import { getCookie } from "../cookie/manager.js"; const resolutionMatch = { "3840": 2160, @@ -15,7 +16,44 @@ const resolutionMatch = { "426": 240 } -const requestApiInfo = (videoId, password) => { +const genericHeaders = { + Accept: 'application/vnd.vimeo.*+json; version=3.4.10', + 'User-Agent': 'com.vimeo.android.videoapp (Google, Pixel 7a, google, Android 16/36 Version 11.8.1) Kotlin VimeoNetworking/3.12.0', + Authorization: 'Basic NzRmYTg5YjgxMWExY2JiNzUwZDg1MjhkMTYzZjQ4YWYyOGEyZGJlMTp4OGx2NFd3QnNvY1lkamI2UVZsdjdDYlNwSDUrdm50YzdNNThvWDcwN1JrenJGZC9tR1lReUNlRjRSVklZeWhYZVpRS0tBcU9YYzRoTGY2Z1dlVkJFYkdJc0dMRHpoZWFZbU0reDRqZ1dkZ1diZmdIdGUrNUM5RVBySlM0VG1qcw==', + 'Accept-Language': 'en', +} + +let bearer = ''; + +const getBearer = async (refresh = false) => { + const cookie = getCookie('vimeo_bearer')?.values?.()?.access_token; + if ((bearer || cookie) && !refresh) return bearer || cookie; + + const oauthResponse = await fetch( + 'https://api.vimeo.com/oauth/authorize/client', + { + method: 'POST', + body: new URLSearchParams({ + scope: 'private public create edit delete interact upload purchased stats', + grant_type: 'client_credentials', + }).toString(), + headers: { + ...genericHeaders, + 'Content-Type': 'application/x-www-form-urlencoded', + } + } + ) + .then(a => a.json()) + .catch(() => {}); + + if (!oauthResponse || !oauthResponse.access_token) { + return; + } + + return bearer = oauthResponse.access_token; +} + +const requestApiInfo = (bearerToken, videoId, password) => { if (password) { videoId += `:${password}` } @@ -24,10 +62,8 @@ const requestApiInfo = (videoId, password) => { `https://api.vimeo.com/videos/${videoId}`, { headers: { - Accept: 'application/vnd.vimeo.*+json; version=3.4.2', - 'User-Agent': 'Vimeo/10.19.0 (com.vimeo; build:101900.57.0; iOS 17.5.1) Alamofire/5.9.0 VimeoNetworking/5.0.0', - Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==', - 'Accept-Language': 'en' + ...genericHeaders, + Authorization: `Bearer ${bearerToken}`, } } ) @@ -151,9 +187,28 @@ export default async function(obj) { if (quality < 240) quality = 240; if (!quality || obj.isAudioOnly) quality = 9000; - const info = await requestApiInfo(obj.id, obj.password); + const bearerToken = await getBearer(); + if (!bearerToken) { + return { error: "fetch.fail" }; + } + + let info = await requestApiInfo(bearerToken, obj.id, obj.password); let response; + // auth error, try to refresh the token + if (info?.error_code === 8003) { + const newBearer = await getBearer(true); + if (!newBearer) { + return { error: "fetch.fail" }; + } + info = await requestApiInfo(newBearer, obj.id, obj.password); + } + + // if there's still no info, then return a generic error + if (!info || info.error_code) { + return { error: "fetch.empty" }; + } + if (obj.isAudioOnly) { response = await getHLS(info.config_url, { ...obj, quality }); } diff --git a/api/src/processing/services/xiaohongshu.js b/api/src/processing/services/xiaohongshu.js index 06de21aa5..f9cbf6804 100644 --- a/api/src/processing/services/xiaohongshu.js +++ b/api/src/processing/services/xiaohongshu.js @@ -6,13 +6,13 @@ const https = (url) => { return url.replace(/^http:/i, 'https:'); } -export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) { +export default async function ({ id, token, shareType, shareId, h265, isAudioOnly, dispatcher }) { let noteId = id; let xsecToken = token; if (!noteId) { const patternMatch = await resolveRedirectingURL( - `https://xhslink.com/a/${shareId}`, + `https://xhslink.com/${shareType}/${shareId}`, dispatcher ); diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 55caa8356..b47c266d3 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -224,7 +224,7 @@ export default async function (o) { let info; try { - info = await yt.getBasicInfo(o.id, innertubeClient); + info = await yt.getBasicInfo(o.id, { client: innertubeClient }); } catch (e) { if (e?.info) { let errorInfo; diff --git a/api/src/processing/url.js b/api/src/processing/url.js index dbbda1cdd..9ac8a3ee9 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -99,7 +99,7 @@ function aliasURL(url) { case "xhslink": if (url.hostname === 'xhslink.com' && parts.length === 3) { - url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`); + url = new URL(`https://www.xiaohongshu.com/${parts[1]}/${parts[2]}`); } break; @@ -147,6 +147,7 @@ function cleanURL(url) { limitQuery('v'); } break; + case "bilibili": case "rutube": if (url.searchParams.get('p')) { limitQuery('p'); diff --git a/api/src/stream/shared.js b/api/src/stream/shared.js index ec06339d8..6d2685641 100644 --- a/api/src/stream/shared.js +++ b/api/src/stream/shared.js @@ -19,6 +19,9 @@ const serviceHeaders = { }, vk: { 'user-agent': vkClientAgent + }, + tiktok: { + referer: 'https://www.tiktok.com/', } } diff --git a/api/src/util/test.js b/api/src/util/test.js index abb9b3cd7..98a9f1319 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -4,7 +4,7 @@ import { env } from "../config.js"; import { runTest } from "../misc/run-test.js"; import { loadJSON } from "../misc/load-from-fs.js"; import { Red, Bright } from "../misc/console-text.js"; -import { setGlobalDispatcher, ProxyAgent } from "undici"; +import { setGlobalDispatcher, EnvHttpProxyAgent, ProxyAgent } from "undici"; import { randomizeCiphers } from "../misc/randomize-ciphers.js"; import { services } from "../processing/service-config.js"; @@ -69,9 +69,10 @@ const printHeader = (service, padLen) => { console.log(service + '='.repeat(50)); } -if (env.externalProxy) { - setGlobalDispatcher(new ProxyAgent(env.externalProxy)); -} +// TODO: remove env.externalProxy in a future version +setGlobalDispatcher( + new EnvHttpProxyAgent({ httpProxy: env.externalProxy || undefined }) +); env.streamLifespan = 10000; env.apiURL = 'http://x/'; diff --git a/api/src/util/tests/bilibili.json b/api/src/util/tests/bilibili.json index c67202952..88b42cdf5 100644 --- a/api/src/util/tests/bilibili.json +++ b/api/src/util/tests/bilibili.json @@ -56,5 +56,14 @@ "code": 200, "status": "tunnel" } + }, + { + "name": "bilibili.com link with part id", + "url": "https://www.bilibili.com/video/BV1uo4y1K72s?spm_id_from=333.788.videopod.episodes&p=6", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } } ] diff --git a/api/src/util/tests/newgrounds.json b/api/src/util/tests/newgrounds.json new file mode 100644 index 000000000..e0c9c83d8 --- /dev/null +++ b/api/src/util/tests/newgrounds.json @@ -0,0 +1,42 @@ +[ + { + "name": "regular video", + "url": "https://www.newgrounds.com/portal/view/938050", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (audio only)", + "url": "https://www.newgrounds.com/portal/view/938050", + "params": { + "downloadMode": "audio" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular video (muted)", + "url": "https://www.newgrounds.com/portal/view/938050", + "params": { + "downloadMode": "mute" + }, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "regular music", + "url": "https://www.newgrounds.com/audio/listen/500476", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + } +] diff --git a/api/src/util/tests/soundcloud.json b/api/src/util/tests/soundcloud.json index e7f54a3b6..67434cbde 100644 --- a/api/src/util/tests/soundcloud.json +++ b/api/src/util/tests/soundcloud.json @@ -86,6 +86,7 @@ }, { "name": "go+ song, should fail", + "canFail": true, "url": "https://soundcloud.com/dualipa/physical-feat-troye-sivan", "params": {}, "expected": { diff --git a/api/src/util/tests/twitch.json b/api/src/util/tests/twitch.json index fd6b84afc..eda23cb3a 100644 --- a/api/src/util/tests/twitch.json +++ b/api/src/util/tests/twitch.json @@ -29,5 +29,14 @@ "code": 200, "status": "tunnel" } + }, + { + "name": "clip (mobile subdomain)", + "url": "https://m.twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": {}, + "expected": { + "code": 200, + "status": "redirect" + } } ] \ No newline at end of file diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index a169cc23c..ed5238567 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -1,7 +1,7 @@ [ { "name": "video (might have expired)", - "url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=", + "url": "https://www.xiaohongshu.com/explore/685e63e1000000000b02ee3b?xsec_token=ABN8EQJCDMPcFX9RRggeIPSHLIJ8zkGceFDyBewLGUz30=", "canFail": true, "params": {}, "expected": { @@ -11,7 +11,7 @@ }, { "name": "picker with multiple live photos (might have expired)", - "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=", + "url": "https://www.xiaohongshu.com/explore/687128a2000000001203d94c?xsec_token=CBlDi5QDXDWZu2uUmbUrpKwg8lEL3uC10mc59lGf43r9w=", "canFail": true, "params": {}, "expected": { @@ -21,7 +21,7 @@ }, { "name": "one photo (might have expired)", - "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA", + "url": "https://www.xiaohongshu.com/explore/64726b99000000000800e115?xsec_token=ABoD3qPHqVZolCfS-J8UP9QQaPXZ6Z6PVyODrhaiUg27U=", "canFail": true, "params": {}, "expected": { @@ -31,7 +31,7 @@ }, { "name": "short link (might have expired)", - "url": "https://xhslink.com/a/czn4z6c1tic4", + "url": "https://xhslink.com/m/2wAnaTkLRc1", "canFail": true, "params": {}, "expected": { diff --git a/docs/api-env-variables.md b/docs/api-env-variables.md index 5a836bfa8..59cc0df7f 100644 --- a/docs/api-env-variables.md +++ b/docs/api-env-variables.md @@ -21,9 +21,15 @@ this document is not final and will expand over time. feel free to improve it! | name | default | value example | |:--------------------|:----------|:--------------------------------------| | API_LISTEN_ADDRESS | `0.0.0.0` | `127.0.0.1` | -| API_EXTERNAL_PROXY | | `http://user:password@127.0.0.1:8080` | | FREEBIND_CIDR | | `2001:db8::/32` | +#### undici proxy vars +| name | value example | +|:------------|:--------------------------------------| +| HTTP_PROXY | `http://user:password@10.0.0.1:1337/` | +| HTTPS_PROXY | `https://10.0.0.2:1337/` | +| NO_PROXY | `localhost` | + [*view details*](#networking) ### limit vars @@ -123,10 +129,23 @@ defines the local address for the api instance. if you are using a docker contai the value is a local IP address. -### API_EXTERNAL_PROXY -URL of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. +### HTTP_PROXY, HTTPS_PROXY, NO_PROXY +URL of the proxy that will be passed to [`EnvHttpProxyAgent`](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent) for proxying external requests. if some cobalt functionality breaks when using a proxy, please [make a new issue](https://github.com/imputnet/cobalt/issues) about it! + +quoted from [undici docs](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent): +> When `HTTP_PROXY` and `HTTPS_PROXY` are set, `HTTP_PROXY` is used for HTTP requests and `HTTPS_PROXY` is used for HTTPS requests. If only `HTTP_PROXY` is set, `HTTP_PROXY` is used for both HTTP and HTTPS requests. If only `HTTPS_PROXY` is set, it is only used for HTTPS requests. + +> `NO_PROXY` is a comma or space-separated list of hostnames that should not be proxied. The list may contain leading wildcard characters (`*`). If `NO_PROXY` is set, the EnvHttpProxyAgent() will bypass the proxy for requests to hosts that match the list. If `NO_PROXY` is set to `"*"`, the EnvHttpProxyAgent() will bypass the proxy for all requests. + +the value is a string: +- `HTTP_PROXY`/`HTTPS_PROXY`: URL or hostname. +- `NO_PROXY`: comma or space-separated list of hostnames. + +### API_EXTERNAL_PROXY (deprecated) +> [!WARNING] +> this env variable is deprecated and will be removed in a future release. please update your configuration to use `HTTP_PROXY` or `HTTPS_PROXY`, as mentioned above. -if some feature breaks when using a proxy, please make a new issue about it! +URL of the proxy that will be passed to [`EnvHttpProxyAgent`](https://undici.nodejs.org/#/docs/api/EnvHttpProxyAgent) and used for proxying external requests. HTTP(S) only. the value is a URL. diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index d788b2ddd..4c4673566 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -13,5 +13,8 @@ ], "youtube": [ "cookie=; b=" + ], + "vimeo": [ + "access_token=" ] } diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index 04c6823b2..281b3b53d 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -4,9 +4,9 @@ this tutorial will help you run your own cobalt processing instance. if your ins ## using docker compose and package from github (recommended) to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured. -if you need help with installing docker, follow *only the first step* of these tutorials by digitalocean: -- [how to install docker](https://www.digitalocean.com/community/tutorial-collections/how-to-install-and-use-docker) -- [how to install docker compose](https://www.digitalocean.com/community/tutorial-collections/how-to-install-docker-compose) +if you need help with installing docker, you can find more information here: +- [how to install docker](https://docs.docker.com/engine/install/) +- [how to install docker compose](https://docs.docker.com/compose/install/) ## how to run a cobalt docker package: 1. create a folder for cobalt config file, something like this: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d67cc686..99bc803c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,14 +53,14 @@ importers: specifier: 2.6.0 version: 2.6.0 undici: - specifier: ^5.19.1 - version: 5.28.4 + specifier: ^6.21.3 + version: 6.21.3 url-pattern: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: ^14.0.0 - version: 14.0.0 + specifier: 15.1.1 + version: 15.1.1 zod: specifier: ^3.23.8 version: 3.23.8 @@ -1876,6 +1876,7 @@ packages: source-map@0.8.0-beta.0: resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==} engines: {node: '>= 8'} + deprecated: The work that was done in this beta branch won't be included in future versions sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} @@ -2080,6 +2081,10 @@ packages: resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==} engines: {node: '>=14.0'} + undici@6.21.3: + resolution: {integrity: sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==} + engines: {node: '>=18.17'} + unist-util-stringify-position@2.0.3: resolution: {integrity: sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g==} @@ -2181,8 +2186,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@14.0.0: - resolution: {integrity: sha512-KAFttOw+9fwwBUvBc1T7KzMNBLczDOuN/dfote8BA9CABxgx8MPgV+vZWlowdDB6DnHjSUYppv+xvJ4VNBLK9A==} + youtubei.js@15.1.1: + resolution: {integrity: sha512-fuEDj9Ky6cAQg93BrRVCbr+GTYNZQAIFZrx/a3oDRuGc3Mf5bS0dQfoYwwgjtSV7sgAKQEEdGtzRdBzOc8g72Q==} zimmerframe@1.1.2: resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==} @@ -2396,7 +2401,8 @@ snapshots: dependencies: levn: 0.4.1 - '@fastify/busboy@2.1.1': {} + '@fastify/busboy@2.1.1': + optional: true '@fontsource/ibm-plex-mono@5.0.13': {} @@ -3960,6 +3966,9 @@ snapshots: undici@5.28.4: dependencies: '@fastify/busboy': 2.1.1 + optional: true + + undici@6.21.3: {} unist-util-stringify-position@2.0.3: dependencies: @@ -4035,12 +4044,11 @@ snapshots: yocto-queue@0.1.0: {} - youtubei.js@14.0.0: + youtubei.js@15.1.1: dependencies: '@bufbuild/protobuf': 2.2.5 jintr: 3.3.1 - tslib: 2.6.3 - undici: 5.28.4 + undici: 6.21.3 zimmerframe@1.1.2: {} diff --git a/web/changelogs/11.2.md b/web/changelogs/11.2.md index 346292899..f1ee42b76 100644 --- a/web/changelogs/11.2.md +++ b/web/changelogs/11.2.md @@ -6,7 +6,7 @@ banner: alt: "meowth plush in a forest looking at the rising sun between the trees." --- -it's summertime! even though it's been rainy for us lately, the sun is right on the horizon, just like this cobalt update. we improved local processing, added long-awaited features, and improved a ton of other stuff. **downloading from youtube is back, btw**. +it's summertime! even though it's been rainy for us lately, the sun is right on the horizon, just like this cobalt update. we improved local processing, added long-awaited features, and improved a ton of other stuff. here's what's new since 11.0: @@ -31,6 +31,8 @@ downloading from youtube on the main instance is restored! sorry that it took a hopefully it'll last for a while, but we think downloading from youtube will get significantly more annoying/complex in next few weeks-months. **right now is the best time to download everything you've been putting off**, either with cobalt or other tools. +**update**: unfortunately it did not last, youtube is unavailable on the main instance again. we will try one more way soon and update this changelog and post about it on socials accordingly. + we're not trying to scare you; it's our educated guess based on what youtube has been doing lately: - roll out of SABR & related limitations for more clients. SABR is Server ABR, Google's proprietary HLS alternative, controlled by the server. - growing potoken enforcement. @@ -49,6 +51,7 @@ by the way, we also made it possible to [choose any preferred media container](/ - pinterest now returns an appropriate error when a pin is unavailable. - AI dubs on youtube are no longer accidentally selected as default tracks. - youtube HLS preference is now deprecated, but can be enabled on a self-hosted instance via `ENABLE_DEPRECATED_YOUTUBE_HLS` in env. +- downloads from vk are now way faster. ## web app improvements - improved compatibility of local processing & related code with older browsers. diff --git a/web/i18n/en/save.json b/web/i18n/en/save.json index 361361f89..91716ab48 100644 --- a/web/i18n/en/save.json +++ b/web/i18n/en/save.json @@ -10,7 +10,7 @@ "services.title": "supported services", "services.title_show": "show supported services", "services.title_hide": "hide supported services", - "services.disclaimer": "cobalt is not affiliated with any of the services listed above.", + "services.disclaimer": "support for a service does not imply affiliation, endorsement, or any form of support other than technical compatibility.", "tutorial.title": "how to save on ios?", "tutorial.intro": "to save media conveniently on ios, you'll need to use a companion siri shortcut from the share sheet.", diff --git a/web/i18n/en/settings.json b/web/i18n/en/settings.json index dac6b62ab..5caf67f51 100644 --- a/web/i18n/en/settings.json +++ b/web/i18n/en/settings.json @@ -85,6 +85,8 @@ "metadata.filename.preview.video": "Video Title - Video Author", "metadata.filename.preview.audio": "Audio Title - Audio Author", + "filename.preview_desc.video": "video file preview", + "filename.preview_desc.audio": "audio file preview", "metadata.file": "file metadata", "metadata.disable.title": "disable file metadata", diff --git a/web/i18n/ru/a11y/save.json b/web/i18n/ru/a11y/save.json index 5cf074271..831481257 100644 --- a/web/i18n/ru/a11y/save.json +++ b/web/i18n/ru/a11y/save.json @@ -1,12 +1,12 @@ { - "link_area": "область ввода ссылки", + "link_area": "поле ввода ссылки", "clear_input": "очистить поле ввода", "download": "скачать", "download.think": "обрабатываю ссылку...", "download.check": "проверяю загрузку...", "download.done": "загрузка завершена", "download.error": "ошибка загрузки", - "link_area.turnstile": "область ввода ссылки. проверяю, что ты не робот.", + "link_area.turnstile": "поле ввода ссылки. проверяю, что ты не робот.", "tutorial.shortcut.photos": "добавить команду \"в фото\"", "tutorial.shortcut.files": "добавить команду \"в файлы\"" } diff --git a/web/i18n/ru/save.json b/web/i18n/ru/save.json index 60bb2998c..26dedbd60 100644 --- a/web/i18n/ru/save.json +++ b/web/i18n/ru/save.json @@ -10,7 +10,7 @@ "services.title": "поддерживаемые сервисы", "services.title_show": "показать поддерживаемые сервисы", "services.title_hide": "скрыть поддерживаемые сервисы", - "services.disclaimer": "кобальт не аффилирован ни с одним из перечисленных выше сервисов.\n\nдеятельность meta platforms (владельца facebook и instagram) запрещена на территории РФ и признана экстремистской.", + "services.disclaimer": "поддержка сервиса не означает аффилированность, одобрение или любую другую форму поддержки, кроме технической совместимости.\n\nдеятельность владельца facebook и instagram запрещена на территории РФ и признана экстремистской.", "tutorial.step.1": "добавь команды-компаньоны:", "tutorial.step.2": "нажми кнопку \"поделиться\" в диалоге сохранения кобальта.", "tutorial.step.3": "выбери нужную команду в окне обмена.", diff --git a/web/i18n/ru/settings.json b/web/i18n/ru/settings.json index 9794aa47c..044087c2f 100644 --- a/web/i18n/ru/settings.json +++ b/web/i18n/ru/settings.json @@ -45,7 +45,7 @@ "audio.youtube.dub.description": "cobalt будет использовать дублированную аудиодорожку для выбранного языка, если она доступна. в противном случае будет использоваться оригинальная.", "language.auto.description": "если доступен перевод, то кобальт будет использовать язык твоего браузера. в ином случае будет использоваться английский.", "theme.description": "авто тема переключается между светлой и тёмной темой в зависимости от системной темы.", - "page.debug": "инфа для гиков", + "page.debug": "инфа для зануд", "page.appearance": "внешний вид", "page.instances": "инстансы", "page.advanced": "продвинутые", @@ -80,6 +80,8 @@ "video.youtube.hls.title": "предпочитать hls для видео и аудио", "metadata.filename.preview.video": "Название Видео - Автор Видео", "metadata.filename.preview.audio": "Название Аудио - Автор Аудио", + "filename.preview_desc.video": "превью видео файла", + "filename.preview_desc.audio": "превью аудио файла", "saving.description": "предпочтительный способ сохранения файла или ссылки с кобальта. если предпочитаемый метод недоступен или что-то пойдёт не так, кобальт спросит тебя как поступить.", "accessibility.transparency.description": "уменьшает прозрачность поверхностей и выключает эффекты размытия. также может улучшить работу интерфейса на менее мощных устройствах.", "accessibility.transparency.title": "уменьшить визуальную прозрачность", diff --git a/web/package.json b/web/package.json index d31e9590f..7de6f75dc 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "@imput/cobalt-web", - "version": "11.2.2", + "version": "11.3", "type": "module", "private": true, "scripts": { diff --git a/web/src/components/settings/FilenamePreview.svelte b/web/src/components/settings/FilenamePreview.svelte index 4af25be9a..35a7612ff 100644 --- a/web/src/components/settings/FilenamePreview.svelte +++ b/web/src/components/settings/FilenamePreview.svelte @@ -75,7 +75,7 @@
{`${videoFilePreview}.${youtubeVideoExt}`}
-
video file preview
+
{$t("settings.filename.preview_desc.video")}
@@ -84,7 +84,7 @@
{`${audioFilePreview}.${audioFormat}`}
-
audio file preview
+
{$t("settings.filename.preview_desc.audio")}