修改为docker部署
This commit is contained in:
@@ -0,0 +1,118 @@
|
||||
const baseUrl = (process.env.CAPTION_API_BASE_URL || '').replace(/\/+$/, '');
|
||||
const videoIds = (process.env.CAPTION_API_VIDEO_IDS || '')
|
||||
.split(',')
|
||||
.map((id) => id.trim())
|
||||
.filter(Boolean);
|
||||
const lang = process.env.CAPTION_API_LANG || 'en';
|
||||
const timeoutMs = Number(process.env.CAPTION_API_TIMEOUT_MS || 30_000);
|
||||
const attempts = Math.max(
|
||||
1,
|
||||
Math.floor(Number(process.env.CAPTION_API_ATTEMPTS || 3))
|
||||
);
|
||||
|
||||
if (!baseUrl) {
|
||||
throw new Error('CAPTION_API_BASE_URL is required');
|
||||
}
|
||||
|
||||
if (videoIds.length === 0) {
|
||||
throw new Error('CAPTION_API_VIDEO_IDS must include at least one video ID');
|
||||
}
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) throw new Error(message);
|
||||
}
|
||||
|
||||
function validateSubtitle(subtitle, videoId) {
|
||||
assert(
|
||||
subtitle && typeof subtitle === 'object',
|
||||
`${videoId}: first subtitle must be an object`
|
||||
);
|
||||
assert(
|
||||
typeof subtitle.start === 'string' && !Number.isNaN(Number(subtitle.start)),
|
||||
`${videoId}: subtitle.start must be a numeric string`
|
||||
);
|
||||
assert(
|
||||
typeof subtitle.dur === 'string' && !Number.isNaN(Number(subtitle.dur)),
|
||||
`${videoId}: subtitle.dur must be a numeric string`
|
||||
);
|
||||
assert(
|
||||
typeof subtitle.text === 'string' && subtitle.text.trim().length > 0,
|
||||
`${videoId}: subtitle.text must be a non-empty string`
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchJson(url, videoId) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
cache: 'no-store',
|
||||
signal: controller.signal,
|
||||
});
|
||||
const body = await response.text();
|
||||
|
||||
assert(
|
||||
response.ok,
|
||||
`${videoId}: expected 2xx from production API, got ${response.status} ${response.statusText}: ${body.slice(0, 500)}`
|
||||
);
|
||||
|
||||
try {
|
||||
return JSON.parse(body);
|
||||
} catch {
|
||||
throw new Error(`${videoId}: production API did not return valid JSON`);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkVideo(videoId) {
|
||||
const params = new URLSearchParams({ videoID: videoId, lang });
|
||||
const url = `${baseUrl}/api/videoDetails?${params.toString()}`;
|
||||
let payload;
|
||||
let lastError;
|
||||
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
try {
|
||||
payload = await fetchJson(url, videoId);
|
||||
break;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt === attempts) break;
|
||||
console.warn(
|
||||
`${videoId}: attempt ${attempt} failed, retrying - ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, attempt * 1000));
|
||||
}
|
||||
}
|
||||
|
||||
if (!payload) throw lastError;
|
||||
|
||||
const details = payload.videoDetails;
|
||||
|
||||
assert(
|
||||
details && typeof details === 'object',
|
||||
`${videoId}: response must include videoDetails`
|
||||
);
|
||||
assert(
|
||||
typeof details.title === 'string' && details.title.trim().length > 0,
|
||||
`${videoId}: videoDetails.title must be non-empty`
|
||||
);
|
||||
assert(
|
||||
Array.isArray(details.subtitles) && details.subtitles.length > 0,
|
||||
`${videoId}: videoDetails.subtitles must be a non-empty array`
|
||||
);
|
||||
|
||||
validateSubtitle(details.subtitles[0], videoId);
|
||||
|
||||
console.log(
|
||||
`${videoId}: OK (${details.subtitles.length} captions) - ${details.title}`
|
||||
);
|
||||
}
|
||||
|
||||
for (const videoId of videoIds) {
|
||||
await checkVideo(videoId);
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Always-run: build + type-check + surface-level tests + tarball validation.
|
||||
# No network calls to YouTube — works on any runner.
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: [20, 22]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: 'npm'
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- run: npm run build
|
||||
|
||||
# Tests are split: surface-level tests run here; live-network tests
|
||||
# are gated by YOUTUBE_LIVE=1 (see live-check.yml).
|
||||
- run: npm test
|
||||
|
||||
- name: Verify published tarball contents
|
||||
run: |
|
||||
npm pack --dry-run 2>&1 | tee pack-output.txt
|
||||
grep -q "dist/index.js" pack-output.txt
|
||||
grep -q "dist/index.d.ts" pack-output.txt
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Production API canary
|
||||
|
||||
# Best-effort nightly probe against the deployed production API. This checks the
|
||||
# real hosted path users hit instead of asking YouTube directly from GitHub
|
||||
# Actions, which is commonly blocked by YouTube's bot detection.
|
||||
#
|
||||
# This workflow does NOT fail the build. It opens / comments on a tracking
|
||||
# issue if a run fails, so maintainers can investigate.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * *' # 06:00 UTC daily
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
prod:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
env:
|
||||
CAPTION_API_BASE_URL: https://youtube-caption-extractor.vercel.app
|
||||
CAPTION_API_VIDEO_IDS: 0Gb1z-2SjHY,D37Ijn2o5U0,g9JIUM0MHgQ,6BB6exR8Zd8,55pTFVoclvE
|
||||
CAPTION_API_LANG: en
|
||||
CAPTION_API_ATTEMPTS: 3
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Check production API
|
||||
id: prod
|
||||
run: node .github/scripts/check-production-api.mjs
|
||||
|
||||
- name: Open / update tracking issue on failure
|
||||
if: failure() && steps.prod.outcome == 'failure'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
TITLE="Nightly production API canary failed"
|
||||
EXISTING=$(gh issue list --state open --search "$TITLE in:title" --json number --jq '.[0].number // empty')
|
||||
BODY=$(cat <<EOF
|
||||
The nightly production API check failed.
|
||||
|
||||
This workflow calls the deployed Vercel app at
|
||||
\`$CAPTION_API_BASE_URL/api/videoDetails\` and validates that it
|
||||
returns non-empty metadata and captions for dashboard sample videos.
|
||||
|
||||
**When to act:** if this issue keeps reopening, investigate the
|
||||
deployed API path first: Vercel environment variables, proxy routing,
|
||||
runtime errors, rate limits, or upstream YouTube/proxy behavior.
|
||||
|
||||
**Failed run:** $RUN_URL
|
||||
EOF
|
||||
)
|
||||
if [ -n "$EXISTING" ]; then
|
||||
gh issue comment "$EXISTING" --body "Failed again: $RUN_URL"
|
||||
else
|
||||
gh issue create --title "$TITLE" --body "$BODY"
|
||||
fi
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
node_modules
|
||||
dist
|
||||
|
||||
*.log
|
||||
.DS_Store
|
||||
.env*.local
|
||||
|
||||
|
||||
/sample/.next/
|
||||
/sample/out/
|
||||
/sample/build/
|
||||
/sample/node_modules/
|
||||
/sample/server/node_modules/
|
||||
/sample/server/.wrangler/
|
||||
/sample/server/.dev.vars*
|
||||
|
||||
/docs
|
||||
@@ -0,0 +1,213 @@
|
||||
# YouTube Caption Extractor — Docker 部署教程
|
||||
|
||||
将 `youtube-caption-extractor` 部署到自有云服务器，通过 Web 浏览器访问使用。
|
||||
|
||||
## 架构概览
|
||||
|
||||
```
|
||||
浏览器 ──→ Nginx(可选) ──→ caption-web (Next.js :3000) ──→ caption-api (Hono :8080) ──→ YouTube
|
||||
```
|
||||
|
||||
- **caption-web**：Next.js 前端，提供用户界面
|
||||
- **caption-api**：Hono 后端，内置缓存、重试、代理支持，负责与 YouTube 通信
|
||||
- 两个服务通过 Docker Compose 编排，在内部网络互访
|
||||
|
||||
## 前置要求
|
||||
|
||||
- 云服务器（Linux，推荐 Ubuntu 20.04+）
|
||||
- Docker 和 Docker Compose 已安装
|
||||
- Git 已安装
|
||||
|
||||
## 第一步：克隆仓库
|
||||
|
||||
```bash
|
||||
git clone https://github.com/devhims/youtube-caption-extractor.git
|
||||
cd youtube-caption-extractor
|
||||
```
|
||||
|
||||
## 第二步：准备部署文件
|
||||
|
||||
在项目根目录创建 `docker-compose.yml`：
|
||||
|
||||
```yaml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
api:
|
||||
build:
|
||||
context: ./sample/server
|
||||
dockerfile: Dockerfile
|
||||
container_name: caption-api
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8765:8080" # 宿主机端口:容器端口，按需修改左侧
|
||||
environment:
|
||||
- CACHE_TTL_SECONDS=21600
|
||||
- EXTRACTION_ATTEMPTS=4
|
||||
- ALLOWED_ORIGINS=*
|
||||
|
||||
web:
|
||||
build:
|
||||
context: ./sample
|
||||
dockerfile: Dockerfile
|
||||
container_name: caption-web
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3456:3000" # 宿主机端口:容器端口，按需修改左侧
|
||||
environment:
|
||||
- CAPTION_API_BASE_URL=http://api:8080
|
||||
depends_on:
|
||||
- api
|
||||
```
|
||||
|
||||
在 `sample/` 目录下创建前端 Dockerfile（`sample/Dockerfile`）：
|
||||
|
||||
```dockerfile
|
||||
FROM node:22-alpine AS base
|
||||
|
||||
FROM base AS deps
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm install
|
||||
|
||||
FROM base AS builder
|
||||
WORKDIR /app
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
FROM base AS runner
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=3000
|
||||
|
||||
RUN addgroup --system --gid 1001 nodejs
|
||||
RUN adduser --system --uid 1001 nextjs
|
||||
|
||||
COPY --from=builder /app/public ./public
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
|
||||
|
||||
USER nextjs
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["node", "server.js"]
|
||||
```
|
||||
|
||||
修改 `sample/next.config.mjs`，添加 `output: 'standalone'`：
|
||||
|
||||
```js
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname } from 'node:path';
|
||||
|
||||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {
|
||||
output: 'standalone',
|
||||
turbopack: {
|
||||
root: dirname(fileURLToPath(import.meta.url)),
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
```
|
||||
|
||||
> 如果 `npm ci` 构建时报 lock 文件不同步，将 Dockerfile 中的 `npm ci` 改为 `npm install`。
|
||||
|
||||
## 第三步：启动服务
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
首次构建约 2-3 分钟。完成后检查状态：
|
||||
|
||||
```bash
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
两个容器应均为 `Up` 状态。
|
||||
|
||||
## 第四步：验证
|
||||
|
||||
```bash
|
||||
# API 健康检查
|
||||
curl http://localhost:8765/health
|
||||
# 应返回 {"status":"ok","runtime":"hono-node-container"}
|
||||
|
||||
# 前端可用性
|
||||
curl -I http://localhost:3456
|
||||
# 应返回 HTTP/1.1 200 OK
|
||||
```
|
||||
|
||||
## 访问
|
||||
|
||||
浏览器打开 `http://你的服务器IP:3456`，输入 YouTube 视频链接即可提取字幕。
|
||||
|
||||
## 端口冲突处理
|
||||
|
||||
修改 `docker-compose.yml` 中 `ports` 左侧的宿主机端口即可：
|
||||
|
||||
```yaml
|
||||
ports:
|
||||
- "你的端口:8080" # API
|
||||
- "你的端口:3000" # Web
|
||||
```
|
||||
|
||||
## 配置 Nginx 反向代理（可选）
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name your-domain.com;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:3456;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
配合 SSL 证书（Let's Encrypt / certbot）即可实现 HTTPS 访问。
|
||||
|
||||
## API 环境变量说明
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `CACHE_TTL_SECONDS` | `21600` | 内存缓存有效期（秒），6小时 |
|
||||
| `EXTRACTION_ATTEMPTS` | `4` | 提取失败重试次数 |
|
||||
| `EXTRACTION_RETRY_BASE_DELAY_MS` | `300` | 重试基础延迟（毫秒） |
|
||||
| `OUTBOUND_PROXY_URL` | 无 | 代理地址，用于绕过 YouTube 数据中心 IP 封锁 |
|
||||
| `ALLOWED_ORIGINS` | `*` | CORS 允许的来源，逗号分隔 |
|
||||
|
||||
## YouTube 封锁数据中心 IP 的应对
|
||||
|
||||
如果 API 返回 `youtube_blocked_datacenter_ip`，说明 YouTube 封锁了服务器出口 IP。配置代理：
|
||||
|
||||
```yaml
|
||||
api:
|
||||
environment:
|
||||
- OUTBOUND_PROXY_URL=http://your-proxy:port
|
||||
```
|
||||
|
||||
## 常用命令
|
||||
|
||||
```bash
|
||||
# 查看日志
|
||||
docker compose logs -f
|
||||
|
||||
# 重启服务
|
||||
docker compose restart
|
||||
|
||||
# 停止服务
|
||||
docker compose down
|
||||
|
||||
# 更新后重新构建
|
||||
docker compose up -d --build
|
||||
|
||||
# 清理旧镜像
|
||||
docker image prune -f
|
||||
```
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Himanshu Gupta
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
@@ -0,0 +1,446 @@
|
||||
<div align="center">
|
||||
<h1>youtube-caption-extractor</h1>
|
||||
<p><strong>Turn public YouTube videos into clean, timestamped transcripts.</strong></p>
|
||||
<p>
|
||||
Extract YouTube captions, subtitles, auto-generated transcripts, and video metadata
|
||||
with a tiny TypeScript library: ~10.8 kB packed, ~31.1 kB unpacked.
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://www.npmjs.com/package/youtube-caption-extractor"><img alt="npm version" src="https://img.shields.io/npm/v/youtube-caption-extractor?color=cb3837"></a>
|
||||
<a href="https://www.npmjs.com/package/youtube-caption-extractor"><img alt="npm downloads" src="https://img.shields.io/npm/dm/youtube-caption-extractor"></a>
|
||||
<a href="./LICENSE"><img alt="license" src="https://img.shields.io/npm/l/youtube-caption-extractor"></a>
|
||||
<a href="https://www.typescriptlang.org/"><img alt="TypeScript ready" src="https://img.shields.io/badge/TypeScript-ready-3178c6"></a>
|
||||
<a href="https://nodejs.org/"><img alt="Node.js 18+" src="https://img.shields.io/badge/Node.js-18%2B-43853d"></a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="#try-it-quickly">Quickstart</a>
|
||||
· <a href="https://youtube-caption-extractor.vercel.app/">Live demo</a>
|
||||
· <a href="./sample">Sample app</a>
|
||||
· <a href="#api">API</a>
|
||||
· <a href="#deployment-notes">Deployment notes</a>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## Why use it?
|
||||
|
||||
- **One-call transcript extraction** — `getSubtitles()` returns timestamped segments ready for search, summarization, indexing, RAG, slide ready research notes, or export.
|
||||
- **Metadata included** — `getVideoDetails()` returns title, description, and captions in one response.
|
||||
- **Manual + auto captions** — prefers exact language matches, then gracefully falls back to available tracks.
|
||||
- **Tiny install** — ~10.8 kB packed on npm, with only two runtime dependencies.
|
||||
- **Runtime-friendly** — uses global `fetch`, with an optional custom transport for retries, caching, regional routing, or proxies.
|
||||
- **Production-aware sample** — includes a Next.js demo plus a token-protected Cloudflare Container API.
|
||||
|
||||
## Built for
|
||||
|
||||
- YouTube caption extraction, subtitle extraction, and timestamped transcript data.
|
||||
- AI summaries, search indexes, RAG pipelines, and agent workflows that need clean video text.
|
||||
- Slide ready notes, presentation research, and content workflows built from public YouTube videos.
|
||||
- Lightweight video metadata enrichment without pulling in a large SDK.
|
||||
|
||||
```ts
|
||||
import { getSubtitles } from 'youtube-caption-extractor';
|
||||
|
||||
const subtitles = await getSubtitles({ videoID: '7GeFt8suV8E', lang: 'en' });
|
||||
// → [
|
||||
// { start: '1.12', dur: '4.56', text: 'This scraper can scrape almost anything' },
|
||||
// { start: '3.36', dur: '5.84', text: 'on the internet and you will be' },
|
||||
// { start: '5.68', dur: '6.64', text: 'surprised how easy it is to use it.' },
|
||||
// ...
|
||||
// ]
|
||||
```
|
||||
|
||||
## Try it quickly
|
||||
|
||||
```sh
|
||||
npm install youtube-caption-extractor
|
||||
```
|
||||
|
||||
```ts
|
||||
import { getVideoDetails } from 'youtube-caption-extractor';
|
||||
|
||||
const video = await getVideoDetails({
|
||||
videoID: '7GeFt8suV8E',
|
||||
lang: 'en',
|
||||
});
|
||||
|
||||
console.log(video.title);
|
||||
console.log(video.subtitles.map((s) => s.text).join('\n'));
|
||||
```
|
||||
|
||||
Want to click around first? Try the hosted demo:
|
||||
[youtube-caption-extractor.vercel.app](https://youtube-caption-extractor.vercel.app/).
|
||||
|
||||
Want a full app example? See [`sample/`](./sample), which includes:
|
||||
|
||||
- A polished Next.js UI
|
||||
- Local API testing with your machine's network egress
|
||||
- A Dockerized Hono API deployed through Cloudflare Containers
|
||||
- A server-side token-protected proxy so the container API is not publicly open
|
||||
|
||||
## Installation
|
||||
|
||||
```sh
|
||||
npm install youtube-caption-extractor
|
||||
```
|
||||
|
||||
Requires **Node.js ≥ 18** when running on Node.js because the library uses the
|
||||
global `fetch` API. It also works in Bun, Deno, Cloudflare Workers, and other
|
||||
modern JavaScript runtimes that provide `fetch`. See
|
||||
[Deployment notes](#deployment-notes) for tips on keeping calls reliable from
|
||||
your runtime of choice.
|
||||
|
||||
## API
|
||||
|
||||
The library exports two functions and three types.
|
||||
|
||||
### `getSubtitles({ videoID, lang?, fetch? })`
|
||||
|
||||
Returns the caption track as an array of timed segments.
|
||||
|
||||
| Param | Type | Default | Notes |
|
||||
| --------- | -------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `videoID` | `string` | (required) | The 11-character YouTube video ID, e.g. `7GeFt8suV8E`. Not the full URL. |
|
||||
| `lang` | `string` | `'en'` | ISO language code (`'en'`, `'es'`, `'fr'`, `'ja'`, …). Manual captions are preferred over auto-generated, and an exact match is preferred over a partial match. |
|
||||
| `fetch` | `typeof fetch` | global `fetch` | Custom fetch implementation. Useful for adding caching, custom retries, or routing through a proxy. See [Customizing the transport](#customizing-the-transport). |
|
||||
|
||||
Resolves to `Subtitle[]`. Returns an empty array if the video plays but has no caption track in the requested language. **Throws** if the video is unavailable on any extraction path (see [Error handling](#error-handling)).
|
||||
|
||||
### `getVideoDetails({ videoID, lang?, fetch? })`
|
||||
|
||||
Same arguments as `getSubtitles`. Returns title, description, and the same subtitle array:
|
||||
|
||||
```ts
|
||||
const details = await getVideoDetails({ videoID: '7GeFt8suV8E', lang: 'en' });
|
||||
// → {
|
||||
// title: 'Master Web Scraping with Firecrawl!',
|
||||
// description: 'Get started with Firecrawl here: https://firecrawl.link/…',
|
||||
// subtitles: [{ start: '1.12', dur: '4.56', text: 'This scraper can scrape almost anything' }, …],
|
||||
// }
|
||||
```
|
||||
|
||||
If subtitles fail to extract but the video metadata is available, `subtitles` will be an empty array and the call still resolves (rather than throwing). This way you can always show title/description even when captions aren't available.
|
||||
|
||||
### Types
|
||||
|
||||
```ts
|
||||
interface Subtitle {
|
||||
start: string; // Segment start time, seconds
|
||||
dur: string; // Segment duration, seconds
|
||||
text: string; // Decoded text content
|
||||
}
|
||||
|
||||
interface VideoDetails {
|
||||
title: string;
|
||||
description: string;
|
||||
subtitles: Subtitle[];
|
||||
}
|
||||
|
||||
interface Options {
|
||||
videoID: string;
|
||||
lang?: string;
|
||||
fetch?: typeof fetch;
|
||||
}
|
||||
```
|
||||
|
||||
All three are exported by name and can be imported directly:
|
||||
|
||||
```ts
|
||||
import type {
|
||||
Subtitle,
|
||||
VideoDetails,
|
||||
Options,
|
||||
} from 'youtube-caption-extractor';
|
||||
```
|
||||
|
||||
## Languages
|
||||
|
||||
The `lang` argument is a hint, not a strict filter. Track selection precedence:
|
||||
|
||||
1. **Manual captions in the requested language** (`vssId === '.<lang>'`)
|
||||
2. **Auto-generated captions in the requested language** (`vssId === 'a.<lang>'`)
|
||||
3. **Any track whose `languageCode` matches** the requested code
|
||||
4. **Any track whose `vssId` contains the requested code** (partial match)
|
||||
5. **The first available track** as a final fallback
|
||||
|
||||
If you pass `lang: 'en'` and the video only has Spanish manual captions, you'll get those — the library prefers _some_ output over none. If you pass a code that doesn't exist on the video, you'll typically get the video's primary language track. To check whether you got what you asked for, inspect the first segment's text or compare against `VideoDetails.title` / `description`.
|
||||
|
||||
## Error handling
|
||||
|
||||
The library throws a regular `Error` when no extraction path succeeds — for instance, when the video is private, deleted, or YouTube didn't return a usable response.
|
||||
|
||||
The error message has a stable, parseable structure listing each client that was attempted along with the status YouTube returned for it:
|
||||
|
||||
```
|
||||
Video not playable on any client. Attempts:
|
||||
ios: LOGIN_REQUIRED - Sign in to confirm you're not a bot
|
||||
android_vr: LOGIN_REQUIRED - Sign in to confirm you're not a bot
|
||||
mweb: LOGIN_REQUIRED - Sign in to confirm you're not a bot
|
||||
```
|
||||
|
||||
A common pattern for classifying errors and surfacing them gracefully:
|
||||
|
||||
```ts
|
||||
try {
|
||||
const subtitles = await getSubtitles({ videoID, lang });
|
||||
return subtitles;
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (msg.includes('LOGIN_REQUIRED') || msg.includes('not a bot')) {
|
||||
// Transient — usually worth retrying. See "Deployment notes" for details.
|
||||
throw new Error('transient_extraction_failure');
|
||||
}
|
||||
if (msg.includes('Video unavailable') || msg.includes('private')) {
|
||||
throw new Error('video_not_accessible');
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## Deployment notes
|
||||
|
||||
The library calls YouTube directly, so reliability depends partly on the network
|
||||
egress of the process making the request.
|
||||
|
||||
Local development and self-hosted servers tend to work out of the box. Shared
|
||||
serverless, container, and edge IP ranges can sometimes be rate-limited or gated
|
||||
by YouTube's bot checks. That is not a library API issue; it is an egress
|
||||
reputation issue. For production, use the patterns below.
|
||||
|
||||
### Recommended app architecture
|
||||
|
||||
Keep YouTube extraction server-side. Do not call YouTube directly from browser
|
||||
code.
|
||||
|
||||
```txt
|
||||
Browser → your app API route → youtube-caption-extractor → YouTube
|
||||
```
|
||||
|
||||
If you use a separate API service, protect it with a server-side token:
|
||||
|
||||
```txt
|
||||
Browser → your app API route → token-protected caption API → YouTube
|
||||
```
|
||||
|
||||
The included [`sample/`](./sample) demonstrates this pattern with:
|
||||
|
||||
- Next.js API routes as the public browser-facing API
|
||||
- A Cloudflare Worker that rejects requests without `Authorization: Bearer <token>`
|
||||
- A Cloudflare Container running a Hono/Node API
|
||||
- `CAPTION_API_TOKEN` kept server-side only, never in `NEXT_PUBLIC_*`
|
||||
|
||||
### Building resilient calls
|
||||
|
||||
A small retry wrapper handles transient failures gracefully:
|
||||
|
||||
```ts
|
||||
import { getSubtitles, type Subtitle } from 'youtube-caption-extractor';
|
||||
|
||||
async function getSubtitlesWithRetry(
|
||||
videoID: string,
|
||||
lang = 'en',
|
||||
maxAttempts = 3,
|
||||
): Promise<Subtitle[]> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
return await getSubtitles({ videoID, lang });
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
// Don't retry on permanent errors (private/deleted video, etc.)
|
||||
if (msg.includes('Video unavailable') || msg.includes('private')) {
|
||||
throw err;
|
||||
}
|
||||
// Small backoff between attempts
|
||||
await new Promise((r) => setTimeout(r, 200 * attempt));
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
```
|
||||
|
||||
### Customizing the transport
|
||||
|
||||
The optional `fetch` argument lets you supply any custom transport — useful for adding caching, custom headers, regional routing, or proxying through another service:
|
||||
|
||||
```ts
|
||||
import { getSubtitles } from 'youtube-caption-extractor';
|
||||
import { ProxyAgent, fetch as undiciFetch } from 'undici';
|
||||
|
||||
const dispatcher = new ProxyAgent(process.env.OUTBOUND_PROXY_URL!);
|
||||
|
||||
const proxied: typeof fetch = (input, init) =>
|
||||
undiciFetch(input, { ...init, dispatcher }) as unknown as Promise<Response>;
|
||||
|
||||
const subtitles = await getSubtitles({
|
||||
videoID: '7GeFt8suV8E',
|
||||
lang: 'en',
|
||||
fetch: proxied,
|
||||
});
|
||||
```
|
||||
|
||||
Common uses for a custom `fetch`:
|
||||
|
||||
- **Caching layers** — wrap the global fetch with an LRU or in-memory cache
|
||||
- **Authenticated proxies** — add `Authorization` headers via a wrapper
|
||||
- **Regional routing** — direct outbound traffic through a specific region or provider
|
||||
|
||||
### Local vs hosted behavior
|
||||
|
||||
If extraction works locally but fails in a hosted environment with a message like
|
||||
`LOGIN_REQUIRED` or "Sign in to confirm you're not a bot", the hosted provider's
|
||||
egress IP is likely being challenged by YouTube. Your options are:
|
||||
|
||||
1. Run the extraction API somewhere with reliable egress for your workload.
|
||||
2. Use the `fetch` option to route outbound YouTube requests through a trusted proxy.
|
||||
3. Cache successful results aggressively so fewer requests reach YouTube.
|
||||
4. Treat these failures as transient and retry with backoff where appropriate.
|
||||
|
||||
## Usage examples
|
||||
|
||||
### Next.js (App Router)
|
||||
|
||||
```ts
|
||||
// app/api/captions/route.ts
|
||||
import { NextResponse, type NextRequest } from 'next/server';
|
||||
import { getVideoDetails } from 'youtube-caption-extractor';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const videoID = request.nextUrl.searchParams.get('videoID');
|
||||
const lang = request.nextUrl.searchParams.get('lang') ?? 'en';
|
||||
|
||||
if (!videoID) {
|
||||
return NextResponse.json({ error: 'Missing videoID' }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
const details = await getVideoDetails({ videoID, lang });
|
||||
return NextResponse.json(details);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return NextResponse.json({ error: msg }, { status: 500 });
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Call from a client component with `fetch('/api/captions?videoID=...')`. This avoids the browser CORS issue and keeps the YouTube call server-side.
|
||||
|
||||
### Express
|
||||
|
||||
```ts
|
||||
import express from 'express';
|
||||
import { getSubtitles } from 'youtube-caption-extractor';
|
||||
|
||||
const app = express();
|
||||
|
||||
app.get('/captions/:videoID', async (req, res) => {
|
||||
try {
|
||||
const subtitles = await getSubtitles({
|
||||
videoID: req.params.videoID,
|
||||
lang: (req.query.lang as string) ?? 'en',
|
||||
});
|
||||
res.json({ subtitles });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Cloudflare Workers
|
||||
|
||||
```ts
|
||||
import { getSubtitles } from 'youtube-caption-extractor';
|
||||
|
||||
export default {
|
||||
async fetch(request: Request): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
const videoID = url.searchParams.get('videoID');
|
||||
if (!videoID) return new Response('Missing videoID', { status: 400 });
|
||||
|
||||
try {
|
||||
const subtitles = await getSubtitles({ videoID, lang: 'en' });
|
||||
return Response.json({ subtitles });
|
||||
} catch (err) {
|
||||
return Response.json({ error: (err as Error).message }, { status: 500 });
|
||||
}
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
Add `compatibility_flags: ["nodejs_compat"]` in your `wrangler.jsonc` so the library's `he` and `striptags` dependencies resolve. For production workloads, wrap the call in the retry helper from [Building resilient calls](#building-resilient-calls).
|
||||
|
||||
## Debug logging
|
||||
|
||||
The library is silent by default. To see what's happening internally — which client returned what, where it fell back, what URL was hit — set the `DEBUG` env var:
|
||||
|
||||
```sh
|
||||
DEBUG=youtube-caption-extractor node your-script.js
|
||||
|
||||
# Cloudflare Workers
|
||||
DEBUG=youtube-caption-extractor wrangler dev
|
||||
|
||||
# Or DEBUG=* for everything
|
||||
```
|
||||
|
||||
The logger uses only `console.log` and `process.env` (read defensively), so it works in any runtime that provides those — no `debug` package dependency.
|
||||
|
||||
## TypeScript
|
||||
|
||||
The package ships type definitions; no `@types/*` install needed. All three types (`Subtitle`, `VideoDetails`, `Options`) are exported:
|
||||
|
||||
```ts
|
||||
import {
|
||||
getSubtitles,
|
||||
getVideoDetails,
|
||||
type Subtitle,
|
||||
type VideoDetails,
|
||||
type Options,
|
||||
} from 'youtube-caption-extractor';
|
||||
|
||||
async function transcript(opts: Options): Promise<Subtitle[]> {
|
||||
return await getSubtitles(opts);
|
||||
}
|
||||
```
|
||||
|
||||
## Changelog
|
||||
|
||||
### v1.10.2
|
||||
|
||||
- Added a production-ready sample API path using a Dockerized Hono server on Cloudflare Containers.
|
||||
- Added server-side token protection for the sample Cloudflare Worker API.
|
||||
- Updated the sample app so browser requests go through Next.js API routes instead of exposing API secrets client-side.
|
||||
- Refreshed README quickstart and deployment guidance to make local testing, hosted demos, and production egress tradeoffs clearer.
|
||||
|
||||
### v1.10.1
|
||||
|
||||
- **Streamlined the internal client fallback chain.** Removed an outdated client that was no longer contributing successful extractions, and reordered the remaining clients with the most reliable one first.
|
||||
- **Faster successful calls** — one fewer round-trip in the common case (~150 ms saved per request).
|
||||
- No API changes; fully backward-compatible.
|
||||
|
||||
### v1.10.0
|
||||
|
||||
- Caption extraction is reliable again — fixes a regression where `getSubtitles` would silently return `[]` for many videos.
|
||||
- Multi-path extraction with automatic fallback across clients; gracefully degrades when one path is unavailable.
|
||||
- json3-based subtitle parser replaces the legacy XML regex, fixing multi-line and special-character edge cases.
|
||||
- New optional `fetch` option for routing through a residential proxy.
|
||||
- `Options` interface now exported.
|
||||
- `engines.node` bumped to `>=18.0.0`.
|
||||
- Slimmer install — published tarball is ~85% smaller.
|
||||
|
||||
### v1.9.0
|
||||
|
||||
- `Subtitle` interface exported.
|
||||
- Universal debug logger that works in Node.js, Cloudflare Workers, and edge runtimes.
|
||||
- Library is silent by default in production.
|
||||
|
||||
### v1.4.2
|
||||
|
||||
- TypeScript definitions shipped with the package.
|
||||
- Node.js and edge runtime support.
|
||||
- New `getVideoDetails` API for title + description + subtitles in one call.
|
||||
|
||||
## License
|
||||
|
||||
[MIT](./LICENSE)
|
||||
@@ -0,0 +1,28 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
api:
|
||||
build:
|
||||
context: ./sample/server
|
||||
dockerfile: Dockerfile
|
||||
container_name: caption-api
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8765:8080"
|
||||
environment:
|
||||
- CACHE_TTL_SECONDS=21600
|
||||
- EXTRACTION_ATTEMPTS=4
|
||||
- ALLOWED_ORIGINS=*
|
||||
|
||||
web:
|
||||
build:
|
||||
context: ./sample
|
||||
dockerfile: Dockerfile
|
||||
container_name: caption-web
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3456:3000"
|
||||
environment:
|
||||
- CAPTION_API_BASE_URL=http://api:8080
|
||||
depends_on:
|
||||
- api
|
||||
@@ -0,0 +1,8 @@
|
||||
// jest.config.ts
|
||||
const config = {
|
||||
verbose: true,
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
};
|
||||
|
||||
export default config;
|
||||
Generated
+3519
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"name": "youtube-caption-extractor",
|
||||
"version": "1.10.2",
|
||||
"description": "Extract clean, timestamped YouTube captions, subtitles, transcripts, and video metadata for AI summaries, RAG, search, and slide-ready workflows.",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:debug": "DEBUG=youtube-caption-extractor vitest run",
|
||||
"lint": "eslint src --ext .ts",
|
||||
"prepublishOnly": "npm run build"
|
||||
},
|
||||
"keywords": [
|
||||
"youtube",
|
||||
"youtube-captions",
|
||||
"youtube-subtitles",
|
||||
"youtube-transcript",
|
||||
"captions",
|
||||
"subtitles",
|
||||
"transcript",
|
||||
"transcripts",
|
||||
"caption-extractor",
|
||||
"subtitle-extractor",
|
||||
"transcript-extractor",
|
||||
"video-transcript",
|
||||
"timestamped-transcript",
|
||||
"video-metadata",
|
||||
"youtube-metadata",
|
||||
"slide-ready",
|
||||
"slides",
|
||||
"presentation-ready",
|
||||
"research-notes",
|
||||
"rag",
|
||||
"llm",
|
||||
"ai",
|
||||
"summarization",
|
||||
"scraper",
|
||||
"extract",
|
||||
"parser",
|
||||
"video"
|
||||
],
|
||||
"author": {
|
||||
"name": "Himanshu Gupta",
|
||||
"email": "thinktank.himanshu@gmail.com",
|
||||
"url": "https://github.com/devhims"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/devhims/youtube-caption-extractor.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/devhims/youtube-caption-extractor/issues"
|
||||
},
|
||||
"homepage": "https://github.com/devhims/youtube-caption-extractor",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"he": "^1.2.0",
|
||||
"striptags": "^3.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/he": "^1.2.0",
|
||||
"@types/node": "^20.11.0",
|
||||
"@typescript-eslint/eslint-plugin": "^6.18.1",
|
||||
"@typescript-eslint/parser": "^6.18.1",
|
||||
"eslint": "^8.56.0",
|
||||
"typescript": "^5.3.3",
|
||||
"vitest": "^1.1.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
# Browser requests go to the Next.js API routes by default. The Next.js
|
||||
# server can then proxy to a local API, local Wrangler, or deployed Worker.
|
||||
CAPTION_API_BASE_URL=http://localhost:8080
|
||||
CAPTION_API_TOKEN=
|
||||
|
||||
# Local Wrangler Worker/container:
|
||||
# CAPTION_API_BASE_URL=http://localhost:8787
|
||||
|
||||
# Deployed Cloudflare Worker/container:
|
||||
# CAPTION_API_BASE_URL=https://youtube-caption-extractor-api.thinktank-himanshu.workers.dev
|
||||
@@ -0,0 +1,29 @@
|
||||
FROM node:22-alpine AS base
|
||||
|
||||
FROM base AS deps
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm install
|
||||
|
||||
FROM base AS builder
|
||||
WORKDIR /app
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
FROM base AS runner
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=3000
|
||||
|
||||
RUN addgroup --system --gid 1001 nodejs
|
||||
RUN adduser --system --uid 1001 nextjs
|
||||
|
||||
COPY --from=builder /app/public ./public
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
|
||||
|
||||
USER nextjs
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["node", "server.js"]
|
||||
@@ -0,0 +1,83 @@
|
||||
# YouTube Caption Extractor Demo
|
||||
|
||||
This is a Next.js project demonstrating the use of the `youtube-caption-extractor` npm package. It allows users to fetch subtitles and video details from YouTube videos using this package.
|
||||
|
||||
## Features
|
||||
|
||||
- Fetch subtitles from YouTube videos
|
||||
- Retrieve video details including title and description
|
||||
- Support for multiple languages
|
||||
|
||||
## Local development
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000). By default, the UI calls the local Next.js API routes.
|
||||
|
||||
The API app lives in `server/` and can be run separately from Next.js.
|
||||
|
||||
For the fastest local loop, run the Hono API directly and point the Next.js API routes at it:
|
||||
|
||||
```bash
|
||||
npm --prefix server install
|
||||
npm run api:dev
|
||||
CAPTION_API_BASE_URL=http://localhost:8080 npm run dev
|
||||
```
|
||||
|
||||
To test the full local Worker → Cloudflare Container path, run Wrangler with Docker and point the Next.js API routes at Wrangler:
|
||||
|
||||
```bash
|
||||
npm --prefix server install
|
||||
npm run cf:dev
|
||||
CAPTION_API_BASE_URL=http://localhost:8787 CAPTION_API_TOKEN=<token> npm run dev
|
||||
```
|
||||
|
||||
Cloudflare dashboard "live instances" only reflects deployed Cloudflare traffic. It does not change when testing the local Hono API or local Wrangler container.
|
||||
|
||||
## Cloudflare container API
|
||||
|
||||
This sample includes a self-contained Cloudflare Containers app in `server/`. It runs the API as a Dockerized Hono Node server and proxies requests through a Hono Worker.
|
||||
|
||||
```bash
|
||||
npm --prefix server install
|
||||
npm run cf:deploy
|
||||
```
|
||||
|
||||
After deploy, set Vercel/Next.js server-side environment variables to the Worker URL and shared API token:
|
||||
|
||||
```bash
|
||||
CAPTION_API_BASE_URL=https://<your-worker>.<your-subdomain>.workers.dev
|
||||
CAPTION_API_TOKEN=<same-token-configured-on-the-worker>
|
||||
```
|
||||
|
||||
Do not put `CAPTION_API_TOKEN` in a `NEXT_PUBLIC_*` variable. The browser calls the Next.js API routes, and the Next.js server attaches the token when it calls the Worker.
|
||||
|
||||
The container endpoint supports:
|
||||
|
||||
- `GET /health`
|
||||
- `GET /api/subtitles?videoID=<id>&lang=en`
|
||||
- `GET /api/videoDetails?videoID=<id>&lang=en`
|
||||
|
||||
Optional runtime environment variables:
|
||||
|
||||
- `CAPTION_API_TOKEN` — shared bearer token required by the Worker before it proxies to the container.
|
||||
- `OUTBOUND_PROXY_URL` — routes YouTube requests through an HTTP(S) proxy via `undici`.
|
||||
- `CACHE_TTL_SECONDS` — controls the warm in-memory response cache, default `21600`.
|
||||
- `EXTRACTION_ATTEMPTS` — retries transient YouTube egress failures inside the container, default `4`.
|
||||
- `EXTRACTION_RETRY_BASE_DELAY_MS` — linear retry backoff base delay, default `300`.
|
||||
- `ALLOWED_ORIGINS` — comma-separated browser origins for CORS, default `*`.
|
||||
- `CONTAINER_VERSION` — version prefix for container instance names; bump it to force fresh instances during rollouts.
|
||||
|
||||
From `server/`, use `npx wrangler secret put CAPTION_API_TOKEN` to set the shared token on Cloudflare.
|
||||
From `server/`, use `npx wrangler secret put OUTBOUND_PROXY_URL` if the proxy URL contains credentials.
|
||||
|
||||
If the API returns `youtube_blocked_datacenter_ip`, the request reached the Cloudflare container but YouTube blocked the container's outbound datacenter IP. Use direct local API testing (`http://localhost:8080`) for local machine egress, or configure `OUTBOUND_PROXY_URL` with a trusted proxy for deployed Cloudflare container testing.
|
||||
|
||||
## Usage
|
||||
|
||||
1. Enter the YouTube video ID in the "Video ID" field.
|
||||
2. Specify the desired language code in the "Language" field (e.g., 'en', 'es', 'fr') or leave it empty.
|
||||
3. Click "Fetch Data" to retrieve the subtitles and video details.
|
||||
@@ -0,0 +1,79 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
|
||||
const PROXY_HEADERS = ['content-type', 'cache-control', 'x-cache'];
|
||||
const RETRYABLE_PROXY_STATUSES = new Set([502, 503, 504]);
|
||||
|
||||
function proxyAttempts(): number {
|
||||
const configured = Number(process.env.CAPTION_API_PROXY_ATTEMPTS || 3);
|
||||
if (!Number.isFinite(configured)) return 3;
|
||||
return Math.max(1, Math.min(5, Math.floor(configured)));
|
||||
}
|
||||
|
||||
function captionApiBaseUrl(): string {
|
||||
return (process.env.CAPTION_API_BASE_URL || '').replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
function captionApiToken(): string {
|
||||
return process.env.CAPTION_API_TOKEN || '';
|
||||
}
|
||||
|
||||
export async function proxyCaptionApi(
|
||||
path: string,
|
||||
searchParams: URLSearchParams
|
||||
): Promise<NextResponse | null> {
|
||||
const baseUrl = captionApiBaseUrl();
|
||||
if (!baseUrl) return null;
|
||||
|
||||
const upstreamUrl = `${baseUrl}${path}?${searchParams.toString()}`;
|
||||
const token = captionApiToken();
|
||||
const headers = new Headers();
|
||||
|
||||
if (token) {
|
||||
headers.set('Authorization', `Bearer ${token}`);
|
||||
}
|
||||
|
||||
const attempts = proxyAttempts();
|
||||
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
try {
|
||||
const upstream = await fetch(upstreamUrl, {
|
||||
cache: 'no-store',
|
||||
headers,
|
||||
});
|
||||
const body = await upstream.text();
|
||||
const responseHeaders = new Headers();
|
||||
|
||||
for (const header of PROXY_HEADERS) {
|
||||
const value = upstream.headers.get(header);
|
||||
if (value) responseHeaders.set(header, value);
|
||||
}
|
||||
|
||||
if (
|
||||
attempt < attempts &&
|
||||
RETRYABLE_PROXY_STATUSES.has(upstream.status)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return new NextResponse(body, {
|
||||
headers: responseHeaders,
|
||||
status: upstream.status,
|
||||
});
|
||||
} catch (error) {
|
||||
if (attempt < attempts) continue;
|
||||
return NextResponse.json(
|
||||
{
|
||||
code: 'caption_api_unreachable',
|
||||
message:
|
||||
error instanceof Error ? error.message : 'Caption API unreachable',
|
||||
},
|
||||
{ status: 502 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{ code: 'caption_api_unreachable', message: 'Caption API unreachable' },
|
||||
{ status: 502 }
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
|
||||
/**
|
||||
* Maps library errors to HTTP responses. YouTube blocks datacenter IPs
|
||||
* (Vercel, AWS Lambda, Cloudflare Workers) with a bot challenge — the
|
||||
* library will throw a descriptive "Video not playable on any client"
|
||||
* error in that case. We surface that as a 503 with a friendly explanation
|
||||
* instead of a generic 500.
|
||||
*/
|
||||
export function handleApiError(error: unknown): NextResponse {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
const looksLikeBotChallenge =
|
||||
message.includes('LOGIN_REQUIRED') ||
|
||||
message.includes('not a bot') ||
|
||||
message.includes('no longer supported') ||
|
||||
message.includes('Video not playable on any client');
|
||||
|
||||
if (looksLikeBotChallenge) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
code: 'youtube_blocked_datacenter_ip',
|
||||
message:
|
||||
'YouTube is blocking this server egress. Cloud/container hosts often use shared datacenter IP ranges that YouTube gates with a bot challenge. If this persists on Cloudflare Containers, route outbound YouTube requests through the library `fetch` option using a trusted proxy.',
|
||||
debug: message,
|
||||
},
|
||||
{ status: 503 }
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{ code: 'unknown_error', message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { getSubtitles } from 'youtube-caption-extractor';
|
||||
import { NextResponse, type NextRequest } from 'next/server';
|
||||
import { proxyCaptionApi } from '../_lib/captionApiProxy';
|
||||
import { handleApiError } from '../_lib/handleError';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const videoID = searchParams.get('videoID');
|
||||
const lang = searchParams.get('lang') || 'en';
|
||||
|
||||
if (!videoID) {
|
||||
return NextResponse.json(
|
||||
{ code: 'missing_video_id', message: 'Missing videoID' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const proxied = await proxyCaptionApi('/api/subtitles', searchParams);
|
||||
if (proxied) return proxied;
|
||||
|
||||
try {
|
||||
const subtitles = await getSubtitles({ videoID, lang });
|
||||
return NextResponse.json({ subtitles }, { status: 200 });
|
||||
} catch (error) {
|
||||
return handleApiError(error);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { getVideoDetails } from 'youtube-caption-extractor';
|
||||
import { NextResponse, type NextRequest } from 'next/server';
|
||||
import { proxyCaptionApi } from '../_lib/captionApiProxy';
|
||||
import { handleApiError } from '../_lib/handleError';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const videoID = searchParams.get('videoID');
|
||||
const lang = searchParams.get('lang') || 'en';
|
||||
|
||||
if (!videoID) {
|
||||
return NextResponse.json(
|
||||
{ code: 'missing_video_id', message: 'Missing videoID' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const proxied = await proxyCaptionApi('/api/videoDetails', searchParams);
|
||||
if (proxied) return proxied;
|
||||
|
||||
try {
|
||||
const videoDetails = await getVideoDetails({ videoID, lang });
|
||||
return NextResponse.json({ videoDetails }, { status: 200 });
|
||||
} catch (error) {
|
||||
return handleApiError(error);
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
@@ -0,0 +1,66 @@
|
||||
@import "tailwindcss";
|
||||
|
||||
@theme {
|
||||
--font-sans: var(--font-sans), ui-sans-serif, system-ui, sans-serif;
|
||||
--font-serif: var(--font-serif), ui-serif, Georgia, serif;
|
||||
--font-mono: var(--font-mono), ui-monospace, "SF Mono", Menlo, monospace;
|
||||
}
|
||||
|
||||
:root {
|
||||
--paper: #fafaf7;
|
||||
--ink: #0c0a09;
|
||||
}
|
||||
|
||||
html, body {
|
||||
background-color: var(--paper);
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
::selection {
|
||||
background-color: #fde68a;
|
||||
color: #0c0a09;
|
||||
}
|
||||
|
||||
/* Subtle, paper-like scrollbar */
|
||||
.scrollbar-paper {
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: #d6d3d1 transparent;
|
||||
}
|
||||
.scrollbar-paper::-webkit-scrollbar {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
}
|
||||
.scrollbar-paper::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
.scrollbar-paper::-webkit-scrollbar-thumb {
|
||||
background-color: #d6d3d1;
|
||||
border-radius: 999px;
|
||||
border: 2px solid transparent;
|
||||
background-clip: content-box;
|
||||
}
|
||||
.scrollbar-paper::-webkit-scrollbar-thumb:hover {
|
||||
background-color: #a8a29e;
|
||||
background-clip: content-box;
|
||||
border: 2px solid transparent;
|
||||
}
|
||||
|
||||
@utility text-balance {
|
||||
text-wrap: balance;
|
||||
}
|
||||
|
||||
@utility wrap-anywhere {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
/* Highlighter-pen effect: wraps only the text per line, with the color capped
|
||||
to the visible ascender height instead of the heading's larger line box. */
|
||||
.highlight {
|
||||
background-image: linear-gradient(#fde68a, #fde68a);
|
||||
background-repeat: no-repeat;
|
||||
background-position: 0 50%;
|
||||
background-size: 100% 0.72em;
|
||||
-webkit-box-decoration-break: clone;
|
||||
box-decoration-break: clone;
|
||||
padding: 0 0.08em;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import type { Metadata } from 'next';
|
||||
import { Inter, Fraunces, JetBrains_Mono } from 'next/font/google';
|
||||
import { Analytics } from '@vercel/analytics/next';
|
||||
import './globals.css';
|
||||
|
||||
const inter = Inter({
|
||||
subsets: ['latin'],
|
||||
variable: '--font-sans',
|
||||
});
|
||||
|
||||
const fraunces = Fraunces({
|
||||
subsets: ['latin'],
|
||||
variable: '--font-serif',
|
||||
axes: ['opsz', 'SOFT'],
|
||||
});
|
||||
|
||||
const jetbrains = JetBrains_Mono({
|
||||
subsets: ['latin'],
|
||||
variable: '--font-mono',
|
||||
});
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: 'YouTube Caption Extractor',
|
||||
description:
|
||||
'Paste a YouTube URL — get the full transcript with timestamps.',
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
children,
|
||||
}: Readonly<{ children: React.ReactNode }>) {
|
||||
return (
|
||||
<html
|
||||
lang='en'
|
||||
className={`${inter.variable} ${fraunces.variable} ${jetbrains.variable}`}
|
||||
>
|
||||
{/* suppressHydrationWarning silences benign mismatches caused by browser
|
||||
extensions (Grammarly, 1Password, etc.) that mutate <body> attributes
|
||||
after the server-rendered HTML has been sent. */}
|
||||
<body className='font-sans antialiased' suppressHydrationWarning>
|
||||
{children}
|
||||
<Analytics />
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,621 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
|
||||
type Subtitle = { start: string; dur: string; text: string };
|
||||
type VideoDetails = { title?: string; description?: string };
|
||||
type VideoDetailsPayload = VideoDetails & { subtitles?: Subtitle[] };
|
||||
type ErrorState = { message: string; code?: string };
|
||||
|
||||
async function readApiError(res: Response): Promise<ErrorState> {
|
||||
try {
|
||||
const body = await res.json();
|
||||
return {
|
||||
message: body.message ?? body.error ?? `HTTP ${res.status}`,
|
||||
code: body.code,
|
||||
};
|
||||
} catch {
|
||||
return { message: `HTTP ${res.status}` };
|
||||
}
|
||||
}
|
||||
|
||||
const YOUTUBE_ID_REGEX =
|
||||
/(?:youtube\.com\/(?:[^/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/|youtube\.com\/shorts\/)([A-Za-z0-9_-]{11})/;
|
||||
|
||||
function extractVideoId(input: string): string {
|
||||
const t = input.trim();
|
||||
if (!t) return '';
|
||||
if (/^[A-Za-z0-9_-]{11}$/.test(t)) return t;
|
||||
const m = t.match(YOUTUBE_ID_REGEX);
|
||||
return m ? m[1] : t;
|
||||
}
|
||||
|
||||
function formatTime(seconds: number | string): string {
|
||||
const total = Math.floor(Number(seconds));
|
||||
const h = Math.floor(total / 3600);
|
||||
const m = Math.floor((total % 3600) / 60);
|
||||
const s = total % 60;
|
||||
if (h > 0)
|
||||
return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
|
||||
return `${m}:${String(s).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function formatSrtTime(sec: number): string {
|
||||
const total = Math.floor(sec);
|
||||
const ms = Math.round((sec - total) * 1000);
|
||||
const h = Math.floor(total / 3600);
|
||||
const m = Math.floor((total % 3600) / 60);
|
||||
const s = total % 60;
|
||||
return `${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}:${String(
|
||||
s,
|
||||
).padStart(2, '0')},${String(ms).padStart(3, '0')}`;
|
||||
}
|
||||
|
||||
function buildSRT(subs: Subtitle[]): string {
|
||||
return subs
|
||||
.map((sub, i) => {
|
||||
const start = Number(sub.start);
|
||||
const end = start + Number(sub.dur);
|
||||
return `${i + 1}\n${formatSrtTime(start)} --> ${formatSrtTime(end)}\n${
|
||||
sub.text
|
||||
}\n`;
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function download(content: string, filename: string, mime: string) {
|
||||
const blob = new Blob([content], { type: mime });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
const LANGUAGES = [
|
||||
['en', 'English'],
|
||||
['es', 'Spanish'],
|
||||
['fr', 'French'],
|
||||
['de', 'German'],
|
||||
['ja', 'Japanese'],
|
||||
['ko', 'Korean'],
|
||||
['pt', 'Portuguese'],
|
||||
['hi', 'Hindi'],
|
||||
['zh', 'Chinese'],
|
||||
] as const;
|
||||
|
||||
const SAMPLE_IDS = [
|
||||
'0Gb1z-2SjHY',
|
||||
'D37Ijn2o5U0',
|
||||
'g9JIUM0MHgQ',
|
||||
'6BB6exR8Zd8',
|
||||
'55pTFVoclvE',
|
||||
];
|
||||
function apiUrl(path: string, params: URLSearchParams): string {
|
||||
const query = params.toString();
|
||||
return `${path}${query ? `?${query}` : ''}`;
|
||||
}
|
||||
|
||||
export default function HomePage() {
|
||||
const [input, setInput] = useState('');
|
||||
const [lang, setLang] = useState('en');
|
||||
const [subtitles, setSubtitles] = useState<Subtitle[]>([]);
|
||||
const [videoDetails, setVideoDetails] = useState<VideoDetails>({});
|
||||
const [videoId, setVideoId] = useState('');
|
||||
const [error, setError] = useState<ErrorState | null>(null);
|
||||
const [isFetching, setIsFetching] = useState(false);
|
||||
const [query, setQuery] = useState('');
|
||||
const [copied, setCopied] = useState(false);
|
||||
const searchRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const filteredSubs = useMemo(() => {
|
||||
if (!query.trim()) return subtitles;
|
||||
const q = query.toLowerCase();
|
||||
return subtitles.filter((s) => s.text.toLowerCase().includes(q));
|
||||
}, [subtitles, query]);
|
||||
|
||||
const totalDuration = useMemo(() => {
|
||||
if (!subtitles.length) return 0;
|
||||
const last = subtitles[subtitles.length - 1];
|
||||
return Number(last.start) + Number(last.dur);
|
||||
}, [subtitles]);
|
||||
|
||||
const wordCount = useMemo(
|
||||
() =>
|
||||
subtitles.reduce(
|
||||
(acc, s) => acc + s.text.trim().split(/\s+/).filter(Boolean).length,
|
||||
0,
|
||||
),
|
||||
[subtitles],
|
||||
);
|
||||
|
||||
const hasResults = subtitles.length > 0;
|
||||
|
||||
// Cmd/Ctrl+K to focus search when results are visible
|
||||
useEffect(() => {
|
||||
if (!hasResults) return;
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
if ((e.metaKey || e.ctrlKey) && e.key === 'k') {
|
||||
e.preventDefault();
|
||||
searchRef.current?.focus();
|
||||
}
|
||||
};
|
||||
window.addEventListener('keydown', onKey);
|
||||
return () => window.removeEventListener('keydown', onKey);
|
||||
}, [hasResults]);
|
||||
|
||||
const fetchData = async (overrideInput?: string) => {
|
||||
const raw = overrideInput ?? input;
|
||||
const id = extractVideoId(raw);
|
||||
if (!id || id.length !== 11) {
|
||||
setError({
|
||||
message: 'That doesn’t look like a YouTube URL or video ID.',
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (overrideInput) setInput(overrideInput);
|
||||
setIsFetching(true);
|
||||
setError(null);
|
||||
setQuery('');
|
||||
try {
|
||||
const params = new URLSearchParams({ videoID: id, lang });
|
||||
const detailsRes = await fetch(apiUrl('/api/videoDetails', params));
|
||||
|
||||
if (!detailsRes.ok) {
|
||||
setError(await readApiError(detailsRes));
|
||||
setSubtitles([]);
|
||||
setVideoDetails({});
|
||||
return;
|
||||
}
|
||||
|
||||
const detailsData = await detailsRes.json();
|
||||
const details: VideoDetailsPayload = detailsData.videoDetails ?? {};
|
||||
const { subtitles: subs = [], ...metadata } = details;
|
||||
|
||||
setSubtitles(subs);
|
||||
setVideoDetails(metadata);
|
||||
setVideoId(id);
|
||||
if (subs.length === 0) {
|
||||
setError({
|
||||
message: `No captions for ${LANGUAGES.find((l) => l[0] === lang)?.[1] ?? lang}. This video might not be subtitled in that language.`,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
setError({
|
||||
message: err instanceof Error ? err.message : 'Unknown error',
|
||||
});
|
||||
setSubtitles([]);
|
||||
setVideoDetails({});
|
||||
} finally {
|
||||
setIsFetching(false);
|
||||
}
|
||||
};
|
||||
|
||||
const copyTranscript = async () => {
|
||||
const text = subtitles.map((s) => s.text).join('\n');
|
||||
await navigator.clipboard.writeText(text);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 1600);
|
||||
};
|
||||
|
||||
return (
|
||||
<main className='min-h-screen flex flex-col'>
|
||||
<div className='mx-auto w-full max-w-2xl px-5 sm:px-8 pt-14 sm:pt-24 pb-16 flex-1'>
|
||||
{/* Eyebrow */}
|
||||
<div className='font-mono text-[10px] uppercase tracking-[0.22em] text-stone-500'>
|
||||
youtube-caption-extractor
|
||||
</div>
|
||||
|
||||
{/* Headline */}
|
||||
<h1 className='mt-3 font-serif text-[40px] sm:text-[56px] leading-[1.15] tracking-[-0.02em] text-stone-900'>
|
||||
<span className='highlight'>Get clean transcript</span>
|
||||
<br />
|
||||
<span className='highlight italic font-light'>
|
||||
for any{' '}
|
||||
<span className='not-italic font-normal tracking-[-0.01em]'>
|
||||
YouTube
|
||||
</span>{' '}
|
||||
video
|
||||
</span>
|
||||
</h1>
|
||||
|
||||
<p className='mt-6 max-w-md text-stone-600 leading-relaxed'>
|
||||
Paste the video link and read it like an article: searchable,
|
||||
timestamped, exportable.
|
||||
</p>
|
||||
|
||||
{/* Input — single underlined field, language as inline metadata below */}
|
||||
<form
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
if (!isFetching) fetchData();
|
||||
}}
|
||||
className='mt-12'
|
||||
>
|
||||
<div className='relative'>
|
||||
<input
|
||||
type='text'
|
||||
value={input}
|
||||
onChange={(e) => setInput(e.target.value)}
|
||||
placeholder='paste a youtube link'
|
||||
spellCheck={false}
|
||||
autoFocus
|
||||
disabled={isFetching}
|
||||
className='w-full bg-transparent border-b border-stone-300 pb-3 pr-10 text-lg text-stone-900 placeholder:text-stone-400 focus:outline-none focus:border-stone-900 transition-colors disabled:opacity-50'
|
||||
/>
|
||||
<button
|
||||
type='submit'
|
||||
disabled={isFetching || !input.trim()}
|
||||
aria-label='Extract transcript'
|
||||
className='absolute right-0 bottom-3 text-stone-400 hover:text-stone-900 disabled:opacity-30 disabled:cursor-not-allowed transition-colors'
|
||||
>
|
||||
{isFetching ? (
|
||||
<Spinner />
|
||||
) : (
|
||||
<svg
|
||||
viewBox='0 0 24 24'
|
||||
className='size-5'
|
||||
fill='none'
|
||||
stroke='currentColor'
|
||||
strokeWidth='1.75'
|
||||
strokeLinecap='round'
|
||||
strokeLinejoin='round'
|
||||
aria-hidden='true'
|
||||
>
|
||||
<path d='M5 12h14M13 5l7 7-7 7' />
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className='mt-4 flex flex-wrap items-baseline justify-between gap-x-6 gap-y-3 text-xs text-stone-500'>
|
||||
<label className='inline-flex items-baseline gap-2'>
|
||||
<span>in</span>
|
||||
<span className='relative'>
|
||||
<select
|
||||
value={lang}
|
||||
onChange={(e) => setLang(e.target.value)}
|
||||
className='appearance-none bg-transparent pr-4 text-stone-900 border-b border-dotted border-stone-400 hover:border-stone-900 focus:outline-none focus:border-stone-900 transition-colors cursor-pointer'
|
||||
aria-label='Language'
|
||||
>
|
||||
{LANGUAGES.map(([code, label]) => (
|
||||
<option key={code} value={code}>
|
||||
{label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<span
|
||||
className='absolute right-0 top-1/2 -translate-y-1/2 text-stone-400 pointer-events-none text-[10px]'
|
||||
aria-hidden='true'
|
||||
>
|
||||
▾
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
{!hasResults && !isFetching && (
|
||||
<>
|
||||
<div className='flex min-w-0 flex-wrap items-baseline gap-x-3 gap-y-2 sm:hidden'>
|
||||
<span className='shrink-0'>or try</span>
|
||||
{SAMPLE_IDS.slice(0, 2).map((id) => (
|
||||
<SampleIdButton
|
||||
key={id}
|
||||
id={id}
|
||||
onClick={() => fetchData(id)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<div className='hidden min-w-0 flex-wrap items-baseline justify-end gap-x-3 gap-y-2 sm:flex'>
|
||||
<span className='shrink-0'>or try</span>
|
||||
{SAMPLE_IDS.map((id) => (
|
||||
<SampleIdButton
|
||||
key={id}
|
||||
id={id}
|
||||
onClick={() => fetchData(id)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{/* Error / notice */}
|
||||
{error &&
|
||||
(error.code === 'youtube_blocked_datacenter_ip' ? (
|
||||
<aside className='mt-8 rounded-md border border-amber-200 bg-amber-50/60 px-5 py-4'>
|
||||
<div className='font-mono text-[10px] uppercase tracking-[0.22em] text-amber-700 mb-2'>
|
||||
Live demo limitation
|
||||
</div>
|
||||
<p className='text-sm text-stone-700 leading-relaxed wrap-anywhere'>
|
||||
{error.message}
|
||||
</p>
|
||||
<div className='mt-3 flex flex-wrap gap-x-5 gap-y-1 text-xs'>
|
||||
<a
|
||||
href='https://github.com/devhims/youtube-caption-extractor#deployment-environments'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='text-amber-800 hover:text-amber-900 underline decoration-amber-300 hover:decoration-amber-700 decoration-1 underline-offset-[3px] transition-colors'
|
||||
>
|
||||
why this happens
|
||||
</a>
|
||||
<a
|
||||
href='https://github.com/devhims/youtube-caption-extractor'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='text-amber-800 hover:text-amber-900 underline decoration-amber-300 hover:decoration-amber-700 decoration-1 underline-offset-[3px] transition-colors'
|
||||
>
|
||||
run locally
|
||||
</a>
|
||||
</div>
|
||||
</aside>
|
||||
) : (
|
||||
<p className='mt-6 text-sm text-stone-600 border-l-2 border-stone-300 pl-3 wrap-anywhere'>
|
||||
{error.message}
|
||||
</p>
|
||||
))}
|
||||
|
||||
{/* Loading */}
|
||||
{isFetching && !hasResults && (
|
||||
<div className='mt-16 space-y-6'>
|
||||
<div className='flex gap-5'>
|
||||
<div className='shrink-0 w-32 aspect-video bg-stone-100 rounded animate-pulse' />
|
||||
<div className='flex-1 space-y-2 pt-1'>
|
||||
<div className='h-5 bg-stone-100 rounded animate-pulse w-3/4' />
|
||||
<div className='h-3 bg-stone-100 rounded animate-pulse w-1/3' />
|
||||
</div>
|
||||
</div>
|
||||
<hr className='border-stone-200' />
|
||||
<div className='space-y-3'>
|
||||
{Array.from({ length: 6 }).map((_, i) => (
|
||||
<div key={i} className='flex gap-4'>
|
||||
<div className='w-10 h-3 bg-stone-100 rounded animate-pulse mt-1.5' />
|
||||
<div
|
||||
className='flex-1 h-3 bg-stone-100 rounded animate-pulse'
|
||||
style={{ width: `${60 + ((i * 7) % 40)}%` }}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Results */}
|
||||
{hasResults && (
|
||||
<article className='mt-16'>
|
||||
{/* Video header */}
|
||||
<header className='flex flex-col gap-4 sm:flex-row sm:gap-5 mb-10'>
|
||||
<a
|
||||
href={`https://youtu.be/${videoId}`}
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='group block w-full aspect-video rounded overflow-hidden bg-stone-100 border border-stone-200 sm:w-32 sm:shrink-0'
|
||||
>
|
||||
<img
|
||||
src={`https://img.youtube.com/vi/${videoId}/hqdefault.jpg`}
|
||||
alt={videoDetails.title ?? 'Video thumbnail'}
|
||||
className='w-full h-full object-cover group-hover:scale-[1.03] transition-transform duration-500'
|
||||
/>
|
||||
</a>
|
||||
<div className='flex-1 min-w-0'>
|
||||
<h2 className='font-serif text-2xl leading-[1.15] tracking-[-0.01em] text-stone-900 text-balance wrap-anywhere'>
|
||||
{videoDetails.title || `youtu.be/${videoId}`}
|
||||
</h2>
|
||||
<a
|
||||
href={`https://youtu.be/${videoId}`}
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='mt-2 inline-flex max-w-full items-center gap-1 text-xs font-mono text-stone-500 hover:text-stone-900 transition-colors'
|
||||
>
|
||||
<span className='min-w-0 truncate'>youtu.be/{videoId}</span>
|
||||
<span aria-hidden='true'>↗</span>
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<hr className='border-stone-200' />
|
||||
|
||||
{/* Stats */}
|
||||
<dl className='mt-6 grid grid-cols-2 gap-x-8 gap-y-5 text-sm sm:flex sm:items-baseline sm:gap-8'>
|
||||
<Stat label='segments' value={subtitles.length.toString()} />
|
||||
<Stat label='length' value={formatTime(totalDuration)} />
|
||||
<Stat label='words' value={wordCount.toLocaleString()} />
|
||||
<Stat label='language' value={lang} />
|
||||
</dl>
|
||||
|
||||
{/* Description */}
|
||||
{videoDetails.description && (
|
||||
<details className='mt-6 group'>
|
||||
<summary className='cursor-pointer text-xs uppercase tracking-[0.18em] text-stone-500 hover:text-stone-900 transition-colors inline-flex items-center gap-1.5'>
|
||||
<span>description</span>
|
||||
<span className='group-open:rotate-90 transition-transform inline-block'>
|
||||
→
|
||||
</span>
|
||||
</summary>
|
||||
<p className='mt-3 text-sm text-stone-600 whitespace-pre-wrap leading-relaxed max-h-60 overflow-y-auto scrollbar-paper pr-2 wrap-anywhere'>
|
||||
{videoDetails.description}
|
||||
</p>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{/* Transcript section */}
|
||||
<section className='mt-14'>
|
||||
<div className='flex flex-wrap items-baseline justify-between gap-4 mb-5'>
|
||||
<h3 className='font-serif text-2xl tracking-[-0.01em] text-stone-900'>
|
||||
Transcript
|
||||
</h3>
|
||||
<div className='flex items-center gap-4 text-xs'>
|
||||
<button
|
||||
onClick={copyTranscript}
|
||||
className='text-stone-500 hover:text-stone-900 transition-colors'
|
||||
>
|
||||
{copied ? 'copied' : 'copy'}
|
||||
</button>
|
||||
<button
|
||||
onClick={() =>
|
||||
download(
|
||||
subtitles.map((s) => s.text).join('\n'),
|
||||
`${videoId}.txt`,
|
||||
'text/plain',
|
||||
)
|
||||
}
|
||||
className='text-stone-500 hover:text-stone-900 transition-colors'
|
||||
>
|
||||
.txt
|
||||
</button>
|
||||
<button
|
||||
onClick={() =>
|
||||
download(
|
||||
buildSRT(subtitles),
|
||||
`${videoId}.srt`,
|
||||
'application/x-subrip',
|
||||
)
|
||||
}
|
||||
className='text-stone-500 hover:text-stone-900 transition-colors'
|
||||
>
|
||||
.srt
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Search */}
|
||||
<div className='relative mb-6'>
|
||||
<input
|
||||
ref={searchRef}
|
||||
type='search'
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
placeholder='Search transcript'
|
||||
className='w-full bg-transparent border-b border-stone-200 py-2 pr-16 text-sm placeholder:text-stone-400 focus:outline-none focus:border-stone-900 transition-colors'
|
||||
/>
|
||||
<kbd className='absolute right-0 top-1/2 -translate-y-1/2 text-[10px] font-mono text-stone-400 border border-stone-200 rounded px-1.5 py-0.5'>
|
||||
⌘K
|
||||
</kbd>
|
||||
{query.trim() && (
|
||||
<div className='mt-2 text-xs text-stone-500'>
|
||||
{filteredSubs.length === 0
|
||||
? `no matches`
|
||||
: `${filteredSubs.length} of ${subtitles.length}`}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Segments */}
|
||||
<ol className='space-y-2'>
|
||||
{filteredSubs.map((sub, i) => (
|
||||
<li key={`${sub.start}-${i}`} className='flex gap-4 sm:gap-5 group'>
|
||||
<a
|
||||
href={`https://youtu.be/${videoId}?t=${Math.floor(
|
||||
Number(sub.start),
|
||||
)}s`}
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='shrink-0 w-10 sm:w-12 pt-[3px] font-mono text-[11px] tabular-nums text-stone-400 hover:text-stone-900 transition-colors text-right'
|
||||
title='Open at this moment'
|
||||
>
|
||||
{formatTime(sub.start)}
|
||||
</a>
|
||||
<p className='min-w-0 flex-1 text-[15px] leading-[1.7] text-stone-800 wrap-anywhere'>
|
||||
{query.trim() ? highlight(sub.text, query) : sub.text}
|
||||
</p>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
</section>
|
||||
</article>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Footer pinned to the bottom of the viewport on short content, naturally follows on long content */}
|
||||
<footer className='border-t border-stone-200'>
|
||||
<div className='mx-auto w-full max-w-2xl px-5 sm:px-8 py-6 text-xs text-stone-500 flex flex-wrap items-baseline gap-x-6 gap-y-2'>
|
||||
<a
|
||||
href='https://www.npmjs.com/package/youtube-caption-extractor'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='hover:text-stone-900 transition-colors'
|
||||
>
|
||||
npm
|
||||
</a>
|
||||
<a
|
||||
href='https://github.com/devhims/youtube-caption-extractor'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='hover:text-stone-900 transition-colors'
|
||||
>
|
||||
github
|
||||
</a>
|
||||
<span className='ml-auto font-mono text-stone-400'>v1.10.2</span>
|
||||
</div>
|
||||
</footer>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
function Spinner() {
|
||||
return (
|
||||
<svg
|
||||
className='size-3.5 animate-spin'
|
||||
viewBox='0 0 24 24'
|
||||
fill='none'
|
||||
stroke='currentColor'
|
||||
strokeWidth='2.5'
|
||||
strokeLinecap='round'
|
||||
>
|
||||
<path d='M12 3a9 9 0 1 0 9 9' className='opacity-90' />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
function SampleIdButton({
|
||||
id,
|
||||
onClick,
|
||||
}: {
|
||||
id: string;
|
||||
onClick: () => void;
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
type='button'
|
||||
onClick={onClick}
|
||||
className='font-mono text-stone-700 hover:text-stone-900 underline decoration-stone-300 hover:decoration-stone-900 decoration-1 underline-offset-[3px] transition-colors'
|
||||
>
|
||||
{id}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function Stat({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<div className='flex flex-col'>
|
||||
<dt className='text-[10px] uppercase tracking-[0.18em] text-stone-500 mb-1'>
|
||||
{label}
|
||||
</dt>
|
||||
<dd className='font-serif text-xl tabular-nums text-stone-900'>
|
||||
{value}
|
||||
</dd>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function highlight(text: string, query: string): React.ReactNode {
|
||||
const q = query.trim();
|
||||
if (!q) return text;
|
||||
const parts = text.split(new RegExp(`(${escapeRegex(q)})`, 'gi'));
|
||||
return parts.map((part, i) =>
|
||||
part.toLowerCase() === q.toLowerCase() ? (
|
||||
<mark key={i} className='bg-yellow-100 text-stone-900 rounded px-0.5'>
|
||||
{part}
|
||||
</mark>
|
||||
) : (
|
||||
<span key={i}>{part}</span>
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function escapeRegex(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
/// <reference types="next" />
|
||||
/// <reference types="next/image-types/global" />
|
||||
import "./.next/dev/types/routes.d.ts";
|
||||
|
||||
// NOTE: This file should not be edited
|
||||
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
||||
@@ -0,0 +1,12 @@
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname } from 'node:path';
|
||||
|
||||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {
|
||||
output: 'standalone',
|
||||
turbopack: {
|
||||
root: dirname(fileURLToPath(import.meta.url)),
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
Generated
+1786
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "youtube-captions-test",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"api:dev": "npm --prefix server run dev",
|
||||
"build": "next build",
|
||||
"cf:dev": "npm --prefix server run cf:dev",
|
||||
"cf:deploy": "npm --prefix server run cf:deploy",
|
||||
"start": "next start",
|
||||
"lint": "next lint"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vercel/analytics": "^2.0.1",
|
||||
"next": "^16.2.6",
|
||||
"react": "^19.2.6",
|
||||
"react-dom": "^19.2.6",
|
||||
"youtube-caption-extractor": "^1.10.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4.3.0",
|
||||
"@types/node": "^25.7.0",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"postcss": "^8",
|
||||
"tailwindcss": "^4.3.0",
|
||||
"typescript": "^6.0.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/** @type {import('postcss-load-config').Config} */
|
||||
const config = {
|
||||
plugins: {
|
||||
'@tailwindcss/postcss': {},
|
||||
},
|
||||
};
|
||||
|
||||
export default config;
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>
|
||||
|
After Width: | Height: | Size: 629 B |
@@ -0,0 +1,4 @@
|
||||
node_modules
|
||||
.wrangler
|
||||
*.log
|
||||
.DS_Store
|
||||
@@ -0,0 +1,15 @@
|
||||
FROM node:22-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=8080
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm ci --omit=dev
|
||||
|
||||
COPY api.cjs ./api.cjs
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["node", "api.cjs"]
|
||||
@@ -0,0 +1,290 @@
|
||||
const { serve } = require('@hono/node-server');
|
||||
const { Hono } = require('hono');
|
||||
const { getSubtitles, getVideoDetails } = require('youtube-caption-extractor');
|
||||
|
||||
const PORT = Number(process.env.PORT || 8080);
|
||||
const CACHE_TTL_MS = Number(process.env.CACHE_TTL_SECONDS || 21600) * 1000;
|
||||
const SUCCESS_CACHE_CONTROL =
|
||||
process.env.SUCCESS_CACHE_CONTROL ||
|
||||
'public, max-age=3600, s-maxage=21600, stale-while-revalidate=86400';
|
||||
const OUTBOUND_PROXY_URL = process.env.OUTBOUND_PROXY_URL;
|
||||
const EXTRACTION_ATTEMPTS = boundedInteger(
|
||||
process.env.EXTRACTION_ATTEMPTS,
|
||||
4,
|
||||
1,
|
||||
8
|
||||
);
|
||||
const EXTRACTION_RETRY_BASE_DELAY_MS = boundedInteger(
|
||||
process.env.EXTRACTION_RETRY_BASE_DELAY_MS,
|
||||
300,
|
||||
0,
|
||||
5000
|
||||
);
|
||||
|
||||
const app = new Hono();
|
||||
const cache = new Map();
|
||||
let extractorFetchPromise;
|
||||
|
||||
function boundedInteger(value, fallback, min, max) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed)) return fallback;
|
||||
return Math.max(min, Math.min(max, Math.floor(parsed)));
|
||||
}
|
||||
|
||||
function parseAllowedOrigins() {
|
||||
const raw = process.env.ALLOWED_ORIGINS || '*';
|
||||
return raw
|
||||
.split(',')
|
||||
.map((origin) => origin.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
const allowedOrigins = parseAllowedOrigins();
|
||||
|
||||
function allowOriginFor(c) {
|
||||
const origin = c.req.header('origin');
|
||||
if (allowedOrigins.includes('*') || !origin) return '*';
|
||||
return allowedOrigins.includes(origin) ? origin : allowedOrigins[0];
|
||||
}
|
||||
|
||||
function normalizeApiError(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const looksLikeBotChallenge =
|
||||
message.includes('LOGIN_REQUIRED') ||
|
||||
message.includes('not a bot') ||
|
||||
message.includes('no longer supported') ||
|
||||
message.includes('Video not playable on any client');
|
||||
|
||||
if (looksLikeBotChallenge) {
|
||||
return {
|
||||
status: 503,
|
||||
body: {
|
||||
code: 'youtube_blocked_datacenter_ip',
|
||||
message:
|
||||
'YouTube is blocking this server egress. Cloud/container hosts often use shared datacenter IP ranges that YouTube gates with a bot challenge. If this persists on Cloudflare Containers, route outbound YouTube requests through the library `fetch` option using a trusted proxy.',
|
||||
debug: message,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 500,
|
||||
body: { code: 'unknown_error', message },
|
||||
};
|
||||
}
|
||||
|
||||
function isRetryableExtractionError(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const lower = message.toLowerCase();
|
||||
|
||||
if (
|
||||
lower.includes('private') ||
|
||||
lower.includes('video unavailable') ||
|
||||
lower.includes('this video is unavailable') ||
|
||||
lower.includes('removed by')
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
message.includes('LOGIN_REQUIRED') ||
|
||||
lower.includes('not a bot') ||
|
||||
lower.includes('no longer supported') ||
|
||||
lower.includes('video not playable on any client') ||
|
||||
lower.includes('innertube /player failed') ||
|
||||
lower.includes('caption fetch failed: 429') ||
|
||||
lower.includes('caption fetch failed: 500') ||
|
||||
lower.includes('caption fetch failed: 502') ||
|
||||
lower.includes('caption fetch failed: 503') ||
|
||||
lower.includes('caption fetch failed: 504') ||
|
||||
lower.includes('fetch failed') ||
|
||||
lower.includes('econnreset') ||
|
||||
lower.includes('etimedout') ||
|
||||
lower.includes('und_err')
|
||||
);
|
||||
}
|
||||
|
||||
function delay(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function withExtractionRetry(operation, options = {}) {
|
||||
const attempts = boundedInteger(
|
||||
options.attempts,
|
||||
EXTRACTION_ATTEMPTS,
|
||||
1,
|
||||
8
|
||||
);
|
||||
const baseDelayMs = boundedInteger(
|
||||
options.baseDelayMs,
|
||||
EXTRACTION_RETRY_BASE_DELAY_MS,
|
||||
0,
|
||||
5000
|
||||
);
|
||||
let lastError;
|
||||
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt === attempts || !isRetryableExtractionError(error)) {
|
||||
throw error;
|
||||
}
|
||||
if (baseDelayMs > 0) {
|
||||
await delay(baseDelayMs * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
function getCached(key) {
|
||||
const entry = cache.get(key);
|
||||
if (!entry) return null;
|
||||
if (Date.now() > entry.expiresAt) {
|
||||
cache.delete(key);
|
||||
return null;
|
||||
}
|
||||
return entry.value;
|
||||
}
|
||||
|
||||
function setCached(key, value) {
|
||||
if (CACHE_TTL_MS <= 0) return;
|
||||
cache.set(key, {
|
||||
value,
|
||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||
});
|
||||
}
|
||||
|
||||
async function getExtractorFetch() {
|
||||
if (!OUTBOUND_PROXY_URL) return undefined;
|
||||
if (!extractorFetchPromise) {
|
||||
extractorFetchPromise = import('undici').then(({ ProxyAgent, fetch }) => {
|
||||
const dispatcher = new ProxyAgent(OUTBOUND_PROXY_URL);
|
||||
return (input, init) => fetch(input, { ...init, dispatcher });
|
||||
});
|
||||
}
|
||||
return extractorFetchPromise;
|
||||
}
|
||||
|
||||
function cacheHeaders(cacheState) {
|
||||
return {
|
||||
'Cache-Control': SUCCESS_CACHE_CONTROL,
|
||||
'X-Cache': cacheState,
|
||||
};
|
||||
}
|
||||
|
||||
function missingVideoId(c) {
|
||||
return c.json(
|
||||
{ code: 'missing_video_id', message: 'Missing videoID' },
|
||||
400
|
||||
);
|
||||
}
|
||||
|
||||
function methodNotAllowed(c) {
|
||||
return c.json({ code: 'method_not_allowed', message: 'Use GET' }, 405);
|
||||
}
|
||||
|
||||
app.use('*', async (c, next) => {
|
||||
c.header('Access-Control-Allow-Origin', allowOriginFor(c));
|
||||
c.header('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
||||
c.header('Access-Control-Allow-Headers', 'Content-Type');
|
||||
c.header('Vary', 'Origin');
|
||||
|
||||
if (c.req.method === 'OPTIONS') {
|
||||
return c.body(null, 204);
|
||||
}
|
||||
|
||||
await next();
|
||||
});
|
||||
|
||||
app.get('/health', (c) =>
|
||||
c.json({ status: 'ok', runtime: 'hono-node-container' })
|
||||
);
|
||||
app.all('/health', methodNotAllowed);
|
||||
|
||||
app.get('/api/subtitles', async (c) => {
|
||||
const videoID = c.req.query('videoID');
|
||||
const lang = c.req.query('lang') || 'en';
|
||||
|
||||
if (!videoID) return missingVideoId(c);
|
||||
|
||||
try {
|
||||
const cacheKey = `subtitles:${videoID}:${lang}`;
|
||||
const cached = getCached(cacheKey);
|
||||
if (cached) {
|
||||
return c.json(cached, 200, cacheHeaders('HIT'));
|
||||
}
|
||||
|
||||
const fetchImpl = await getExtractorFetch();
|
||||
const body = {
|
||||
subtitles: await withExtractionRetry(() =>
|
||||
getSubtitles({ videoID, lang, fetch: fetchImpl })
|
||||
),
|
||||
};
|
||||
setCached(cacheKey, body);
|
||||
return c.json(body, 200, cacheHeaders('MISS'));
|
||||
} catch (error) {
|
||||
const normalized = normalizeApiError(error);
|
||||
return c.json(normalized.body, normalized.status);
|
||||
}
|
||||
});
|
||||
app.all('/api/subtitles', methodNotAllowed);
|
||||
|
||||
app.get('/api/videoDetails', async (c) => {
|
||||
const videoID = c.req.query('videoID');
|
||||
const lang = c.req.query('lang') || 'en';
|
||||
|
||||
if (!videoID) return missingVideoId(c);
|
||||
|
||||
try {
|
||||
const cacheKey = `videoDetails:${videoID}:${lang}`;
|
||||
const cached = getCached(cacheKey);
|
||||
if (cached) {
|
||||
return c.json(cached, 200, cacheHeaders('HIT'));
|
||||
}
|
||||
|
||||
const fetchImpl = await getExtractorFetch();
|
||||
const body = {
|
||||
videoDetails: await withExtractionRetry(() =>
|
||||
getVideoDetails({ videoID, lang, fetch: fetchImpl })
|
||||
),
|
||||
};
|
||||
setCached(cacheKey, body);
|
||||
return c.json(body, 200, cacheHeaders('MISS'));
|
||||
} catch (error) {
|
||||
const normalized = normalizeApiError(error);
|
||||
return c.json(normalized.body, normalized.status);
|
||||
}
|
||||
});
|
||||
app.all('/api/videoDetails', methodNotAllowed);
|
||||
|
||||
app.notFound((c) => c.json({ code: 'not_found', message: 'Not found' }, 404));
|
||||
|
||||
app.onError((error, c) => {
|
||||
const normalized = normalizeApiError(error);
|
||||
return c.json(normalized.body, normalized.status);
|
||||
});
|
||||
|
||||
if (require.main === module) {
|
||||
serve(
|
||||
{
|
||||
fetch: app.fetch,
|
||||
hostname: '0.0.0.0',
|
||||
port: PORT,
|
||||
},
|
||||
() => {
|
||||
console.log(`Caption API listening on :${PORT}`);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
app,
|
||||
boundedInteger,
|
||||
isRetryableExtractionError,
|
||||
normalizeApiError,
|
||||
withExtractionRetry,
|
||||
};
|
||||
@@ -0,0 +1,64 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const test = require('node:test');
|
||||
|
||||
const {
|
||||
isRetryableExtractionError,
|
||||
normalizeApiError,
|
||||
withExtractionRetry,
|
||||
} = require('./api.cjs');
|
||||
|
||||
test('withExtractionRetry retries transient YouTube egress failures', async () => {
|
||||
let calls = 0;
|
||||
|
||||
const value = await withExtractionRetry(
|
||||
async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
throw new Error('ios: LOGIN_REQUIRED - Sign in to confirm you are not a bot');
|
||||
}
|
||||
return { subtitles: [{ text: 'ok' }] };
|
||||
},
|
||||
{ attempts: 3, baseDelayMs: 0 }
|
||||
);
|
||||
|
||||
assert.equal(calls, 2);
|
||||
assert.deepEqual(value, { subtitles: [{ text: 'ok' }] });
|
||||
});
|
||||
|
||||
test('withExtractionRetry does not retry permanent video errors', async () => {
|
||||
let calls = 0;
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
withExtractionRetry(
|
||||
async () => {
|
||||
calls += 1;
|
||||
throw new Error('Video unavailable');
|
||||
},
|
||||
{ attempts: 3, baseDelayMs: 0 }
|
||||
),
|
||||
/Video unavailable/
|
||||
);
|
||||
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('normalizeApiError maps bot challenges to the public egress error', () => {
|
||||
const normalized = normalizeApiError(
|
||||
new Error('mweb: LOGIN_REQUIRED - Sign in to confirm you are not a bot')
|
||||
);
|
||||
|
||||
assert.equal(normalized.status, 503);
|
||||
assert.equal(normalized.body.code, 'youtube_blocked_datacenter_ip');
|
||||
});
|
||||
|
||||
test('isRetryableExtractionError recognises transient statuses', () => {
|
||||
assert.equal(
|
||||
isRetryableExtractionError(new Error('Caption fetch failed: 503')),
|
||||
true
|
||||
);
|
||||
assert.equal(
|
||||
isRetryableExtractionError(new Error('This video is unavailable')),
|
||||
false
|
||||
);
|
||||
});
|
||||
Generated
+1576
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "youtube-caption-extractor-api",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "node api.cjs",
|
||||
"test": "node --test api.test.cjs worker-routing.test.mjs",
|
||||
"cf:dev": "wrangler dev",
|
||||
"cf:deploy": "wrangler deploy",
|
||||
"cf:dry-run": "wrangler deploy --dry-run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^2.0.2",
|
||||
"hono": "^4.12.19",
|
||||
"undici": "^8.3.0",
|
||||
"youtube-caption-extractor": "^1.10.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@cloudflare/containers": "^0.0.28",
|
||||
"wrangler": "^4.34.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
export function routingKeyFromUrl(rawUrl) {
|
||||
try {
|
||||
const url = new URL(rawUrl);
|
||||
const videoID = url.searchParams.get('videoID') || '';
|
||||
const lang = url.searchParams.get('lang') || 'en';
|
||||
|
||||
if (url.pathname.startsWith('/api/')) {
|
||||
return `${url.pathname}:${videoID}:${lang}`;
|
||||
}
|
||||
|
||||
return url.pathname;
|
||||
} catch {
|
||||
return rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function stableHash(input) {
|
||||
let hash = 2166136261;
|
||||
for (let i = 0; i < input.length; i += 1) {
|
||||
hash ^= input.charCodeAt(i);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
export function containerSlotOrderForUrl(rawUrl, instanceCount) {
|
||||
const count = Math.max(1, Math.floor(Number(instanceCount) || 1));
|
||||
const primary = stableHash(routingKeyFromUrl(rawUrl)) % count;
|
||||
const slots = [primary];
|
||||
|
||||
for (let offset = 1; offset < count; offset += 1) {
|
||||
slots.push((primary + offset) % count);
|
||||
}
|
||||
|
||||
return slots;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
containerSlotOrderForUrl,
|
||||
routingKeyFromUrl,
|
||||
} from './worker-routing.mjs';
|
||||
|
||||
test('routingKeyFromUrl ignores unrelated cache-busting query params', () => {
|
||||
assert.equal(
|
||||
routingKeyFromUrl(
|
||||
'https://example.com/api/videoDetails?videoID=55pTFVoclvE&lang=en&t=1'
|
||||
),
|
||||
routingKeyFromUrl(
|
||||
'https://example.com/api/videoDetails?lang=en&videoID=55pTFVoclvE&t=2'
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test('containerSlotOrderForUrl returns a stable primary and all fallback slots', () => {
|
||||
const url =
|
||||
'https://example.com/api/videoDetails?videoID=55pTFVoclvE&lang=en';
|
||||
const first = containerSlotOrderForUrl(url, 4);
|
||||
const second = containerSlotOrderForUrl(url, 4);
|
||||
|
||||
assert.deepEqual(first, second);
|
||||
assert.equal(first.length, 4);
|
||||
assert.deepEqual([...new Set(first)].sort(), [0, 1, 2, 3]);
|
||||
});
|
||||
|
||||
test('containerSlotOrderForUrl handles invalid instance counts', () => {
|
||||
assert.deepEqual(containerSlotOrderForUrl('/api/videoDetails', 0), [0]);
|
||||
assert.deepEqual(containerSlotOrderForUrl('/api/videoDetails', 'nope'), [0]);
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import { Container, getContainer } from '@cloudflare/containers';
|
||||
import { env as workerEnv } from 'cloudflare:workers';
|
||||
import { Hono } from 'hono';
|
||||
import { containerSlotOrderForUrl } from './worker-routing.mjs';
|
||||
|
||||
const app = new Hono();
|
||||
const RETRYABLE_CONTAINER_STATUSES = new Set([502, 503, 504]);
|
||||
|
||||
export class CaptionApiContainer extends Container {
|
||||
defaultPort = 8080;
|
||||
sleepAfter = '30m';
|
||||
enableInternet = true;
|
||||
envVars = {
|
||||
NODE_ENV: 'production',
|
||||
OUTBOUND_PROXY_URL: workerEnv.OUTBOUND_PROXY_URL || '',
|
||||
CACHE_TTL_SECONDS: workerEnv.CACHE_TTL_SECONDS || '21600',
|
||||
EXTRACTION_ATTEMPTS: workerEnv.EXTRACTION_ATTEMPTS || '4',
|
||||
EXTRACTION_RETRY_BASE_DELAY_MS:
|
||||
workerEnv.EXTRACTION_RETRY_BASE_DELAY_MS || '300',
|
||||
ALLOWED_ORIGINS: workerEnv.ALLOWED_ORIGINS || '*',
|
||||
};
|
||||
}
|
||||
|
||||
function instanceCountFor(c) {
|
||||
const instanceCount = Number.parseInt(c.env.API_INSTANCE_COUNT || '2', 10);
|
||||
return Number.isFinite(instanceCount) && instanceCount > 0
|
||||
? instanceCount
|
||||
: 1;
|
||||
}
|
||||
|
||||
function getCaptionApiContainer(c, instanceSlot) {
|
||||
const containerVersion = c.env.CONTAINER_VERSION || 'default';
|
||||
return getContainer(
|
||||
c.env.CAPTION_API,
|
||||
`${containerVersion}-${instanceSlot}`
|
||||
);
|
||||
}
|
||||
|
||||
async function proxyToContainer(c) {
|
||||
const slots = containerSlotOrderForUrl(c.req.url, instanceCountFor(c));
|
||||
const requestPath = new URL(c.req.url).pathname;
|
||||
let lastError;
|
||||
|
||||
for (let i = 0; i < slots.length; i += 1) {
|
||||
try {
|
||||
const container = getCaptionApiContainer(c, slots[i]);
|
||||
const response = await container.fetch(new Request(c.req.raw));
|
||||
|
||||
if (
|
||||
!requestPath.startsWith('/api/') ||
|
||||
!RETRYABLE_CONTAINER_STATUSES.has(response.status) ||
|
||||
i === slots.length - 1
|
||||
) {
|
||||
return response;
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (i === slots.length - 1) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Caption API container unavailable');
|
||||
}
|
||||
|
||||
async function requireApiToken(c, next) {
|
||||
const expectedToken = c.env.CAPTION_API_TOKEN;
|
||||
|
||||
if (!expectedToken) {
|
||||
return c.json(
|
||||
{
|
||||
code: 'auth_misconfigured',
|
||||
message: 'CAPTION_API_TOKEN is not configured',
|
||||
},
|
||||
500
|
||||
);
|
||||
}
|
||||
|
||||
const authorization = c.req.header('authorization') || '';
|
||||
const token = authorization.startsWith('Bearer ')
|
||||
? authorization.slice('Bearer '.length).trim()
|
||||
: '';
|
||||
|
||||
if (token !== expectedToken) {
|
||||
c.header('WWW-Authenticate', 'Bearer');
|
||||
return c.json({ code: 'unauthorized', message: 'Unauthorized' }, 401);
|
||||
}
|
||||
|
||||
await next();
|
||||
}
|
||||
|
||||
function methodNotAllowed(c) {
|
||||
return c.json({ code: 'method_not_allowed', message: 'Use GET' }, 405);
|
||||
}
|
||||
|
||||
app.get('/', (c) =>
|
||||
c.json({
|
||||
service: 'youtube-caption-extractor-api',
|
||||
runtime: 'cloudflare-worker+hono',
|
||||
backend: 'cloudflare-container+hono',
|
||||
endpoints: ['/health', '/api/subtitles', '/api/videoDetails'],
|
||||
})
|
||||
);
|
||||
|
||||
app.use('/health', requireApiToken);
|
||||
app.use('/api/*', requireApiToken);
|
||||
|
||||
app.get('/health', proxyToContainer);
|
||||
app.get('/api/*', proxyToContainer);
|
||||
app.all('/health', methodNotAllowed);
|
||||
app.all('/api/*', methodNotAllowed);
|
||||
|
||||
app.notFound((c) => c.json({ code: 'not_found', message: 'Not found' }, 404));
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"$schema": "./node_modules/wrangler/config-schema.json",
|
||||
"name": "youtube-caption-extractor-api",
|
||||
"main": "worker.js",
|
||||
"compatibility_date": "2026-05-16",
|
||||
"vars": {
|
||||
"API_INSTANCE_COUNT": "2",
|
||||
"CONTAINER_VERSION": "server-v2",
|
||||
"CACHE_TTL_SECONDS": "21600",
|
||||
"EXTRACTION_ATTEMPTS": "4",
|
||||
"EXTRACTION_RETRY_BASE_DELAY_MS": "300",
|
||||
"ALLOWED_ORIGINS": "*"
|
||||
},
|
||||
"containers": [
|
||||
{
|
||||
"class_name": "CaptionApiContainer",
|
||||
"image": "./Dockerfile",
|
||||
"instance_type": "lite",
|
||||
"max_instances": 4
|
||||
}
|
||||
],
|
||||
"durable_objects": {
|
||||
"bindings": [
|
||||
{
|
||||
"name": "CAPTION_API",
|
||||
"class_name": "CaptionApiContainer"
|
||||
}
|
||||
]
|
||||
},
|
||||
"migrations": [
|
||||
{
|
||||
"tag": "v1",
|
||||
"new_sqlite_classes": ["CaptionApiContainer"]
|
||||
}
|
||||
],
|
||||
"observability": {
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": [
|
||||
"dom",
|
||||
"dom.iterable",
|
||||
"esnext"
|
||||
],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "react-jsx",
|
||||
"incremental": true,
|
||||
"plugins": [
|
||||
{
|
||||
"name": "next"
|
||||
}
|
||||
],
|
||||
"paths": {
|
||||
"@/*": [
|
||||
"./*"
|
||||
]
|
||||
},
|
||||
"target": "ES2017"
|
||||
},
|
||||
"include": [
|
||||
"next-env.d.ts",
|
||||
"**/*.ts",
|
||||
"**/*.tsx",
|
||||
".next/types/**/*.ts",
|
||||
".next/dev/types/**/*.ts"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import {
|
||||
getSubtitles,
|
||||
getVideoDetails,
|
||||
Options,
|
||||
Subtitle,
|
||||
VideoDetails,
|
||||
} from './index';
|
||||
|
||||
// Surface-level sanity tests — always run, no network.
|
||||
describe('public API surface', () => {
|
||||
test('exports the documented functions and types', () => {
|
||||
expect(typeof getSubtitles).toBe('function');
|
||||
expect(typeof getVideoDetails).toBe('function');
|
||||
});
|
||||
});
|
||||
|
||||
// Live-network integration tests. YouTube blocks most datacenter IP ranges
|
||||
// (including GitHub Actions, AWS, etc.) with a "Sign in to confirm you're
|
||||
// not a bot" challenge that no client version bypasses. So these tests are
|
||||
// gated behind YOUTUBE_LIVE=1 and only run by default in local development
|
||||
// (where the requester IP is residential).
|
||||
const liveTestsEnabled =
|
||||
!process.env.CI || process.env.YOUTUBE_LIVE === '1';
|
||||
const describeLive = liveTestsEnabled ? describe : describe.skip;
|
||||
|
||||
const TEST_VIDEOS = {
|
||||
// "I Stopped Building MCP Servers. Here's Why." — ASR English only
|
||||
asr: 'Q0RFb53ExfY',
|
||||
// Manual English captions
|
||||
manual: 'fKxLbERmB4U',
|
||||
};
|
||||
|
||||
const NETWORK_TIMEOUT = 20_000;
|
||||
|
||||
describeLive('getSubtitles (live)', () => {
|
||||
test(
|
||||
'extracts auto-generated English captions',
|
||||
async () => {
|
||||
const subtitles = await getSubtitles({
|
||||
videoID: TEST_VIDEOS.asr,
|
||||
lang: 'en',
|
||||
});
|
||||
expect(Array.isArray(subtitles)).toBe(true);
|
||||
expect(subtitles.length).toBeGreaterThan(0);
|
||||
|
||||
const first = subtitles[0];
|
||||
expect(first).toHaveProperty('start');
|
||||
expect(first).toHaveProperty('dur');
|
||||
expect(first).toHaveProperty('text');
|
||||
expect(typeof first.text).toBe('string');
|
||||
expect(first.text.length).toBeGreaterThan(0);
|
||||
expect(Number(first.start)).not.toBeNaN();
|
||||
expect(Number(first.dur)).not.toBeNaN();
|
||||
},
|
||||
NETWORK_TIMEOUT
|
||||
);
|
||||
|
||||
test(
|
||||
'extracts manual English captions',
|
||||
async () => {
|
||||
const subtitles = await getSubtitles({
|
||||
videoID: TEST_VIDEOS.manual,
|
||||
lang: 'en',
|
||||
});
|
||||
expect(subtitles.length).toBeGreaterThan(0);
|
||||
},
|
||||
NETWORK_TIMEOUT
|
||||
);
|
||||
});
|
||||
|
||||
describeLive('getVideoDetails (live)', () => {
|
||||
let videoDetails: VideoDetails;
|
||||
|
||||
beforeAll(async () => {
|
||||
const options: Options = { videoID: TEST_VIDEOS.asr, lang: 'en' };
|
||||
videoDetails = await getVideoDetails(options);
|
||||
}, NETWORK_TIMEOUT);
|
||||
|
||||
test('returns title, description, and subtitles', () => {
|
||||
expect(videoDetails).toHaveProperty('title');
|
||||
expect(videoDetails).toHaveProperty('description');
|
||||
expect(videoDetails).toHaveProperty('subtitles');
|
||||
});
|
||||
|
||||
test('title and description are non-empty strings', () => {
|
||||
expect(typeof videoDetails.title).toBe('string');
|
||||
expect(videoDetails.title.length).toBeGreaterThan(0);
|
||||
expect(videoDetails.title).not.toBe('No title found');
|
||||
expect(typeof videoDetails.description).toBe('string');
|
||||
expect(videoDetails.description.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('subtitles is a non-empty array of Subtitle objects', () => {
|
||||
expect(Array.isArray(videoDetails.subtitles)).toBe(true);
|
||||
expect(videoDetails.subtitles.length).toBeGreaterThan(0);
|
||||
const s: Subtitle = videoDetails.subtitles[0];
|
||||
expect(s.text.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
+386
@@ -0,0 +1,386 @@
|
||||
import he from 'he';
|
||||
import striptags from 'striptags';
|
||||
|
||||
// Universal logger that works in all environments (Node.js, Cloudflare Workers, etc.)
|
||||
const createLogger = (namespace: string) => {
|
||||
const isDebugEnabled = () => {
|
||||
try {
|
||||
const env =
|
||||
typeof process !== 'undefined' && process.env ? process.env : {};
|
||||
const debugEnv = env.DEBUG || '';
|
||||
return debugEnv === '*' || debugEnv.includes(namespace);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
return (message: string, ...args: any[]) => {
|
||||
if (isDebugEnabled()) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const logMessage = `${timestamp} ${namespace} ${message}`;
|
||||
if (args.length > 0) {
|
||||
console.log(logMessage, ...args);
|
||||
} else {
|
||||
console.log(logMessage);
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const debug = createLogger('youtube-caption-extractor');
|
||||
|
||||
export interface Subtitle {
|
||||
start: string;
|
||||
dur: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface Options {
|
||||
videoID: string;
|
||||
lang?: string;
|
||||
/**
|
||||
* Custom fetch implementation. Useful for routing requests through a
|
||||
* residential proxy (undici's ProxyAgent, node-fetch + https-proxy-agent, etc.)
|
||||
* when deploying to environments where YouTube blocks datacenter IPs
|
||||
* (Vercel, AWS Lambda, Cloudflare Workers). Defaults to global fetch.
|
||||
*/
|
||||
fetch?: typeof fetch;
|
||||
}
|
||||
|
||||
export interface VideoDetails {
|
||||
title: string;
|
||||
description: string;
|
||||
subtitles: Subtitle[];
|
||||
}
|
||||
|
||||
interface CaptionTrack {
|
||||
baseUrl: string;
|
||||
vssId?: string;
|
||||
languageCode?: string;
|
||||
kind?: string;
|
||||
}
|
||||
|
||||
interface PlayerResponse {
|
||||
playabilityStatus?: {
|
||||
status?: string;
|
||||
reason?: string;
|
||||
};
|
||||
videoDetails?: {
|
||||
title?: string;
|
||||
shortDescription?: string;
|
||||
};
|
||||
captions?: {
|
||||
playerCaptionsTracklistRenderer?: {
|
||||
captionTracks?: CaptionTrack[];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// InnerTube client profiles, tried in order until one returns playable data
|
||||
// with caption tracks. IOS is first because it's the most reliable client
|
||||
// when YouTube isn't actively filtering the source IP; ANDROID_VR and MWEB
|
||||
// follow as fallbacks with slightly different fingerprints.
|
||||
//
|
||||
// TVHTML5_SIMPLY_EMBEDDED_PLAYER v2.0 used to lead the chain but as of
|
||||
// v1.10.1 it's been removed — direct InnerTube probes (May 2026, see
|
||||
// docs/PR notes) showed YouTube globally rejects it with "no longer
|
||||
// supported in this application or device" regardless of source IP.
|
||||
//
|
||||
// When CI's nightly canary starts failing, the typical fix is to bump
|
||||
// clientVersion strings to current values (track yt-dlp's recent commits).
|
||||
interface ClientProfile {
|
||||
name: string;
|
||||
clientName: string;
|
||||
clientVersion: string;
|
||||
clientNameHeader: string;
|
||||
userAgent: string;
|
||||
context: Record<string, unknown>;
|
||||
}
|
||||
|
||||
const CLIENT_PROFILES: ClientProfile[] = [
|
||||
{
|
||||
name: 'ios',
|
||||
clientName: 'IOS',
|
||||
clientVersion: '20.10.4',
|
||||
clientNameHeader: '5',
|
||||
userAgent:
|
||||
'com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)',
|
||||
context: {
|
||||
deviceMake: 'Apple',
|
||||
deviceModel: 'iPhone16,2',
|
||||
platform: 'MOBILE',
|
||||
osName: 'iOS',
|
||||
osVersion: '18.3.2.22D82',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'android_vr',
|
||||
clientName: 'ANDROID_VR',
|
||||
clientVersion: '1.62.20',
|
||||
clientNameHeader: '28',
|
||||
userAgent:
|
||||
'com.google.android.apps.youtube.vr.oculus/1.62.20 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
|
||||
context: {
|
||||
deviceMake: 'Oculus',
|
||||
deviceModel: 'Quest 3',
|
||||
platform: 'MOBILE',
|
||||
osName: 'Android',
|
||||
osVersion: '12L',
|
||||
androidSdkVersion: 32,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'mweb',
|
||||
clientName: 'MWEB',
|
||||
clientVersion: '2.20251209.01.00',
|
||||
clientNameHeader: '2',
|
||||
userAgent:
|
||||
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1',
|
||||
context: {
|
||||
platform: 'MOBILE',
|
||||
osName: 'iOS',
|
||||
osVersion: '17.5.1',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const INNERTUBE_ENDPOINT =
|
||||
'https://youtubei.googleapis.com/youtubei/v1/player?prettyPrint=false';
|
||||
|
||||
async function fetchPlayerWithClient(
|
||||
videoID: string,
|
||||
client: ClientProfile,
|
||||
fetchImpl: typeof fetch
|
||||
): Promise<PlayerResponse> {
|
||||
const body = {
|
||||
context: {
|
||||
client: {
|
||||
clientName: client.clientName,
|
||||
clientVersion: client.clientVersion,
|
||||
hl: 'en',
|
||||
gl: 'US',
|
||||
...client.context,
|
||||
},
|
||||
user: { lockedSafetyMode: false },
|
||||
request: { useSsl: true },
|
||||
},
|
||||
videoId: videoID,
|
||||
contentCheckOk: true,
|
||||
racyCheckOk: true,
|
||||
};
|
||||
|
||||
debug(`Calling InnerTube /player with ${client.name} client`);
|
||||
|
||||
const response = await fetchImpl(INNERTUBE_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Accept: '*/*',
|
||||
'User-Agent': client.userAgent,
|
||||
'X-YouTube-Client-Name': client.clientNameHeader,
|
||||
'X-YouTube-Client-Version': client.clientVersion,
|
||||
Origin: 'https://www.youtube.com',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`InnerTube /player failed (${client.name}): ${response.status} ${response.statusText}`
|
||||
);
|
||||
}
|
||||
|
||||
return (await response.json()) as PlayerResponse;
|
||||
}
|
||||
|
||||
// Try every client in order. Return the first response that has captionTracks;
|
||||
// fall back to the first OK response (so getVideoDetails still gets title/desc);
|
||||
// throw a combined error only if every client fails.
|
||||
//
|
||||
// We do NOT bail early on individual error statuses. YouTube uses the same
|
||||
// `ERROR` status for both "video is private/deleted" *and* "this client
|
||||
// version is too old" — and different clients see different states for the
|
||||
// same video. Trying them all is cheap and the only reliable signal that a
|
||||
// video is truly inaccessible is when no client returns OK.
|
||||
async function fetchPlayer(
|
||||
videoID: string,
|
||||
fetchImpl: typeof fetch
|
||||
): Promise<PlayerResponse> {
|
||||
let firstPlayable: PlayerResponse | null = null;
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const client of CLIENT_PROFILES) {
|
||||
try {
|
||||
const data = await fetchPlayerWithClient(videoID, client, fetchImpl);
|
||||
const status = data.playabilityStatus?.status;
|
||||
debug(`${client.name} client returned playabilityStatus=${status}`);
|
||||
|
||||
if (status && status !== 'OK') {
|
||||
const reason = data.playabilityStatus?.reason;
|
||||
failures.push(
|
||||
`${client.name}: ${status}${reason ? ` - ${reason}` : ''}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!firstPlayable) firstPlayable = data;
|
||||
const tracks =
|
||||
data.captions?.playerCaptionsTracklistRenderer?.captionTracks;
|
||||
if (tracks && tracks.length > 0) {
|
||||
return data;
|
||||
}
|
||||
failures.push(`${client.name}: OK but no caption tracks`);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
failures.push(`${client.name}: ${msg}`);
|
||||
debug(`${client.name} client error: ${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (firstPlayable) return firstPlayable;
|
||||
throw new Error(
|
||||
`Video not playable on any client. Attempts:\n${failures.join('\n')}`
|
||||
);
|
||||
}
|
||||
|
||||
function pickCaptionTrack(
|
||||
tracks: CaptionTrack[],
|
||||
lang: string
|
||||
): CaptionTrack | null {
|
||||
if (!tracks.length) return null;
|
||||
return (
|
||||
tracks.find((t) => t.vssId === `.${lang}`) || // manual captions in requested lang
|
||||
tracks.find((t) => t.vssId === `a.${lang}`) || // auto-generated in requested lang
|
||||
tracks.find((t) => t.languageCode === lang) ||
|
||||
tracks.find((t) => t.vssId?.includes(`.${lang}`)) ||
|
||||
tracks[0]
|
||||
);
|
||||
}
|
||||
|
||||
interface Json3Segment {
|
||||
utf8?: string;
|
||||
}
|
||||
|
||||
interface Json3Event {
|
||||
tStartMs?: number;
|
||||
dDurationMs?: number;
|
||||
segs?: Json3Segment[];
|
||||
aAppend?: number;
|
||||
}
|
||||
|
||||
interface Json3Transcript {
|
||||
events?: Json3Event[];
|
||||
}
|
||||
|
||||
async function fetchCaptionTrack(
|
||||
track: CaptionTrack,
|
||||
fetchImpl: typeof fetch
|
||||
): Promise<Subtitle[]> {
|
||||
// Force json3 — structured, stable across edge cases, no regex parsing needed
|
||||
let url = track.baseUrl.replace(/&fmt=[^&]+/, '');
|
||||
url += '&fmt=json3';
|
||||
|
||||
debug(`Fetching caption track from ${url.split('?')[0]}?…`);
|
||||
|
||||
const response = await fetchImpl(url, {
|
||||
headers: {
|
||||
'User-Agent': CLIENT_PROFILES[0].userAgent,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Caption fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
if (!text.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
let data: Json3Transcript;
|
||||
try {
|
||||
data = JSON.parse(text) as Json3Transcript;
|
||||
} catch {
|
||||
throw new Error('Caption response was not valid JSON');
|
||||
}
|
||||
|
||||
const events = data.events ?? [];
|
||||
const subtitles: Subtitle[] = [];
|
||||
|
||||
for (const event of events) {
|
||||
if (!event.segs || event.aAppend === 1) continue;
|
||||
const raw = event.segs.map((s) => s.utf8 ?? '').join('');
|
||||
const text = he.decode(striptags(raw)).trim();
|
||||
if (!text) continue;
|
||||
const startMs = event.tStartMs ?? 0;
|
||||
const durMs = event.dDurationMs ?? 0;
|
||||
subtitles.push({
|
||||
start: (startMs / 1000).toString(),
|
||||
dur: (durMs / 1000).toString(),
|
||||
text,
|
||||
});
|
||||
}
|
||||
|
||||
debug(`Parsed ${subtitles.length} caption events from json3`);
|
||||
return subtitles;
|
||||
}
|
||||
|
||||
async function extractSubtitles(
|
||||
playerData: PlayerResponse,
|
||||
lang: string,
|
||||
fetchImpl: typeof fetch
|
||||
): Promise<Subtitle[]> {
|
||||
const tracks =
|
||||
playerData.captions?.playerCaptionsTracklistRenderer?.captionTracks;
|
||||
if (!tracks || tracks.length === 0) {
|
||||
debug('No caption tracks available on this video');
|
||||
return [];
|
||||
}
|
||||
|
||||
const track = pickCaptionTrack(tracks, lang);
|
||||
if (!track?.baseUrl) {
|
||||
debug(`No matching caption track for lang=${lang}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
debug(`Selected caption track: ${track.vssId ?? track.languageCode}`);
|
||||
return fetchCaptionTrack(track, fetchImpl);
|
||||
}
|
||||
|
||||
export const getSubtitles = async ({
|
||||
videoID,
|
||||
lang = 'en',
|
||||
fetch: customFetch,
|
||||
}: Options): Promise<Subtitle[]> => {
|
||||
const fetchImpl = customFetch ?? fetch;
|
||||
debug(`getSubtitles videoID=${videoID} lang=${lang}`);
|
||||
const playerData = await fetchPlayer(videoID, fetchImpl);
|
||||
return extractSubtitles(playerData, lang, fetchImpl);
|
||||
};
|
||||
|
||||
export const getVideoDetails = async ({
|
||||
videoID,
|
||||
lang = 'en',
|
||||
fetch: customFetch,
|
||||
}: Options): Promise<VideoDetails> => {
|
||||
const fetchImpl = customFetch ?? fetch;
|
||||
debug(`getVideoDetails videoID=${videoID} lang=${lang}`);
|
||||
const playerData = await fetchPlayer(videoID, fetchImpl);
|
||||
|
||||
const title = playerData.videoDetails?.title ?? 'No title found';
|
||||
const description =
|
||||
playerData.videoDetails?.shortDescription ?? 'No description found';
|
||||
|
||||
let subtitles: Subtitle[] = [];
|
||||
try {
|
||||
subtitles = await extractSubtitles(playerData, lang, fetchImpl);
|
||||
} catch (err) {
|
||||
debug(
|
||||
`Subtitle extraction failed: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
return { title, description, subtitles };
|
||||
};
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2018",
|
||||
"module": "CommonJS",
|
||||
"declaration": true,
|
||||
"outDir": "./dist",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"moduleResolution": "node"
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/*.test.ts"]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
include: ['src/**/*.test.ts'],
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user