# YouTubeTranscript.dev Robots.txt
# https://www.youtubetranscript.dev
#
# AI assistants: see https://www.youtubetranscript.dev/llms.txt (overview)
# and https://www.youtubetranscript.dev/llms-full.txt (full documentation).
# Machine-readable API reference: https://www.youtubetranscript.dev/api/v2/openapi.yaml

# ===========================================
# AI CRAWLERS — ALLOWED ON ALL PUBLIC CONTENT
# (Grouped so they get the same private-area
#  exclusions as general crawlers.)
# ===========================================

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: CCBot
User-agent: Bytespider
User-agent: cohere-ai
Allow: /
# Machine-readable docs stay accessible (longest-match Allow overrides /api/ Disallow)
Allow: /api/docs/
Allow: /api/og
Allow: /api/v1/openapi.json
Allow: /api/v1/openapi.yaml
Allow: /api/v2/openapi.json
Allow: /api/v2/openapi.yaml
Disallow: /api/
Disallow: /dashboard/
Disallow: /*/dashboard/
Disallow: /auth/
Disallow: /*/auth/
Disallow: /account/
Disallow: /*/account/

# ===========================================
# BING / MICROSOFT EXPLICIT ALLOW
# ===========================================

User-agent: Bingbot
User-agent: MSNBot
Allow: /
Allow: /api/og
Disallow: /api/
Disallow: /dashboard/
Disallow: /*/dashboard/
Disallow: /auth/
Disallow: /*/auth/
Disallow: /account/
Disallow: /*/account/
Disallow: /transcript/
Disallow: /*/transcript/

# ===========================================
# GENERAL CRAWLERS
# ===========================================

User-agent: *
Allow: /
Allow: /api/docs/
Allow: /api/og
Allow: /api/v1/openapi.json
Allow: /api/v1/openapi.yaml
Allow: /api/v2/openapi.json
Allow: /api/v2/openapi.yaml
Disallow: /api/
Disallow: /dashboard/
Disallow: /*/dashboard/
Disallow: /auth/
Disallow: /*/auth/
Disallow: /account/
Disallow: /*/account/
Disallow: /transcript/
Disallow: /*/transcript/
# Allow _next/static (JS/CSS bundles) so crawlers can render pages,
# but block _next/data and internal build artifacts
Disallow: /_next/data/
Allow: /_next/static/
Allow: /_next/image

# ===========================================
# SITEMAPS
# ===========================================

Sitemap: https://www.youtubetranscript.dev/sitemap.xml