# robots.txt — Pika Resume
#
# General policy:
#   - Marketing/SEO/content surfaces: ALLOW everything
#   - Authenticated app surfaces (dashboard, resume editor, auth, referral,
#     API) are off-limits to ALL crawlers including AI bots; they require
#     login and would otherwise leak per-user noise into search/training
#     indexes.
#
# AI/LLM crawler policy (Generative Engine Optimization):
#   - Every major LLM training crawler is explicitly allowed on the
#     marketing surfaces. This is intentional — getting indexed into
#     Common Crawl (CCBot), the OpenAI training pool (GPTBot), and the
#     Anthropic / Google / Apple / Perplexity / Amazon / ByteDance training
#     pools is the prerequisite for being cited by their respective
#     consumer LLMs.
#   - Live-fetch crawlers (ChatGPT-User, OAI-SearchBot, PerplexityBot) are
#     allowed too so real-time citations resolve.

User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard
Disallow: /my-resumes
Disallow: /resume/
Disallow: /auth
Disallow: /referral
Disallow: /api/

# --- Training-data crawlers (feed future model knowledge) ---

# OpenAI training crawler.
User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /my-resumes
Disallow: /resume/
Disallow: /auth
Disallow: /referral
Disallow: /api/

# Common Crawl — the open dataset that nearly every public LLM
# (including GPT, Claude, Llama, Mistral, Gemini base layers) trains on.
# Allowing CCBot is the single highest-leverage opt-in for long-term LLM
# citation reach.
User-agent: CCBot
Allow: /
Disallow: /dashboard
Disallow: /my-resumes
Disallow: /resume/
Disallow: /auth
Disallow: /referral
Disallow: /api/

# Anthropic training crawler.
User-agent: anthropic-ai
Allow: /

# Anthropic live-fetch crawler (cited in Claude answers when browsing).
User-agent: ClaudeBot
Allow: /

# Google's separate AI-training crawler (Gemini, Bard). Distinct from
# Googlebot, which handles classic web indexing.
User-agent: Google-Extended
Allow: /

# Apple Intelligence training crawler. Powers Siri, Mail / system writing
# tools, and any future Apple Search Generative experience.
User-agent: Applebot-Extended
Allow: /

# Amazon training/indexing crawler (Alexa Plus, Rufus, internal LLMs).
User-agent: Amazonbot
Allow: /

# ByteDance / Doubao (TikTok's parent's LLM). Significant in Asia.
User-agent: Bytespider
Allow: /

# Diffbot — knowledge-graph extraction service used by enterprise LLM
# pipelines and search products to ground facts.
User-agent: Diffbot
Allow: /

# --- Live-fetch crawlers (real-time citation surfaces) ---

# ChatGPT's live-browsing user agent (cited in ChatGPT answers when
# browsing is invoked).
User-agent: ChatGPT-User
Allow: /

# OpenAI's search-index crawler (powers ChatGPT Search citations).
User-agent: OAI-SearchBot
Allow: /

# Perplexity's live-fetch crawler.
User-agent: PerplexityBot
Allow: /

Sitemap: https://pikaresume.com/sitemap.xml