# ApplyArc Job Tracker - robots.txt
# https://applyarc.com
# Updated: 2026-05-26

# ============================================================================
# DEFAULT RULES — All crawlers
# ============================================================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
# OAuth tokens, in any query position. Each param needs BOTH forms: `?x=` for
# first position and `?*&x=` for later. The bare `/*code=*` was tried and
# rejected — it also blocks ?postcode=, ?promo_code=, ?discount_code= and
# ?application_state=. `/*&code=*` alone matches nothing under a Google-spec
# parser. Verified with Protego: blocks every token position, allows every
# legitimate param (utm_*, ref, tool, canceled, postcode, promo_code).
Disallow: /*?code=*
Disallow: /*?*&code=*
Disallow: /*?state=*
Disallow: /*?*&state=*
Disallow: /*?session_state=*
Disallow: /*?*&session_state=*
# NOTE: /dashboard, /track/*, /upgrade* are NOT disallowed here.
# They use X-Robots-Tag: noindex, declared once in
# scripts/page-compiler/url-states.mjs (PRIVATE) and generated into
# staticwebapp.config.json.
# Per Google: noindex requires crawl access, so robots.txt Disallow would
# silently break the noindex and these URLs could still be indexed via
# external links. https://developers.google.com/search/docs/crawling-indexing/block-indexing
# Public query variants must stay crawlable so Google can see canonical signals.
# Do not use robots.txt for canonicalization.

# ============================================================================
# AI CRAWLER RULES — Explicitly allow AI bots to index content
# These bots power ChatGPT, Claude, Perplexity, Gemini, Copilot, etc.
#
# WHY ONE GROUP: per the Robots Exclusion Protocol, "a user agent can match
# only one rule set, which is the first, most specific group that matches"
# (https://developers.google.com/crawling/docs/robots-txt/create-robots-txt).
# Named groups do NOT inherit from `User-agent: *`. When each bot had its own
# group, all 32 silently bypassed the global /admin/ and OAuth query-param
# disallows — Bingbot had a bare `Allow: /` and was exempt from every one.
# Listing the agents together keeps the "AI crawlers welcome" signal explicit
# while giving them exactly the same baseline as `*`. Multiple User-agent
# lines per group are valid and documented by Google (see link above).
# Add new bots to this list — do NOT give a bot its own group unless it
# genuinely needs DIFFERENT rules.
# ============================================================================

# OpenAI — GPTBot (training), OAI-SearchBot (ChatGPT search citation),
# ChatGPT-User (user-initiated browsing)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic — Claude crawler, web fetcher, search index, user browsing
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: Claude-User
# Perplexity — index crawler + user-initiated queries
User-agent: PerplexityBot
User-agent: Perplexity-User
# Google — Google-Extended gates Gemini app responses, Vertex AI grounding and
# future Gemini model training. It does NOT control Google Search crawling or
# AI Overviews; Googlebot does, and Googlebot is covered by `User-agent: *`.
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Gemini-Deep-Research
User-agent: Google-CloudVertexBot
# Microsoft — Bing index (also serves DuckDuckGo, Ecosia, MSN) + Azure AI
User-agent: Bingbot
User-agent: AzureAI-SearchBot
# Apple — Siri, Apple Intelligence
User-agent: Applebot-Extended
# Meta — Meta AI / Llama retrieval
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: meta-webindexer
User-agent: meta-externalfetcher
# Amazon — Alexa AI, Rufus
User-agent: Amazonbot
# DuckDuckGo AI answers
User-agent: DuckAssistBot
# Other AI search + RAG
User-agent: cohere-ai
User-agent: CCBot
User-agent: YouBot
User-agent: PhindBot
User-agent: DeepSeekBot
User-agent: MistralAI-User
User-agent: Bytespider
User-agent: FirecrawlAgent
User-agent: ExaBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Disallow: /*?code=*
Disallow: /*?*&code=*
Disallow: /*?state=*
Disallow: /*?*&state=*
Disallow: /*?session_state=*
Disallow: /*?*&session_state=*

# ============================================================================
# SITEMAPS & LLM FILES
# ============================================================================
# Primary entry — sitemap-index (Google + Bing recurse this)
Sitemap: https://applyarc.com/sitemap.xml
# Explicit sub-sitemaps for bots that don't follow sitemap-index
# (Yandex, Naver, Seznam, FirecrawlAgent, most CLI AI scrapers)
Sitemap: https://applyarc.com/pages-sitemap.xml
Sitemap: https://applyarc.com/blog-sitemap.xml
Sitemap: https://applyarc.com/compare-sitemap.xml
Sitemap: https://applyarc.com/guides-sitemap.xml

# LLM / AI agent identity files
# See https://llmstxt.org for spec
# llms.txt = concise summary for quick context
# llms-full.txt = complete product reference with all pages
# ai.txt = AI training & citation policy (Spawning ai.txt v1)
# applyarc-facts = verified facts to prevent AI hallucination
# https://applyarc.com/llms.txt
# https://applyarc.com/llms-full.txt
# https://applyarc.com/ai.txt