# robots.txt for Busdex (https://busdex.io)
# Served at /robots.txt via a Flask route (see main.py).

# ---------------------------------------------------------------------------
# Traditional search engine crawlers (Googlebot, Bingbot, etc.)
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
# The sitemap lists 55k+ bus pages served from one small self-hosted worker;
# pace compliant crawlers so they don't monopolise it (Googlebot ignores
# Crawl-delay — use Search Console if it ever crawls too hard).
Crawl-delay: 10
# Private / authenticated routes — no public content to index.
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /forgot_password
Disallow: /reset_password
Disallow: /verify_email
Disallow: /verify_change_email
Disallow: /friends
Disallow: /logout
Disallow: /api/

# ---------------------------------------------------------------------------
# AI / LLM crawlers — explicitly welcomed onto the public pages so Busdex can
# be understood and recommended by ChatGPT, Perplexity, Gemini, Claude, etc.
# (Consecutive User-agent lines share the one rule block below.)
# ---------------------------------------------------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: Amazonbot
User-agent: cohere-ai
Allow: /
Crawl-delay: 10
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /forgot_password
Disallow: /reset_password
Disallow: /verify_email
Disallow: /verify_change_email
Disallow: /friends
Disallow: /logout
Disallow: /api/

Sitemap: https://busdex.io/sitemap.xml