# My Dev Day — crawler policy.
#
# Goal: stay discoverable to real search engines while opting out of AI
# training / AI-answer crawlers. Polite crawlers honor this file; abusive ones
# ignore it, which is why Cloudflare bot rules sit in front of this (belt and
# braces). See also the X-Robots-Tag: noai, noimageai header set in index.php.

# --- AI training & AI-answer crawlers: keep out of everything ---------------

User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Timpibot
Disallow: /

# --- Everyone else (real search engines, humans, feed readers): welcome -----
# Googlebot, Bingbot, DuckDuckBot, Applebot, etc. fall through to this default.
# Members-only and utility paths are excluded — crawlers only get redirects or
# JSON there anyway.

# ($-anchored where needed: project slugs live at the root, and a bare prefix
# rule like "/new" would also catch a slug that happens to start with "new".)

User-agent: *
Disallow: /api/v1/
Disallow: /mcp$
Disallow: /feed$
Disallow: /settings
Disallow: /billing$
Disallow: /bookmarks$
Disallow: /admin$
Disallow: /new$
Disallow: /logout$
Disallow: /verify
Disallow: /auth/
Disallow: /connections/
Disallow: /stripe/
Disallow: /day/
Disallow: /share-status
Disallow: /video/

Sitemap: https://mydev.day/sitemap.xml