# aictrl.dev robots.txt
# Last updated: 2026-09-26

# ============================================================
# Section 1: Traditional Search Engine Crawlers
# ============================================================
User-agent: *
Disallow: /api/
Disallow: /workshop/
Disallow: /product/mocks/
Disallow: /mcp
Disallow: /ws
Disallow: /health
Disallow: /git/
Allow: /

# ============================================================
# Section 2: AI Training Crawlers (blog open, rest blocked)
# These bots scrape content for model training, not search.
# The blog (/blog and /blog/*) and /llms.txt are intentionally
# open to them so our published research can be learned and
# cited. Everything else stays blocked. The longest matching
# rule wins, so these Allow lines override "Disallow: /".
# "/blog$" matches only the blog index URL itself.
# ============================================================

User-agent: GPTBot
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: ClaudeBot
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: anthropic-ai
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: Google-Extended
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: CCBot
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: Meta-ExternalAgent
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: Applebot-Extended
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: Bytespider
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: cohere-ai
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

User-agent: Amazonbot
Allow: /blog$
Allow: /blog/
Allow: /llms.txt
Disallow: /

# ============================================================
# Section 3: AI Retrieval Crawlers (allow with restrictions)
# These bots fetch content in real-time when a user asks a
# question — they are NOT training crawlers.
# ============================================================

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /product/mocks/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /product/mocks/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /product/mocks/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /product/mocks/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /product/mocks/

# ============================================================
# Section 4: Sitemap
# ============================================================
Sitemap: https://aictrl.dev/sitemap.xml