# =====================================================================
# MemClaw — robots.txt
# Public docs, landing, and marketing pages are open to all crawlers
# (including AI answer engines). Authenticated/app routes are disallowed.
# =====================================================================

User-agent: *
Allow: /
Allow: /about
Allow: /architecture
Allow: /blog
Allow: /pricing
Allow: /docs
Allow: /guide
Allow: /for-agents
Allow: /terms
Disallow: /demo
Disallow: /fleet
Disallow: /graph
Disallow: /ingest
Disallow: /manage
# /mcp$ + /mcp/ = the MCP protocol endpoint only. A bare "Disallow: /mcp"
# prefix would also swallow /mcp-server (the ads landing page — must stay
# crawlable for Quality Score) and is why these are anchored.
Disallow: /mcp$
Disallow: /mcp/
Disallow: /mcp-tools
Allow: /mcp-server
Disallow: /prism
Disallow: /superadmin
Disallow: /signin
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /api/

# --------------------------------------------------------------------
# AI answer engines — explicitly allowed. We want AEO surface area.
# --------------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

Sitemap: https://memclaw.net/sitemap.xml
# Note: sitemap.xml is dynamically generated by Next.js from app/sitemap.ts