# https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt
# Machine-readable policy: all well-behaved crawlers are welcome on HTML, the
# Markdown alternates, and the public AAIO surface (llms.txt, openapi.json,
# ai-plugin.json, ai-dataset.json, sitemap.xml, /api/agent).
#
# Authenticated endpoints (everything under /api/ except /api/agent) are
# disallowed because they require an API key and are not useful to crawl.

User-agent: *
Allow: /
Allow: /api/agent
Disallow: /api/

# Explicit allow for major LLM and AI search agents. Same policy as the
# wildcard above; listed separately so operators of these agents can see we
# have considered them.
User-agent: GPTBot
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: Claude-Web
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: Google-Extended
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: CCBot
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: Bytespider
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: cohere-ai
Allow: /
Allow: /api/agent
Disallow: /api/

User-agent: MistralAI-User
Allow: /
Allow: /api/agent
Disallow: /api/

# Sitemap (marketing pages, Markdown alternates, and machine-readable
# discovery URLs).
# Agentmap is omitted on purpose. Lighthouse 13.5 treats any directive outside
# its safelist as "Unknown directive" and fails the SEO robots-txt audit.
# Agents find the catalog from <link rel="ai-catalog"> and the Link header.
Sitemap: https://www.chatstack.app/sitemap.xml