# Stalwart Labs (https://stalw.art)
# Documentation, blog, and product pages are open to crawlers and AI agents.
# Index: /llms.txt (curated, machine-readable index per llmstxt.org).

User-agent: *
Allow: /
Crawl-delay: 1

# Frequently misbehaving paths to avoid hammering.
Disallow: /tags/
Disallow: /blog/authors/

# AI / LLM crawlers: explicitly permitted, same rules as everyone else.
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: Bingbot
Allow: /
User-agent: Mistralbot
Allow: /
User-agent: cohere-ai
Allow: /
User-agent: meta-externalagent
Allow: /

Sitemap: https://stalw.art/sitemap-index.xml
