# https://instantclaw.co
# Verified Search Engine Bots - Protect from aggressive blocking
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# AI Crawlers
# Explicit allow for AI citation crawlers (ChatGPT, Perplexity, Claude)
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /deploy/provisioning

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /deploy/provisioning

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /deploy/provisioning

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /deploy/provisioning

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /deploy/provisioning

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /deploy/provisioning
Crawl-delay: 1

# Sitemap
Sitemap: https://instantclaw.co/sitemap.xml

# RSS feed for resources
# Feed: https://instantclaw.co/feed.xml

# LLM-friendly content (llmstxt.org)
# LLMs may fetch /llms.txt and /llms-full.txt for context

# Disallow API routes and auth callbacks from indexing (no SEO value, may leak info)
# General crawlers (with crawl delay to be nice to servers)
User-agent: *
Allow: /
Crawl-delay: 10
Disallow: /api/
Disallow: /auth/
Disallow: /deploy/provisioning
Disallow: /admin/
Disallow: /dashboard/
Disallow: /*?*