# robots.txt for thesparkedgroup.com
# Last reviewed: 2026-05-06
#
# Search engines: free crawl. AI agents: explicitly welcomed for both
# the marketing site and The Spark blog. Index lives in /llms.txt with
# a deeper context document at /llms-full.txt. Sitemap at /sitemap.xml.

# Default policy: open to everyone
User-agent: *
Allow: /
Disallow: /admin
Disallow: /api/

# Search engines
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /api/

User-agent: DuckDuckBot
Allow: /

# AI agents: explicitly allowed. We want our content findable in
# Claude, ChatGPT, Perplexity, Gemini, etc. See /llms.txt for the
# structured index, /llms-full.txt for full context.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Bytespider
Allow: /

# Sitemap and AI index
Sitemap: https://thesparkedgroup.com/sitemap.xml

# AI-friendly hints (non-spec but commonly read)
# llms.txt:      https://thesparkedgroup.com/llms.txt
# llms-full.txt: https://thesparkedgroup.com/llms-full.txt