# robots.txt for dailybuilt.co
#
# dailybuilt wants to be discoverable by AI crawlers, not blocked from
# them. We list well-known AI / LLM user-agents explicitly so the intent
# is unambiguous — a bare `User-agent: *` would also allow them, but
# several operators only honor rules under their specific token.
# AI-facing summaries: https://dailybuilt.co/llms.txt (index) and https://dailybuilt.co/llms-full.txt (full content).

User-Agent: *
Allow: /

# Core search indexers
User-Agent: Googlebot
Allow: /

User-Agent: Bingbot
Allow: /

# Training and general AI crawlers
User-Agent: GPTBot
Allow: /

User-Agent: ClaudeBot
Allow: /

User-Agent: CCBot
Allow: /

User-Agent: PerplexityBot
Allow: /

User-Agent: Bytespider
Disallow: /

User-Agent: cohere-ai
Allow: /

User-Agent: cohere-training-data-crawler
Allow: /

User-Agent: Meta-ExternalAgent
Allow: /

User-Agent: FacebookBot
Allow: /

User-Agent: Amazonbot
Allow: /

User-Agent: Diffbot
Allow: /

User-Agent: AI2Bot
Allow: /

User-Agent: Timpibot
Allow: /

User-Agent: omgilibot
Allow: /

User-Agent: YouBot
Allow: /

User-Agent: DuckAssistBot
Allow: /

User-Agent: ImagesiftBot
Allow: /

User-Agent: Kangaroo Bot
Allow: /

User-Agent: PanguBot
Allow: /

User-Agent: MistralAI-User
Allow: /

# On-demand / search-side fetchers used by AI assistants
User-Agent: OAI-SearchBot
Allow: /

User-Agent: ChatGPT-User
Allow: /

User-Agent: Claude-User
Allow: /

User-Agent: Claude-SearchBot
Allow: /

User-Agent: Perplexity-User
Allow: /

# Opt-in tokens (no crawl of their own — they signal training consent
# for the parent vendor's models)
User-Agent: Google-Extended
Allow: /

User-Agent: Applebot-Extended
Allow: /

Sitemap: https://dailybuilt.co/sitemap.xml