# robots.txt for https://toddmorourke.com
# ---------------------------------------------------------------------------
# Content Signals (https://contentsignals.org,
# IETF draft-romm-aipref-contentsignals). The "Content-Signal" lines express
# how this site's content may be used. They are USAGE PREFERENCES, not access
# controls -- crawling is governed by the Allow/Disallow rules in each group.
#   search    = yes  -> may be indexed for search and linked back to
#   ai-train  = no   -> may NOT be used to train or fine-tune AI / LLM models
#   ai-input  = yes  -> may be used as real-time input to AI answers
#                       (retrieval / grounding / RAG) that cite this site
# ---------------------------------------------------------------------------

# All crawlers: full access. Covers search engines (Googlebot, Bingbot, ...)
# and any AI crawler not named in its own group below.
User-agent: *
Content-Signal: search=yes, ai-train=no, ai-input=yes
Disallow:

# === AI search / answer engines (real-time retrieval + citation) ===========
# Crawling allowed so this site can be cited in AI answers (ai-input=yes).

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Claude-Web
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Claude-User
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Applebot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

# === AI training / dataset crawlers ========================================

User-agent: GPTBot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: ClaudeBot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: anthropic-ai
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Google-Extended
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: CCBot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Amazonbot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: Bytespider
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

User-agent: cohere-ai
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /

# === File-level directives (agentic discovery + sitemap) ===================
Sitemap: https://toddmorourke.com/sitemap.xml
Agentmap: https://toddmorourke.com/.well-known/ai-catalog.json
Entitymap: https://toddmorourke.com/entitymap.json