# FreeMyTools — every page is public and every crawler is welcome.
#
# The business model is being found. There is nothing here worth gating: no
# user data, no paywall, no proprietary corpus — just tool pages that only earn
# anything when someone arrives at them. A citation inside an AI answer is worth
# more to us than a blocked crawler, so AI crawlers are allowed explicitly
# below rather than left to the wildcard. See the note at the bottom.

User-agent: *
Allow: /

# ── AI / LLM crawlers, named explicitly ─────────────────────────────────────
# Named rather than left to `User-agent: *` because several of these bots read
# the most specific matching group ONLY. A future `Disallow` added to the
# wildcard group would silently not apply to them, and vice versa — being
# explicit means the intent survives the next edit of this file.

# OpenAI — training crawler, ChatGPT live browsing, and ChatGPT Search
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google. Googlebot itself is covered by `*`; Google-Extended is a separate
# opt-out control for Gemini and AI Overviews grounding. Disallowing it would
# remove us from AI Overviews, which is a placement we actively want.
User-agent: Google-Extended
Allow: /

# Microsoft / Bing — Bingbot also feeds Copilot answers.
User-agent: Bingbot
Allow: /

# Apple. Applebot-Extended governs Apple Intelligence training specifically.
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Common Crawl — the corpus a large share of open models are trained on.
User-agent: CCBot
Allow: /

# Meta, Amazon, ByteDance, Mistral, Cohere, DuckDuckGo
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: DuckAssistBot
Allow: /

# ── Sitemaps ────────────────────────────────────────────────────────────────
# Astro's sitemap integration emits sitemap-index.xml. /sitemap.xml is our own
# endpoint (src/pages/sitemap.xml.js) covering the same URLs in one file, for
# the many tools that only ever try the conventional path.
# Verify both with: curl -s https://freemytools.com/sitemap-index.xml
Sitemap: https://freemytools.com/sitemap-index.xml
Sitemap: https://freemytools.com/sitemap.xml

# LLM-facing index, per the llmstxt.org convention. Not part of the robots.txt
# spec — listed here because it is where an agent looks first.
# https://freemytools.com/llms.txt
# https://freemytools.com/llms-full.txt

# ── The trade-off, recorded deliberately ────────────────────────────────────
# Allowing training crawlers (GPTBot, CCBot, Applebot-Extended, Google-Extended)
# means this site's copy can end up in model weights with no attribution and no
# click. That is a real cost. It is accepted because:
#   1. The content is generic utility copy, not a defensible asset.
#   2. The retrieval crawlers that DO cite (OAI-SearchBot, PerplexityBot,
#      ClaudeBot, Bingbot) are largely the same infrastructure; blocking the
#      training half tends to cost citations from the retrieval half too.
#   3. A brand-new domain's scarcest resource is being known about at all.
# Revisit if the hub ever carries genuinely proprietary data.