# Love Tales — robots.txt
# Content signals per https://contentsignals.org/
# ai-train=yes   → major AI engines may use this content (incl. training)
# ai-input=yes   → allow real-time agent browsing / grounding
# search=yes     → allow classic search indexing
# Only bulk data-scraper bots are blocked (see section below).

User-agent: *
Content-Signal: search=yes, ai-train=yes, ai-input=yes
Allow: /
# Critical: do NOT disallow /_next/ — Googlebot needs CSS, JS, and
# fonts to render pages. Blocking them breaks rendering and hurts
# indexing. The explicit Allow below documents this for future edits.
Allow: /_next/static/
Disallow: /app/
Disallow: /auth/
Disallow: /api/
Disallow: /llms.txt.backup

# Major AI engines (OpenAI GPTBot, Anthropic ClaudeBot/anthropic-ai,
# Google-Extended) are intentionally NOT listed here — they fall under
# the global `User-agent: * / Allow: /` above. We allow them so the
# models build brand familiarity from training in addition to the
# search/grounding variants below. Do not re-add Disallow blocks for
# GPTBot / ClaudeBot / anthropic-ai / Google-Extended: their separate
# search/citation bots were never the thing being blocked, and blocking
# the training bots only cost us model brand-memory with zero citation
# upside.

# Bulk data-scraper bots — block. These crawl at scale for dataset
# resale / aggregation with weak citation value and a reputation for
# aggressive crawling, so we opt out of them specifically.
User-agent: CCBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Bytespider
Disallow: /

# AI search / grounding crawlers — explicitly allow. Redundant with
# `User-agent: *` above but documents intent so future contributors
# don't accidentally lump these in with the training-bot blocks.
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

Sitemap: https://lovetales.ai/sitemap.xml