# promptteardown.com/robots.txt
# Policy: index everything public. AI crawlers are welcome. Being cited
# beats being invisible. Block only bots that cost bandwidth and send nothing.

# ---------------------------------------------------------------
# Search engines. Full access (listed explicitly so the decision is on
# the record and survives any future platform default)
# ---------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# ---------------------------------------------------------------
# AI / answer engines. Full access, deliberately
# ---------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: cohere-ai
Allow: /

# ---------------------------------------------------------------
# Link-preview bots. Needed for cards to render when posts are shared
# ---------------------------------------------------------------
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: Discordbot
Allow: /

# ---------------------------------------------------------------
# Blocked. Abandoned or abusive crawlers. No traffic, no citations,
# just bandwidth. Safe to drop.
# ---------------------------------------------------------------
User-agent: BLEXBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ---------------------------------------------------------------
# Everyone else. Allowed, minus the subscribe endpoint (POST only,
# nothing to index). Tracking parameters are handled by the canonical
# tag on every page, not blocked here.
# ---------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/

Sitemap: https://promptteardown.com/sitemap-index.xml

# Files for AI agents, explained at https://promptteardown.com/for-agents/
# Site summary: https://promptteardown.com/llms.txt
# Full text of every post: https://promptteardown.com/llms-full.txt
# Every page as JSON: https://promptteardown.com/content-index.json