# FlashFeed — robots policy
# Search engines and AI/LLM crawlers are welcome. Sitemaps at the bottom.

User-agent: *
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /profile
Disallow: /bookmarks
Disallow: /admin
Disallow: /search
Disallow: /comments
Disallow: /puls
Disallow: /pulse
Disallow: /feed/load_more
Disallow: /read_events
Disallow: /reactions
Disallow: /dismissals
Disallow: /push_subscriptions
# Localized variants of private/utility pages
Disallow: /*/profile
Disallow: /*/bookmarks
Disallow: /*/search
Crawl-delay: 5

# ─── Aggressive SEO/backlink scrapers — heavy crawl load, no value to readers ───
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# High-volume crawlers observed in production; also blocked by Rack::Attack.
User-agent: Bytespider
User-agent: PetalBot
Disallow: /

# ─── AI / LLM crawlers — explicitly welcome (GEO: be cited in AI answers) ───
# Listed separately so intent is explicit; same private-path rules apply.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bingbot
User-agent: DuckAssistBot
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: cohere-ai
User-agent: YouBot
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /profile
Disallow: /bookmarks
Disallow: /admin
Disallow: /search
Disallow: /comments
Disallow: /puls
Disallow: /pulse
# Welcome ≠ unmetered: AI crawlers dominate request volume (Applebot alone was
# ~half of all traffic in Render logs), so ask the ones that honor crawl-delay
# (ClaudeBot, PetalBot, Bingbot…) to pace themselves like everyone else.
Crawl-delay: 5

Sitemap: https://flashfeed.pl/sitemap.xml
Sitemap: https://flashfeed.pl/news-sitemap.xml