# Robots.txt for Agni Folio - Portfolio Tracker & Wealth Management
# Updated: 2026-05-08
# https://www.robotstxt.org/robotstxt.html
# AI assistants: see https://agnifolio.com/llms.txt for a structured product summary.

# Default rules for all bots
User-agent: *
Allow: /

# Content Signals (https://contentsignals.org) — AI usage preferences
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Block private/protected areas
Disallow: /static/
Disallow: /js/
Disallow: /css/
Disallow: /portfolio/
Disallow: /settings/
Disallow: /accounts/
Disallow: /api/
Disallow: /admin/

# Explicitly allow ALL public content
Allow: /fire-calculator
Allow: /tools
Allow: /tools/fire-calculator
Allow: /tools/death-readiness
Allow: /tools/networth-ranking
Allow: /tools/high-interest
Allow: /blog
Allow: /blog/
Allow: /blog/*
Allow: /legal
Allow: /faq
Allow: /features
Allow: /contact
Allow: /contact-us
Allow: /changelog
Allow: /about
Allow: /leave-a-review
Allow: /signin
Allow: /signup

# Sitemaps
Sitemap: https://agnifolio.com/sitemap.xml

# Major Search Engines
User-agent: Googlebot
Allow: /
Allow: /features
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /
Allow: /features
Crawl-delay: 0

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 0

User-agent: Baiduspider
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /
Crawl-delay: 1

# AI Crawlers - Allow all public content
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

# New AI Crawlers - Grok, Gemini, Claude, Anthropic
User-agent: GrokBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: GoogleAI
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: Gemini
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: Claude
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

User-agent: Anthropic
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /blog/*
Allow: /features
Allow: /faq
Allow: /changelog
Allow: /contact-us
Allow: /tools/*
Allow: /fire-calculator
Allow: /about
Allow: /leave-a-review

# Social Media Bots
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Block aggressive crawlers
User-agent: MJ12bot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

# Default crawl delay
User-agent: *
Crawl-delay: 1