# Robots.txt for Rivioo - App Review Export Tool
# This file tells search engines which pages they can and cannot crawl

# Allow all search engines to crawl the site
User-agent: *
Allow: /

# Prevent crawling of sensitive or temporary files
Disallow: /downloads/
Disallow: /public/downloads/
Disallow: /api/
Disallow: /node_modules/
Disallow: /src/
Disallow: *.json
Disallow: *.log
Disallow: /test-analytics.html

# Allow crawling of main pages and assets
Allow: /index.html
Allow: /blog
Allow: /blog/
Allow: /*.css
Allow: /*.js
Allow: /*.png
Allow: /*.jpg
Allow: /*.ico

# Sitemap location
Sitemap: https://www.rivioo.app/sitemap.xml

# Crawl delay for polite crawling (optional)
Crawl-delay: 1

# Specific instructions for major search engines
User-agent: Googlebot
Allow: /
Disallow: /downloads/
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /downloads/
Disallow: /api/

# AI search engine crawlers — explicitly allowed for GEO (Generative Engine Optimization)
User-agent: GPTBot
Allow: /
Disallow: /downloads/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /downloads/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /downloads/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /downloads/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /downloads/
Disallow: /api/

# Note: AhrefsBot, MJ12bot, and SemrushBot are SEO analysis tools
# Allowing them helps with backlink discovery and competitor research