# Robots.txt for Sendbot.co
# Version: 2.0
# Last Updated: 2024

# Allow all crawlers to access most content
User-agent: *
Allow: /
Allow: /api/health
Allow: /api/performance

# Block sensitive API endpoints
Disallow: /api/contact
Disallow: /api/error-tracking
Disallow: /api/auth/
Disallow: /api/admin/
Disallow: /admin/
Disallow: /_next/
Disallow: /.next/
Disallow: /private/
Disallow: /tmp/

# Block common bot traps
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# Special rules for specific crawlers
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Block aggressive crawlers
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Sitemaps
Sitemap: https://sendbot.co/sitemap.xml
Sitemap: https://sendbot.co/sitemap-images.xml
Sitemap: https://sendbot.co/sitemap-news.xml

# Cache directive
# Crawlers should respect this
Host: sendbot.co