# robots.txt for Remove Pest Control (https://removepest.in)
# Optimized for Search Engine Crawling
# Last Updated: 2025-12-16

# Allow all search engines to crawl everything
User-agent: *
Allow: /
Disallow: /cgi-bin/
Disallow: /sendemail.php

# Crawl-delay for polite crawling (optional, helps with server load)
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: Slurp
Allow: /

# Block bad bots (optional)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# Sitemap location (important!)
Sitemap: https://removepest.in/sitemap.xml

# Additional sitemaps (if you create them in future)
# Sitemap: https://removepest.in/sitemap-images.xml
# Sitemap: https://removepest.in/sitemap-blog.xml