# robots.txt for https://mytownroofing.com
# Last updated: 2026-07-09
# --------------------------------
# Global rules (WordPress-friendly)
# --------------------------------
User-agent: *
# Block internal/action endpoints & thin search pages
Disallow: /calendar/action*
Disallow: /events/action*
Disallow: /?s=*
Disallow: /search/*
# Block junk parameter URLs (crawl budget protection)
Disallow: /*?nocache=*
Disallow: /*?jet_blog_ajax=*
Disallow: /*?utm_source=*
Disallow: /*?utm_medium=*
Disallow: /*?utm_campaign=*
# Block fake pagination URLs
Disallow: /*/1000
# Block paginated category/blog URLs beyond page 1
Disallow: /*/page/*
# Block Elementor system pages
Disallow: /elementor-library-about-us
# Standard WP hygiene
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# Keep media and assets fully crawlable (images/CSS/JS needed for proper indexing)
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-includes/js/
# Crawl pacing (Google ignores; Bing/others may respect)
Crawl-delay: 3
# --------------------------------
# Google (web, images, previews)
# --------------------------------
User-agent: Googlebot
Allow: /
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Video
Allow: /
User-agent: Google-Extended     # Controls some Gemini/Vertex AI uses (not AI Overviews inclusion)
Allow: /
# --------------------------------
# Microsoft / Bing ecosystem
# --------------------------------
User-agent: Bingbot
Allow: /
User-agent: BingPreview
Allow: /
# --------------------------------
# Apple (Siri/Spotlight/Apple Intelligence)
# --------------------------------
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended   # Controls Apple AI training usage
Allow: /
# --------------------------------
# OpenAI / Anthropic / Perplexity / Common Crawl
# --------------------------------
User-agent: OAI-SearchBot       # ChatGPT search/citations
Allow: /
User-agent: ChatGPT-User        # On-demand retrieval in ChatGPT
Allow: /
User-agent: GPTBot              # OpenAI training/ingestion
Allow: /
User-agent: ClaudeBot           # Anthropic/Claude
Allow: /
User-agent: PerplexityBot       # Perplexity AI
Allow: /
User-agent: CCBot               # Common Crawl (downstream for many AIs)
Allow: /
# --------------------------------
# Sitemaps
# --------------------------------
Sitemap: https://mytownroofing.com/sitemaps.xml
Sitemap: https://mytownroofing.com/post-sitemap1.xml
Sitemap: https://mytownroofing.com/page-sitemap1.xml
Sitemap: https://mytownroofing.com/service-areas-sitemap1.xml
Sitemap: https://mytownroofing.com/county-sitemap1.xml
Sitemap: https://mytownroofing.com/state-sitemap1.xml
Sitemap: https://mytownroofing.com/portfolio-sitemap1.xml
Sitemap: https://mytownroofing.com/portfolio-categories-sitemap1.xml
Sitemap: https://mytownroofing.com/team-members-sitemap1.xml
Sitemap: https://mytownroofing.com/category-sitemap1.xml
Sitemap: https://mytownroofing.com/video1.xml
# --------------------------------
# AI policy reference (optional but recommended)
# --------------------------------
# Clarifies how AI systems may access/reference your content.
# AI-Policy: https://mytownroofing.com/ai-policy/