# =========================================================
# Advanced robots.txt for Soulful Dental Care (2026 SEO + AI Ready)
# Website: https://www.soulfuldental.com/
# Optimized for Google, Bing, AI Crawlers & Crawl Budget
# =========================================================

User-agent: *
Allow: /

# =========================
# CORE WORDPRESS BLOCKS
# =========================
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# =========================
# INTERNAL SEARCH & FILTERS
# =========================
Disallow: /search/
Disallow: /*?s=
Disallow: /*?filter=
Disallow: /*?sort=
Disallow: /*?orderby=
Disallow: /*?replytocom=
Disallow: /*?amp=
Disallow: /*?share=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?campaign=
Disallow: /*?sessionid=

# =========================
# LOW VALUE URLS
# =========================
Disallow: /tag/
Disallow: /author/
Disallow: /comments/
Disallow: */trackback/
Disallow: */feed/
Disallow: */embed/

# =========================
# SYSTEM / TEMP FILES
# =========================
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /cache/
Disallow: /core/
Disallow: /backup/
Disallow: /old/
Disallow: /test/
Disallow: /dev/

# =========================
# BLOCK THIN / DUPLICATE PAGES
# =========================
Disallow: /page/
Disallow: */page/*
Disallow: /*?page=

# =========================
# IMPORTANT ASSETS ALLOWED
# =========================
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$

# =========================
# GOOGLEBOT
# =========================
User-agent: Googlebot
Allow: /

# =========================
# BINGBOT
# =========================
User-agent: Bingbot
Allow: /

# =========================
# GOOGLE IMAGE BOT
# =========================
User-agent: Googlebot-Image
Allow: /

# =========================
# AI SEARCH & AI TRAINING BOTS
# =========================

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# TikTok AI crawler
User-agent: Bytespider
Disallow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# =========================
# SEO TOOL CRAWL CONTROL
# =========================

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 15

User-agent: DotBot
Crawl-delay: 15

# =========================
# BAD / AGGRESSIVE BOTS
# =========================

User-agent: HTTrack
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: BlackWidow
Disallow: /

# =========================
# HOST
# =========================
Host: https://www.soulfuldental.com/

# =========================
# XML SITEMAPS
# =========================

Sitemap: https://www.soulfuldental.com/sitemap.xml

# =========================================================
# END OF FILE
# =========================================================