# ===========================================
# Elephant Garden - robots.txt
# Last updated: 2026-03-03
# ===========================================
#
# NOTE — robots.txt precedence:
# A bot with its own explicit User-agent block follows ONLY
# that block — it does NOT inherit rules from User-agent: *.
# All "welcome" bots are therefore intentionally left unlisted
# so they fall through to the wildcard and inherit all Disallow
# rules. Only bots with truly different access (blocked bots)
# get their own explicit blocks.
#
# Bots covered by User-agent: * below:
#
# — Search Engines —
#   Googlebot, Bingbot, DuckDuckBot, YandexBot, Applebot
#
# — OpenAI / ChatGPT —
#   OAI-SearchBot    (ChatGPT Search indexing — most important)
#   GPTBot           (GPT model training)
#   ChatGPT-User     (real-time user-triggered browsing)
#
# — Anthropic / Claude —
#   ClaudeBot        (Claude model training)
#   Claude-SearchBot (Claude Search indexing)
#   Claude-User      (real-time user-triggered browsing)
#   anthropic-ai     (legacy ClaudeBot variant)
#
# — xAI / Grok —
#   GrokBot          (Grok training & indexing)
#   xAI-Grok         (Grok Search)
#   Grok-DeepSearch  (Grok DeepSearch)
#
# — Google AI —
#   Google-Extended      (Gemini / AI Overviews training)
#   Gemini-Deep-Research (Gemini Deep Research agent)
#
# — Perplexity —
#   PerplexityBot    (Perplexity search indexing)
#   Perplexity-User  (real-time user query fetching)
#
# — Other AI —
#   Meta-ExternalAgent  (Meta AI / Llama training)
#   Meta-WebIndexer     (Meta AI search)
#   Applebot-Extended   (Apple Intelligence — separate from Applebot)
#   DuckAssistBot       (DuckDuckGo AI / DuckAssist)
#   MistralAI-User      (Mistral Le Chat)
#   CCBot               (Common Crawl — base training data for many LLMs)
#   Amazonbot           (Amazon Alexa AI)
# ===========================================


# -------------------------------------------
# GLOBAL RULES
# Applies to all crawlers not explicitly listed
# -------------------------------------------
User-agent: *
Allow: /

# Block WordPress admin & internals
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /wp-json/

# Block WooCommerce private/dynamic pages
Disallow: /cart-2/
Disallow: /cart/
Disallow: /checkout-2/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /garden-gate/
Disallow: /garden-gates/


# Block duplicate content from query parameters
Disallow: /*?add-to-cart=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?remove_item=*
Disallow: /*?removed_item=*
Disallow: /*?wc-ajax=*

# Block internal search results (thin content)
Disallow: /?s=
Disallow: /search/

# Block feeds (reduces crawl waste)
Disallow: /comments/feed/
Disallow: /*/feed/

# Tag archives: ALLOWED to crawl. Handled via Yoast noindex
# (noindex-tax-post_tag = true). Robots.txt block was wrong:
# robots.txt-blocked + linked-internally creates BlockedByRobotsTxt
# noise without letting Bing see the noindex signal. Allow crawl
# + noindex is the SEO-correct end-state.

# Author archives: same — handled via Yoast noindex
# (noindex-author-wpseo = true).


# -------------------------------------------
# BLOCKED — Bad bots, SEO scrapers, data harvesters
# These get their own explicit blocks so they are
# fully disallowed regardless of wildcard rules.
# -------------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ByteDance (TikTok parent) — aggressive scraper, no SEO/AI value
User-agent: Bytespider
Disallow: /


# -------------------------------------------
Sitemap: https://elephantgarden.co/sitemap_index.xml
Sitemap: https://elephantgarden.co/location-sitemap.xml