#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed


# Block AJAX endpoint from crawling. Drupal core's Views AJAX controller is an
# internal callback; polite crawlers that index it only fragment the page cache
# with no SEO value. The second rule covers language-prefixed paths
# (/en/views/ajax, /de/views/ajax, etc.).
Disallow: /views/ajax
Disallow: /*/views/ajax

# Block search routes from crawling. Search results pages are thin/duplicate
# content that hurts SEO and creates a vector for bot abuse. Sites that want
# search indexed can override with an Allow rule.
Disallow: /search

# Block Commerce cart and checkout from crawling. Checkout URLs carry an order
# ID and are access-checked against the visitor's session, so a crawler only
# ever collects 403s — observed as ~10k Googlebot 403s in three weeks on
# /checkout/{order}/{step} (sisal, Sep 2026). The cart is per-visitor state with
# nothing to index. /cart is anchored rather than prefix-matched so paths that
# merely begin with "cart" (a /cartography page, a carton product) stay crawlable.
Disallow: /checkout/
Disallow: /cart$
Disallow: /cart?
Disallow: /cart/

# Block faceted search URLs from crawling. Each unique facet combination is a
# distinct URL that bots can multiply to fragment the cache.
Disallow: /*?*f[0]=
Disallow: /*?*f%5B0%5D=

# Throttle AhrefsSiteAudit. Observed in the wild driving a 5-hour 499/502
# cascade by hammering uncacheable form pages with a distributed crawler swarm.
# Crawl-delay is interpreted per-bot, so 60s × ~30 concurrent crawlers caps
# aggregate near 0.5 req/sec — enough headroom for Pantheon's PHP workers.
# Real-time AhrefsBot (search index crawler) is intentionally not throttled.
# To run a legitimate site audit, raise the delay or remove this block.
User-agent: AhrefsSiteAudit
Crawl-delay: 60

# Block IbouBot. Self-identifying scraper with no SEO value, observed
# hammering form/auth pages.
User-agent: IbouBot
Disallow: /

# Block YisouSpider. Shenma/Yisou's mobile search crawler — observed driving
# 28% of total request volume on a US augustash site (MSP, May 2026) with
# no measurable organic referrals. Yisou is a Chinese mobile search engine
# dwarfed by Baidu even in its home market; US sites with no Chinese-specific
# content get the load with none of the benefit. Worst hit-to-value ratio
# observed in the wild.
User-agent: YisouSpider
Disallow: /

# Block Barkrowler. Babbar.tech's backlink/SEO analysis crawler — useful
# only if running an active Babbar audit, which augustash sites generally
# are not. Observed driving ~3% of traffic with no SEO benefit.
User-agent: Barkrowler
Disallow: /

# Block SleepBot. Self-identifying as "SleepBot" with no documented owner
# or purpose. Observed driving ~1% of traffic with no organic referrals.
User-agent: SleepBot
Disallow: /

# Reset to wildcard so any subsequent appends (e.g. site-level robots-additions)
# inherit User-agent: * cleanly instead of attaching to the last UA above.
User-agent: *