# https://www.hymnallibrary.org/robots.txt
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /hymns/hymns/
# Autocomplete / heavy filtered search — polite crawlers should skip

# Known aggressive SEO / AI scrapers (polite ones honor this; others need WAF)
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Crawl-delay: 10

# Prefer the www host (also enforced by CanonicalHostMiddleware)
Host: www.hymnallibrary.org

Sitemap: https://www.hymnallibrary.org/sitemap.xml