# Slowing crawlers down on a small server.
# Google ignores Crawl-delay entirely: there the rate is set in Search Console.
# Bing, Yandex and Seznam do honour it. The value is seconds between requests.

User-agent: Bingbot
Crawl-delay: 10

User-agent: Yandex
Crawl-delay: 5

User-agent: SeznamBot
Crawl-delay: 10

User-agent: *
Crawl-delay: 20
Disallow: /search
Disallow: /export/

Sitemap: https://example.com/sitemap.xml