# Crawlers
User-agent: SeznamBot
Disallow: /
User-agent: ahrefs
Disallow: /
User-agent: Biano
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: petalbot
Disallow: /
User-agent: aspiegel
Disallow: /
User-agent: rogerbot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: OpenLinkProfiler
Disallow: /
User-agent: Blexbot
Disallow: /
User-agent: Ltx71
Disallow: /
User-agent: Dotbot
Disallow: /
User-agent: Leipzig Corpora Collection
Disallow: /
User-agent: Linguee
Disallow: /
User-agent: HubSpot
Disallow: /
User-agent: Quant
Disallow: /
User-agent: Seoscanners
Disallow: /
User-agent: Vagabondo
Disallow: /
User-agent: AlphaBot
Disallow: /
User-agent: Mail.ru
Disallow: /
User-agent: Magebee
Disallow: /
User-agent: Domain Reanimator
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: seoscanners.net/1
Disallow: /
User-agent: seoscanners.net
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Baiduspider/2.0
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: *
Allow: /
Crawl-delay: 5