Author: Pedro Lucas Porcellis <porcellis@eletrotupi.com>
robots: Disable a bunch of aggressive robots
public/robots.txt | 87 +++++++++++++++++++++++++++++++++++++++++++++++++
diff --git a/public/robots.txt b/public/robots.txt index 1d056b390ecb4e86d3894800b64cde192871458e..5e3a1747e069a7d9f8fa854a1b99bfc24b67225e 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -7,3 +7,90 @@ Disallow: /search Disallow: /tag Disallow: /search*query=* Disallow: /cat/* + +# The policy is to block any SEO/Marketing bot that's too aggressive +# If you annoy our sysadmins you will be blocked. +# Reach out to porcellis@eletrotupi.com for any questions. + +# Too aggressive, marketing/SEO +User-agent: SemrushBot +Disallow: / + +# Too aggressive, marketing/SEO +User-agent: SemrushBot-SA +Disallow: / + +# Marketing/SEO +User-agent: AhrefsBot +Disallow: / + +# Marketing/SEO +User-agent: dotbot +Disallow: / + +# Marketing/SEO +User-agent: rogerbot +Disallow: / + +# Too aggressive +User-agent: bingbot +Disallow: / + +User-agent: Bingbot +Disallow: / + +# Too aggressive +User-agent: Yandex +Disallow: / + +# Too aggressive +User-agent: "The Knowledge AI" +Disallow: / + +# Too aggressive +User-agent: "YandexBot" +Disallow: / + +# Dickhead, too aggressive +# Knowledge AI +User-agent: "Knowledge" +Disallow: / + +User-agent: msnbot +Disallow: / + +User-agent: Purebot +Disallow: / + +User-agent: Baiduspider +Disallow: / + +User-agent: Lipperhey +Disallow: / + +User-agent: Mail.Ru +Disallow: / + +User-agent: scrapbot +Disallow: / + +User-agent: MJ12bot +Disallow: / + +User-agent: BDCbot +Disallow: / + +User-agent: MegaIndex +Disallow: / + +User-agent: UniLeipzigASV +Disallow: / + +User-agent: DotBot +Disallow: / + +User-agent: Typhoeus +Disallow: / + +User-agent: PetalBot +Disallow: /