cirandas.net

commit e0b22ec2ca7cf80a7bd50aecee8cc84cca6aaebe

Author: Pedro Lucas Porcellis <porcellis@eletrotupi.com>

robots: Disable a bunch of aggressive robots

 public/robots.txt | 87 +++++++++++++++++++++++++++++++++++++++++++++++++


diff --git a/public/robots.txt b/public/robots.txt
index 1d056b390ecb4e86d3894800b64cde192871458e..5e3a1747e069a7d9f8fa854a1b99bfc24b67225e 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -7,3 +7,90 @@ Disallow: /search
 Disallow: /tag
 Disallow: /search*query=*
 Disallow: /cat/*
+
+# The policy is to block any SEO/Marketing bot that's too aggressive
+# If you annoy our sysadmins you will be blocked.
+# Reach out to porcellis@eletrotupi.com for any questions.
+
+# Too aggressive, marketing/SEO
+User-agent: SemrushBot
+Disallow: /
+
+# Too aggressive, marketing/SEO
+User-agent: SemrushBot-SA
+Disallow: /
+
+# Marketing/SEO
+User-agent: AhrefsBot
+Disallow: /
+
+# Marketing/SEO
+User-agent: dotbot
+Disallow: /
+
+# Marketing/SEO
+User-agent: rogerbot
+Disallow: /
+
+# Too aggressive
+User-agent: bingbot
+Disallow: /
+
+User-agent: Bingbot
+Disallow: /
+
+# Too aggressive
+User-agent: Yandex
+Disallow: /
+
+# Too aggressive
+User-agent: "The Knowledge AI"
+Disallow: /
+
+# Too aggressive
+User-agent: "YandexBot"
+Disallow: /
+
+# Dickhead, too aggressive
+# Knowledge AI
+User-agent: "Knowledge"
+Disallow: /
+
+User-agent: msnbot
+Disallow: /
+
+User-agent: Purebot
+Disallow: /
+
+User-agent: Baiduspider
+Disallow: /
+
+User-agent: Lipperhey
+Disallow: /
+
+User-agent: Mail.Ru
+Disallow: /
+
+User-agent: scrapbot
+Disallow: /
+
+User-agent: MJ12bot
+Disallow: /
+
+User-agent: BDCbot
+Disallow: /
+
+User-agent: MegaIndex
+Disallow: /
+
+User-agent: UniLeipzigASV
+Disallow: /
+
+User-agent: DotBot
+Disallow: /
+
+User-agent: Typhoeus
+Disallow: /
+
+User-agent: PetalBot
+Disallow: /