Disallow some more crawlers via robots.txt

rubymonsters · Nov 4, 2024 · 11dc39b · 11dc39b
1 parent bf4d786
commit 11dc39b
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -4,6 +4,15 @@
 User-Agent: *
 Disallow: /topics
 
+User-Agent: Seobility
+Disallow: /
+
+User-Agent: crawl
+Disallow: /
+
+User-Agent: MJ12bot
+Disallow: /
+
 User-Agent: SemrushBot
 Disallow: /