diff --git a/public/robots.txt b/public/robots.txt
index 3d1e9388..eeda15b2 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,5 +1,16 @@
-# no caching
-User-agent: ia_archiver
+# block all spiders by default
+User-agent: *
 Disallow: /
-User-agent: archive.org_bot
+
+# but allow major ones
+User-agent: Googlebot
+Allow: /
+
+User-agent: Slurp
+Allow: /
+
+User-Agent: msnbot
+Disallow: 
+
+User-agent: Baiduspider
 Disallow: /