robots: tell spiders not to crawl pages

2017-06-05 09:51:29 -05:00 · 2017-06-05 09:51:29 -05:00 · 51f03832f2
parent 32aa58fc98
commit 51f03832f2
1 changed files with 6 additions and 0 deletions
--- a/public/robots.txt
+++ b/public/robots.txt
@ -1,2 +1,8 @@
 User-agent: *
 Disallow: /search
+
+User-agent: *
+Disallow: /page/
+
+User-agent: *
+Disallow: /comments/page/