ask polite bots to slow down

2020-11-29 10:24:57 -06:00 · 2020-11-29 10:24:57 -06:00 · 13072e05ae
parent c4440695e2
commit 13072e05ae
1 changed files with 1 additions and 4 deletions
--- a/public/robots.txt
+++ b/public/robots.txt
@ -1,8 +1,5 @@
 User-agent: *
+Crawl-delay: 1
 Disallow: /search
-
-User-agent: *
 Disallow: /page/
-
-User-agent: *
 Disallow: /comments/page/