diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 1477af46..f4b4c97c 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -8,7 +8,6 @@ - <% if @meta_tags %> diff --git a/public/robots.txt b/public/robots.txt index aa94cad4..87a97c07 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,19 +1,2 @@ -# block all spiders by default -User-agent: * -Disallow: / - -# but allow major ones -User-agent: Googlebot -Allow: / - -User-agent: Slurp -Allow: / - -User-Agent: msnbot -Disallow: - -User-agent: Baiduspider -Disallow: / - User-agent: * Disallow: /search