generate sitemaps

Google's indexing of the site is very spotty, and this seems a lot easier than
improving our builtin search engine.

.gitignore

@@ -19,6 +19,7 @@
 vendor/bundle
 public/assets
 public/avatars
+public/sitemap*
 node_modules/
 yarn-error.log
 yarn.lock

Gemfile

@@ -33,11 +33,9 @@ gem "nokogiri", ">= 1.7.2"
 gem "htmlentities"
 gem "commonmarker", "~> 0.14"
 
-# for twitter-posting bot
-gem "oauth"
-
-# for parsing incoming mail
-gem "mail"
+gem "oauth" # for twitter-posting bot
+gem "mail" # for parsing incoming mail
+gem "sitemap_generator" # for better search engine indexing
 
 group :test, :development do
   gem 'bullet'

Gemfile.lock

@@ -190,6 +190,8 @@ GEM
     scenic-mysql_adapter (1.0.1)
       mysql2
       scenic (>= 1.4.0)
+    sitemap_generator (6.0.1)
+      builder (~> 3.0)
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -245,6 +247,7 @@ DEPENDENCIES
   rubocop
   scenic
   scenic-mysql_adapter
+  sitemap_generator
   sqlite3
   uglifier (>= 1.3.0)
   unicorn

config/sitemap.rb

@@ -0,0 +1,29 @@
+SitemapGenerator::Sitemap.default_host = "https://lobste.rs"
+
+check_hourly = 4.days.ago
+check_daily = 2.weeks.ago
+top_score = Story.all.maximum('upvotes')
+
+SitemapGenerator::Sitemap.create do
+  %w{/about /chat}.each do |path|
+    add path, changefreq: 'monthly', lastmod: nil
+  end
+
+  add recent_path, changefreq: 'always', priority: 1
+  add newest_path, changefreq: 'always', priority: 1
+  add comments_path, changefreq: 'always', priority: 1
+
+  Story.order('id desc').find_each do |story|
+    last_comment = story.comments.order('id desc').first
+
+    lastmod = story.created_at
+    lastmod = last_comment.updated_at if last_comment && last_comment.updated_at > lastmod
+
+    changefreq = 'monthly'
+    changefreq = 'daily'   if lastmod >= check_daily
+    changefreq = 'hourly'  if lastmod >= check_hourly
+
+    priority = 1.0 * story.upvotes / top_score
+    add story.comments_path, lastmod: lastmod, changefreq: changefreq, priority: priority
+  end
+end