diff --git a/.gitignore b/.gitignore index de7c305a..a531473f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ vendor/bundle public/assets public/avatars +public/sitemap* node_modules/ yarn-error.log yarn.lock diff --git a/Gemfile b/Gemfile index 0053cb86..4515c204 100644 --- a/Gemfile +++ b/Gemfile @@ -33,11 +33,9 @@ gem "nokogiri", ">= 1.7.2" gem "htmlentities" gem "commonmarker", "~> 0.14" -# for twitter-posting bot -gem "oauth" - -# for parsing incoming mail -gem "mail" +gem "oauth" # for twitter-posting bot +gem "mail" # for parsing incoming mail +gem "sitemap_generator" # for better search engine indexing group :test, :development do gem 'bullet' diff --git a/Gemfile.lock b/Gemfile.lock index 023ce34b..dd66a6b1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -190,6 +190,8 @@ GEM scenic-mysql_adapter (1.0.1) mysql2 scenic (>= 1.4.0) + sitemap_generator (6.0.1) + builder (~> 3.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -245,6 +247,7 @@ DEPENDENCIES rubocop scenic scenic-mysql_adapter + sitemap_generator sqlite3 uglifier (>= 1.3.0) unicorn diff --git a/config/sitemap.rb b/config/sitemap.rb new file mode 100644 index 00000000..0d21ea2f --- /dev/null +++ b/config/sitemap.rb @@ -0,0 +1,29 @@ +SitemapGenerator::Sitemap.default_host = "https://lobste.rs" + +check_hourly = 4.days.ago +check_daily = 2.weeks.ago +top_score = Story.all.maximum('upvotes') + +SitemapGenerator::Sitemap.create do + %w{/about /chat}.each do |path| + add path, changefreq: 'monthly', lastmod: nil + end + + add recent_path, changefreq: 'always', priority: 1 + add newest_path, changefreq: 'always', priority: 1 + add comments_path, changefreq: 'always', priority: 1 + + Story.order('id desc').find_each do |story| + last_comment = story.comments.order('id desc').first + + lastmod = story.created_at + lastmod = last_comment.updated_at if last_comment && last_comment.updated_at > lastmod + + changefreq = 'monthly' + changefreq = 'daily' if lastmod >= check_daily + changefreq = 'hourly' if lastmod >= check_hourly + + priority = 1.0 * story.upvotes / top_score + add story.comments_path, lastmod: lastmod, changefreq: changefreq, priority: priority + end +end