From feb651d698edbee62ade5f5d7b4fb7b63d24b568 Mon Sep 17 00:00:00 2001
From: nervuri <seva@nervuri.net>
Date: Fri, 4 Jun 2021 11:20:22 +0000
Subject: [PATCH] add hosts from Lupa

---
 README.md    |  2 +-
 get-hosts.sh | 37 ++++++++++++++++++++-----------------
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 53cd518..5339ef5 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ Geminispace is (currently) small enough that we can afford to download all known
 
 This repo contains scripts for:
 
-1. downloading a list of hosts from gemini://geminispace.info/known-hosts
+1. downloading a list of hosts from [geminispace.info](gemini://geminispace.info/known-hosts) and [Lupa](gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt)
 2. downloading the TLS certificates of those hosts
 3. generating a table containing details about each certificate
 4. generating trust stores for various Gemini clients, currently:
diff --git a/get-hosts.sh b/get-hosts.sh
index af810f1..f2d0534 100755
--- a/get-hosts.sh
+++ b/get-hosts.sh
@@ -1,6 +1,8 @@
 #!/bin/sh
 
-# Download a list of Gemini hosts from gemini://geminispace.info/known-hosts
+# Download and merge lists of Gemini hosts from:
+# gemini://geminispace.info/known-hosts
+# gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt
 
 set -o errexit  # (-e) exit immediately if any command has a non-zero exit status
 set -o nounset  # (-u) don't accept undefined variables
@@ -12,33 +14,34 @@ cd "$(dirname "$0")" || exit
 # If Agunua is installed, use it.
 if command -v agunua >/dev/null; then
   # Using Agunua is more secure, because it does certificate pinning.
-  hosts=$(agunua --insecure --binary gemini://geminispace.info/known-hosts \
+  # The --insecure option just makes it accept certificates that are
+  # not signed by a (known) CA.
+  hosts1=$(agunua --insecure --binary gemini://geminispace.info/known-hosts 2>/dev/null \
     | grep "gemini://" | cut -d ' ' -f 3)
+  hosts2=$(agunua --insecure --binary gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt 2>/dev/null)
 else
   # If Agunua is not installed, pipe the request into OpenSSL s_client.
-  hosts=$(printf "gemini://geminispace.info/known-hosts\r\n" \
+  hosts1=$(printf "gemini://geminispace.info/known-hosts\r\n" \
     | timeout 5 openssl s_client -quiet -connect "geminispace.info:1965" 2>/dev/null \
     | grep "gemini://" | cut -d ' ' -f 3)
+  hosts2=$(printf "gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt\r\n" \
+    | timeout 5 openssl s_client -quiet -connect "gemini.bortzmeyer.org:1965" 2>/dev/null \
+    | tail -n +2)
 fi
 
+# Concatenate the two files.
+hosts="$hosts1
+$hosts2"
+
+# Convert punycode to unicode; sort entries; remove duplicates; remove empty lines.
+hosts=$(echo "$hosts" | idn --allow-unassigned --idna-to-unicode | sort -fu | awk NF)
+
 if [ -z "$hosts" ]; then
-  >&2 echo "hosts file download failed."
+  >&2 echo "hosts file downloads failed."
   exit 1
 fi
 
-# Add a few hosts that are missing from geminispace.info
-hosts="$hosts
-campaignwiki.org
-feeds.drewdevault.com
-gem.adele.work
-makeworld.gq
-qwertqwefsday.eu:80
-simplynews.metalune.xyz"
-
-# Remove a host which changes its cert every few minutes.
-hosts=$(echo "$hosts" | grep -v 'tofu-tester.random-projects.net:1966')
-
 # Save to file.
-echo "$hosts" | sort | uniq > hosts
+echo "$hosts" > hosts
 
 echo OK