trust-store-generators/get-hosts.sh

#!/bin/sh

# Download and merge lists of Gemini hosts from:
# gemini://geminispace.info/known-hosts
# gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt

set -o errexit  # (-e) exit immediately if any command has a non-zero exit status
set -o nounset  # (-u) don't accept undefined variables
#set -o xtrace  # for debugging

# Go where this script is.
cd "$(dirname "$0")" || exit

# If Agunua is installed, use it.
if command -v agunua >/dev/null; then
  # Using Agunua is more secure, because it does certificate pinning.
  # The --insecure option just makes it accept certificates that are
  # not signed by a (known) CA.
  hosts1=$(agunua --insecure --binary --maximum-time 20 \
    gemini://geminispace.info/known-hosts 2>/dev/null \
    | grep "gemini://" | cut -d ' ' -f 3)
  if [ -z "$hosts1" ]; then
    >&2 echo "geminispace.info/known-hosts download failed."
    exit 1
  fi
  hosts2=$(agunua --insecure --binary --maximum-time 20 \
    gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt 2>/dev/null)
  if [ -z "$hosts2" ]; then
    >&2 echo "lupa-capsules.txt download failed."
    exit 1
  fi
else
  # If Agunua is not installed, pipe the request into OpenSSL s_client.
  hosts1=$(printf "gemini://geminispace.info/known-hosts\r\n" \
    | timeout 20 openssl s_client -quiet -connect "geminispace.info:1965" 2>/dev/null \
    | grep "gemini://" | cut -d ' ' -f 3)
  if [ -z "$hosts1" ]; then
    >&2 echo "geminispace.info/known-hosts download failed."
    exit 1
  fi
  hosts2=$(printf "gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt\r\n" \
    | timeout 20 openssl s_client -quiet -connect "gemini.bortzmeyer.org:1965" 2>/dev/null \
    | tail -n +2)
  if [ -z "$hosts2" ]; then
    >&2 echo "lupa-capsules.txt download failed."
    exit 1
  fi
fi

# Concatenate the two files.
hosts="$hosts1
$hosts2"

# Remove empty lines; convert punycode to unicode; sort entries; remove duplicates.
hosts=$(echo "$hosts" | awk NF | idn --allow-unassigned --idna-to-unicode | sort -fu)

# Remove hosts which contain neither "." nor ":", such as "localhost".
hosts=$(echo "$hosts" | grep '\.\|:')

# Remove explicitly excluded hosts.
hosts=$(echo "$hosts" | grep -vxEf excluded-hosts)

if [ -z "$hosts" ]; then
  >&2 echo "hosts file downloads failed."
  exit 1
fi

# Save to file.
echo "$hosts" > hosts

echo OK