2021-04-28 09:20:18 +00:00
|
|
|
#!/bin/sh
|
|
|
|
|
2021-06-04 11:20:22 +00:00
|
|
|
# Download and merge lists of Gemini hosts from:
|
|
|
|
# gemini://geminispace.info/known-hosts
|
|
|
|
# gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt
|
2021-04-28 09:20:18 +00:00
|
|
|
|
|
|
|
set -o errexit # (-e) exit immediately if any command has a non-zero exit status
|
|
|
|
set -o nounset # (-u) don't accept undefined variables
|
|
|
|
#set -o xtrace # for debugging
|
|
|
|
|
|
|
|
# Go where this script is.
|
|
|
|
cd "$(dirname "$0")" || exit
|
|
|
|
|
|
|
|
# If Agunua is installed, use it.
|
|
|
|
if command -v agunua >/dev/null; then
|
|
|
|
# Using Agunua is more secure, because it does certificate pinning.
|
2021-09-24 09:35:45 +00:00
|
|
|
hosts1=$(agunua --binary --maximum-time 20 \
|
2021-08-22 15:18:08 +00:00
|
|
|
gemini://geminispace.info/known-hosts 2>/dev/null \
|
2021-04-28 09:20:18 +00:00
|
|
|
| grep "gemini://" | cut -d ' ' -f 3)
|
2021-08-22 15:18:08 +00:00
|
|
|
if [ -z "$hosts1" ]; then
|
|
|
|
>&2 echo "geminispace.info/known-hosts download failed."
|
|
|
|
exit 1
|
|
|
|
fi
|
2021-09-24 09:35:45 +00:00
|
|
|
hosts2=$(agunua --binary --maximum-time 20 \
|
2021-08-22 15:18:08 +00:00
|
|
|
gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt 2>/dev/null)
|
|
|
|
if [ -z "$hosts2" ]; then
|
|
|
|
>&2 echo "lupa-capsules.txt download failed."
|
|
|
|
exit 1
|
|
|
|
fi
|
2021-04-28 09:20:18 +00:00
|
|
|
else
|
|
|
|
# If Agunua is not installed, pipe the request into OpenSSL s_client.
|
2021-06-04 11:20:22 +00:00
|
|
|
hosts1=$(printf "gemini://geminispace.info/known-hosts\r\n" \
|
2021-08-30 19:13:01 +00:00
|
|
|
| timeout 20 openssl s_client -quiet -connect "geminispace.info:1965" 2>/dev/null \
|
2021-04-28 09:20:18 +00:00
|
|
|
| grep "gemini://" | cut -d ' ' -f 3)
|
2021-08-22 15:18:08 +00:00
|
|
|
if [ -z "$hosts1" ]; then
|
|
|
|
>&2 echo "geminispace.info/known-hosts download failed."
|
|
|
|
exit 1
|
|
|
|
fi
|
2021-06-04 11:20:22 +00:00
|
|
|
hosts2=$(printf "gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt\r\n" \
|
2021-08-30 19:13:01 +00:00
|
|
|
| timeout 20 openssl s_client -quiet -connect "gemini.bortzmeyer.org:1965" 2>/dev/null \
|
2021-06-04 11:20:22 +00:00
|
|
|
| tail -n +2)
|
2021-08-22 15:18:08 +00:00
|
|
|
if [ -z "$hosts2" ]; then
|
|
|
|
>&2 echo "lupa-capsules.txt download failed."
|
|
|
|
exit 1
|
|
|
|
fi
|
2021-04-28 09:20:18 +00:00
|
|
|
fi
|
|
|
|
|
2021-06-04 11:20:22 +00:00
|
|
|
# Concatenate the two files.
|
|
|
|
hosts="$hosts1
|
|
|
|
$hosts2"
|
|
|
|
|
2021-08-22 15:18:08 +00:00
|
|
|
# Remove empty lines; convert punycode to unicode; sort entries; remove duplicates.
|
|
|
|
hosts=$(echo "$hosts" | awk NF | idn --allow-unassigned --idna-to-unicode | sort -fu)
|
2021-06-04 11:20:22 +00:00
|
|
|
|
2021-06-30 17:57:14 +00:00
|
|
|
# Remove hosts which contain neither "." nor ":", such as "localhost".
|
|
|
|
hosts=$(echo "$hosts" | grep '\.\|:')
|
|
|
|
|
|
|
|
# Remove explicitly excluded hosts.
|
|
|
|
hosts=$(echo "$hosts" | grep -vxEf excluded-hosts)
|
|
|
|
|
2021-09-16 17:46:43 +00:00
|
|
|
if ! echo "$hosts" | grep -qE '\.onion(:.*)?$'; then
|
|
|
|
>&2 echo "The .onions are missing!"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2021-04-28 09:20:18 +00:00
|
|
|
if [ -z "$hosts" ]; then
|
2021-06-04 11:20:22 +00:00
|
|
|
>&2 echo "hosts file downloads failed."
|
2021-04-28 09:20:18 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2021-09-24 12:18:34 +00:00
|
|
|
# Save to temporary file.
|
|
|
|
tempfile=$(mktemp)
|
|
|
|
echo "$hosts" > "$tempfile"
|
|
|
|
|
|
|
|
# Delete temporary file on exit.
|
|
|
|
finish() {
|
|
|
|
rm -f "$tempfile"
|
|
|
|
}
|
|
|
|
trap finish EXIT
|
|
|
|
|
|
|
|
# Test if removed hosts are still online.
|
|
|
|
echo "Testing removed hosts..."
|
|
|
|
for removed_host in $(diff hosts "$tempfile" | grep ^\< | cut -c 3-); do
|
|
|
|
printf "%s" "$removed_host"
|
|
|
|
# If direct connection fails, try to connect through Tor.
|
|
|
|
if agunua --no-tofu --maximum-time 20 "$removed_host" >/dev/null 2>&1 || \
|
|
|
|
agunua --socks 127.0.0.1:9050 --no-tofu --maximum-time 20 "$removed_host" >/dev/null 2>&1; then
|
|
|
|
echo " - ONLINE"
|
|
|
|
# Add removed host back.
|
|
|
|
hosts="$hosts
|
|
|
|
$removed_host"
|
|
|
|
else
|
|
|
|
echo " - offilne"
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
|
|
|
# Sort entries again.
|
|
|
|
hosts=$(echo "$hosts" | sort)
|
|
|
|
|
2021-04-28 09:20:18 +00:00
|
|
|
# Save to file.
|
2021-06-04 11:20:22 +00:00
|
|
|
echo "$hosts" > hosts
|
2021-04-28 09:20:18 +00:00
|
|
|
|
|
|
|
echo OK
|