check dependencies

README.md

@@ -17,9 +17,9 @@ Trust stores generated by these scripts can be found at https://tildegit.org/ner
 
 ## Dependencies
 
-Required: [OpenSSL](https://www.openssl.org/), [idn](https://www.gnu.org/software/libidn/)
+Required: [OpenSSL](https://www.openssl.org/), [idn](https://www.gnu.org/software/libidn/), [Agunua](https://framagit.org/bortzmeyer/agunua).
 
-Optional: [torsocks](https://packages.debian.org/buster/torsocks) (for .onion capsules and for double-checking certificates using a different network perspective) and [Agunua](https://framagit.org/bortzmeyer/agunua) (for downloading host lists more securely).
+Optional: [torsocks](https://packages.debian.org/buster/torsocks) (for .onion capsules and for double-checking certificates using a different network perspective).
 
 To install them in Debian, run: `sudo apt install openssl idn torsocks` and `pip3 install agunua`.
 

excluded-hosts

@@ -13,6 +13,7 @@ localhost(:[0-9]+)?
 \[::1\](:[0-9]+)?
 # Reserved TLDs
 # https://tools.ietf.org/id/draft-chapin-additional-reserved-tlds-01.html
+# https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xml#special-use-domain
 .*\.test(:[0-9]+)?
 .*\.example(:[0-9]+)?
 .*\.invalid(:[0-9]+)?

get-certs.sh

@@ -107,6 +107,7 @@ while read -r host; do
       >&2 echo "$host_and_port - Tor connection failed"
     elif [ -n "$cert" ] && [ "$cert" != "$cert_via_tor" ]; then
       # Mismatch.
+      >&2 echo  # empty line
       >&2 echo "$host_and_port - Tor VERIFICATION FAILED (certs don't match)!!!"
       # In this case, don't save any certificate to file.
       # Output both certificates to stderr instead.

get-hosts.sh

@@ -11,38 +11,18 @@ set -o nounset  # (-u) don't accept undefined variables
 # Go where this script is.
 cd "$(dirname "$0")" || exit
 
-# If Agunua is installed, use it.
-if command -v agunua >/dev/null; then
-  # Using Agunua is more secure, because it does certificate pinning.
-  hosts1=$(agunua --binary --maximum-time 20 \
-    gemini://geminispace.info/known-hosts 2>/dev/null \
-    | grep "gemini://" | cut -d ' ' -f 3)
-  if [ -z "$hosts1" ]; then
-    >&2 echo "geminispace.info/known-hosts download failed."
-    exit 1
-  fi
-  hosts2=$(agunua --binary --maximum-time 20 \
-    gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt 2>/dev/null)
-  if [ -z "$hosts2" ]; then
-    >&2 echo "lupa-capsules.txt download failed."
-    exit 1
-  fi
-else
-  # If Agunua is not installed, pipe the request into OpenSSL s_client.
-  hosts1=$(printf "gemini://geminispace.info/known-hosts\r\n" \
-    | timeout 20 openssl s_client -quiet -connect "geminispace.info:1965" 2>/dev/null \
-    | grep "gemini://" | cut -d ' ' -f 3)
-  if [ -z "$hosts1" ]; then
-    >&2 echo "geminispace.info/known-hosts download failed."
-    exit 1
-  fi
-  hosts2=$(printf "gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt\r\n" \
-    | timeout 20 openssl s_client -quiet -connect "gemini.bortzmeyer.org:1965" 2>/dev/null \
-    | tail -n +2)
-  if [ -z "$hosts2" ]; then
-    >&2 echo "lupa-capsules.txt download failed."
-    exit 1
-  fi
+hosts1=$(agunua --binary --maximum-time 20 \
+  gemini://geminispace.info/known-hosts 2>/dev/null \
+  | grep "gemini://" | cut -d ' ' -f 3)
+if [ -z "$hosts1" ]; then
+  >&2 echo "geminispace.info/known-hosts download failed."
+  exit 1
+fi
+hosts2=$(agunua --binary --maximum-time 20 \
+  gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt 2>/dev/null)
+if [ -z "$hosts2" ]; then
+  >&2 echo "lupa-capsules.txt download failed."
+  exit 1
 fi
 
 # Concatenate the two files.
@@ -79,20 +59,22 @@ finish() {
 trap finish EXIT
 
 # Test if removed hosts are still online.
-echo "Testing removed hosts..."
-for removed_host in $(diff hosts "$tempfile" | grep ^\< | cut -c 3-); do
-  printf "%s" "$removed_host"
-  # If direct connection fails, try to connect through Tor.
-  if agunua --no-tofu --maximum-time 20 "$removed_host" >/dev/null 2>&1 || \
-      agunua --socks 127.0.0.1:9050 --no-tofu --maximum-time 20 "$removed_host" >/dev/null 2>&1; then
-    echo " - ONLINE"
-    # Add removed host back.
-    hosts="$hosts
+if [ -f hosts ]; then
+  echo "Testing removed hosts..."
+  for removed_host in $(diff hosts "$tempfile" | grep ^\< | cut -c 3-); do
+    printf "%s" "$removed_host"
+    # If direct connection fails, try to connect through Tor.
+    if agunua --no-tofu --maximum-time 20 "$removed_host" >/dev/null 2>&1 || \
+        agunua --socks 127.0.0.1:9050 --no-tofu --maximum-time 20 "$removed_host" >/dev/null 2>&1; then
+      echo " - ONLINE"
+      # Add removed host back.
+      hosts="$hosts
 $removed_host"
-  else
-    echo " - offline"
-  fi
-done
+    else
+      echo " - offline"
+    fi
+  done
+fi
 
 # Sort entries again.
 hosts=$(echo "$hosts" | sort)

main.sh

@@ -4,6 +4,23 @@
 
 set -o errexit  # (-e) exit immediately if any command has a non-zero exit status
 
+# Check if dependencies are installed.
+if ! command -v openssl >/dev/null; then
+  >&2 echo '"openssl" not installed! [required]'
+  exit 1
+fi
+if ! command -v idn >/dev/null; then
+  >&2 echo '"idn" not installed! [required]'
+  exit 1
+fi
+if ! command -v agunua >/dev/null; then
+  >&2 echo '"agunua" not installed! [required]'
+  exit 1
+fi
+if ! command -v torsocks >/dev/null; then
+  >&2 echo '"torsocks" not installed! [optional]'
+fi
+
 # Go where this script is.
 cd "$(dirname "$0")" || exit