nervuri
/
trust-store-generators
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add hosts from Lupa

This commit is contained in:
nervuri 2021-06-04 11:20:22 +00:00
parent 73ee829328
commit feb651d698
Signed by: nervuri
GPG Key ID: C4769EEA7BA61672
2 changed files with 21 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -4,7 +4,7 @@ Geminispace is (currently) small enough that we can afford to download all known
This repo contains scripts for: This repo contains scripts for:
1. downloading a list of hosts from gemini://geminispace.info/known-hosts 1. downloading a list of hosts from [geminispace.info](gemini://geminispace.info/known-hosts) and [Lupa](gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt)
2. downloading the TLS certificates of those hosts 2. downloading the TLS certificates of those hosts
3. generating a table containing details about each certificate 3. generating a table containing details about each certificate
4. generating trust stores for various Gemini clients, currently: 4. generating trust stores for various Gemini clients, currently:

37
get-hosts.sh
View File

@ -1,6 +1,8 @@
#!/bin/sh #!/bin/sh
# Download a list of Gemini hosts from gemini://geminispace.info/known-hosts # Download and merge lists of Gemini hosts from:
# gemini://geminispace.info/known-hosts
# gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt
set -o errexit # (-e) exit immediately if any command has a non-zero exit status set -o errexit # (-e) exit immediately if any command has a non-zero exit status
set -o nounset # (-u) don't accept undefined variables set -o nounset # (-u) don't accept undefined variables
@ -12,33 +14,34 @@ cd "$(dirname "$0")" || exit
# If Agunua is installed, use it. # If Agunua is installed, use it.
if command -v agunua >/dev/null; then if command -v agunua >/dev/null; then
# Using Agunua is more secure, because it does certificate pinning. # Using Agunua is more secure, because it does certificate pinning.
hosts=$(agunua --insecure --binary gemini://geminispace.info/known-hosts \ # The --insecure option just makes it accept certificates that are
# not signed by a (known) CA.
hosts1=$(agunua --insecure --binary gemini://geminispace.info/known-hosts 2>/dev/null \
| grep "gemini://" | cut -d ' ' -f 3) | grep "gemini://" | cut -d ' ' -f 3)
hosts2=$(agunua --insecure --binary gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt 2>/dev/null)
else else
# If Agunua is not installed, pipe the request into OpenSSL s_client. # If Agunua is not installed, pipe the request into OpenSSL s_client.
hosts=$(printf "gemini://geminispace.info/known-hosts\r\n" \ hosts1=$(printf "gemini://geminispace.info/known-hosts\r\n" \
| timeout 5 openssl s_client -quiet -connect "geminispace.info:1965" 2>/dev/null \ | timeout 5 openssl s_client -quiet -connect "geminispace.info:1965" 2>/dev/null \
| grep "gemini://" | cut -d ' ' -f 3) | grep "gemini://" | cut -d ' ' -f 3)
hosts2=$(printf "gemini://gemini.bortzmeyer.org/software/lupa/lupa-capsules.txt\r\n" \
| timeout 5 openssl s_client -quiet -connect "gemini.bortzmeyer.org:1965" 2>/dev/null \
| tail -n +2)
fi fi
# Concatenate the two files.
hosts="$hosts1
$hosts2"
# Convert punycode to unicode; sort entries; remove duplicates; remove empty lines.
hosts=$(echo "$hosts" | idn --allow-unassigned --idna-to-unicode | sort -fu | awk NF)
if [ -z "$hosts" ]; then if [ -z "$hosts" ]; then
>&2 echo "hosts file download failed." >&2 echo "hosts file downloads failed."
exit 1 exit 1
fi fi
# Add a few hosts that are missing from geminispace.info
hosts="$hosts
campaignwiki.org
feeds.drewdevault.com
gem.adele.work
makeworld.gq
qwertqwefsday.eu:80
simplynews.metalune.xyz"
# Remove a host which changes its cert every few minutes.
hosts=$(echo "$hosts" | grep -v 'tofu-tester.random-projects.net:1966')
# Save to file. # Save to file.
echo "$hosts" | sort | uniq > hosts echo "$hosts" > hosts
echo OK echo OK