From 39ce27a9d84cb386efd9e9ea1168de8cdeb997f9 Mon Sep 17 00:00:00 2001
From: nervuri <seva@nervuri.net>
Date: Wed, 30 Jun 2021 17:57:14 +0000
Subject: [PATCH] add exclusion list for specific hosts, private IPs and
 reserved TLDs

---
 excluded-hosts | 25 +++++++++++++++++++++++++
 get-hosts.sh   |  6 ++++++
 2 files changed, 31 insertions(+)
 create mode 100644 excluded-hosts

diff --git a/excluded-hosts b/excluded-hosts
new file mode 100644
index 0000000..3cd87c7
--- /dev/null
+++ b/excluded-hosts
@@ -0,0 +1,25 @@
+# Specific hosts
+tofu-tester\.random-projects\.net:1966
+localhost(:[0-9]+)?
+# Private IP addresses
+# https://en.wikipedia.org/wiki/Private_network
+# TODO: 100.64.0.0/10, fc00::/8 and fd00::/8
+127\.0\.0\.1(:[0-9]+)?
+10\.[0-9]+\.[0-9]+\.[0-9]+(:[0-9]+)?
+192\.168\.[0-9]+\.[0-9]+(:[0-9]+)?
+172\.((1[6-9])|(2[0-9])|(3[0-1]))\.[0-9]+\.[0-9]+(:[0-9]+)?
+169\.254\.[0-9]+\.[0-9]+(:[0-9]+)?
+::1
+\[::1\](:[0-9]+)?
+# Reserved TLDs
+# https://tools.ietf.org/id/draft-chapin-additional-reserved-tlds-01.html
+.*\.test(:[0-9]+)?
+.*\.example(:[0-9]+)?
+.*\.invalid(:[0-9]+)?
+.*\.localhost(:[0-9]+)?
+.*\.localdomain(:[0-9]+)?
+.*\.domain(:[0-9]+)?
+.*\.lan(:[0-9]+)?
+.*\.home(:[0-9]+)?
+.*\.corp(:[0-9]+)?
+.*\.mail(:[0-9]+)?
diff --git a/get-hosts.sh b/get-hosts.sh
index f2d0534..f7d497a 100755
--- a/get-hosts.sh
+++ b/get-hosts.sh
@@ -36,6 +36,12 @@ $hosts2"
 # Convert punycode to unicode; sort entries; remove duplicates; remove empty lines.
 hosts=$(echo "$hosts" | idn --allow-unassigned --idna-to-unicode | sort -fu | awk NF)
 
+# Remove hosts which contain neither "." nor ":", such as "localhost".
+hosts=$(echo "$hosts" | grep '\.\|:')
+
+# Remove explicitly excluded hosts.
+hosts=$(echo "$hosts" | grep -vxEf excluded-hosts)
+
 if [ -z "$hosts" ]; then
   >&2 echo "hosts file downloads failed."
   exit 1