add exclusion list for specific hosts, private IPs and reserved TLDs

This commit is contained in:
nervuri 2021-06-30 17:57:14 +00:00
parent 6a6e8ae643
commit 39ce27a9d8
Signed by: nervuri
GPG Key ID: C4769EEA7BA61672
2 changed files with 31 additions and 0 deletions

25
excluded-hosts Normal file
View File

@ -0,0 +1,25 @@
# Specific hosts
tofu-tester\.random-projects\.net:1966
localhost(:[0-9]+)?
# Private IP addresses
# https://en.wikipedia.org/wiki/Private_network
# TODO: 100.64.0.0/10, fc00::/8 and fd00::/8
127\.0\.0\.1(:[0-9]+)?
10\.[0-9]+\.[0-9]+\.[0-9]+(:[0-9]+)?
192\.168\.[0-9]+\.[0-9]+(:[0-9]+)?
172\.((1[6-9])|(2[0-9])|(3[0-1]))\.[0-9]+\.[0-9]+(:[0-9]+)?
169\.254\.[0-9]+\.[0-9]+(:[0-9]+)?
::1
\[::1\](:[0-9]+)?
# Reserved TLDs
# https://tools.ietf.org/id/draft-chapin-additional-reserved-tlds-01.html
.*\.test(:[0-9]+)?
.*\.example(:[0-9]+)?
.*\.invalid(:[0-9]+)?
.*\.localhost(:[0-9]+)?
.*\.localdomain(:[0-9]+)?
.*\.domain(:[0-9]+)?
.*\.lan(:[0-9]+)?
.*\.home(:[0-9]+)?
.*\.corp(:[0-9]+)?
.*\.mail(:[0-9]+)?

View File

@ -36,6 +36,12 @@ $hosts2"
# Convert punycode to unicode; sort entries; remove duplicates; remove empty lines.
hosts=$(echo "$hosts" | idn --allow-unassigned --idna-to-unicode | sort -fu | awk NF)
# Remove hosts which contain neither "." nor ":", such as "localhost".
hosts=$(echo "$hosts" | grep '\.\|:')
# Remove explicitly excluded hosts.
hosts=$(echo "$hosts" | grep -vxEf excluded-hosts)
if [ -z "$hosts" ]; then
>&2 echo "hosts file downloads failed."
exit 1