Experimental VF-1-derived client for the "Gemini" protocol
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1655 lines
62 KiB

#!/usr/bin/env python3
# AV-98 Gemini client
# Dervied from VF-1 (https://github.com/solderpunk/VF-1),
# (C) 2019, 2020 Solderpunk <solderpunk@sdf.org>
# With contributions from:
# - danceka <hannu.hartikainen@gmail.com>
# - <jprjr@tilde.club>
# - <vee@vnsf.xyz>
# - Klaus Alexander Seistrup <klaus@seistrup.dk>
# - govynnus <govynnus@sdf.org>
import argparse
import cmd
import cgi
import codecs
import collections
import datetime
import fnmatch
import getpass
import glob
import hashlib
import io
import mimetypes
import os
import os.path
import random
import shlex
import shutil
import socket
import sqlite3
import ssl
from ssl import CertificateError
import subprocess
import sys
import tempfile
import time
import urllib.parse
import uuid
import webbrowser
import ansiwrap as textwrap
except ModuleNotFoundError:
import textwrap
from cryptography import x509
from cryptography.hazmat.backends import default_backend
_BACKEND = default_backend()
except ModuleNotFoundError:
_VERSION = "1.0.2dev"
# Command abbreviations
"a": "add",
"b": "back",
"bb": "blackbox",
"bm": "bookmarks",
"book": "bookmarks",
"f": "fold",
"fo": "forward",
"g": "go",
"h": "history",
"hist": "history",
"l": "less",
"n": "next",
"p": "previous",
"prev": "previous",
"q": "quit",
"r": "reload",
"s": "save",
"se": "search",
"/": "search",
"t": "tour",
"u": "up",
"application/pdf": "xpdf %s",
"audio/mpeg": "mpg123 %s",
"audio/ogg": "ogg123 %s",
"image/*": "feh %s",
"text/html": "lynx -dump -force_html %s",
"text/*": "cat %s",
# monkey-patch Gemini support in urllib.parse
# see https://github.com/python/cpython/blob/master/Lib/urllib/parse.py
def fix_ipv6_url(url):
if not url.count(":") > 2: # Best way to detect them?
return url
# If there's a pair of []s in there, it's probably fine as is.
if "[" in url and "]" in url:
return url
# Easiest case is a raw address, no schema, no path.
# Just wrap it in square brackets and whack a slash on the end
if "/" not in url:
return "[" + url + "]/"
# Now the trickier cases...
if "://" in url:
schema, schemaless = url.split("://")
schema, schemaless = None, url
if "/" in schemaless:
netloc, rest = schemaless.split("/",1)
schemaless = "[" + netloc + "]" + "/" + rest
if schema:
return schema + "://" + schemaless
return schemaless
standard_ports = {
"gemini": 1965,
"gopher": 70,
class GeminiItem():
def __init__(self, url, name=""):
if "://" not in url:
url = "gemini://" + url
self.url = fix_ipv6_url(url)
self.name = name
parsed = urllib.parse.urlparse(self.url)
self.scheme = parsed.scheme
self.host = parsed.hostname
self.port = parsed.port or standard_ports.get(self.scheme, 0)
self.path = parsed.path
def root(self):
return GeminiItem(self._derive_url("/"))
def up(self):
pathbits = list(os.path.split(self.path.rstrip('/')))
# Don't try to go higher than root
if len(pathbits) == 1:
return self
# Get rid of bottom component
new_path = os.path.join(*pathbits)
return GeminiItem(self._derive_url(new_path))
def query(self, query):
query = urllib.parse.quote(query)
return GeminiItem(self._derive_url(query=query))
def _derive_url(self, path="", query=""):
A thin wrapper around urlunparse which avoids inserting standard ports
into URLs just to keep things clean.
return urllib.parse.urlunparse((self.scheme,
self.host if self.port == standard_ports[self.scheme] else self.host + ":" + str(self.port),
path or self.path, "", query, ""))
def absolutise_url(self, relative_url):
Convert a relative URL to an absolute URL by using the URL of this
GeminiItem as a base.
return urllib.parse.urljoin(self.url, relative_url)
def to_map_line(self, name=None):
if name or self.name:
return "=> {} {}\n".format(self.url, name or self.name)
return "=> {}\n".format(self.url)
def from_map_line(cls, line, origin_gi):
assert line.startswith("=>")
assert line[2:].strip()
bits = line[2:].strip().split(maxsplit=1)
bits[0] = origin_gi.absolutise_url(bits[0])
return cls(*bits)
CRLF = '\r\n'
# Cheap and cheerful URL detector
def looks_like_url(word):
return "." in word and word.startswith("gemini://")
class UserAbortException(Exception):
# GeminiClient Decorators
def needs_gi(inner):
def outer(self, *args, **kwargs):
if not self.gi:
print("You need to 'go' somewhere, first")
return None
return inner(self, *args, **kwargs)
outer.__doc__ = inner.__doc__
return outer
def restricted(inner):
def outer(self, *args, **kwargs):
if self.restricted:
print("Sorry, this command is not available in restricted mode!")
return None
return inner(self, *args, **kwargs)
outer.__doc__ = inner.__doc__
return outer
class GeminiClient(cmd.Cmd):
def __init__(self, restricted=False):
# Set umask so that nothing we create can be read by anybody else.
# The certificate cache and TOFU database contain "browser history"
# type sensitivie information.
2 years ago
# Find config directory
## Look for something pre-existing
for confdir in ("~/.av98/", "~/.config/av98/"):
confdir = os.path.expanduser(confdir)
if os.path.exists(confdir):
self.config_dir = confdir
## Otherwise, make one in .config if it exists
if os.path.exists(os.path.expanduser("~/.config/")):
self.config_dir = os.path.expanduser("~/.config/av98/")
self.config_dir = os.path.expanduser("~/.av98/")
print("Creating config directory {}".format(self.config_dir))
2 years ago
self.no_cert_prompt = "\x1b[38;5;76m" + "AV-98" + "\x1b[38;5;255m" + "> " + "\x1b[0m"
self.cert_prompt = "\x1b[38;5;202m" + "AV-98" + "\x1b[38;5;255m" + "+cert> " + "\x1b[0m"
self.prompt = self.no_cert_prompt
self.gi = None
self.history = []
self.hist_index = 0
self.idx_filename = ""
self.index = []
self.index_index = -1
self.lookup = self.index
self.marks = {}
self.page_index = 0
self.permanent_redirects = {}
self.previous_redirectors = set()
self.restricted = restricted
self.tmp_filename = ""
self.visited_hosts = set()
self.waypoints = []
self.client_certs = {
"active": None
self.active_cert_domains = []
self.active_is_transient = False
self.transient_certs_created = []
self.options = {
"debug" : False,
"ipv6" : True,
"timeout" : 10,
2 years ago
"width" : 80,
"auto_follow_redirects" : True,
"gopher_proxy" : None,
"tls_mode" : "tofu",
"cache" : False
self.log = {
"start_time": time.time(),
"requests": 0,
"ipv4_requests": 0,
"ipv6_requests": 0,
"bytes_recvd": 0,
"ipv4_bytes_recvd": 0,
"ipv6_bytes_recvd": 0,
"dns_failures": 0,
"refused_connections": 0,
"reset_connections": 0,
"timeouts": 0,
"cache_hits": 0,
self.cache = {}
self.cache_timestamps = {}
def _connect_to_tofu_db(self):
db_path = os.path.join(self.config_dir, "tofu.db")
self.db_conn = sqlite3.connect(db_path)
self.db_cur = self.db_conn.cursor()
self.db_cur.execute("""CREATE TABLE IF NOT EXISTS cert_cache
(hostname text, address text, fingerprint text,
first_seen date, last_seen date, count integer)""")
def _go_to_gi(self, gi, update_hist=True, check_cache=True, handle=True):
"""This method might be considered "the heart of AV-98".
Everything involved in fetching a gemini resource happens here:
sending the request over the network, parsing the response if
its a menu, storing the response in a temporary file, choosing
and calling a handler program, and updating the history."""
# Don't try to speak to servers running other protocols
if gi.scheme in ("http", "https"):
elif gi.scheme == "gopher" and not self.options.get("gopher_proxy", None):
print("""AV-98 does not speak Gopher natively.
However, you can use `set gopher_proxy hostname:port` to tell it about a
Gopher-to-Gemini proxy (such as a running Agena instance), in which case
you'll be able to transparently follow links to Gopherspace!""")
elif gi.scheme not in ("gemini", "gopher"):
print("Sorry, no support for {} links.".format(gi.scheme))
# Obey permanent redirects
if gi.url in self.permanent_redirects:
new_gi = GeminiItem(self.permanent_redirects[gi.url], name=gi.name)
# Use cache, or hit the network if resource is not cached
if check_cache and self.options["cache"] and self._is_cached(gi.url):
mime, body, tmpfile = self._get_cached(gi.url)
gi, mime, body, tmpfile = self._fetch_over_network(gi)
except UserAbortException:
except Exception as err:
# Print an error message
if isinstance(err, socket.gaierror):
self.log["dns_failures"] += 1
print("ERROR: DNS error!")
elif isinstance(err, ConnectionRefusedError):
self.log["refused_connections"] += 1
print("ERROR: Connection refused!")
elif isinstance(err, ConnectionResetError):
self.log["reset_connections"] += 1
print("ERROR: Connection reset!")
elif isinstance(err, (TimeoutError, socket.timeout)):
self.log["timeouts"] += 1
print("""ERROR: Connection timed out!
Slow internet connection? Use 'set timeout' to be more patient.""")
print("ERROR: " + str(err))
# Pass file to handler, unless we were asked not to
if handle:
if mime == "text/gemini":
self._handle_gemtext(body, gi)
cmd_str = self._get_handler_cmd(mime)
subprocess.call(shlex.split(cmd_str % tmpfile))
except FileNotFoundError:
print("Handler program %s not found!" % shlex.split(cmd_str)[0])
print("You can use the ! command to specify another handler program or pipeline.")
# Update state
self.gi = gi
self.mime = mime
if update_hist:
def _fetch_over_network(self, gi):
# Be careful with client certificates!
# Are we crossing a domain boundary?
if self.active_cert_domains and gi.host not in self.active_cert_domains:
if self.active_is_transient:
print("Permanently delete currently active transient certificate?")
resp = input("Y/N? ")
if resp.strip().lower() in ("y", "yes"):
print("Destroying certificate.")
print("Staying here.")
raise UserAbortException()
print("PRIVACY ALERT: Deactivate client cert before connecting to a new domain?")
resp = input("Y/N? ")
if resp.strip().lower() in ("n", "no"):
print("Keeping certificate active for {}".format(gi.host))
print("Deactivating certificate.")
# Suggest reactivating previous certs
if not self.client_certs["active"] and gi.host in self.client_certs:
print("PRIVACY ALERT: Reactivate previously used client cert for {}?".format(gi.host))
resp = input("Y/N? ")
if resp.strip().lower() in ("y", "yes"):
2 years ago
print("Remaining unidentified.")
# Is this a local file?
if not gi.host:
address, f = None, open(gi.path, "rb")
address, f = self._send_request(gi)
# Spec dictates <META> should not exceed 1024 bytes,
# so maximum valid header length is 1027 bytes.
header = f.readline(1027)
header = header.decode("UTF-8")
if not header or header[-1] != '\n':
raise RuntimeError("Received invalid header from server!")
header = header.strip()
self._debug("Response header: %s." % header)
# Validate header
status, meta = header.split(maxsplit=1)
if len(meta) > 1024 or len(status) != 2 or not status.isnumeric():
raise RuntimeError("Received invalid header from server!")
# Update redirect loop/maze escaping state
if not status.startswith("3"):
self.previous_redirectors = set()
# Handle non-SUCCESS headers, which don't have a response body
# Inputs
if status.startswith("1"):
if status == "11":
user_input = getpass.getpass("> ")
user_input = input("> ")
return self._fetch_over_network(gi.query(user_input))
# Redirects
elif status.startswith("3"):
new_gi = GeminiItem(gi.absolutise_url(meta))
if new_gi.url == gi.url:
raise RuntimeError("URL redirects to itself!")
elif new_gi.url in self.previous_redirectors:
raise RuntimeError("Caught in redirect loop!")
elif len(self.previous_redirectors) == _MAX_REDIRECTS:
raise RuntimeError("Refusing to follow more than %d consecutive redirects!" % _MAX_REDIRECTS)
# Never follow cross-domain redirects without asking
elif new_gi.host != gi.host:
follow = input("Follow cross-domain redirect to %s? (y/n) " % new_gi.url)
# Never follow cross-protocol redirects without asking
elif new_gi.scheme != gi.scheme:
follow = input("Follow cross-protocol redirect to %s? (y/n) " % new_gi.url)
# Don't follow *any* redirect without asking if auto-follow is off
elif not self.options["auto_follow_redirects"]:
follow = input("Follow redirect to %s? (y/n) " % new_gi.url)
# Otherwise, follow away
follow = "yes"
if follow.strip().lower() not in ("y", "yes"):
raise UserAbortException()
self._debug("Following redirect to %s." % new_gi.url)
self._debug("This is consecutive redirect number %d." % len(self.previous_redirectors))
if status == "31":
# Permanent redirect
self.permanent_redirects[gi.url] = new_gi.url
return self._fetch_over_network(new_gi)
# Errors
elif status.startswith("4") or status.startswith("5"):
raise RuntimeError(meta)
# Client cert
elif status.startswith("6"):
return self._fetch_over_network(gi)
# Invalid status
elif not status.startswith("2"):
raise RuntimeError("Server returned undefined status code %s!" % status)
# If we're here, this must be a success and there's a response body
assert status.startswith("2")
mime = meta
if mime == "":
mime = "text/gemini; charset=utf-8"
mime, mime_options = cgi.parse_header(mime)
if "charset" in mime_options:
except LookupError:
raise RuntimeError("Header declared unknown encoding %s" % value)
# Read the response body over the network
body = f.read()
# Save the result in a temporary file
## Set file mode
if mime.startswith("text/"):
mode = "w"
encoding = mime_options.get("charset", "UTF-8")
body = body.decode(encoding)
except UnicodeError:
raise RuntimeError("Could not decode response body using %s encoding declared in header!" % encoding)
mode = "wb"
encoding = None
## Write
tmpf = tempfile.NamedTemporaryFile(mode, encoding=encoding, delete=False)
size = tmpf.write(body)
self.tmp_filename = tmpf.name
self._debug("Wrote %d byte response to %s." % (size, self.tmp_filename))
# Maintain cache and log
if self.options["cache"]:
self._add_to_cache(gi.url, mime, self.tmp_filename)
self._log_visit(gi, address, size)
return gi, mime, body, self.tmp_filename
def _send_request(self, gi):
"""Send a selector to a given host and port.
Returns the resolved address and binary file with the reply."""
if gi.scheme == "gemini":
# For Gemini requests, connect to the host and port specified in the URL
host, port = gi.host, gi.port
elif gi.scheme == "gopher":
# For Gopher requests, use the configured proxy
host, port = self.options["gopher_proxy"].rsplit(":", 1)
self._debug("Using gopher proxy: " + self.options["gopher_proxy"])
# Do DNS resolution
addresses = self._get_addresses(host, port)
# Prepare TLS context
protocol = ssl.PROTOCOL_TLS if sys.version_info.minor >=6 else ssl.PROTOCOL_TLSv1_2
context = ssl.SSLContext(protocol)
# Use CAs or TOFU
if self.options["tls_mode"] == "ca":
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = True
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
# Impose minimum TLS version
## In 3.7 and above, this is easy...
if sys.version_info.minor >= 7:
context.minimum_version = ssl.TLSVersion.TLSv1_2
## Otherwise, it seems very hard...
## The below is less strict than it ought to be, but trying to disable
## TLS v1.1 here using ssl.OP_NO_TLSv1_1 produces unexpected failures
## with recent versions of OpenSSL. What a mess...
context.options |= ssl.OP_NO_SSLv3
context.options |= ssl.OP_NO_SSLv2
# Try to enforce sensible ciphers
except ssl.SSLError:
# Rely on the server to only support sensible things, I guess...
# Load client certificate if needed
if self.client_certs["active"]:
certfile, keyfile = self.client_certs["active"]
context.load_cert_chain(certfile, keyfile)
# Connect to remote host by any address possible
err = None
for address in addresses:
self._debug("Connecting to: " + str(address[4]))
s = socket.socket(address[0], address[1])
s = context.wrap_socket(s, server_hostname = gi.host)
except OSError as e:
err = e
# If we couldn't connect to *any* of the addresses, just
# bubble up the exception from the last attempt and deny
# knowledge of earlier failures.
raise err
2 years ago
if sys.version_info.minor >=5:
self._debug("Established {} connection.".format(s.version()))
2 years ago
self._debug("Cipher is: {}.".format(s.cipher()))
if self.options["tls_mode"] != "ca":
cert = s.getpeercert(binary_form=True)
self._validate_cert(address[4][0], host, cert)
# Remember that we showed the current cert to this domain...
if self.client_certs["active"]:
self.client_certs[gi.host] = self.client_certs["active"]
# Send request and wrap response in a file descriptor
self._debug("Sending %s<CRLF>" % gi.url)
s.sendall((gi.url + CRLF).encode("UTF-8"))
return address, s.makefile(mode = "rb")
def _get_addresses(self, host, port):
# DNS lookup - will get IPv4 and IPv6 records if IPv6 is enabled
if ":" in host:
# This is likely a literal IPv6 address, so we can *only* ask for
# IPv6 addresses or getaddrinfo will complain
family_mask = socket.AF_INET6
elif socket.has_ipv6 and self.options["ipv6"]:
# Accept either IPv4 or IPv6 addresses
family_mask = 0
# IPv4 only
family_mask = socket.AF_INET
addresses = socket.getaddrinfo(host, port, family=family_mask,
# Sort addresses so IPv6 ones come first
addresses.sort(key=lambda add: add[0] == socket.AF_INET6, reverse=True)
return addresses
def _is_cached(self, url):
if url not in self.cache:
return False
now = time.time()
cached = self.cache_timestamps[url]
if now - cached > _MAX_CACHE_AGE_SECS:
self._debug("Expiring old cached copy of resource.")
return False
self._debug("Found cached copy of resource.")
return True
def _remove_from_cache(self, url):
mime, filename = self.cache.pop(url)
def _add_to_cache(self, url, mime, filename):
self.cache_timestamps[url] = time.time()
self.cache[url] = (mime, filename)
if len(self.cache) > _MAX_CACHE_SIZE:
def _trim_cache(self):
# Order cache entries by age
lru = [(t, u) for (u, t) in self.cache_timestamps.items()]
# Drop the oldest entry no matter what
_, url = lru[0]
self._debug("Dropping cached copy of {} from full cache.".format(url))
# Drop other entries if they are older than the limit
now = time.time()
for cached, url in lru[1:]:
if now - cached > _MAX_CACHE_AGE_SECS:
self._debug("Dropping cached copy of {} from full cache.".format(url))
def _get_cached(self, url):
mime, filename = self.cache[url]
self.log["cache_hits"] += 1
if mime.startswith("text/gemini"):
with open(filename, "r") as fp:
body = fp.read()
return mime, body, filename
return mime, None, filename
def _empty_cache(self):
for mime, filename in self.cache.values():
if os.path.exists(filename):
def _validate_cache(self):
assert self.cache.keys() == self.cache_timestamps.keys()
for _, filename in self.cache.values():
assert os.path.isfile(filename)
def _handle_cert_request(self, meta):
# Don't do client cert stuff in restricted mode, as in principle
# it could be used to fill up the disk by creating a whole lot of
# certificates
if self.restricted:
print("The server is requesting a client certificate.")
print("These are not supported in restricted mode, sorry.")
raise UserAbortException()
print("SERVER SAYS: ", meta)
# Present different messages for different 6x statuses, but
# handle them the same.
if status in ("64", "65"):
print("The server rejected your certificate because it is either expired or not yet valid.")
elif status == "63":
print("The server did not accept your certificate.")
print("You may need to e.g. coordinate with the admin to get your certificate fingerprint whitelisted.")
print("The site {} is requesting a client certificate.".format(gi.host))
print("This will allow the site to recognise you across requests.")
# Give the user choices
print("What do you want to do?")
print("1. Give up.")
print("2. Generate a new transient certificate.")
print("3. Generate a new persistent certificate.")
print("4. Load a previously generated persistent.")
print("5. Load certificate from an external file.")
choice = input("> ").strip()
if choice == "2":
elif choice == "3":
elif choice == "4":
elif choice == "5":
print("Giving up.")
raise UserAbortException()
def _validate_cert(self, address, host, cert):
Validate a TLS certificate in TOFU mode.
If the cryptography module is installed:
- Check the certificate Common Name or SAN matches `host`
- Check the certificate's not valid before date is in the past
- Check the certificate's not valid after date is in the future
Whether the cryptography module is installed or not, check the
certificate's fingerprint against the TOFU database to see if we've
previously encountered a different certificate for this IP address and
now = datetime.datetime.utcnow()
# Using the cryptography module we can get detailed access
# to the properties of even self-signed certs, unlike in
# the standard ssl library...
c = x509.load_der_x509_certificate(cert, _BACKEND)
# Check certificate validity dates
if c.not_valid_before >= now:
raise CertificateError("Certificate not valid until: {}!".format(c.not_valid_before))
elif c.not_valid_after <= now:
raise CertificateError("Certificate expired as of: {})!".format(c.not_valid_after))
# Check certificate hostnames
names = []
common_name = c.subject.get_attributes_for_oid(x509.oid.NameOID.COMMON_NAME)
if common_name:
names.extend([alt.value for alt in c.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME).value])
except x509.ExtensionNotFound:
names = set(names)
for name in names:
ssl._dnsname_match(name, host)
except CertificateError: