#!/usr/bin/env python3 # Offpunk Offline Gemini client # Derived from AV-98 by Solderpunk, # (C) 2021, 2022 Ploum # (C) 2019, 2020 Solderpunk # With contributions from: # - danceka # - # - # - Klaus Alexander Seistrup # - govynnus # - Björn Wärmedal # - import argparse import cmd import cgi import codecs import collections import datetime import fnmatch import getpass import glob import hashlib import io import mimetypes import os import os.path import filecmp import random import shlex import shutil import socket import sqlite3 import ssl from ssl import CertificateError import subprocess import sys import tempfile import time import urllib.parse import uuid import webbrowser try: import ansiwrap as textwrap except ModuleNotFoundError: print("Try installing python-ansiwrap for better rendering") import textwrap try: from cryptography import x509 from cryptography.hazmat.backends import default_backend _HAS_CRYPTOGRAPHY = True _BACKEND = default_backend() except ModuleNotFoundError: _HAS_CRYPTOGRAPHY = False try: import magic _HAS_MAGIC = True except ModuleNotFoundError: _HAS_MAGIC = False try: import requests _DO_HTTP = True except ModuleNotFoundError: _DO_HTTP = False try: from readability import Document from bs4 import BeautifulSoup _DO_HTML = True except ModuleNotFoundError: _DO_HTML = False _VERSION = "0.1~dev" ## Config directories try: import xdg _CACHE_PATH = os.path.join(xdg.xdg_cache_home().resolve(), "offpunk/") _CONFIG_DIR = os.path.join(xdg.xdg_config_home().resolve(), "offpunk/") _DATA_DIR = os.path.join(xdg.xdg_data_home().resolve(), "offpunk/") except ModuleNotFoundError: _CACHE_PATH = os.path.expanduser("~/.cache/offpunk/") _CONFIG_DIR = None ## Look for pre-existing config directory, if any for confdir in ("~/.offpunk/", "~/.config/offpunk/"): confdir = os.path.expanduser(confdir) if os.path.exists(confdir): _CONFIG_DIR = confdir break ## Otherwise, make one in .config if it exists if not _CONFIG_DIR and os.path.exists("~/.config/"): _CONFIG_DIR = os.path.expanduser("~/.config/offpunk/") elif not _CONFIG_DIR: _CONFIG_DIR = os.path.expanduser("~/.offpunk/") _DATA_DIR = _CONFIG_DIR for f in [_CONFIG_DIR, _CACHE_PATH, _DATA_DIR]: if not os.path.exists(f): print("Creating config directory {}".format(f)) os.makedirs(f) _MAX_REDIRECTS = 5 _MAX_CACHE_SIZE = 10 _MAX_CACHE_AGE_SECS = 180 #_DEFAULT_LESS = "less -EXFRfM -PMurl\ lines\ \%lt-\%lb/\%L\ \%Pb\%$ %s" _DEFAULT_LESS = "less -EXFRfM %s" # Command abbreviations _ABBREVS = { "a": "add", "b": "back", "bb": "blackbox", "bm": "bookmarks", "book": "bookmarks", "cp": "copy", "f": "fold", "fo": "forward", "g": "go", "h": "history", "hist": "history", "l": "less", "n": "next", "off": "offline", "on": "online", "p": "previous", "prev": "previous", "q": "quit", "r": "reload", "s": "save", "se": "search", "/": "search", "t": "tour", "u": "up", } _MIME_HANDLERS = { "application/pdf": "zathura %s", "audio/mpeg": "mpg123 %s", "audio/ogg": "ogg123 %s", "image/*": "feh -. %s", #"text/html": "lynx -dump -force_html %s", } # monkey-patch Gemini support in urllib.parse # see https://github.com/python/cpython/blob/master/Lib/urllib/parse.py urllib.parse.uses_relative.append("gemini") urllib.parse.uses_netloc.append("gemini") def fix_ipv6_url(url): if not url.count(":") > 2: # Best way to detect them? return url # If there's a pair of []s in there, it's probably fine as is. if "[" in url and "]" in url: return url # Easiest case is a raw address, no schema, no path. # Just wrap it in square brackets and whack a slash on the end if "/" not in url: return "[" + url + "]/" # Now the trickier cases... if "://" in url: schema, schemaless = url.split("://",maxsplit=1) else: schema, schemaless = None, url if "/" in schemaless: netloc, rest = schemaless.split("/",1) schemaless = "[" + netloc + "]" + "/" + rest if schema: return schema + "://" + schemaless return schemaless standard_ports = { "gemini": 1965, "gopher": 70, "http" : 80, "https" : 443, } # First, we define the gemtext and html renderers, outside of the rest # (They could later be factorized in other files or replaced) # Gemtext Rendering Engine # this method takes the original gemtext and returns # [rendered_text,links_table] def render_gemtext(gemtext, width=80): links = [] preformatted = False rendered_text = "" #This local method takes a line and apply the ansi code given as "color" #The whole line is then wrapped and ansi code are ended. def wrap_line(line,color=None,i_indent="",s_indent=""): wrapped = textwrap.wrap(line,width,initial_indent=i_indent,\ subsequent_indent=s_indent) final = "" for l in wrapped: if color: l = color + l + "\x1b[0m" if l.strip() != "": final += l + "\n" return final def format_link(url,index,name=None): if "://" in url: protocol,adress = url.split("://",maxsplit=1) protocol = " %s" %protocol else: adress = url protocol = "" if "gemini" in protocol: protocol = "" if not name: name = adress line = "[%d%s] %s" % (index, protocol, name) return line for line in gemtext.splitlines(): if line.startswith("```"): preformatted = not preformatted elif preformatted: rendered_text += line + "\n" elif line.startswith("=>"): strippedline = line[2:].strip() if strippedline: links.append(strippedline) splitted = strippedline.split(maxsplit=1) url = splitted[0] name = None if len(splitted) > 1: name = splitted[1] link = format_link(url,len(links),name=name) startpos = link.find("] ") + 2 wrapped = wrap_line(link,s_indent=startpos*" ") rendered_text += wrapped elif line.startswith("* "): line = line[1:].lstrip("\t ") rendered_text += textwrap.fill(line, width, initial_indent = "• ", subsequent_indent=" ") + "\n" elif line.startswith(">"): line = line[1:].lstrip("\t ") rendered_text += textwrap.fill(line,width, initial_indent = "> ", subsequent_indent="> ") + "\n" elif line.startswith("###"): line = line[3:].lstrip("\t ") rendered_text += wrap_line(line, color="\x1b[34m\x1b[2m") elif line.startswith("##"): line = line[2:].lstrip("\t ") rendered_text += wrap_line(line, color="\x1b[34m") elif line.startswith("#"): line = line[1:].lstrip("\t ") rendered_text += wrap_line(line,color="\x1b[1;34m\x1b[4m") else: rendered_text += wrap_line(line).rstrip() + "\n" return rendered_text, links # Our own HTML engine (crazy, isn’t it?) # Return [rendered_body, list_of_links] def render_html(body,width=80): if not _DO_HTML: print("HTML document detected. Please install python-bs4 and python-readability.") return # This method recursively parse the HTML r_body = "" links = [] def recursive_render(element,indent=""): rendered_body = "" #print("rendering %s - %s with indent %s" %(element.name,element.string,indent)) if element.name == "blockquote": for child in element.children: rendered_body += recursive_render(child,indent="\t").rstrip("\t") elif element.name == "div": rendered_body += "\n" for child in element.children: rendered_body += recursive_render(child,indent=indent) elif element.name in ["h1","h2","h3","h4","h5","h6"]: line = element.get_text() if element.name in ["h1","h2"]: rendered_body += "\n"+"\x1b[1;34m\x1b[4m" + line + "\x1b[0m"+"\n" elif element.name in ["h3","h4"]: rendered_body += "\n" + "\x1b[34m" + line + "\x1b[0m" + "\n" else: rendered_body += "\n" + "\x1b[34m\x1b[2m" + line + "\x1b[0m" + "\n" elif element.name == "pre": rendered_body += "\n" for child in element.children: rendered_body += recursive_render(child,indent=indent) rendered_body += "\n\n" elif element.name == "li": line = "" for child in element.children: line += recursive_render(child,indent=indent).strip("\n") #print("in li: ***%s***"%line) rendered_body += " * " + line.strip() + "\n" elif element.name in ["code","em","b","i"]: # we don’t do anything with those markup right now. Maybe later? for child in element.children: rendered_body += recursive_render(child,indent=indent).strip("\n") elif element.name == "p": temp_str = "" for child in element.children: temp_str += recursive_render(child,indent=indent) rendered_body = temp_str + "\n\n" elif element.name == "a": text = element.get_text().strip() link = element.get('href') if link: links.append(link+" "+text) link_id = " [%s] "%(len(links)) rendered_body = "\x1b[34m\x1b[2m " + text + link_id + "\x1b[0m" else: #No real link found rendered_body = text elif element.name == "br": rendered_body = "\n" elif element.string: #print("tag without children:",element.name) #print("string : **%s** "%element.string.strip()) #print("########") rendered_body = element.string.strip("\n").strip("\t") else: #print("tag children:",element.name) for child in element.children: rendered_body += recursive_render(child,indent=indent) #print("body for element %s: %s"%(element.name,rendered_body)) return indent + rendered_body # the real render_html hearth readable = Document(body) title = readable.short_title() summary = readable.summary() r_body += "\x1b[1;34m\x1b[4m" + title + "\x1b[0m""\n" soup = BeautifulSoup(summary, 'html.parser') rendered_body = "" if soup and soup.body : for el in soup.body.contents: rendered_body += recursive_render(el) paragraphs = rendered_body.split("\n\n") for par in paragraphs: lines = par.splitlines() for line in lines: if line.startswith("\t"): i_indent = " " s_indent = i_indent line = line.strip("\t") elif line.startswith(" * "): i_indent = "" # we keep the initial bullet) s_indent = " " else: i_indent = "" s_indent = i_indent if line.strip() != "": wrapped = textwrap.fill(line,width,initial_indent=i_indent, subsequent_indent=s_indent) wrapped += "\n" else: wrapped = "" r_body += wrapped r_body += "\n" return r_body,links # Mapping mimetypes with renderers # (any content with a mimetype text/* not listed here will be rendered with render_gemtext) _FORMAT_RENDERERS = { "text/gemini": render_gemtext, "text/html" : render_html, "text/xml" : render_html } # Offpunk is organized as follow: # - a GeminiClient instance which handles the browsing of GeminiItems (= pages). # - There’s only one GeminiClient. Each page is a GeminiItem (name is historical, as # it could be non-gemini content) # - A GeminiItem is created with an URL from which it will derives content. # - Content include : a title, a body, an ansi-rendered-body and a list of links. # - Each GeminiItem generates a "cache_path" in which it maintains a cached version of its content. class GeminiItem(): def __init__(self, url, name=""): if "://" not in url and ("./" not in url and url[0] != "/"): url = "gemini://" + url self.url = fix_ipv6_url(url).strip() self.name = name self.mime = None self.renderer = None self.links = None parsed = urllib.parse.urlparse(self.url) if "./" in url or url[0] == "/": self.scheme = "localhost" else: self.scheme = parsed.scheme if self.scheme == "localhost": self.local = True self.host = None h = self.url.split('/') self.host = h[0:len(h)-1] self._cache_path = None # localhost:/ is 11 char if self.url.startswith("localhost://"): self.path = self.url[11:] else: self.path = self.url else: self.path = parsed.path if parsed.query: self.path += "/" + parsed.query self.local = False self.host = parsed.hostname #if not local, we create a local cache path. self._cache_path = os.path.expanduser(_CACHE_PATH + self.scheme +\ "/" + self.host + self.path) # FIXME : this is a gross hack to give a name to # index files. This will break if the index is not # index.gmi. I don’t know how to know the real name # of the file. But first, we need to ensure that the domain name # finish by "/". Else, the cache will create a file, not a folder. if self.scheme.startswith("http"): index = "index.html" else: index = "index.gmi" if self.path == "" or os.path.isdir(self._cache_path): if not self._cache_path.endswith("/"): self._cache_path += "/" if not self.url.endswith("/"): self.url += "/" if self._cache_path.endswith("/"): self._cache_path += index self.port = parsed.port or standard_ports.get(self.scheme, 0) def get_title(self): #small intelligence to try to find a good name for a capsule #we try to find eithe ~username or /users/username #else we fallback to hostname if self.scheme == "localhost": if self.name != "": self.title = self.name else: self.title = self.path else: self.title = self.host if "user" in self.path: i = 0 splitted = self.path.split("/") while i < (len(splitted)-1): if splitted[i].startswith("user"): self.title = splitted[i+1] i += 1 if "~" in self.path: for pp in self.path.split("/"): if pp.startswith("~"): self.title = pp[1:] return self.title def is_cache_valid(self,validity=0): # Validity is the acceptable time for # a cache to be valid (in seconds) # If 0, then any cache is considered as valid # (use validity = 1 if you want to refresh everything) if self.local: return True elif self._cache_path : if os.path.exists(self._cache_path): if validity > 0 : last_modification = self.cache_last_modified() now = time.time() age = now - last_modification return age < validity else: return True else: #Cache has not been build return False else: #There’s not even a cache! return False def cache_last_modified(self): if self._cache_path: return os.path.getmtime(self._cache_path) elif self.local: return 0 else: print("ERROR : NO CACHE in cache_last_modified") return None def get_body(self,as_file=False): if self.local: path = self.path elif self.is_cache_valid(): path = self._cache_path else: path = None if path: if as_file: return path else: with open(path) as f: body = f.read() f.close() return body else: print("ERROR: NO CACHE for %s" %self._cache_path) return error # This method is used to load once the list of links in a gi # Links can be followed, after a space, by a description/title def __make_links(self,links): self.links = [] for l in links: #split between link and potential name splitted = l.split(maxsplit=1) url = self.absolutise_url(splitted[0]) if looks_like_url(url): if len(splitted) > 1: newgi = GeminiItem(url,splitted[1]) else: newgi = GeminiItem(url) self.links.append(newgi) def get_links(self): if not self.links: r_body = self.get_rendered_body() return self.links def get_link(self,nb): if not self.links: r_body = self.get_rendered_body() if len(self.links) < nb: print("Index too high! No link %s for %s" %(nb,self.url)) return None else: return self.links[nb-1] # Red title above rendered content def _make_terminal_title(self): title = self.get_title() #FIXME : how do I know that I’m offline_only ? if self.is_cache_valid(): #and self.offline_only and not self.local: last_modification = self.cache_last_modified() str_last = time.ctime(last_modification) title += " \x1b[0;31m(last accessed on %s)"%str_last rendered_title = "\x1b[31m\x1b[1m"+ title + "\x1b[0m" #FIXME: width to replace self.options["width"] wrapped = textwrap.fill(rendered_title,80) return wrapped + "\n" def get_rendered_body(self): if not self.renderer: mime = self.get_mime() if mime in _FORMAT_RENDERERS: self.renderer = _FORMAT_RENDERERS[mime] if self.renderer: body = self.get_body() r_body, links = self.renderer(body) self.__make_links(links) to_return = self._make_terminal_title() + r_body return to_return else: return None def get_filename(self): filename = os.path.basename(self._cache_path) return filename def write_body(self,body,mime): ## body is a copy of the raw gemtext ## Write_body() also create the cache ! # DEFAULT GEMINI MIME if mime == "": mime = "text/gemini; charset=utf-8" mime, mime_options = cgi.parse_header(mime) self.mime = mime if self.mime and self.mime.startswith("text/"): mode = "w" else: mode = "wb" cache_dir = os.path.dirname(self._cache_path) # If the subdirectory already exists as a file (not a folder) # We remove it (happens when accessing URL/subfolder before # URL/subfolder/file.gmi. # This causes loss of data in the cache # proper solution would be to save "sufolder" as "sufolder/index.gmi" if os.path.isfile(cache_dir): os.remove(cache_dir) os.makedirs(cache_dir,exist_ok=True) with open(self._cache_path, mode=mode) as f: f.write(body) f.close() def get_mime(self): if self.is_cache_valid(): if self.local: path = self.path else: path = self._cache_path mime,encoding = mimetypes.guess_type(path,strict=False) #gmi Mimetype is not recognized yet if not mime and path.endswith(".gmi"): mime = "text/gemini" elif not mime and _HAS_MAGIC : mime = magic.from_file(path,mime=True) elif not _HAS_MAGIC : print("Cannot guess the mime type of the file. Install Python-magic") if mime.startswith("text") and mime not in _FORMAT_RENDERERS: #by default, we consider it’s gemini except for html #print("replacing MIME %s by gemini" %mime) mime = "text/gemini" self.mime = mime return self.mime def set_error(self,err): # If we get an error, we want to keep an existing cache # but we need to touch it or to create an empty one # to avoid hitting the error at each refresh if self.is_cache_valid(): os.utime(self._cache_path) else: cache_dir = os.path.dirname(self._cache_path) if os.path.isdir(cache_dir): with open(self._cache_path, "w") as cache: cache.write(str(datetime.datetime.now())+"\n") cache.write("ERROR while caching %s\n" %self.url) cache.write(str(err)) cache.write("\n\nIf you believe this error was temporary, type ""reload"".\n") cache.write("The ressource will be tentatively fetched during next sync.\n") cache.write("\n") cache.close() def root(self): return GeminiItem(self._derive_url("/")) def up(self): pathbits = list(os.path.split(self.path.rstrip('/'))) # Don't try to go higher than root if len(pathbits) == 1: return self # Get rid of bottom component pathbits.pop() new_path = os.path.join(*pathbits) return GeminiItem(self._derive_url(new_path)) def query(self, query): query = urllib.parse.quote(query) return GeminiItem(self._derive_url(query=query)) def _derive_url(self, path="", query=""): """ A thin wrapper around urlunparse which avoids inserting standard ports into URLs just to keep things clean. """ return urllib.parse.urlunparse((self.scheme, self.host if self.port == standard_ports[self.scheme] else self.host + ":" + str(self.port), path or self.path, "", query, "")) def absolutise_url(self, relative_url): """ Convert a relative URL to an absolute URL by using the URL of this GeminiItem as a base. """ abs_url = urllib.parse.urljoin(self.url, relative_url) return abs_url def to_map_line(self, name=None): #SPECIFIC GEMINI if name or self.name: return "=> {} {}\n".format(self.url, name or self.name) else: return "=> {}\n".format(self.url) CRLF = '\r\n' # Cheap and cheerful URL detector def looks_like_url(word): try: url = fix_ipv6_url(word).strip() parsed = urllib.parse.urlparse(url) #sometimes, urllib crashed only when requesting the port port = parsed.port start = word.startswith("gemini://") or word.startswith("http://")\ or word.startswith("https://") if not start: return looks_like_url("gemini://"+word) else: return "." in word except ValueError: return False class UserAbortException(Exception): pass # GeminiClient Decorators def needs_gi(inner): def outer(self, *args, **kwargs): if not self.gi: print("You need to 'go' somewhere, first") return None else: return inner(self, *args, **kwargs) outer.__doc__ = inner.__doc__ return outer def restricted(inner): def outer(self, *args, **kwargs): if self.restricted: print("Sorry, this command is not available in restricted mode!") return None else: return inner(self, *args, **kwargs) outer.__doc__ = inner.__doc__ return outer class GeminiClient(cmd.Cmd): def __init__(self, restricted=False, synconly=False): cmd.Cmd.__init__(self) # Set umask so that nothing we create can be read by anybody else. # The certificate cache and TOFU database contain "browser history" # type sensitivie information. os.umask(0o077) self.no_cert_prompt = "\x1b[38;5;76m" + "ON" + "\x1b[38;5;255m" + "> " + "\x1b[0m" self.cert_prompt = "\x1b[38;5;202m" + "ON" + "\x1b[38;5;255m" self.offline_prompt = "\x1b[38;5;76m" + "OFF" + "\x1b[38;5;255m" + "> " + "\x1b[0m" self.prompt = self.no_cert_prompt self.gi = None self.history = [] self.hist_index = 0 self.idx_filename = "" self.index = [] self.index_index = -1 self.lookup = self.index self.marks = {} self.page_index = 0 self.permanent_redirects = {} self.previous_redirectors = set() # Sync-only mode is restriced by design self.restricted = restricted or synconly self.synconly = synconly self.tmp_filename = "" self.visited_hosts = set() self.offline_only = False self.sync_only = False self.tourfile = os.path.join(_CONFIG_DIR, "tour") self.syncfile = os.path.join(_CONFIG_DIR, "to_fetch") self.client_certs = { "active": None } self.active_cert_domains = [] self.active_is_transient = False self.transient_certs_created = [] self.options = { "debug" : False, "ipv6" : True, "timeout" : 600, "short_timeout" : 5, "width" : 80, "auto_follow_redirects" : True, "gopher_proxy" : None, "tls_mode" : "tofu", "http_proxy": None, } self.log = { "start_time": time.time(), "requests": 0, "ipv4_requests": 0, "ipv6_requests": 0, "bytes_recvd": 0, "ipv4_bytes_recvd": 0, "ipv6_bytes_recvd": 0, "dns_failures": 0, "refused_connections": 0, "reset_connections": 0, "timeouts": 0, "cache_hits": 0, } self._connect_to_tofu_db() def _connect_to_tofu_db(self): db_path = os.path.join(_CONFIG_DIR, "tofu.db") self.db_conn = sqlite3.connect(db_path) self.db_cur = self.db_conn.cursor() self.db_cur.execute("""CREATE TABLE IF NOT EXISTS cert_cache (hostname text, address text, fingerprint text, first_seen date, last_seen date, count integer)""") def _go_to_gi(self, gi, update_hist=True, check_cache=True, handle=True): """This method might be considered "the heart of Offpunk". Everything involved in fetching a gemini resource happens here: sending the request over the network, parsing the response, storing the response in a temporary file, choosing and calling a handler program, and updating the history. Nothing is returned.""" if not gi: return # Don't try to speak to servers running other protocols if gi.scheme == "gopher" and not self.options.get("gopher_proxy", None)\ and not self.sync_only: print("""Offpunk does not speak Gopher natively. However, you can use `set gopher_proxy hostname:port` to tell it about a Gopher-to-Gemini proxy (such as a running Agena instance), in which case you'll be able to transparently follow links to Gopherspace!""") return elif gi.scheme not in ("localhost","gemini", "gopher", "http", "https") and not self.sync_only: print("Sorry, no support for {} links.".format(gi.scheme)) return # Obey permanent redirects if gi.url in self.permanent_redirects: new_gi = GeminiItem(self.permanent_redirects[gi.url], name=gi.name) self._go_to_gi(new_gi) return # Use cache or mark as to_fetch if resource is not cached # Why is this code useful ? It set the mimetype ! if self.offline_only: if not gi.is_cache_valid(): print("%s not available, marked for syncing"%gi.url) with open(self.syncfile,mode='a') as sf: line = gi.url.strip() + '\n' sf.write(line) sf.close() return elif not self.offline_only and not gi.local: try: if gi.scheme in ("http", "https"): if _DO_HTTP: gi = self._fetch_http(gi) else: print("Install python3-requests to handle http requests natively") webbrowser.open_new_tab(gi.url) return else: gi = self._fetch_over_network(gi) except UserAbortException: return except Exception as err: gi.set_error(err) # Print an error message # we fail silently when sync_only print_error = not self.sync_only if isinstance(err, socket.gaierror): self.log["dns_failures"] += 1 if print_error: print("ERROR: DNS error!") elif isinstance(err, ConnectionRefusedError): self.log["refused_connections"] += 1 if print_error: print("ERROR1: Connection refused!") elif isinstance(err, ConnectionResetError): self.log["reset_connections"] += 1 if print_error: print("ERROR2: Connection reset!") elif isinstance(err, (TimeoutError, socket.timeout)): self.log["timeouts"] += 1 if print_error: print("""ERROR3: Connection timed out! Slow internet connection? Use 'set timeout' to be more patient.""") elif isinstance(err, FileExistsError): print("""ERROR5: Trying to create a directory which already exists in the cache : """) print(err) else: if print_error: print("ERROR4: " + str(err)) return # Pass file to handler, unless we were asked not to if gi : rendered_body = gi.get_rendered_body() display = handle and not self.sync_only if rendered_body: self.index = gi.get_links() self.lookup = self.index self.page_index = 0 self.index_index = -1 if display: # We actually put the body in a tmpfile before giving it to less if self.idx_filename: os.unlink(self.idx_filename) tmpf = tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) tmpf.write(rendered_body) tmpf.close() self.idx_filename = tmpf.name cmd_str = self._get_handler_cmd("text/gemini") subprocess.run(shlex.split(cmd_str % tmpf.name)) elif display : cmd_str = self._get_handler_cmd(gi.get_mime()) try: # get tmpfile from gi ! tmpfile = gi.get_body(as_file=True) subprocess.call(shlex.split(cmd_str % tmpfile)) except FileNotFoundError: print("Handler program %s not found!" % shlex.split(cmd_str)[0]) print("You can use the ! command to specify another handler program or pipeline.") # Update state self.gi = gi if update_hist: self._update_history(gi) def _fetch_http(self,gi): response = requests.get(gi.url) mime = response.headers['content-type'] body = response.content if "text/" in mime: body = response.text else: body = response.content gi.write_body(body,mime) return gi # fetch_over_network will modify with gi.write_body(body,mime) # before returning the gi def _fetch_over_network(self, gi): # Be careful with client certificates! # Are we crossing a domain boundary? if self.active_cert_domains and gi.host not in self.active_cert_domains: if self.active_is_transient: print("Permanently delete currently active transient certificate?") resp = input("Y/N? ") if resp.strip().lower() in ("y", "yes"): print("Destroying certificate.") self._deactivate_client_cert() else: print("Staying here.") raise UserAbortException() else: print("PRIVACY ALERT: Deactivate client cert before connecting to a new domain?") resp = input("Y/N? ") if resp.strip().lower() in ("n", "no"): print("Keeping certificate active for {}".format(gi.host)) else: print("Deactivating certificate.") self._deactivate_client_cert() # Suggest reactivating previous certs if not self.client_certs["active"] and gi.host in self.client_certs: print("PRIVACY ALERT: Reactivate previously used client cert for {}?".format(gi.host)) resp = input("Y/N? ") if resp.strip().lower() in ("y", "yes"): self._activate_client_cert(*self.client_certs[gi.host]) else: print("Remaining unidentified.") self.client_certs.pop(gi.host) # Is this a local file? if gi.local: address, f = None, open(gi.path, "rb") else: address, f = self._send_request(gi) # Spec dictates should not exceed 1024 bytes, # so maximum valid header length is 1027 bytes. header = f.readline(1027) header = header.decode("UTF-8") if not header or header[-1] != '\n': raise RuntimeError("Received invalid header from server!") header = header.strip() self._debug("Response header: %s." % header) # Validate header status, meta = header.split(maxsplit=1) if len(meta) > 1024 or len(status) != 2 or not status.isnumeric(): f.close() raise RuntimeError("Received invalid header from server!") # Update redirect loop/maze escaping state if not status.startswith("3"): self.previous_redirectors = set() # Handle non-SUCCESS headers, which don't have a response body # Inputs if status.startswith("1"): if self.sync_only: return None else: print(meta) if status == "11": user_input = getpass.getpass("> ") else: user_input = input("> ") return self._fetch_over_network(gi.query(user_input)) # Redirects elif status.startswith("3"): new_gi = GeminiItem(gi.absolutise_url(meta)) if new_gi.url == gi.url: raise RuntimeError("URL redirects to itself!") elif new_gi.url in self.previous_redirectors: raise RuntimeError("Caught in redirect loop!") elif len(self.previous_redirectors) == _MAX_REDIRECTS: raise RuntimeError("Refusing to follow more than %d consecutive redirects!" % _MAX_REDIRECTS) # Never follow cross-domain redirects without asking elif new_gi.host != gi.host: follow = input("Follow cross-domain redirect to %s? (y/n) " % new_gi.url) # Never follow cross-protocol redirects without asking elif new_gi.scheme != gi.scheme: follow = input("Follow cross-protocol redirect to %s? (y/n) " % new_gi.url) # Don't follow *any* redirect without asking if auto-follow is off elif not self.options["auto_follow_redirects"]: follow = input("Follow redirect to %s? (y/n) " % new_gi.url) # Otherwise, follow away else: follow = "yes" if follow.strip().lower() not in ("y", "yes"): raise UserAbortException() self._debug("Following redirect to %s." % new_gi.url) self._debug("This is consecutive redirect number %d." % len(self.previous_redirectors)) self.previous_redirectors.add(gi.url) if status == "31": # Permanent redirect self.permanent_redirects[gi.url] = new_gi.url return self._fetch_over_network(new_gi) # Errors elif status.startswith("4") or status.startswith("5"): raise RuntimeError(meta) # Client cert elif status.startswith("6"): self._handle_cert_request(meta) return self._fetch_over_network(gi) # Invalid status elif not status.startswith("2"): raise RuntimeError("Server returned undefined status code %s!" % status) # If we're here, this must be a success and there's a response body assert status.startswith("2") mime = meta # Read the response body over the network fbody = f.read() # DEFAULT GEMINI MIME if mime == "": mime = "text/gemini; charset=utf-8" shortmime, mime_options = cgi.parse_header(mime) if "charset" in mime_options: try: codecs.lookup(mime_options["charset"]) except LookupError: raise RuntimeError("Header declared unknown encoding %s" % value) if shortmime.startswith("text/"): #Get the charset and default to UTF-8 in none encoding = mime_options.get("charset", "UTF-8") try: body = fbody.decode(encoding) except UnicodeError: raise RuntimeError("Could not decode response body using %s\ encoding declared in header!" % encoding) else: body = fbody gi.write_body(body,mime) return gi def _send_request(self, gi): """Send a selector to a given host and port. Returns the resolved address and binary file with the reply.""" if gi.scheme == "gemini": # For Gemini requests, connect to the host and port specified in the URL host, port = gi.host, gi.port elif gi.scheme == "gopher": # For Gopher requests, use the configured proxy host, port = self.options["gopher_proxy"].rsplit(":", 1) self._debug("Using gopher proxy: " + self.options["gopher_proxy"]) elif gi.scheme in ("http", "https"): host, port = self.options["http_proxy"].rsplit(":",1) self._debug("Using http proxy: " + self.options["http_proxy"]) # Do DNS resolution addresses = self._get_addresses(host, port) # Prepare TLS context protocol = ssl.PROTOCOL_TLS if sys.version_info.minor >=6 else ssl.PROTOCOL_TLSv1_2 context = ssl.SSLContext(protocol) # Use CAs or TOFU if self.options["tls_mode"] == "ca": context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True context.load_default_certs() else: context.check_hostname = False context.verify_mode = ssl.CERT_NONE # Impose minimum TLS version ## In 3.7 and above, this is easy... if sys.version_info.minor >= 7: context.minimum_version = ssl.TLSVersion.TLSv1_2 ## Otherwise, it seems very hard... ## The below is less strict than it ought to be, but trying to disable ## TLS v1.1 here using ssl.OP_NO_TLSv1_1 produces unexpected failures ## with recent versions of OpenSSL. What a mess... else: context.options |= ssl.OP_NO_SSLv3 context.options |= ssl.OP_NO_SSLv2 # Try to enforce sensible ciphers try: context.set_ciphers("AESGCM+ECDHE:AESGCM+DHE:CHACHA20+ECDHE:CHACHA20+DHE:!DSS:!SHA1:!MD5:@STRENGTH") except ssl.SSLError: # Rely on the server to only support sensible things, I guess... pass # Load client certificate if needed if self.client_certs["active"]: certfile, keyfile = self.client_certs["active"] context.load_cert_chain(certfile, keyfile) # Connect to remote host by any address possible err = None for address in addresses: self._debug("Connecting to: " + str(address[4])) s = socket.socket(address[0], address[1]) if self.sync_only: timeout = self.options["short_timeout"] else: timeout = self.options["timeout"] s.settimeout(timeout) s = context.wrap_socket(s, server_hostname = gi.host) try: s.connect(address[4]) break except OSError as e: err = e else: # If we couldn't connect to *any* of the addresses, just # bubble up the exception from the last attempt and deny # knowledge of earlier failures. raise err if sys.version_info.minor >=5: self._debug("Established {} connection.".format(s.version())) self._debug("Cipher is: {}.".format(s.cipher())) # Do TOFU if self.options["tls_mode"] != "ca": cert = s.getpeercert(binary_form=True) self._validate_cert(address[4][0], host, cert) # Remember that we showed the current cert to this domain... if self.client_certs["active"]: self.active_cert_domains.append(gi.host) self.client_certs[gi.host] = self.client_certs["active"] # Send request and wrap response in a file descriptor self._debug("Sending %s" % gi.url) s.sendall((gi.url + CRLF).encode("UTF-8")) mf= s.makefile(mode = "rb") return address, mf def _get_addresses(self, host, port): # DNS lookup - will get IPv4 and IPv6 records if IPv6 is enabled if ":" in host: # This is likely a literal IPv6 address, so we can *only* ask for # IPv6 addresses or getaddrinfo will complain family_mask = socket.AF_INET6 elif socket.has_ipv6 and self.options["ipv6"]: # Accept either IPv4 or IPv6 addresses family_mask = 0 else: # IPv4 only family_mask = socket.AF_INET addresses = socket.getaddrinfo(host, port, family=family_mask, type=socket.SOCK_STREAM) # Sort addresses so IPv6 ones come first addresses.sort(key=lambda add: add[0] == socket.AF_INET6, reverse=True) return addresses def _handle_cert_request(self, meta): # Don't do client cert stuff in restricted mode, as in principle # it could be used to fill up the disk by creating a whole lot of # certificates if self.restricted: print("The server is requesting a client certificate.") print("These are not supported in restricted mode, sorry.") raise UserAbortException() print("SERVER SAYS: ", meta) # Present different messages for different 6x statuses, but # handle them the same. if status in ("64", "65"): print("The server rejected your certificate because it is either expired or not yet valid.") elif status == "63": print("The server did not accept your certificate.") print("You may need to e.g. coordinate with the admin to get your certificate fingerprint whitelisted.") else: print("The site {} is requesting a client certificate.".format(gi.host)) print("This will allow the site to recognise you across requests.") # Give the user choices print("What do you want to do?") print("1. Give up.") print("2. Generate a new transient certificate.") print("3. Generate a new persistent certificate.") print("4. Load a previously generated persistent.") print("5. Load certificate from an external file.") choice = input("> ").strip() if choice == "2": self._generate_transient_cert_cert() elif choice == "3": self._generate_persistent_client_cert() elif choice == "4": self._choose_client_cert() elif choice == "5": self._load_client_cert() else: print("Giving up.") raise UserAbortException() def _validate_cert(self, address, host, cert): """ Validate a TLS certificate in TOFU mode. If the cryptography module is installed: - Check the certificate Common Name or SAN matches `host` - Check the certificate's not valid before date is in the past - Check the certificate's not valid after date is in the future Whether the cryptography module is installed or not, check the certificate's fingerprint against the TOFU database to see if we've previously encountered a different certificate for this IP address and hostname. """ now = datetime.datetime.utcnow() if _HAS_CRYPTOGRAPHY: # Using the cryptography module we can get detailed access # to the properties of even self-signed certs, unlike in # the standard ssl library... c = x509.load_der_x509_certificate(cert, _BACKEND) sha = hashlib.sha256() sha.update(cert) fingerprint = sha.hexdigest() # Have we been here before? self.db_cur.execute("""SELECT fingerprint, first_seen, last_seen, count FROM cert_cache WHERE hostname=? AND address=?""", (host, address)) cached_certs = self.db_cur.fetchall() # If so, check for a match if cached_certs: max_count = 0 most_frequent_cert = None for cached_fingerprint, first, last, count in cached_certs: if count > max_count: max_count = count most_frequent_cert = cached_fingerprint if fingerprint == cached_fingerprint: # Matched! self._debug("TOFU: Accepting previously seen ({} times) certificate {}".format(count, fingerprint)) self.db_cur.execute("""UPDATE cert_cache SET last_seen=?, count=? WHERE hostname=? AND address=? AND fingerprint=?""", (now, count+1, host, address, fingerprint)) self.db_conn.commit() break else: if _HAS_CRYPTOGRAPHY: # Load the most frequently seen certificate to see if it has # expired certdir = os.path.join(_CONFIG_DIR, "cert_cache") with open(os.path.join(certdir, most_frequent_cert+".crt"), "rb") as fp: previous_cert = fp.read() previous_cert = x509.load_der_x509_certificate(previous_cert, _BACKEND) previous_ttl = previous_cert.not_valid_after - now print(previous_ttl) self._debug("TOFU: Unrecognised certificate {}! Raising the alarm...".format(fingerprint)) print("****************************************") print("[SECURITY WARNING] Unrecognised certificate!") print("The certificate presented for {} ({}) has never been seen before.".format(host, address)) print("This MIGHT be a Man-in-the-Middle attack.") print("A different certificate has previously been seen {} times.".format(max_count)) if _HAS_CRYPTOGRAPHY: if previous_ttl < datetime.timedelta(): print("That certificate has expired, which reduces suspicion somewhat.") else: print("That certificate is still valid for: {}".format(previous_ttl)) print("****************************************") print("Attempt to verify the new certificate fingerprint out-of-band:") print(fingerprint) choice = input("Accept this new certificate? Y/N ").strip().lower() if choice in ("y", "yes"): self.db_cur.execute("""INSERT INTO cert_cache VALUES (?, ?, ?, ?, ?, ?)""", (host, address, fingerprint, now, now, 1)) self.db_conn.commit() with open(os.path.join(certdir, fingerprint+".crt"), "wb") as fp: fp.write(cert) else: raise Exception("TOFU Failure!") # If not, cache this cert else: self._debug("TOFU: Blindly trusting first ever certificate for this host!") self.db_cur.execute("""INSERT INTO cert_cache VALUES (?, ?, ?, ?, ?, ?)""", (host, address, fingerprint, now, now, 1)) self.db_conn.commit() certdir = os.path.join(_CONFIG_DIR, "cert_cache") if not os.path.exists(certdir): os.makedirs(certdir) with open(os.path.join(certdir, fingerprint+".crt"), "wb") as fp: fp.write(cert) def _get_handler_cmd(self, mimetype): # Now look for a handler for this mimetype # Consider exact matches before wildcard matches exact_matches = [] wildcard_matches = [] for handled_mime, cmd_str in _MIME_HANDLERS.items(): if "*" in handled_mime: wildcard_matches.append((handled_mime, cmd_str)) else: exact_matches.append((handled_mime, cmd_str)) for handled_mime, cmd_str in exact_matches + wildcard_matches: if fnmatch.fnmatch(mimetype, handled_mime): break else: if mimetype.startswith("text/"): cmd_str = _DEFAULT_LESS else: # Use "xdg-open" as a last resort. cmd_str = "xdg-open %s" self._debug("Using handler: %s" % cmd_str) return cmd_str #TODO: remove format_geminiitem def _format_geminiitem(self, index, gi, url=False): protocol = "" if gi.scheme == "gemini" else " %s" % gi.scheme line = "[%d%s] %s" % (index, protocol, gi.name or gi.url) if gi.name and url: line += " (%s)" % gi.url return line def _show_lookup(self, offset=0, end=None, url=False): for n, gi in enumerate(self.lookup[offset:end]): print(self._format_geminiitem(n+offset+1, gi, url)) def _update_history(self, gi): # Don't duplicate if self.history and self.history[self.hist_index] == gi: return self.history = self.history[0:self.hist_index+1] self.history.append(gi) self.hist_index = len(self.history) - 1 def _log_visit(self, gi, address, size): if not address: return self.log["requests"] += 1 self.log["bytes_recvd"] += size self.visited_hosts.add(address) if address[0] == socket.AF_INET: self.log["ipv4_requests"] += 1 self.log["ipv4_bytes_recvd"] += size elif address[0] == socket.AF_INET6: self.log["ipv6_requests"] += 1 self.log["ipv6_bytes_recvd"] += size def _get_active_tmpfile(self): if self.gi.get_mime() in _FORMAT_RENDERERS: return self.idx_filename else: return self.tmp_filename def _debug(self, debug_text): if not self.options["debug"]: return debug_text = "\x1b[0;32m[DEBUG] " + debug_text + "\x1b[0m" print(debug_text) def _load_client_cert(self): """ Interactively load a TLS client certificate from the filesystem in PEM format. """ print("Loading client certificate file, in PEM format (blank line to cancel)") certfile = input("Certfile path: ").strip() if not certfile: print("Aborting.") return certfile = os.path.expanduser(certfile) if not os.path.isfile(certfile): print("Certificate file {} does not exist.".format(certfile)) return print("Loading private key file, in PEM format (blank line to cancel)") keyfile = input("Keyfile path: ").strip() if not keyfile: print("Aborting.") return keyfile = os.path.expanduser(keyfile) if not os.path.isfile(keyfile): print("Private key file {} does not exist.".format(keyfile)) return self._activate_client_cert(certfile, keyfile) def _generate_transient_cert_cert(self): """ Use `openssl` command to generate a new transient client certificate with 24 hours of validity. """ certdir = os.path.join(_CONFIG_DIR, "transient_certs") name = str(uuid.uuid4()) self._generate_client_cert(certdir, name, transient=True) self.active_is_transient = True self.transient_certs_created.append(name) def _generate_persistent_client_cert(self): """ Interactively use `openssl` command to generate a new persistent client certificate with one year of validity. """ certdir = os.path.join(_CONFIG_DIR, "client_certs") print("What do you want to name this new certificate?") print("Answering `mycert` will create `{0}/mycert.crt` and `{0}/mycert.key`".format(certdir)) name = input("> ") if not name.strip(): print("Aborting.") return self._generate_client_cert(certdir, name) def _generate_client_cert(self, certdir, basename, transient=False): """ Use `openssl` binary to generate a client certificate (which may be transient or persistent) and save the certificate and private key to the specified directory with the specified basename. """ if not os.path.exists(certdir): os.makedirs(certdir) certfile = os.path.join(certdir, basename+".crt") keyfile = os.path.join(certdir, basename+".key") cmd = "openssl req -x509 -newkey rsa:2048 -days {} -nodes -keyout {} -out {}".format(1 if transient else 365, keyfile, certfile) if transient: cmd += " -subj '/CN={}'".format(basename) os.system(cmd) self._activate_client_cert(certfile, keyfile) def _choose_client_cert(self): """ Interactively select a previously generated client certificate and activate it. """ certdir = os.path.join(_CONFIG_DIR, "client_certs") certs = glob.glob(os.path.join(certdir, "*.crt")) if len(certs) == 0: print("There are no previously generated certificates.") return certdir = {} for n, cert in enumerate(certs): certdir[str(n+1)] = (cert, os.path.splitext(cert)[0] + ".key") print("{}. {}".format(n+1, os.path.splitext(os.path.basename(cert))[0])) choice = input("> ").strip() if choice in certdir: certfile, keyfile = certdir[choice] self._activate_client_cert(certfile, keyfile) else: print("What?") def _activate_client_cert(self, certfile, keyfile): self.client_certs["active"] = (certfile, keyfile) self.active_cert_domains = [] self.prompt = self.cert_prompt + "+" + os.path.basename(certfile).replace('.crt','') + "> " + "\x1b[0m" self._debug("Using ID {} / {}.".format(*self.client_certs["active"])) def _deactivate_client_cert(self): if self.active_is_transient: for filename in self.client_certs["active"]: os.remove(filename) for domain in self.active_cert_domains: self.client_certs.pop(domain) self.client_certs["active"] = None self.active_cert_domains = [] self.prompt = self.no_cert_prompt self.active_is_transient = False # Cmd implementation follows def default(self, line): if line.strip() == "EOF": return self.onecmd("quit") elif line.strip() == "..": return self.do_up() elif line.startswith("/"): return self.do_search(line[1:]) # Expand abbreviated commands first_word = line.split()[0].strip() if first_word in _ABBREVS: full_cmd = _ABBREVS[first_word] expanded = line.replace(first_word, full_cmd, 1) return self.onecmd(expanded) # Try to access it like an URL if looks_like_url(line): return self.do_go(line) # Try to parse numerical index for lookup table try: n = int(line.strip()) except ValueError: print("What?") return try: gi = self.lookup[n-1] except IndexError: print ("Index too high!") return self.index_index = n self._go_to_gi(gi) ### Settings @restricted def do_set(self, line): """View or set various options.""" if not line.strip(): # Show all current settings for option in sorted(self.options.keys()): print("%s %s" % (option, self.options[option])) elif len(line.split()) == 1: # Show current value of one specific setting option = line.strip() if option in self.options: print("%s %s" % (option, self.options[option])) else: print("Unrecognised option %s" % option) else: # Set value of one specific setting option, value = line.split(" ", 1) if option not in self.options: print("Unrecognised option %s" % option) return # Validate / convert values if option == "gopher_proxy": if ":" not in value: value += ":1965" else: host, port = value.rsplit(":",1) if not port.isnumeric(): print("Invalid proxy port %s" % port) return elif option == "tls_mode": if value.lower() not in ("ca", "tofu"): print("TLS mode must be `ca` or `tofu`!") return elif value.isnumeric(): value = int(value) elif value.lower() == "false": value = False elif value.lower() == "true": value = True else: try: value = float(value) except ValueError: pass self.options[option] = value @restricted def do_cert(self, line): """Manage client certificates""" print("Managing client certificates") if self.client_certs["active"]: print("Active certificate: {}".format(self.client_certs["active"][0])) print("1. Deactivate client certificate.") print("2. Generate new certificate.") print("3. Load previously generated certificate.") print("4. Load externally created client certificate from file.") print("Enter blank line to exit certificate manager.") choice = input("> ").strip() if choice == "1": print("Deactivating client certificate.") self._deactivate_client_cert() elif choice == "2": self._generate_persistent_client_cert() elif choice == "3": self._choose_client_cert() elif choice == "4": self._load_client_cert() else: print("Aborting.") @restricted def do_handler(self, line): """View or set handler commands for different MIME types.""" if not line.strip(): # Show all current handlers for mime in sorted(_MIME_HANDLERS.keys()): print("%s %s" % (mime, _MIME_HANDLERS[mime])) elif len(line.split()) == 1: mime = line.strip() if mime in _MIME_HANDLERS: print("%s %s" % (mime, _MIME_HANDLERS[mime])) else: print("No handler set for MIME type %s" % mime) else: mime, handler = line.split(" ", 1) _MIME_HANDLERS[mime] = handler if "%s" not in handler: print("Are you sure you don't want to pass the filename to the handler?") def do_abbrevs(self, *args): """Print all Offpunk command abbreviations.""" header = "Command Abbreviations:" self.stdout.write("\n{}\n".format(str(header))) if self.ruler: self.stdout.write("{}\n".format(str(self.ruler * len(header)))) for k, v in _ABBREVS.items(): self.stdout.write("{:<7} {}\n".format(k, v)) self.stdout.write("\n") def do_offline(self, *args): """Use Offpunk offline by only accessing cached content""" if self.offline_only: print("Offline and undisturbed.") else: self.offline_only = True self.prompt = self.offline_prompt print("Offpunk is now offline and will only access cached content") def do_online(self, *args): """Use Offpunk online with a direct connection""" if self.offline_only: self.offline_only = False self.prompt = self.no_cert_prompt print("Offpunk is online and will access the network") else: print("Already online. Try offline.") def do_copy(self, *args): """Copy the content of the last visited page as gemtext in the clipboard. Use with "url" as argument to only copy the adress. Use with "raw" to copy the content as seen in your terminal (not gemtext)""" if self.gi: if shutil.which('xsel'): if args and args[0] == "url": subprocess.call(("echo %s |xsel -b -i" % self.gi.url), shell=True) elif args and args[0] == "raw": subprocess.call(("cat %s |xsel -b -i" % self._get_active_tmpfile()), shell=True) else: subprocess.call(("cat %s |xsel -b -i" % self.gi.get_body(as_file=True)), shell=True) else: print("Please install xsel to use copy") else: print("No content to copy, visit a page first") ### Stuff for getting around def do_go(self, line): """Go to a gemini URL or marked item.""" line = line.strip() if not line: if shutil.which('xsel'): clipboards = [] urls = [] clipboards.append(subprocess.check_output(['xsel','-p'],text=True)) clipboards.append(subprocess.check_output(['xsel','-s'],text=True)) clipboards.append(subprocess.check_output(['xsel','-b'],text=True)) for u in clipboards: if looks_like_url(u) : urls.append(u) if len(urls) > 1: self.lookup = [] for u in urls: self.lookup.append(GeminiItem(u)) print("Where do you want to go today?") self._show_lookup() elif len(urls) == 1: self.do_go(urls[0]) else: print("Go where? (hint: simply copy an URL)") else: print("Go where? (hint: install xsel to go to copied URLs)") # First, check for possible marks elif line in self.marks: gi = self.marks[line] self._go_to_gi(gi) # or a local file elif os.path.exists(os.path.expanduser(line)): self._go_to_gi(GeminiItem(line)) # If this isn't a mark, treat it as a URL elif looks_like_url(line): self._go_to_gi(GeminiItem(line)) @needs_gi def do_reload(self, *args): """Reload the current URL.""" if self.offline_only: with open(self.syncfile,mode='a') as sf: line = self.gi.url + '\n' sf.write(line) sf.close() print("%s marked for syncing" %self.gi.url) else: self._go_to_gi(self.gi, check_cache=False) @needs_gi def do_up(self, *args): """Go up one directory in the path.""" self._go_to_gi(self.gi.up()) def do_back(self, *args): """Go back to the previous gemini item.""" if not self.history or self.hist_index == 0: return self.hist_index -= 1 gi = self.history[self.hist_index] self._go_to_gi(gi, update_hist=False) def do_forward(self, *args): """Go forward to the next gemini item.""" if not self.history or self.hist_index == len(self.history) - 1: return self.hist_index += 1 gi = self.history[self.hist_index] self._go_to_gi(gi, update_hist=False) def do_next(self, *args): """Go to next item after current in index.""" return self.onecmd(str(self.index_index+1)) def do_previous(self, *args): """Go to previous item before current in index.""" self.lookup = self.index return self.onecmd(str(self.index_index-1)) @needs_gi def do_root(self, *args): """Go to root selector of the server hosting current item.""" self._go_to_gi(self.gi.root()) def do_tour(self, line): """Add index items as waypoints on a tour, which is basically a FIFO queue of gemini items. Items can be added with `tour 1 2 3 4` or ranges like `tour 1-4`. All items in current menu can be added with `tour *`. Current item can be added back to the end of the tour with `tour .`. Current tour can be listed with `tour ls` and scrubbed with `tour clear`.""" def add_to_tourfile(url): with open(self.tourfile,'a') as f: l = url.strip()+"\n" f.write(l) f.close def pop_from_tourfile(): l = None lines = [] with open(self.tourfile,'r') as f: l = f.readline() lines = f.readlines() f.close() with open(self.tourfile,'w') as f: if len(lines) > 0: f.writelines(lines) f.close() return l def list_tourfile(): lines = [] with open(self.tourfile,'r') as f: lines = f.readlines() f.close() return lines def clear_tourfile(): with open(self.tourfile,'w') as f: f.close() line = line.strip() if not line: # Fly to next waypoint on tour nexttour = pop_from_tourfile() if not nexttour: print("End of tour.") else: self._go_to_gi(GeminiItem(nexttour)) elif line == "ls": old_lookup = self.lookup self.lookup = [] for u in list_tourfile(): self.lookup.append(GeminiItem(u)) self._show_lookup() self.lookup = old_lookup elif line == "clear": clear_tourfile() elif line == "*": for l in self.lookup: add_to_tourfile(l.url) elif line == ".": add_to_tourfile(self.gi.url) elif looks_like_url(line): add_to_tourfile(line) else: for index in line.split(): try: pair = index.split('-') if len(pair) == 1: # Just a single index n = int(index) gi = self.lookup[n-1] add_to_tourfile(gi.url) elif len(pair) == 2: # Two endpoints for a range of indices if int(pair[0]) < int(pair[1]): for n in range(int(pair[0]), int(pair[1]) + 1): gi = self.lookup[n-1] add_to_tourfile(gi.url) else: for n in range(int(pair[0]), int(pair[1]) - 1, -1): gi = self.lookup[n-1] add_to_tourfile(gi.url) else: # Syntax error print("Invalid use of range syntax %s, skipping" % index) except ValueError: print("Non-numeric index %s, skipping." % index) except IndexError: print("Invalid index %d, skipping." % n) @needs_gi def do_mark(self, line): """Mark the current item with a single letter. This letter can then be passed to the 'go' command to return to the current item later. Think of it like marks in vi: 'mark a'='ma' and 'go a'=''a'.""" line = line.strip() if not line: for mark, gi in self.marks.items(): print("[%s] %s (%s)" % (mark, gi.name, gi.url)) elif line.isalpha() and len(line) == 1: self.marks[line] = self.gi else: print("Invalid mark, must be one letter") def do_version(self, line): """Display version information.""" print("Offpunk " + _VERSION) ### Stuff that modifies the lookup table def do_ls(self, line): """List contents of current index. Use 'ls -l' to see URLs.""" self.lookup = self.index self._show_lookup(url = "-l" in line) self.page_index = 0 def do_gus(self, line): """Submit a search query to the geminispace.info search engine.""" gus = GeminiItem("gemini://geminispace.info/search") self._go_to_gi(gus.query(line)) def do_history(self, *args): """Display history.""" self.lookup = self.history self._show_lookup(url=True) self.page_index = 0 def do_search(self, searchterm): """Search index (case insensitive).""" results = [ gi for gi in self.lookup if searchterm.lower() in gi.name.lower()] if results: self.lookup = results self._show_lookup() self.page_index = 0 else: print("No results found.") def emptyline(self): """Page through index ten lines at a time.""" i = self.page_index if i > len(self.lookup): return self._show_lookup(offset=i, end=i+10) self.page_index += 10 ### Stuff that does something to most recently viewed item @needs_gi def do_cat(self, *args): """Run most recently visited item through "cat" command.""" subprocess.call(shlex.split("cat %s" % self._get_active_tmpfile())) @needs_gi def do_less(self, *args): """Run most recently visited item through "less" command.""" cmd_str = self._get_handler_cmd(self.gi.get_mime()) cmd_str = cmd_str % self._get_active_tmpfile() subprocess.call("%s | less -RM" % cmd_str, shell=True) @needs_gi def do_fold(self, *args): """Run most recently visited item through "fold" command.""" cmd_str = self._get_handler_cmd(self.gi.get_mime()) cmd_str = cmd_str % self._get_active_tmpfile() subprocess.call("%s | fold -w 70 -s" % cmd_str, shell=True) @restricted @needs_gi def do_shell(self, line): """'cat' most recently visited item through a shell pipeline.""" subprocess.call(("cat %s |" % self._get_active_tmpfile()) + line, shell=True) @restricted @needs_gi def do_save(self, line): """Save an item to the filesystem. 'save n filename' saves menu item n to the specified filename. 'save filename' saves the last viewed item to the specified filename. 'save n' saves menu item n to an automagic filename.""" args = line.strip().split() # First things first, figure out what our arguments are if len(args) == 0: # No arguments given at all # Save current item, if there is one, to a file whose name is # inferred from the gemini path if not self.gi.is_cache_valid(): print("You cannot save if not cached!") return else: index = None filename = None elif len(args) == 1: # One argument given # If it's numeric, treat it as an index, and infer the filename try: index = int(args[0]) filename = None # If it's not numeric, treat it as a filename and # save the current item except ValueError: index = None filename = os.path.expanduser(args[0]) elif len(args) == 2: # Two arguments given # Treat first as an index and second as filename index, filename = args try: index = int(index) except ValueError: print("First argument is not a valid item index!") return filename = os.path.expanduser(filename) else: print("You must provide an index, a filename, or both.") return # Next, fetch the item to save, if it's not the current one. if index: last_gi = self.gi try: gi = self.lookup[index-1] self._go_to_gi(gi, update_hist = False, handle = False) except IndexError: print ("Index too high!") self.gi = last_gi return else: gi = self.gi # Derive filename from current GI's path, if one hasn't been set if not filename: filename = gi.get_filename() # Check for filename collisions and actually do the save if safe if os.path.exists(filename): print("File %s already exists!" % filename) else: # Don't use _get_active_tmpfile() here, because we want to save the # "source code" of menus, not the rendered view - this way Offpunk # can navigate to it later. print("Saved to %s" % filename) shutil.copyfile(gi.get_body(as_file=True), filename) # Restore gi if necessary if index != None: self._go_to_gi(last_gi, handle=False) @needs_gi def do_url(self, *args): """Print URL of most recently visited item.""" print(self.gi.url) ### Bookmarking stuff @restricted @needs_gi def do_add(self, line): """Add the current URL to the list specied as argument. If no argument given, URL is added to Bookmarks.""" args = line.split() if len(args) < 1 : with open(os.path.join(_CONFIG_DIR, "bookmarks.gmi"), "a") as fp: fp.write(self.gi.to_map_line()) else: self.list_add_line(args[0]) def do_bookmarks(self, line): """Show or access the bookmarks menu. 'bookmarks' shows all bookmarks. 'bookmarks n' navigates immediately to item n in the bookmark menu. Bookmarks are stored using the 'add' command.""" bm_file = os.path.join(_CONFIG_DIR, "bookmarks.gmi") if not os.path.exists(bm_file): print("You need to 'add' some bookmarks, first!") return args = line.strip() if len(args.split()) > 1 or (args and not args.isnumeric()): print("bookmarks command takes a single integer argument!") return gi = GeminiItem("localhost:/" + bm_file,"Bookmarks") display = not self.sync_only if args: gi = gi.get_link(int(args)) self._go_to_gi(gi,handle=display) def list_add_line(self,list): list_path = os.path.join(_DATA_DIR, "lists/%s.gmi"%list) if not os.path.exists(list_path): print("List %s does not exist. Create it with ""list create %s"""%(list,list)) return else: with open(list_path,"a") as l_file: l_file.write(self.gi.to_map_line()) l_file.close() print("Page added to your %s list" %list) def list_rm_line(self,line,list=None): list_path = os.path.join(_DATA_DIR, "lists/%s.gmi"%list) print("removing %s from %s not yet implemented"%(line,list)) def list_go_to_line(self,line,list): list_path = os.path.join(_DATA_DIR, "lists/%s.gmi"%list) if not os.path.exists(list_path): print("List %s does not exist. Create it with ""list create %s"""%(list,list)) return None elif not line.isnumeric(): print("go_to_line requires a number as parameter") return None else: gi = GeminiItem("localhost:/" + list_path,list) gi = gi.get_link(line) display = not self.sync_only self._go_to_gi(gi,handle=display) def list_show(self,list): list_path = os.path.join(_DATA_DIR, "lists/%s.gmi"%list) if not os.path.exists(list_path): print("List %s does not exist. Create it with ""list create %s"""%(list,list)) else: gi = GeminiItem("localhost:/" + list_path,list) display = not self.sync_only self._go_to_gi(gi,handle=display) def list_create(self,list,title=None): listdir = os.path.join(_DATA_DIR,"lists") os.makedirs(listdir,exist_ok=True) list_path = os.path.join(listdir, "%s.gmi"%list) if not os.path.exists(list_path): with open(list_path,"a") as lfile: if title: lfile.write("# %s\n"%title) else: lfile.write("# %s\n"%list) lfile.close() print("list created. Display with `list %s`"%list) else: print("list %s already exists" %list) def do_list(self,arg): """Manage list of bookmarked pages. - list : display the list of available lists - list $LIST : display pages in list $LIST - list create $NEWLIST : create a list Also see : - add $NEWLIST (to add current page to list) """ listdir = os.path.join(_DATA_DIR,"lists") os.makedirs(listdir,exist_ok=True) if not arg: lists = os.listdir(listdir) # TODO : make a true gemini page if len(lists) > 0: print("# Available lists:") for l in lists: print("=> %s" %l) else: print("No lists yet. Use `list create`") else: args = arg.split() if len(args) == 1: self.list_show(args[0]) elif args[0] == "create": if len(args) > 2: name = " ".join(args[2:]) else: name = None self.list_create(args[1],title=name) else: print("%s is not a valid list command") def do_help(self, arg): """ALARM! Recursion detected! ALARM! Prepare to eject!""" if arg == "!": print("! is an alias for 'shell'") elif arg == "?": print("? is an alias for 'help'") elif arg in _ABBREVS: full_cmd = _ABBREVS[arg] print("%s is aan alias for '%s'" %(arg,full_cmd)) print("See the list of aliases with 'abbrevs'") print("'help %s':"%full_cmd) cmd.Cmd.do_help(self, full_cmd) else: cmd.Cmd.do_help(self, arg) ### Flight recorder def do_blackbox(self, *args): """Display contents of flight recorder, showing statistics for the current gemini browsing session.""" lines = [] # Compute flight time now = time.time() delta = now - self.log["start_time"] hours, remainder = divmod(delta, 3600) minutes, seconds = divmod(remainder, 60) # Count hosts ipv4_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET]) ipv6_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET6]) # Assemble lines lines.append(("Patrol duration", "%02d:%02d:%02d" % (hours, minutes, seconds))) lines.append(("Requests sent:", self.log["requests"])) lines.append((" IPv4 requests:", self.log["ipv4_requests"])) lines.append((" IPv6 requests:", self.log["ipv6_requests"])) lines.append(("Bytes received:", self.log["bytes_recvd"])) lines.append((" IPv4 bytes:", self.log["ipv4_bytes_recvd"])) lines.append((" IPv6 bytes:", self.log["ipv6_bytes_recvd"])) lines.append(("Unique hosts visited:", len(self.visited_hosts))) lines.append((" IPv4 hosts:", ipv4_hosts)) lines.append((" IPv6 hosts:", ipv6_hosts)) lines.append(("DNS failures:", self.log["dns_failures"])) lines.append(("Timeouts:", self.log["timeouts"])) lines.append(("Refused connections:", self.log["refused_connections"])) lines.append(("Reset connections:", self.log["reset_connections"])) lines.append(("Cache hits:", self.log["cache_hits"])) # Print for key, value in lines: print(key.ljust(24) + str(value).rjust(8)) ### The end! def do_quit(self, *args): """Exit Offpunk.""" # Close TOFU DB self.db_conn.commit() self.db_conn.close() # Clean up after ourself if self.tmp_filename and os.path.exists(self.tmp_filename): os.unlink(self.tmp_filename) if self.idx_filename and os.path.exists(self.idx_filename): os.unlink(self.idx_filename) for cert in self.transient_certs_created: for ext in (".crt", ".key"): certfile = os.path.join(_CONFIG_DIR, "transient_certs", cert+ext) if os.path.exists(certfile): os.remove(certfile) print() print("You can close your screen!") sys.exit() do_exit = do_quit # Main function def main(): # Parse args parser = argparse.ArgumentParser(description='A command line gemini client.') parser.add_argument('--bookmarks', action='store_true', help='start with your list of bookmarks') parser.add_argument('--tls-cert', metavar='FILE', help='TLS client certificate file') parser.add_argument('--tls-key', metavar='FILE', help='TLS client certificate private key file') parser.add_argument('--restricted', action="store_true", help='Disallow shell, add, and save commands') parser.add_argument('--sync', action='store_true', help='run non-interactively to build cache by exploring bookmarks') parser.add_argument('--fetch-later', action='store_true', help='run non-interactively with an URL as argument to fetch it later') parser.add_argument('--cache-validity', help='duration for which a cache is valid before sync (seconds)') parser.add_argument('--version', action='store_true', help='display version information and quit') parser.add_argument('url', metavar='URL', nargs='*', help='start with this URL') args = parser.parse_args() # Handle --version if args.version: print("Offpunk " + _VERSION) sys.exit() # Instantiate client gc = GeminiClient(restricted=args.restricted,synconly=args.sync) # Process config file rcfile = os.path.join(_CONFIG_DIR, "offpunkrc") if os.path.exists(rcfile): print("Using config %s" % rcfile) with open(rcfile, "r") as fp: for line in fp: line = line.strip() if ((args.bookmarks or args.url) and any((line.startswith(x) for x in ("go", "g", "tour", "t"))) ): if args.bookmarks: print("Skipping rc command \"%s\" due to --bookmarks option." % line) else: print("Skipping rc command \"%s\" due to provided URLs." % line) continue gc.cmdqueue.append(line) # Say hi if not args.sync and not args.fetch_later: print("Welcome to Offpunk!") if args.restricted: print("Restricted mode engaged!") print("Enjoy your patrol through Geminispace...") # Act on args if args.tls_cert: # If tls_key is None, python will attempt to load the key from tls_cert. gc._activate_client_cert(args.tls_cert, args.tls_key) if args.bookmarks: gc.cmdqueue.append("bookmarks") elif args.url: if len(args.url) == 1: gc.cmdqueue.append("go %s" % args.url[0]) else: for url in args.url: if not url.startswith("gemini://"): url = "gemini://" + url gc.cmdqueue.append("tour %s" % url) gc.cmdqueue.append("tour") # Endless interpret loop if args.fetch_later: if args.url: # we go offline to fetch later and in sync-only to not display anything gc.onecmd("offline") gc.sync_only = True for u in args.url: gc.onecmd("go %s"%u) if gc.gi and u in gc.gi.url and gc.gi.is_cache_valid(): # forcing re-fetch in case an old catch already exists # FIXME : this will be fetched too many times, # should be handled on the to_fetch level gc.onecmd("reload") else: print("--netch-later requires an URL (or a list of URLS) as argument") elif args.sync: # fetch_cache is the core of the sync algorithm. # It takes as input : # - a GeminiItem to be fetched # - depth : the degree of recursion to build the cache (0 means no recursion) # - validity : the age, in seconds, existing caches need to have before # being refreshed (0 = never refreshed if it already exists) # - savetotour : if True, newly cached items are added to tour def add_to_tour(gitem): if gitem.is_cache_valid(): print(" -> adding to tour: %s" %gitem.url) with open(gc.tourfile,mode='a') as tf: line = gitem.url.strip() + "\n" tf.write(line) tf.close() def fetch_cache(gitem,depth=0,validity=0,savetotour=False,count=[0,0],strin=""): #savetotour = True will save to tour newly cached content # else, do not save to tour #regardless of valitidy if not gitem.is_cache_valid(validity=validity): if strin != "": endline = '\r' else: endline = None #Did we already had a cache (even an old one) ? isnew = not gitem.is_cache_valid() print("%s [%s/%s] Fetch "%(strin,count[0],count[1]),gitem.url,end=endline) gc.onecmd("go %s" %gitem.url) if savetotour and isnew and gitem.is_cache_valid(): #we add to the next tour only if we managed to cache #the ressource add_to_tour(gitem) if depth > 0: d = depth - 1 temp_lookup = set(gc.lookup) subcount = [0,len(temp_lookup)] for k in temp_lookup: #recursive call substri = strin + " -->" subcount[0] += 1 fetch_cache(k,depth=d,validity=0,savetotour=savetotour,\ count=subcount,strin=substri) if args.cache_validity: refresh_time = int(args.cache_validity) else: # if no refresh time, a default of 0 is used (which means "infinite") refresh_time = 0 gc.sync_only = True # We start by syncing the bookmarks gc.onecmd("bookmarks") original_lookup = gc.lookup end = len(original_lookup) counter = 0 print(" * * * %s bookmarks to fetch * * *" %end) for j in original_lookup: #refreshing the bookmarks counter += 1 fetch_cache(j,depth=1,validity=refresh_time,savetotour=True,count=[counter,end]) ## Second we get ressources in the tour lines_lookup = [] if os.path.exists(gc.tourfile): with open(gc.tourfile,mode='r') as tf: lines_lookup += tf.readlines() tf.close tot = len(set(lines_lookup)) counter = 0 if tot > 0: print(" * * * %s to fetch from your tour browsing * * *" %tot) for l in set(lines_lookup): #always get to_fetch and tour, regarless of refreshtime #we don’t save to tour (it’s already there) counter += 1 if l.startswith("gemini://") or l.startswith("http"): fetch_cache(GeminiItem(l.strip()),depth=1,validity=refresh_time,\ savetotour=False,count=[counter,tot]) # Then we get ressources from syncfile lines_lookup = [] if os.path.exists(gc.syncfile): with open(gc.syncfile,mode='r') as sf: lines_lookup += sf.readlines() sf.close() tot = len(set(lines_lookup)) counter = 0 if tot > 0: print(" * * * %s to fetch from your offline browsing * * *" %tot) for l in set(lines_lookup): #always fetch the cache (we allows only a 10 minutes time # to avoid multiple fetch in the same sync run) #then add to tour counter += 1 gitem = GeminiItem(l.strip()) if l.startswith("gemini://") or l.startswith("http"): fetch_cache(gitem,depth=1,validity=600,\ savetotour=False,count=[counter,tot]) add_to_tour(gitem) # let’s empty the sync file once the fetch is successful if os.path.exists(gc.syncfile): open(gc.syncfile,mode='w').close() gc.onecmd("blackbox") else: while True: try: gc.cmdloop() except KeyboardInterrupt: print("") if __name__ == '__main__': main()