2019-06-22 12:58:21 +00:00
|
|
|
|
#!/usr/bin/env python3
|
2021-12-30 15:03:08 +00:00
|
|
|
|
# Offpunk Offline Gemini client
|
|
|
|
|
# Derived from AV-98 by Solderpunk,
|
2022-01-08 21:18:54 +00:00
|
|
|
|
# (C) 2021, 2022 Ploum <offpunk@ploum.eu>
|
2020-05-17 15:57:34 +00:00
|
|
|
|
# (C) 2019, 2020 Solderpunk <solderpunk@sdf.org>
|
|
|
|
|
# With contributions from:
|
2020-06-02 20:57:48 +00:00
|
|
|
|
# - danceka <hannu.hartikainen@gmail.com>
|
2020-05-17 15:57:34 +00:00
|
|
|
|
# - <jprjr@tilde.club>
|
2020-06-02 20:57:48 +00:00
|
|
|
|
# - <vee@vnsf.xyz>
|
2020-06-07 17:06:39 +00:00
|
|
|
|
# - Klaus Alexander Seistrup <klaus@seistrup.dk>
|
2020-08-14 20:29:21 +00:00
|
|
|
|
# - govynnus <govynnus@sdf.org>
|
2021-08-25 06:07:18 +00:00
|
|
|
|
# - Björn Wärmedal <bjorn.warmedal@gmail.com>
|
|
|
|
|
# - <jake@rmgr.dev>
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import cmd
|
2019-06-22 13:29:14 +00:00
|
|
|
|
import cgi
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import codecs
|
|
|
|
|
import collections
|
2020-05-16 16:58:53 +00:00
|
|
|
|
import datetime
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import fnmatch
|
2020-06-07 17:13:00 +00:00
|
|
|
|
import getpass
|
2020-05-11 21:27:48 +00:00
|
|
|
|
import glob
|
2020-05-16 16:58:53 +00:00
|
|
|
|
import hashlib
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import io
|
|
|
|
|
import mimetypes
|
2020-05-10 15:25:03 +00:00
|
|
|
|
import os
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import os.path
|
2021-12-13 12:49:12 +00:00
|
|
|
|
import filecmp
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import random
|
|
|
|
|
import shlex
|
|
|
|
|
import shutil
|
|
|
|
|
import socket
|
2020-05-16 16:58:53 +00:00
|
|
|
|
import sqlite3
|
2020-05-11 20:22:24 +00:00
|
|
|
|
import ssl
|
2020-05-17 18:38:06 +00:00
|
|
|
|
from ssl import CertificateError
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
|
|
|
|
import tempfile
|
|
|
|
|
import time
|
2020-05-11 20:22:24 +00:00
|
|
|
|
import urllib.parse
|
|
|
|
|
import uuid
|
2019-11-05 17:58:04 +00:00
|
|
|
|
import webbrowser
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2020-05-10 13:02:24 +00:00
|
|
|
|
try:
|
|
|
|
|
import ansiwrap as textwrap
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
import textwrap
|
|
|
|
|
|
2020-05-17 18:38:06 +00:00
|
|
|
|
try:
|
|
|
|
|
from cryptography import x509
|
|
|
|
|
from cryptography.hazmat.backends import default_backend
|
|
|
|
|
_HAS_CRYPTOGRAPHY = True
|
|
|
|
|
_BACKEND = default_backend()
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_HAS_CRYPTOGRAPHY = False
|
|
|
|
|
|
2021-12-30 14:43:00 +00:00
|
|
|
|
try:
|
|
|
|
|
import magic
|
|
|
|
|
_HAS_MAGIC = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_HAS_MAGIC = False
|
|
|
|
|
|
2022-01-10 10:19:29 +00:00
|
|
|
|
try:
|
|
|
|
|
import requests
|
|
|
|
|
_DO_HTTP = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_DO_HTTP = False
|
|
|
|
|
|
2022-01-10 11:49:24 +00:00
|
|
|
|
try:
|
|
|
|
|
from readability import Document
|
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
_DO_HTML = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_DO_HTML = False
|
2022-01-01 21:05:02 +00:00
|
|
|
|
_VERSION = "0.1"
|
2020-05-10 12:34:48 +00:00
|
|
|
|
|
2019-10-13 17:42:04 +00:00
|
|
|
|
_MAX_REDIRECTS = 5
|
2020-08-30 21:17:21 +00:00
|
|
|
|
_MAX_CACHE_SIZE = 10
|
|
|
|
|
_MAX_CACHE_AGE_SECS = 180
|
2021-12-04 20:01:11 +00:00
|
|
|
|
# TODO : use XDG spec for cache
|
2021-12-30 15:03:08 +00:00
|
|
|
|
_CACHE_PATH = "~/.cache/offpunk/"
|
2022-01-06 12:44:02 +00:00
|
|
|
|
#_DEFAULT_LESS = "less -EXFRfM -PMurl\ lines\ \%lt-\%lb/\%L\ \%Pb\%$ %s"
|
|
|
|
|
_DEFAULT_LESS = "less -EXFRfM %s"
|
2019-10-13 17:42:04 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Command abbreviations
|
|
|
|
|
_ABBREVS = {
|
|
|
|
|
"a": "add",
|
|
|
|
|
"b": "back",
|
|
|
|
|
"bb": "blackbox",
|
|
|
|
|
"bm": "bookmarks",
|
|
|
|
|
"book": "bookmarks",
|
2022-01-05 20:12:59 +00:00
|
|
|
|
"cp": "copy",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"f": "fold",
|
|
|
|
|
"fo": "forward",
|
|
|
|
|
"g": "go",
|
|
|
|
|
"h": "history",
|
|
|
|
|
"hist": "history",
|
|
|
|
|
"l": "less",
|
|
|
|
|
"n": "next",
|
2022-01-05 20:12:59 +00:00
|
|
|
|
"off": "offline",
|
|
|
|
|
"on": "online",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"p": "previous",
|
|
|
|
|
"prev": "previous",
|
|
|
|
|
"q": "quit",
|
|
|
|
|
"r": "reload",
|
|
|
|
|
"s": "save",
|
|
|
|
|
"se": "search",
|
|
|
|
|
"/": "search",
|
|
|
|
|
"t": "tour",
|
|
|
|
|
"u": "up",
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
_MIME_HANDLERS = {
|
2021-12-19 14:51:59 +00:00
|
|
|
|
"application/pdf": "zathura %s",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"audio/mpeg": "mpg123 %s",
|
|
|
|
|
"audio/ogg": "ogg123 %s",
|
2022-01-08 20:32:25 +00:00
|
|
|
|
"image/*": "feh -. %s",
|
2022-01-10 14:53:41 +00:00
|
|
|
|
#"text/html": "lynx -dump -force_html %s",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
2021-12-06 15:43:16 +00:00
|
|
|
|
|
2020-06-01 07:21:55 +00:00
|
|
|
|
# monkey-patch Gemini support in urllib.parse
|
|
|
|
|
# see https://github.com/python/cpython/blob/master/Lib/urllib/parse.py
|
|
|
|
|
urllib.parse.uses_relative.append("gemini")
|
|
|
|
|
urllib.parse.uses_netloc.append("gemini")
|
|
|
|
|
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def fix_ipv6_url(url):
|
2019-08-13 16:56:15 +00:00
|
|
|
|
if not url.count(":") > 2: # Best way to detect them?
|
|
|
|
|
return url
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# If there's a pair of []s in there, it's probably fine as is.
|
|
|
|
|
if "[" in url and "]" in url:
|
|
|
|
|
return url
|
|
|
|
|
# Easiest case is a raw address, no schema, no path.
|
|
|
|
|
# Just wrap it in square brackets and whack a slash on the end
|
|
|
|
|
if "/" not in url:
|
|
|
|
|
return "[" + url + "]/"
|
|
|
|
|
# Now the trickier cases...
|
|
|
|
|
if "://" in url:
|
2021-12-20 15:32:54 +00:00
|
|
|
|
schema, schemaless = url.split("://",maxsplit=1)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
schema, schemaless = None, url
|
|
|
|
|
if "/" in schemaless:
|
|
|
|
|
netloc, rest = schemaless.split("/",1)
|
|
|
|
|
schemaless = "[" + netloc + "]" + "/" + rest
|
|
|
|
|
if schema:
|
|
|
|
|
return schema + "://" + schemaless
|
|
|
|
|
return schemaless
|
|
|
|
|
|
2019-08-13 16:56:15 +00:00
|
|
|
|
standard_ports = {
|
|
|
|
|
"gemini": 1965,
|
|
|
|
|
"gopher": 70,
|
2022-01-12 09:05:52 +00:00
|
|
|
|
"http" : 80,
|
|
|
|
|
"https" : 443,
|
2019-08-13 16:56:15 +00:00
|
|
|
|
}
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2019-08-13 16:56:15 +00:00
|
|
|
|
class GeminiItem():
|
|
|
|
|
|
|
|
|
|
def __init__(self, url, name=""):
|
2021-11-18 11:02:00 +00:00
|
|
|
|
if "://" not in url and ("./" not in url and url[0] != "/"):
|
2019-08-13 16:56:15 +00:00
|
|
|
|
url = "gemini://" + url
|
2021-12-20 15:32:54 +00:00
|
|
|
|
self.url = fix_ipv6_url(url).strip()
|
2019-08-13 16:56:15 +00:00
|
|
|
|
self.name = name
|
2022-01-08 13:16:55 +00:00
|
|
|
|
self.mime = None
|
2019-08-13 16:56:15 +00:00
|
|
|
|
parsed = urllib.parse.urlparse(self.url)
|
2022-01-08 21:18:54 +00:00
|
|
|
|
if "./" in url or url[0] == "/":
|
|
|
|
|
self.scheme = "localhost"
|
|
|
|
|
else:
|
|
|
|
|
self.scheme = parsed.scheme
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if self.scheme == "localhost":
|
|
|
|
|
self.local = True
|
|
|
|
|
self.host = None
|
2021-11-18 11:02:00 +00:00
|
|
|
|
h = self.url.split('/')
|
|
|
|
|
self.host = h[0:len(h)-1]
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self._cache_path = None
|
|
|
|
|
# localhost:/ is 11 char
|
2022-01-08 21:18:54 +00:00
|
|
|
|
if self.url.startswith("localhost://"):
|
|
|
|
|
self.path = self.url[11:]
|
|
|
|
|
else:
|
|
|
|
|
self.path = self.url
|
2022-01-08 20:32:25 +00:00
|
|
|
|
else:
|
|
|
|
|
self.path = parsed.path
|
|
|
|
|
self.local = False
|
|
|
|
|
self.host = parsed.hostname
|
2021-12-04 18:52:08 +00:00
|
|
|
|
#if not local, we create a local cache path.
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self._cache_path = os.path.expanduser(_CACHE_PATH + self.scheme + "/" + self.host + self.path)
|
2021-12-04 18:52:08 +00:00
|
|
|
|
# FIXME : this is a gross hack to give a name to
|
|
|
|
|
# index files. This will break if the index is not
|
2021-12-07 12:08:43 +00:00
|
|
|
|
# index.gmi. I don’t know how to know the real name
|
2021-12-16 09:43:25 +00:00
|
|
|
|
# of the file. But first, we need to ensure that the domain name
|
2021-12-20 15:32:54 +00:00
|
|
|
|
# finish by "/". Else, the cache will create a file, not a folder.
|
2022-01-09 19:41:47 +00:00
|
|
|
|
if self.scheme.startswith("http"):
|
|
|
|
|
index = "index.html"
|
|
|
|
|
else:
|
|
|
|
|
index = "index.gmi"
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if self.path == "" or os.path.isdir(self._cache_path):
|
|
|
|
|
if not self._cache_path.endswith("/"):
|
|
|
|
|
self._cache_path += "/"
|
2021-12-17 14:50:20 +00:00
|
|
|
|
if not self.url.endswith("/"):
|
|
|
|
|
self.url += "/"
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if self._cache_path.endswith("/"):
|
2022-01-09 19:41:47 +00:00
|
|
|
|
self._cache_path += index
|
2022-01-12 08:19:08 +00:00
|
|
|
|
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self.port = parsed.port or standard_ports.get(self.scheme, 0)
|
2022-01-10 10:19:29 +00:00
|
|
|
|
|
|
|
|
|
def get_title(self):
|
2022-01-08 20:32:25 +00:00
|
|
|
|
#small intelligence to try to find a good name for a capsule
|
|
|
|
|
#we try to find eithe ~username or /users/username
|
|
|
|
|
#else we fallback to hostname
|
2022-01-10 10:19:29 +00:00
|
|
|
|
if self.scheme == "localhost":
|
|
|
|
|
if self.name != "":
|
|
|
|
|
self.title = self.name
|
|
|
|
|
else:
|
|
|
|
|
self.title = self.path
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
self.title = self.host
|
|
|
|
|
if "user" in self.path:
|
|
|
|
|
i = 0
|
|
|
|
|
splitted = self.path.split("/")
|
|
|
|
|
while i < (len(splitted)-1):
|
|
|
|
|
if splitted[i].startswith("user"):
|
|
|
|
|
self.title = splitted[i+1]
|
|
|
|
|
i += 1
|
|
|
|
|
if "~" in self.path:
|
|
|
|
|
for pp in self.path.split("/"):
|
|
|
|
|
if pp.startswith("~"):
|
|
|
|
|
self.title = pp[1:]
|
|
|
|
|
return self.title
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
2021-12-18 09:16:19 +00:00
|
|
|
|
def is_cache_valid(self,validity=0):
|
2021-12-16 12:10:55 +00:00
|
|
|
|
# Validity is the acceptable time for
|
|
|
|
|
# a cache to be valid (in seconds)
|
2021-12-18 09:16:19 +00:00
|
|
|
|
# If 0, then any cache is considered as valid
|
|
|
|
|
# (use validity = 1 if you want to refresh everything)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if self.local:
|
|
|
|
|
return True
|
|
|
|
|
elif self._cache_path :
|
|
|
|
|
if os.path.exists(self._cache_path):
|
2021-12-18 09:16:19 +00:00
|
|
|
|
if validity > 0 :
|
2022-01-08 20:32:25 +00:00
|
|
|
|
last_modification = self.cache_last_modified()
|
2021-12-16 12:10:55 +00:00
|
|
|
|
now = time.time()
|
2021-12-17 11:08:27 +00:00
|
|
|
|
age = now - last_modification
|
2021-12-16 12:10:55 +00:00
|
|
|
|
return age < validity
|
|
|
|
|
else:
|
|
|
|
|
return True
|
2021-12-13 12:49:12 +00:00
|
|
|
|
else:
|
|
|
|
|
#Cache has not been build
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
#There’s not even a cache!
|
|
|
|
|
return False
|
|
|
|
|
|
2022-01-08 20:32:25 +00:00
|
|
|
|
def cache_last_modified(self):
|
|
|
|
|
if self._cache_path:
|
|
|
|
|
return os.path.getmtime(self._cache_path)
|
|
|
|
|
elif self.local:
|
|
|
|
|
return 0
|
|
|
|
|
else:
|
|
|
|
|
print("ERROR : NO CACHE in cache_last_modified")
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
def get_body(self,as_file=False):
|
|
|
|
|
if self.local:
|
|
|
|
|
path = self.path
|
|
|
|
|
elif self.is_cache_valid():
|
|
|
|
|
path = self._cache_path
|
|
|
|
|
else:
|
|
|
|
|
path = None
|
|
|
|
|
if path:
|
|
|
|
|
if as_file:
|
|
|
|
|
return path
|
|
|
|
|
else:
|
|
|
|
|
with open(path) as f:
|
|
|
|
|
body = f.read()
|
|
|
|
|
f.close()
|
2022-01-10 11:49:24 +00:00
|
|
|
|
return body
|
2022-01-08 20:32:25 +00:00
|
|
|
|
else:
|
|
|
|
|
print("ERROR: NO CACHE for %s" %self._cache_path)
|
|
|
|
|
return FIXME
|
2022-01-08 20:46:57 +00:00
|
|
|
|
|
|
|
|
|
def get_filename(self):
|
|
|
|
|
filename = os.path.basename(self._cache_path)
|
|
|
|
|
return filename
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
2022-01-09 14:27:02 +00:00
|
|
|
|
def write_body(self,body,mime):
|
2022-01-08 20:32:25 +00:00
|
|
|
|
## body is a copy of the raw gemtext
|
2022-01-09 10:35:09 +00:00
|
|
|
|
## Write_body() also create the cache !
|
2022-01-09 14:53:14 +00:00
|
|
|
|
# DEFAULT GEMINI MIME
|
|
|
|
|
if mime == "":
|
|
|
|
|
mime = "text/gemini; charset=utf-8"
|
|
|
|
|
mime, mime_options = cgi.parse_header(mime)
|
2022-01-09 15:12:07 +00:00
|
|
|
|
self.mime = mime
|
2022-01-09 14:27:02 +00:00
|
|
|
|
if self.mime and self.mime.startswith("text/"):
|
|
|
|
|
mode = "w"
|
|
|
|
|
else:
|
|
|
|
|
mode = "wb"
|
2022-01-08 20:32:25 +00:00
|
|
|
|
cache_dir = os.path.dirname(self._cache_path)
|
|
|
|
|
# If the subdirectory already exists as a file (not a folder)
|
|
|
|
|
# We remove it (happens when accessing URL/subfolder before
|
|
|
|
|
# URL/subfolder/file.gmi.
|
|
|
|
|
# This causes loss of data in the cache
|
|
|
|
|
# proper solution would be to save "sufolder" as "sufolder/index.gmi"
|
|
|
|
|
if os.path.isfile(cache_dir):
|
|
|
|
|
os.remove(cache_dir)
|
|
|
|
|
os.makedirs(cache_dir,exist_ok=True)
|
2022-01-09 14:27:02 +00:00
|
|
|
|
with open(self._cache_path, mode=mode) as f:
|
|
|
|
|
f.write(body)
|
|
|
|
|
f.close()
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_mime(self):
|
|
|
|
|
if self.is_cache_valid():
|
|
|
|
|
mime,encoding = mimetypes.guess_type(self._cache_path,strict=False)
|
|
|
|
|
#gmi Mimetype is not recognized yet
|
|
|
|
|
if not mime and _HAS_MAGIC :
|
|
|
|
|
mime = magic.from_file(self._cache_path,mime=True)
|
|
|
|
|
elif not _HAS_MAGIC :
|
|
|
|
|
print("Cannot guess the mime type of the file. Install Python-magic")
|
|
|
|
|
if mime.startswith("text"):
|
2022-01-09 19:41:47 +00:00
|
|
|
|
#by default, we consider it’s gemini except for html
|
|
|
|
|
if "html" not in mime:
|
|
|
|
|
mime = "text/gemini"
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self.mime = mime
|
|
|
|
|
return self.mime
|
|
|
|
|
|
|
|
|
|
def set_error(self,err):
|
|
|
|
|
# If we get an error, we want to keep an existing cache
|
|
|
|
|
# but we need to touch it or to create an empty one
|
|
|
|
|
# to avoid hitting the error at each refresh
|
|
|
|
|
if self.is_cache_valid():
|
|
|
|
|
os.utime(self._cache_path)
|
|
|
|
|
else:
|
|
|
|
|
cache_dir = os.path.dirname(self._cache_path)
|
|
|
|
|
if os.path.isdir(cache_dir):
|
|
|
|
|
with open(self._cache_path, "w") as cache:
|
|
|
|
|
cache.write(str(datetime.datetime.now())+"\n")
|
|
|
|
|
cache.write("ERROR while caching %s\n" %self.url)
|
|
|
|
|
cache.write(str(err))
|
|
|
|
|
cache.write("\n")
|
|
|
|
|
cache.close()
|
|
|
|
|
|
|
|
|
|
|
2019-08-13 17:36:58 +00:00
|
|
|
|
def root(self):
|
|
|
|
|
return GeminiItem(self._derive_url("/"))
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
2019-08-13 17:36:58 +00:00
|
|
|
|
def up(self):
|
2019-09-28 07:28:01 +00:00
|
|
|
|
pathbits = list(os.path.split(self.path.rstrip('/')))
|
2019-08-13 16:56:15 +00:00
|
|
|
|
# Don't try to go higher than root
|
|
|
|
|
if len(pathbits) == 1:
|
2019-09-28 07:22:01 +00:00
|
|
|
|
return self
|
2019-08-13 16:56:15 +00:00
|
|
|
|
# Get rid of bottom component
|
|
|
|
|
pathbits.pop()
|
|
|
|
|
new_path = os.path.join(*pathbits)
|
2019-08-13 17:36:58 +00:00
|
|
|
|
return GeminiItem(self._derive_url(new_path))
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
2019-08-13 17:36:58 +00:00
|
|
|
|
def query(self, query):
|
2020-05-12 20:00:17 +00:00
|
|
|
|
query = urllib.parse.quote(query)
|
2019-08-13 17:24:00 +00:00
|
|
|
|
return GeminiItem(self._derive_url(query=query))
|
|
|
|
|
|
|
|
|
|
def _derive_url(self, path="", query=""):
|
|
|
|
|
"""
|
|
|
|
|
A thin wrapper around urlunparse which avoids inserting standard ports
|
|
|
|
|
into URLs just to keep things clean.
|
|
|
|
|
"""
|
2019-08-13 17:39:55 +00:00
|
|
|
|
return urllib.parse.urlunparse((self.scheme,
|
2019-08-13 17:17:40 +00:00
|
|
|
|
self.host if self.port == standard_ports[self.scheme] else self.host + ":" + str(self.port),
|
2019-08-13 17:24:00 +00:00
|
|
|
|
path or self.path, "", query, ""))
|
2019-08-13 17:17:40 +00:00
|
|
|
|
|
2019-08-13 16:56:15 +00:00
|
|
|
|
def absolutise_url(self, relative_url):
|
|
|
|
|
"""
|
|
|
|
|
Convert a relative URL to an absolute URL by using the URL of this
|
|
|
|
|
GeminiItem as a base.
|
|
|
|
|
"""
|
2021-12-15 10:05:38 +00:00
|
|
|
|
abs_url = urllib.parse.urljoin(self.url, relative_url)
|
|
|
|
|
return abs_url
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
|
|
|
|
def to_map_line(self, name=None):
|
2022-01-08 13:16:55 +00:00
|
|
|
|
#SPECIFIC GEMINI
|
2019-08-13 16:56:15 +00:00
|
|
|
|
if name or self.name:
|
2019-09-28 06:04:45 +00:00
|
|
|
|
return "=> {} {}\n".format(self.url, name or self.name)
|
2019-08-13 16:56:15 +00:00
|
|
|
|
else:
|
2019-09-28 06:04:45 +00:00
|
|
|
|
return "=> {}\n".format(self.url)
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
|
|
|
|
@classmethod
|
2022-01-12 11:21:11 +00:00
|
|
|
|
#Create a GeminiItem based on a gemini line
|
|
|
|
|
#also makes relative URL absolute
|
2019-08-13 16:56:15 +00:00
|
|
|
|
def from_map_line(cls, line, origin_gi):
|
|
|
|
|
assert line.startswith("=>")
|
|
|
|
|
assert line[2:].strip()
|
|
|
|
|
bits = line[2:].strip().split(maxsplit=1)
|
|
|
|
|
bits[0] = origin_gi.absolutise_url(bits[0])
|
2022-01-12 11:21:11 +00:00
|
|
|
|
if looks_like_url(bits[0]):
|
|
|
|
|
try:
|
|
|
|
|
newgi = cls(*bits)
|
|
|
|
|
return newgi
|
|
|
|
|
except:
|
|
|
|
|
return None
|
|
|
|
|
else:
|
|
|
|
|
return None
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
|
|
|
|
CRLF = '\r\n'
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
# Cheap and cheerful URL detector
|
|
|
|
|
def looks_like_url(word):
|
2022-01-11 13:04:20 +00:00
|
|
|
|
try:
|
|
|
|
|
url = fix_ipv6_url(word).strip()
|
|
|
|
|
#print("looks_like_url before %s"%word)
|
|
|
|
|
#print("looks_like_url after %s"%url)
|
2022-01-12 08:19:08 +00:00
|
|
|
|
parsed = urllib.parse.urlparse(url)
|
|
|
|
|
#sometimes, urllib crashed only when requesting the port
|
|
|
|
|
port = parsed.port
|
2022-01-11 13:04:20 +00:00
|
|
|
|
start = word.startswith("gemini://") or word.startswith("http://")\
|
|
|
|
|
or word.startswith("https://")
|
|
|
|
|
return "." in word and start
|
|
|
|
|
except ValueError:
|
|
|
|
|
return False
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2020-08-31 19:18:15 +00:00
|
|
|
|
class UserAbortException(Exception):
|
|
|
|
|
pass
|
|
|
|
|
|
2019-08-13 16:56:15 +00:00
|
|
|
|
# GeminiClient Decorators
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def needs_gi(inner):
|
|
|
|
|
def outer(self, *args, **kwargs):
|
|
|
|
|
if not self.gi:
|
|
|
|
|
print("You need to 'go' somewhere, first")
|
|
|
|
|
return None
|
|
|
|
|
else:
|
|
|
|
|
return inner(self, *args, **kwargs)
|
|
|
|
|
outer.__doc__ = inner.__doc__
|
|
|
|
|
return outer
|
|
|
|
|
|
2020-04-07 20:46:05 +00:00
|
|
|
|
def restricted(inner):
|
|
|
|
|
def outer(self, *args, **kwargs):
|
|
|
|
|
if self.restricted:
|
|
|
|
|
print("Sorry, this command is not available in restricted mode!")
|
|
|
|
|
return None
|
|
|
|
|
else:
|
|
|
|
|
return inner(self, *args, **kwargs)
|
|
|
|
|
outer.__doc__ = inner.__doc__
|
|
|
|
|
return outer
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
class GeminiClient(cmd.Cmd):
|
|
|
|
|
|
2021-12-09 14:12:32 +00:00
|
|
|
|
def __init__(self, restricted=False, synconly=False):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
cmd.Cmd.__init__(self)
|
2020-05-10 20:51:33 +00:00
|
|
|
|
|
2020-05-23 11:17:12 +00:00
|
|
|
|
# Set umask so that nothing we create can be read by anybody else.
|
|
|
|
|
# The certificate cache and TOFU database contain "browser history"
|
|
|
|
|
# type sensitivie information.
|
2020-05-23 11:24:39 +00:00
|
|
|
|
os.umask(0o077)
|
2020-05-23 11:17:12 +00:00
|
|
|
|
|
2020-05-10 20:51:33 +00:00
|
|
|
|
# Find config directory
|
|
|
|
|
## Look for something pre-existing
|
2021-12-30 15:03:08 +00:00
|
|
|
|
for confdir in ("~/.offpunk/", "~/.config/offpunk/"):
|
2020-05-10 20:51:33 +00:00
|
|
|
|
confdir = os.path.expanduser(confdir)
|
|
|
|
|
if os.path.exists(confdir):
|
|
|
|
|
self.config_dir = confdir
|
|
|
|
|
break
|
|
|
|
|
## Otherwise, make one in .config if it exists
|
|
|
|
|
else:
|
|
|
|
|
if os.path.exists(os.path.expanduser("~/.config/")):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
self.config_dir = os.path.expanduser("~/.config/offpunk/")
|
2020-05-10 20:51:33 +00:00
|
|
|
|
else:
|
2021-12-30 15:03:08 +00:00
|
|
|
|
self.config_dir = os.path.expanduser("~/.offpunk/")
|
2020-05-23 11:18:37 +00:00
|
|
|
|
print("Creating config directory {}".format(self.config_dir))
|
2020-05-22 21:24:49 +00:00
|
|
|
|
os.makedirs(self.config_dir)
|
2020-05-10 20:51:33 +00:00
|
|
|
|
|
2021-12-30 15:03:08 +00:00
|
|
|
|
self.no_cert_prompt = "\x1b[38;5;76m" + "ON" + "\x1b[38;5;255m" + "> " + "\x1b[0m"
|
|
|
|
|
self.cert_prompt = "\x1b[38;5;202m" + "ON" + "\x1b[38;5;255m"
|
|
|
|
|
self.offline_prompt = "\x1b[38;5;76m" + "OFF" + "\x1b[38;5;255m" + "> " + "\x1b[0m"
|
2020-05-10 10:35:46 +00:00
|
|
|
|
self.prompt = self.no_cert_prompt
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.gi = None
|
|
|
|
|
self.history = []
|
|
|
|
|
self.hist_index = 0
|
|
|
|
|
self.idx_filename = ""
|
|
|
|
|
self.index = []
|
|
|
|
|
self.index_index = -1
|
|
|
|
|
self.lookup = self.index
|
|
|
|
|
self.marks = {}
|
|
|
|
|
self.page_index = 0
|
2019-10-15 19:12:32 +00:00
|
|
|
|
self.permanent_redirects = {}
|
2019-10-13 17:42:04 +00:00
|
|
|
|
self.previous_redirectors = set()
|
2021-12-09 14:12:32 +00:00
|
|
|
|
# Sync-only mode is restriced by design
|
|
|
|
|
self.restricted = restricted or synconly
|
|
|
|
|
self.synconly = synconly
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.tmp_filename = ""
|
|
|
|
|
self.visited_hosts = set()
|
2021-12-06 15:43:16 +00:00
|
|
|
|
self.offline_only = False
|
2021-12-09 14:12:32 +00:00
|
|
|
|
self.sync_only = False
|
2021-12-14 14:26:37 +00:00
|
|
|
|
self.tourfile = os.path.join(self.config_dir, "tour")
|
2021-12-14 15:07:02 +00:00
|
|
|
|
self.syncfile = os.path.join(self.config_dir, "to_fetch")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2020-05-10 10:59:26 +00:00
|
|
|
|
self.client_certs = {
|
|
|
|
|
"active": None
|
|
|
|
|
}
|
2020-05-10 11:44:40 +00:00
|
|
|
|
self.active_cert_domains = []
|
2020-05-11 20:22:24 +00:00
|
|
|
|
self.active_is_transient = False
|
|
|
|
|
self.transient_certs_created = []
|
2020-05-10 10:59:26 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.options = {
|
|
|
|
|
"debug" : False,
|
2019-08-13 10:04:39 +00:00
|
|
|
|
"ipv6" : True,
|
2021-04-27 07:53:58 +00:00
|
|
|
|
"timeout" : 600,
|
2021-12-10 10:27:48 +00:00
|
|
|
|
"short_timeout" : 5,
|
2020-02-03 20:34:17 +00:00
|
|
|
|
"width" : 80,
|
2019-10-14 16:47:02 +00:00
|
|
|
|
"auto_follow_redirects" : True,
|
2020-05-12 19:20:36 +00:00
|
|
|
|
"gopher_proxy" : None,
|
2020-05-19 21:14:09 +00:00
|
|
|
|
"tls_mode" : "tofu",
|
2021-08-25 06:03:12 +00:00
|
|
|
|
"http_proxy": None,
|
2021-12-13 13:30:40 +00:00
|
|
|
|
"offline_web" : None
|
2019-06-22 12:58:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self.log = {
|
|
|
|
|
"start_time": time.time(),
|
|
|
|
|
"requests": 0,
|
|
|
|
|
"ipv4_requests": 0,
|
|
|
|
|
"ipv6_requests": 0,
|
|
|
|
|
"bytes_recvd": 0,
|
|
|
|
|
"ipv4_bytes_recvd": 0,
|
|
|
|
|
"ipv6_bytes_recvd": 0,
|
|
|
|
|
"dns_failures": 0,
|
|
|
|
|
"refused_connections": 0,
|
|
|
|
|
"reset_connections": 0,
|
|
|
|
|
"timeouts": 0,
|
2020-09-01 19:14:17 +00:00
|
|
|
|
"cache_hits": 0,
|
2019-06-22 12:58:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
self._connect_to_tofu_db()
|
|
|
|
|
|
|
|
|
|
def _connect_to_tofu_db(self):
|
|
|
|
|
|
|
|
|
|
db_path = os.path.join(self.config_dir, "tofu.db")
|
|
|
|
|
self.db_conn = sqlite3.connect(db_path)
|
|
|
|
|
self.db_cur = self.db_conn.cursor()
|
|
|
|
|
|
|
|
|
|
self.db_cur.execute("""CREATE TABLE IF NOT EXISTS cert_cache
|
|
|
|
|
(hostname text, address text, fingerprint text,
|
|
|
|
|
first_seen date, last_seen date, count integer)""")
|
|
|
|
|
|
2020-09-01 21:27:59 +00:00
|
|
|
|
def _go_to_gi(self, gi, update_hist=True, check_cache=True, handle=True):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""This method might be considered "the heart of Offpunk".
|
2019-06-22 12:58:21 +00:00
|
|
|
|
Everything involved in fetching a gemini resource happens here:
|
2022-01-08 13:16:55 +00:00
|
|
|
|
sending the request over the network, parsing the response,
|
|
|
|
|
storing the response in a temporary file, choosing
|
|
|
|
|
and calling a handler program, and updating the history.
|
|
|
|
|
Nothing is returned."""
|
2019-08-14 18:16:58 +00:00
|
|
|
|
# Don't try to speak to servers running other protocols
|
2022-01-10 10:19:29 +00:00
|
|
|
|
if gi.scheme == "gopher" and not self.options.get("gopher_proxy", None)\
|
2021-12-10 10:27:48 +00:00
|
|
|
|
and not self.sync_only:
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("""Offpunk does not speak Gopher natively.
|
2020-05-12 19:20:36 +00:00
|
|
|
|
However, you can use `set gopher_proxy hostname:port` to tell it about a
|
|
|
|
|
Gopher-to-Gemini proxy (such as a running Agena instance), in which case
|
|
|
|
|
you'll be able to transparently follow links to Gopherspace!""")
|
|
|
|
|
return
|
2021-11-18 11:02:00 +00:00
|
|
|
|
elif gi.local:
|
|
|
|
|
if os.path.exists(gi.path):
|
|
|
|
|
with open(gi.path,'r') as f:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self._handle_gemtext(gi)
|
2021-11-18 11:02:00 +00:00
|
|
|
|
self.gi = gi
|
|
|
|
|
self._update_history(gi)
|
|
|
|
|
return
|
|
|
|
|
else:
|
2022-01-08 21:18:54 +00:00
|
|
|
|
print("Sorry, file %s does not exist."%gi.path)
|
2021-11-18 11:02:00 +00:00
|
|
|
|
return
|
2022-01-10 10:19:29 +00:00
|
|
|
|
elif gi.scheme not in ("gemini", "gopher", "http", "https") and not self.sync_only:
|
2020-06-09 20:13:42 +00:00
|
|
|
|
print("Sorry, no support for {} links.".format(gi.scheme))
|
2019-08-14 18:16:58 +00:00
|
|
|
|
return
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2019-10-15 19:12:32 +00:00
|
|
|
|
# Obey permanent redirects
|
|
|
|
|
if gi.url in self.permanent_redirects:
|
|
|
|
|
new_gi = GeminiItem(self.permanent_redirects[gi.url], name=gi.name)
|
|
|
|
|
self._go_to_gi(new_gi)
|
|
|
|
|
return
|
2020-05-10 20:51:33 +00:00
|
|
|
|
|
2022-01-08 20:32:25 +00:00
|
|
|
|
# Use cache or mark as to_fetch if resource is not cached
|
|
|
|
|
# Why is this code useful ? It set the mimetype !
|
|
|
|
|
if self.offline_only:
|
|
|
|
|
|
|
|
|
|
if not gi.is_cache_valid():
|
2022-01-10 20:29:19 +00:00
|
|
|
|
print("%s not available, marked for syncing"%gi.url)
|
2021-12-14 15:07:02 +00:00
|
|
|
|
with open(self.syncfile,mode='a') as sf:
|
2021-12-20 15:32:54 +00:00
|
|
|
|
line = gi.url.strip() + '\n'
|
2021-12-14 15:07:02 +00:00
|
|
|
|
sf.write(line)
|
|
|
|
|
sf.close()
|
2021-12-06 16:03:22 +00:00
|
|
|
|
return
|
|
|
|
|
|
2022-01-08 20:32:25 +00:00
|
|
|
|
elif not self.offline_only:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
try:
|
2022-01-09 15:12:07 +00:00
|
|
|
|
if gi.scheme in ("http", "https"):
|
2022-01-10 10:19:29 +00:00
|
|
|
|
if _DO_HTTP:
|
|
|
|
|
gi = self._fetch_http(gi)
|
|
|
|
|
else:
|
|
|
|
|
print("Install python3-requests to handle http requests natively")
|
|
|
|
|
webbrowser.open_new_tab(gi.url)
|
|
|
|
|
return
|
2022-01-09 15:12:07 +00:00
|
|
|
|
else:
|
|
|
|
|
gi = self._fetch_over_network(gi)
|
2020-08-31 19:18:15 +00:00
|
|
|
|
except UserAbortException:
|
|
|
|
|
return
|
2020-08-30 18:21:15 +00:00
|
|
|
|
except Exception as err:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
gi.set_error(err)
|
2020-08-30 18:21:15 +00:00
|
|
|
|
# Print an error message
|
2022-01-08 13:16:55 +00:00
|
|
|
|
# we fail silently when sync_only
|
2021-12-14 12:29:09 +00:00
|
|
|
|
print_error = not self.sync_only
|
2020-08-30 18:21:15 +00:00
|
|
|
|
if isinstance(err, socket.gaierror):
|
|
|
|
|
self.log["dns_failures"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
|
|
|
|
print("ERROR: DNS error!")
|
2020-08-30 18:21:15 +00:00
|
|
|
|
elif isinstance(err, ConnectionRefusedError):
|
|
|
|
|
self.log["refused_connections"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
|
|
|
|
print("ERROR1: Connection refused!")
|
2020-08-30 18:21:15 +00:00
|
|
|
|
elif isinstance(err, ConnectionResetError):
|
|
|
|
|
self.log["reset_connections"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
|
|
|
|
print("ERROR2: Connection reset!")
|
2020-08-30 18:21:15 +00:00
|
|
|
|
elif isinstance(err, (TimeoutError, socket.timeout)):
|
|
|
|
|
self.log["timeouts"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
2021-12-10 10:27:48 +00:00
|
|
|
|
print("""ERROR3: Connection timed out!
|
|
|
|
|
Slow internet connection? Use 'set timeout' to be more patient.""")
|
2021-12-16 09:43:25 +00:00
|
|
|
|
elif isinstance(err, FileExistsError):
|
|
|
|
|
print("""ERROR5: Trying to create a directory which already exists
|
|
|
|
|
in the cache : """)
|
|
|
|
|
print(err)
|
2020-08-30 18:21:15 +00:00
|
|
|
|
else:
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
2021-12-10 10:27:48 +00:00
|
|
|
|
print("ERROR4: " + str(err))
|
2020-08-30 18:21:15 +00:00
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
# Pass file to handler, unless we were asked not to
|
2022-01-08 13:16:55 +00:00
|
|
|
|
#SPECIFIC GEMINI : default handler should be provided by the GI.
|
2022-01-10 08:45:21 +00:00
|
|
|
|
if gi and handle :
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if gi.get_mime() == "text/gemini":
|
|
|
|
|
self._handle_gemtext(gi, display=not self.sync_only)
|
2022-01-10 11:49:24 +00:00
|
|
|
|
elif gi.get_mime() == "text/html":
|
|
|
|
|
self._handle_html(gi,display=not self.sync_only)
|
2021-12-14 12:29:09 +00:00
|
|
|
|
elif not self.sync_only :
|
2022-01-08 20:32:25 +00:00
|
|
|
|
cmd_str = self._get_handler_cmd(gi.get_mime())
|
2020-08-30 18:21:15 +00:00
|
|
|
|
try:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
# get tmpfile from gi !
|
|
|
|
|
tmpfile = gi.get_body(as_file=True)
|
2020-08-31 19:17:06 +00:00
|
|
|
|
subprocess.call(shlex.split(cmd_str % tmpfile))
|
2020-08-30 18:21:15 +00:00
|
|
|
|
except FileNotFoundError:
|
|
|
|
|
print("Handler program %s not found!" % shlex.split(cmd_str)[0])
|
|
|
|
|
print("You can use the ! command to specify another handler program or pipeline.")
|
|
|
|
|
|
|
|
|
|
# Update state
|
|
|
|
|
self.gi = gi
|
|
|
|
|
if update_hist:
|
|
|
|
|
self._update_history(gi)
|
|
|
|
|
|
2022-01-09 14:53:14 +00:00
|
|
|
|
|
|
|
|
|
def _fetch_http(self,gi):
|
2022-01-09 15:12:07 +00:00
|
|
|
|
response = requests.get(gi.url)
|
2022-01-09 14:53:14 +00:00
|
|
|
|
mime = response.headers['content-type']
|
|
|
|
|
body = response.content
|
2022-01-10 10:19:29 +00:00
|
|
|
|
if "text/" in mime:
|
2022-01-09 14:53:14 +00:00
|
|
|
|
body = response.text
|
|
|
|
|
else:
|
|
|
|
|
body = response.content
|
|
|
|
|
gi.write_body(body,mime)
|
2022-01-09 15:12:07 +00:00
|
|
|
|
return gi
|
2022-01-09 14:53:14 +00:00
|
|
|
|
|
2022-01-09 19:41:47 +00:00
|
|
|
|
# fetch_over_network will modify with gi.write_body(body,mime)
|
2022-01-09 10:35:09 +00:00
|
|
|
|
# before returning the gi
|
2020-08-30 18:21:15 +00:00
|
|
|
|
def _fetch_over_network(self, gi):
|
2021-12-06 16:03:22 +00:00
|
|
|
|
|
2020-05-11 20:22:24 +00:00
|
|
|
|
# Be careful with client certificates!
|
|
|
|
|
# Are we crossing a domain boundary?
|
2020-05-10 11:44:40 +00:00
|
|
|
|
if self.active_cert_domains and gi.host not in self.active_cert_domains:
|
2020-05-10 20:51:33 +00:00
|
|
|
|
if self.active_is_transient:
|
|
|
|
|
print("Permanently delete currently active transient certificate?")
|
|
|
|
|
resp = input("Y/N? ")
|
|
|
|
|
if resp.strip().lower() in ("y", "yes"):
|
|
|
|
|
print("Destroying certificate.")
|
|
|
|
|
self._deactivate_client_cert()
|
|
|
|
|
else:
|
|
|
|
|
print("Staying here.")
|
2020-08-31 19:18:15 +00:00
|
|
|
|
raise UserAbortException()
|
2020-05-10 11:44:40 +00:00
|
|
|
|
else:
|
2020-05-10 20:51:33 +00:00
|
|
|
|
print("PRIVACY ALERT: Deactivate client cert before connecting to a new domain?")
|
|
|
|
|
resp = input("Y/N? ")
|
|
|
|
|
if resp.strip().lower() in ("n", "no"):
|
|
|
|
|
print("Keeping certificate active for {}".format(gi.host))
|
|
|
|
|
else:
|
|
|
|
|
print("Deactivating certificate.")
|
|
|
|
|
self._deactivate_client_cert()
|
|
|
|
|
|
2020-05-10 12:19:12 +00:00
|
|
|
|
# Suggest reactivating previous certs
|
|
|
|
|
if not self.client_certs["active"] and gi.host in self.client_certs:
|
|
|
|
|
print("PRIVACY ALERT: Reactivate previously used client cert for {}?".format(gi.host))
|
|
|
|
|
resp = input("Y/N? ")
|
|
|
|
|
if resp.strip().lower() in ("y", "yes"):
|
2020-05-10 15:00:30 +00:00
|
|
|
|
self._activate_client_cert(*self.client_certs[gi.host])
|
2020-05-10 12:19:12 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Remaining unidentified.")
|
|
|
|
|
self.client_certs.pop(gi.host)
|
|
|
|
|
|
2020-08-30 18:21:15 +00:00
|
|
|
|
# Is this a local file?
|
2021-11-18 11:02:00 +00:00
|
|
|
|
if gi.local:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
address, f = None, open(gi.path, "rb")
|
|
|
|
|
else:
|
|
|
|
|
address, f = self._send_request(gi)
|
|
|
|
|
|
|
|
|
|
# Spec dictates <META> should not exceed 1024 bytes,
|
|
|
|
|
# so maximum valid header length is 1027 bytes.
|
|
|
|
|
header = f.readline(1027)
|
|
|
|
|
header = header.decode("UTF-8")
|
|
|
|
|
if not header or header[-1] != '\n':
|
|
|
|
|
raise RuntimeError("Received invalid header from server!")
|
|
|
|
|
header = header.strip()
|
|
|
|
|
self._debug("Response header: %s." % header)
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Validate header
|
2019-08-13 17:00:15 +00:00
|
|
|
|
status, meta = header.split(maxsplit=1)
|
2020-05-16 13:59:05 +00:00
|
|
|
|
if len(meta) > 1024 or len(status) != 2 or not status.isnumeric():
|
2019-08-13 15:09:29 +00:00
|
|
|
|
f.close()
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Received invalid header from server!")
|
2019-08-13 15:09:29 +00:00
|
|
|
|
|
2019-10-13 17:42:04 +00:00
|
|
|
|
# Update redirect loop/maze escaping state
|
|
|
|
|
if not status.startswith("3"):
|
|
|
|
|
self.previous_redirectors = set()
|
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Handle non-SUCCESS headers, which don't have a response body
|
|
|
|
|
# Inputs
|
|
|
|
|
if status.startswith("1"):
|
2021-12-09 17:05:46 +00:00
|
|
|
|
if self.sync_only:
|
|
|
|
|
return None
|
2020-06-07 17:13:00 +00:00
|
|
|
|
else:
|
2021-12-09 17:05:46 +00:00
|
|
|
|
print(meta)
|
|
|
|
|
if status == "11":
|
|
|
|
|
user_input = getpass.getpass("> ")
|
|
|
|
|
else:
|
|
|
|
|
user_input = input("> ")
|
|
|
|
|
return self._fetch_over_network(gi.query(user_input))
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Redirects
|
2019-08-08 18:23:58 +00:00
|
|
|
|
elif status.startswith("3"):
|
2019-08-13 17:00:15 +00:00
|
|
|
|
new_gi = GeminiItem(gi.absolutise_url(meta))
|
2020-08-18 19:06:12 +00:00
|
|
|
|
if new_gi.url == gi.url:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("URL redirects to itself!")
|
2020-08-18 19:06:12 +00:00
|
|
|
|
elif new_gi.url in self.previous_redirectors:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Caught in redirect loop!")
|
2019-10-13 17:42:04 +00:00
|
|
|
|
elif len(self.previous_redirectors) == _MAX_REDIRECTS:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Refusing to follow more than %d consecutive redirects!" % _MAX_REDIRECTS)
|
2020-05-23 10:53:20 +00:00
|
|
|
|
# Never follow cross-domain redirects without asking
|
|
|
|
|
elif new_gi.host != gi.host:
|
|
|
|
|
follow = input("Follow cross-domain redirect to %s? (y/n) " % new_gi.url)
|
2020-05-31 12:06:23 +00:00
|
|
|
|
# Never follow cross-protocol redirects without asking
|
|
|
|
|
elif new_gi.scheme != gi.scheme:
|
|
|
|
|
follow = input("Follow cross-protocol redirect to %s? (y/n) " % new_gi.url)
|
2020-05-31 12:23:30 +00:00
|
|
|
|
# Don't follow *any* redirect without asking if auto-follow is off
|
2019-10-15 19:12:32 +00:00
|
|
|
|
elif not self.options["auto_follow_redirects"]:
|
|
|
|
|
follow = input("Follow redirect to %s? (y/n) " % new_gi.url)
|
2020-05-31 12:23:30 +00:00
|
|
|
|
# Otherwise, follow away
|
|
|
|
|
else:
|
2020-05-31 12:24:23 +00:00
|
|
|
|
follow = "yes"
|
2020-05-31 12:06:23 +00:00
|
|
|
|
if follow.strip().lower() not in ("y", "yes"):
|
2020-08-31 19:18:15 +00:00
|
|
|
|
raise UserAbortException()
|
2020-05-31 12:06:23 +00:00
|
|
|
|
self._debug("Following redirect to %s." % new_gi.url)
|
|
|
|
|
self._debug("This is consecutive redirect number %d." % len(self.previous_redirectors))
|
|
|
|
|
self.previous_redirectors.add(gi.url)
|
|
|
|
|
if status == "31":
|
|
|
|
|
# Permanent redirect
|
|
|
|
|
self.permanent_redirects[gi.url] = new_gi.url
|
2020-08-30 18:21:15 +00:00
|
|
|
|
return self._fetch_over_network(new_gi)
|
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Errors
|
2019-08-08 18:23:58 +00:00
|
|
|
|
elif status.startswith("4") or status.startswith("5"):
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError(meta)
|
|
|
|
|
|
2019-08-11 19:26:30 +00:00
|
|
|
|
# Client cert
|
|
|
|
|
elif status.startswith("6"):
|
2020-08-31 19:18:15 +00:00
|
|
|
|
self._handle_cert_request(meta)
|
|
|
|
|
return self._fetch_over_network(gi)
|
2020-08-18 19:41:51 +00:00
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Invalid status
|
|
|
|
|
elif not status.startswith("2"):
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Server returned undefined status code %s!" % status)
|
2019-08-13 15:09:29 +00:00
|
|
|
|
|
|
|
|
|
# If we're here, this must be a success and there's a response body
|
|
|
|
|
assert status.startswith("2")
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
2019-08-13 17:00:15 +00:00
|
|
|
|
mime = meta
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Read the response body over the network
|
2022-01-09 15:12:07 +00:00
|
|
|
|
fbody = f.read()
|
|
|
|
|
# DEFAULT GEMINI MIME
|
|
|
|
|
if mime == "":
|
|
|
|
|
mime = "text/gemini; charset=utf-8"
|
|
|
|
|
shortmime, mime_options = cgi.parse_header(mime)
|
|
|
|
|
if "charset" in mime_options:
|
|
|
|
|
try:
|
|
|
|
|
codecs.lookup(mime_options["charset"])
|
|
|
|
|
except LookupError:
|
|
|
|
|
raise RuntimeError("Header declared unknown encoding %s" % value)
|
|
|
|
|
if shortmime.startswith("text/"):
|
2022-01-11 21:26:57 +00:00
|
|
|
|
#Get the charset and default to UTF-8 in none
|
2022-01-09 15:12:07 +00:00
|
|
|
|
encoding = mime_options.get("charset", "UTF-8")
|
|
|
|
|
try:
|
|
|
|
|
body = fbody.decode(encoding)
|
|
|
|
|
except UnicodeError:
|
|
|
|
|
raise RuntimeError("Could not decode response body using %s\
|
|
|
|
|
encoding declared in header!" % encoding)
|
2022-01-13 09:06:10 +00:00
|
|
|
|
else:
|
|
|
|
|
body = fbody
|
2022-01-09 14:27:02 +00:00
|
|
|
|
gi.write_body(body,mime)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
return gi
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def _send_request(self, gi):
|
|
|
|
|
"""Send a selector to a given host and port.
|
|
|
|
|
Returns the resolved address and binary file with the reply."""
|
2019-08-12 14:14:42 +00:00
|
|
|
|
if gi.scheme == "gemini":
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# For Gemini requests, connect to the host and port specified in the URL
|
|
|
|
|
host, port = gi.host, gi.port
|
2019-08-12 14:14:42 +00:00
|
|
|
|
elif gi.scheme == "gopher":
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# For Gopher requests, use the configured proxy
|
|
|
|
|
host, port = self.options["gopher_proxy"].rsplit(":", 1)
|
|
|
|
|
self._debug("Using gopher proxy: " + self.options["gopher_proxy"])
|
2021-08-25 06:03:12 +00:00
|
|
|
|
elif gi.scheme in ("http", "https"):
|
|
|
|
|
host, port = self.options["http_proxy"].rsplit(":",1)
|
|
|
|
|
self._debug("Using http proxy: " + self.options["http_proxy"])
|
2020-05-10 10:35:46 +00:00
|
|
|
|
# Do DNS resolution
|
2019-08-13 10:04:07 +00:00
|
|
|
|
addresses = self._get_addresses(host, port)
|
2020-05-10 10:35:46 +00:00
|
|
|
|
|
|
|
|
|
# Prepare TLS context
|
|
|
|
|
protocol = ssl.PROTOCOL_TLS if sys.version_info.minor >=6 else ssl.PROTOCOL_TLSv1_2
|
|
|
|
|
context = ssl.SSLContext(protocol)
|
2020-05-19 21:14:09 +00:00
|
|
|
|
# Use CAs or TOFU
|
|
|
|
|
if self.options["tls_mode"] == "ca":
|
|
|
|
|
context.verify_mode = ssl.CERT_REQUIRED
|
|
|
|
|
context.check_hostname = True
|
|
|
|
|
context.load_default_certs()
|
|
|
|
|
else:
|
|
|
|
|
context.check_hostname = False
|
|
|
|
|
context.verify_mode = ssl.CERT_NONE
|
2020-05-10 10:35:46 +00:00
|
|
|
|
# Impose minimum TLS version
|
|
|
|
|
## In 3.7 and above, this is easy...
|
|
|
|
|
if sys.version_info.minor >= 7:
|
|
|
|
|
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
|
|
|
## Otherwise, it seems very hard...
|
|
|
|
|
## The below is less strict than it ought to be, but trying to disable
|
|
|
|
|
## TLS v1.1 here using ssl.OP_NO_TLSv1_1 produces unexpected failures
|
|
|
|
|
## with recent versions of OpenSSL. What a mess...
|
|
|
|
|
else:
|
|
|
|
|
context.options |= ssl.OP_NO_SSLv3
|
|
|
|
|
context.options |= ssl.OP_NO_SSLv2
|
|
|
|
|
# Try to enforce sensible ciphers
|
|
|
|
|
try:
|
2020-06-04 14:21:11 +00:00
|
|
|
|
context.set_ciphers("AESGCM+ECDHE:AESGCM+DHE:CHACHA20+ECDHE:CHACHA20+DHE:!DSS:!SHA1:!MD5:@STRENGTH")
|
2020-05-10 10:35:46 +00:00
|
|
|
|
except ssl.SSLError:
|
|
|
|
|
# Rely on the server to only support sensible things, I guess...
|
|
|
|
|
pass
|
|
|
|
|
# Load client certificate if needed
|
2020-05-10 10:59:26 +00:00
|
|
|
|
if self.client_certs["active"]:
|
|
|
|
|
certfile, keyfile = self.client_certs["active"]
|
|
|
|
|
context.load_cert_chain(certfile, keyfile)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Connect to remote host by any address possible
|
|
|
|
|
err = None
|
|
|
|
|
for address in addresses:
|
|
|
|
|
self._debug("Connecting to: " + str(address[4]))
|
|
|
|
|
s = socket.socket(address[0], address[1])
|
2021-12-10 10:27:48 +00:00
|
|
|
|
if self.sync_only:
|
|
|
|
|
timeout = self.options["short_timeout"]
|
|
|
|
|
else:
|
|
|
|
|
timeout = self.options["timeout"]
|
|
|
|
|
s.settimeout(timeout)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
s = context.wrap_socket(s, server_hostname = gi.host)
|
|
|
|
|
try:
|
|
|
|
|
s.connect(address[4])
|
|
|
|
|
break
|
|
|
|
|
except OSError as e:
|
|
|
|
|
err = e
|
|
|
|
|
else:
|
|
|
|
|
# If we couldn't connect to *any* of the addresses, just
|
|
|
|
|
# bubble up the exception from the last attempt and deny
|
|
|
|
|
# knowledge of earlier failures.
|
|
|
|
|
raise err
|
2019-08-18 19:59:49 +00:00
|
|
|
|
|
2020-04-12 19:20:29 +00:00
|
|
|
|
if sys.version_info.minor >=5:
|
|
|
|
|
self._debug("Established {} connection.".format(s.version()))
|
2019-08-18 19:59:49 +00:00
|
|
|
|
self._debug("Cipher is: {}.".format(s.cipher()))
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
# Do TOFU
|
2020-05-19 21:14:09 +00:00
|
|
|
|
if self.options["tls_mode"] != "ca":
|
|
|
|
|
cert = s.getpeercert(binary_form=True)
|
|
|
|
|
self._validate_cert(address[4][0], host, cert)
|
2020-05-16 16:58:53 +00:00
|
|
|
|
|
2020-05-10 11:44:40 +00:00
|
|
|
|
# Remember that we showed the current cert to this domain...
|
|
|
|
|
if self.client_certs["active"]:
|
|
|
|
|
self.active_cert_domains.append(gi.host)
|
|
|
|
|
self.client_certs[gi.host] = self.client_certs["active"]
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Send request and wrap response in a file descriptor
|
2019-08-13 16:56:15 +00:00
|
|
|
|
self._debug("Sending %s<CRLF>" % gi.url)
|
|
|
|
|
s.sendall((gi.url + CRLF).encode("UTF-8"))
|
2022-01-08 20:32:25 +00:00
|
|
|
|
mf= s.makefile(mode = "rb")
|
|
|
|
|
return address, mf
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2019-08-12 14:14:42 +00:00
|
|
|
|
def _get_addresses(self, host, port):
|
|
|
|
|
# DNS lookup - will get IPv4 and IPv6 records if IPv6 is enabled
|
|
|
|
|
if ":" in host:
|
|
|
|
|
# This is likely a literal IPv6 address, so we can *only* ask for
|
|
|
|
|
# IPv6 addresses or getaddrinfo will complain
|
|
|
|
|
family_mask = socket.AF_INET6
|
|
|
|
|
elif socket.has_ipv6 and self.options["ipv6"]:
|
|
|
|
|
# Accept either IPv4 or IPv6 addresses
|
|
|
|
|
family_mask = 0
|
|
|
|
|
else:
|
|
|
|
|
# IPv4 only
|
|
|
|
|
family_mask = socket.AF_INET
|
|
|
|
|
addresses = socket.getaddrinfo(host, port, family=family_mask,
|
|
|
|
|
type=socket.SOCK_STREAM)
|
|
|
|
|
# Sort addresses so IPv6 ones come first
|
|
|
|
|
addresses.sort(key=lambda add: add[0] == socket.AF_INET6, reverse=True)
|
|
|
|
|
|
|
|
|
|
return addresses
|
|
|
|
|
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2020-08-31 19:18:15 +00:00
|
|
|
|
def _handle_cert_request(self, meta):
|
|
|
|
|
|
|
|
|
|
# Don't do client cert stuff in restricted mode, as in principle
|
|
|
|
|
# it could be used to fill up the disk by creating a whole lot of
|
|
|
|
|
# certificates
|
|
|
|
|
if self.restricted:
|
|
|
|
|
print("The server is requesting a client certificate.")
|
|
|
|
|
print("These are not supported in restricted mode, sorry.")
|
|
|
|
|
raise UserAbortException()
|
|
|
|
|
|
|
|
|
|
print("SERVER SAYS: ", meta)
|
|
|
|
|
# Present different messages for different 6x statuses, but
|
|
|
|
|
# handle them the same.
|
|
|
|
|
if status in ("64", "65"):
|
|
|
|
|
print("The server rejected your certificate because it is either expired or not yet valid.")
|
|
|
|
|
elif status == "63":
|
|
|
|
|
print("The server did not accept your certificate.")
|
|
|
|
|
print("You may need to e.g. coordinate with the admin to get your certificate fingerprint whitelisted.")
|
|
|
|
|
else:
|
|
|
|
|
print("The site {} is requesting a client certificate.".format(gi.host))
|
|
|
|
|
print("This will allow the site to recognise you across requests.")
|
|
|
|
|
|
|
|
|
|
# Give the user choices
|
|
|
|
|
print("What do you want to do?")
|
|
|
|
|
print("1. Give up.")
|
|
|
|
|
print("2. Generate a new transient certificate.")
|
|
|
|
|
print("3. Generate a new persistent certificate.")
|
|
|
|
|
print("4. Load a previously generated persistent.")
|
|
|
|
|
print("5. Load certificate from an external file.")
|
|
|
|
|
choice = input("> ").strip()
|
|
|
|
|
if choice == "2":
|
|
|
|
|
self._generate_transient_cert_cert()
|
|
|
|
|
elif choice == "3":
|
|
|
|
|
self._generate_persistent_client_cert()
|
|
|
|
|
elif choice == "4":
|
|
|
|
|
self._choose_client_cert()
|
|
|
|
|
elif choice == "5":
|
|
|
|
|
self._load_client_cert()
|
|
|
|
|
else:
|
|
|
|
|
print("Giving up.")
|
|
|
|
|
raise UserAbortException()
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
def _validate_cert(self, address, host, cert):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Validate a TLS certificate in TOFU mode.
|
|
|
|
|
|
|
|
|
|
If the cryptography module is installed:
|
|
|
|
|
- Check the certificate Common Name or SAN matches `host`
|
|
|
|
|
- Check the certificate's not valid before date is in the past
|
|
|
|
|
- Check the certificate's not valid after date is in the future
|
2020-05-17 18:38:06 +00:00
|
|
|
|
|
2020-06-07 18:42:19 +00:00
|
|
|
|
Whether the cryptography module is installed or not, check the
|
|
|
|
|
certificate's fingerprint against the TOFU database to see if we've
|
|
|
|
|
previously encountered a different certificate for this IP address and
|
|
|
|
|
hostname.
|
|
|
|
|
"""
|
2020-05-28 19:01:04 +00:00
|
|
|
|
now = datetime.datetime.utcnow()
|
2020-05-17 18:38:06 +00:00
|
|
|
|
if _HAS_CRYPTOGRAPHY:
|
|
|
|
|
# Using the cryptography module we can get detailed access
|
|
|
|
|
# to the properties of even self-signed certs, unlike in
|
|
|
|
|
# the standard ssl library...
|
|
|
|
|
c = x509.load_der_x509_certificate(cert, _BACKEND)
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
sha = hashlib.sha256()
|
|
|
|
|
sha.update(cert)
|
|
|
|
|
fingerprint = sha.hexdigest()
|
|
|
|
|
|
|
|
|
|
# Have we been here before?
|
|
|
|
|
self.db_cur.execute("""SELECT fingerprint, first_seen, last_seen, count
|
|
|
|
|
FROM cert_cache
|
|
|
|
|
WHERE hostname=? AND address=?""", (host, address))
|
|
|
|
|
cached_certs = self.db_cur.fetchall()
|
|
|
|
|
|
|
|
|
|
# If so, check for a match
|
|
|
|
|
if cached_certs:
|
|
|
|
|
max_count = 0
|
2020-05-23 10:53:02 +00:00
|
|
|
|
most_frequent_cert = None
|
2020-05-16 16:58:53 +00:00
|
|
|
|
for cached_fingerprint, first, last, count in cached_certs:
|
|
|
|
|
if count > max_count:
|
|
|
|
|
max_count = count
|
2020-05-23 10:53:02 +00:00
|
|
|
|
most_frequent_cert = cached_fingerprint
|
2020-05-16 16:58:53 +00:00
|
|
|
|
if fingerprint == cached_fingerprint:
|
|
|
|
|
# Matched!
|
|
|
|
|
self._debug("TOFU: Accepting previously seen ({} times) certificate {}".format(count, fingerprint))
|
|
|
|
|
self.db_cur.execute("""UPDATE cert_cache
|
|
|
|
|
SET last_seen=?, count=?
|
|
|
|
|
WHERE hostname=? AND address=? AND fingerprint=?""",
|
|
|
|
|
(now, count+1, host, address, fingerprint))
|
2020-05-17 12:02:36 +00:00
|
|
|
|
self.db_conn.commit()
|
2020-05-16 16:58:53 +00:00
|
|
|
|
break
|
|
|
|
|
else:
|
2020-05-23 10:53:02 +00:00
|
|
|
|
if _HAS_CRYPTOGRAPHY:
|
|
|
|
|
# Load the most frequently seen certificate to see if it has
|
|
|
|
|
# expired
|
|
|
|
|
certdir = os.path.join(self.config_dir, "cert_cache")
|
|
|
|
|
with open(os.path.join(certdir, most_frequent_cert+".crt"), "rb") as fp:
|
|
|
|
|
previous_cert = fp.read()
|
|
|
|
|
previous_cert = x509.load_der_x509_certificate(previous_cert, _BACKEND)
|
|
|
|
|
previous_ttl = previous_cert.not_valid_after - now
|
|
|
|
|
print(previous_ttl)
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
self._debug("TOFU: Unrecognised certificate {}! Raising the alarm...".format(fingerprint))
|
|
|
|
|
print("****************************************")
|
|
|
|
|
print("[SECURITY WARNING] Unrecognised certificate!")
|
|
|
|
|
print("The certificate presented for {} ({}) has never been seen before.".format(host, address))
|
|
|
|
|
print("This MIGHT be a Man-in-the-Middle attack.")
|
2020-05-23 10:53:02 +00:00
|
|
|
|
print("A different certificate has previously been seen {} times.".format(max_count))
|
|
|
|
|
if _HAS_CRYPTOGRAPHY:
|
|
|
|
|
if previous_ttl < datetime.timedelta():
|
|
|
|
|
print("That certificate has expired, which reduces suspicion somewhat.")
|
|
|
|
|
else:
|
|
|
|
|
print("That certificate is still valid for: {}".format(previous_ttl))
|
2020-05-16 16:58:53 +00:00
|
|
|
|
print("****************************************")
|
|
|
|
|
print("Attempt to verify the new certificate fingerprint out-of-band:")
|
|
|
|
|
print(fingerprint)
|
|
|
|
|
choice = input("Accept this new certificate? Y/N ").strip().lower()
|
|
|
|
|
if choice in ("y", "yes"):
|
|
|
|
|
self.db_cur.execute("""INSERT INTO cert_cache
|
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
|
|
|
(host, address, fingerprint, now, now, 1))
|
2020-05-17 12:02:36 +00:00
|
|
|
|
self.db_conn.commit()
|
2020-05-17 16:35:35 +00:00
|
|
|
|
with open(os.path.join(certdir, fingerprint+".crt"), "wb") as fp:
|
|
|
|
|
fp.write(cert)
|
2020-05-16 16:58:53 +00:00
|
|
|
|
else:
|
|
|
|
|
raise Exception("TOFU Failure!")
|
|
|
|
|
|
|
|
|
|
# If not, cache this cert
|
|
|
|
|
else:
|
|
|
|
|
self._debug("TOFU: Blindly trusting first ever certificate for this host!")
|
|
|
|
|
self.db_cur.execute("""INSERT INTO cert_cache
|
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
|
|
|
(host, address, fingerprint, now, now, 1))
|
2020-05-17 12:02:36 +00:00
|
|
|
|
self.db_conn.commit()
|
2020-05-17 16:35:35 +00:00
|
|
|
|
certdir = os.path.join(self.config_dir, "cert_cache")
|
|
|
|
|
if not os.path.exists(certdir):
|
|
|
|
|
os.makedirs(certdir)
|
|
|
|
|
with open(os.path.join(certdir, fingerprint+".crt"), "wb") as fp:
|
|
|
|
|
fp.write(cert)
|
2020-05-16 16:58:53 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def _get_handler_cmd(self, mimetype):
|
|
|
|
|
# Now look for a handler for this mimetype
|
|
|
|
|
# Consider exact matches before wildcard matches
|
|
|
|
|
exact_matches = []
|
|
|
|
|
wildcard_matches = []
|
|
|
|
|
for handled_mime, cmd_str in _MIME_HANDLERS.items():
|
|
|
|
|
if "*" in handled_mime:
|
|
|
|
|
wildcard_matches.append((handled_mime, cmd_str))
|
|
|
|
|
else:
|
|
|
|
|
exact_matches.append((handled_mime, cmd_str))
|
|
|
|
|
for handled_mime, cmd_str in exact_matches + wildcard_matches:
|
|
|
|
|
if fnmatch.fnmatch(mimetype, handled_mime):
|
|
|
|
|
break
|
|
|
|
|
else:
|
2022-01-06 12:44:02 +00:00
|
|
|
|
if mimetype.startswith("text/"):
|
|
|
|
|
cmd_str = _DEFAULT_LESS
|
|
|
|
|
else:
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Use "xdg-open" as a last resort.
|
2022-01-06 12:44:02 +00:00
|
|
|
|
cmd_str = "xdg-open %s"
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self._debug("Using handler: %s" % cmd_str)
|
|
|
|
|
return cmd_str
|
|
|
|
|
|
2022-01-10 10:19:29 +00:00
|
|
|
|
|
|
|
|
|
# Red title above rendered content
|
|
|
|
|
def _make_terminal_title(self,gi):
|
|
|
|
|
title = gi.get_title()
|
|
|
|
|
if gi.is_cache_valid() and self.offline_only and not gi.local:
|
|
|
|
|
last_modification = gi.cache_last_modified()
|
|
|
|
|
str_last = time.ctime(last_modification)
|
|
|
|
|
title += " \x1b[0;31m(last accessed on %s)"%str_last
|
2022-01-12 09:41:21 +00:00
|
|
|
|
rendered_title = "\x1b[31m\x1b[1m"+ title + "\x1b[0m"
|
|
|
|
|
wrapped = textwrap.fill(rendered_title,self.options["width"])
|
|
|
|
|
return wrapped + "\n"
|
2022-01-10 11:49:24 +00:00
|
|
|
|
# Our own HTML engine (crazy, isn’t it?)
|
|
|
|
|
def _handle_html(self,gi,display=True):
|
|
|
|
|
if not _DO_HTML:
|
|
|
|
|
print("HTML document detected. Please install python-bs4 and python readability.")
|
|
|
|
|
return
|
2022-01-10 14:30:23 +00:00
|
|
|
|
# This method recursively parse the HTML
|
2022-01-12 12:23:22 +00:00
|
|
|
|
def recursive_render(element,indent=""):
|
2022-01-10 14:30:23 +00:00
|
|
|
|
rendered_body = ""
|
2022-01-12 12:23:22 +00:00
|
|
|
|
#print("rendering %s - %s with indent %s" %(element.name,element.string,indent))
|
|
|
|
|
if element.name == "blockquote":
|
|
|
|
|
for child in element.children:
|
|
|
|
|
rendered_body += recursive_render(child,indent="\t").rstrip("\t")
|
|
|
|
|
elif element.name == "div":
|
2022-01-10 14:30:23 +00:00
|
|
|
|
rendered_body += "\n"
|
|
|
|
|
for child in element.children:
|
2022-01-12 12:23:22 +00:00
|
|
|
|
rendered_body += recursive_render(child,indent=indent)
|
2022-01-10 15:48:35 +00:00
|
|
|
|
elif element.name in ["h1","h2","h3","h4","h5","h6"]:
|
|
|
|
|
line = element.get_text()
|
|
|
|
|
if element.name in ["h1","h2"]:
|
|
|
|
|
rendered_body += "\n"+"\x1b[1;34m\x1b[4m" + line + "\x1b[0m"+"\n"
|
|
|
|
|
elif element.name in ["h3","h4"]:
|
|
|
|
|
rendered_body += "\n" + "\x1b[34m" + line + "\x1b[0m" + "\n"
|
|
|
|
|
else:
|
|
|
|
|
rendered_body += "\n" + "\x1b[34m\x1b[2m" + line + "\x1b[0m" + "\n"
|
2022-01-10 15:35:37 +00:00
|
|
|
|
elif element.name == "pre":
|
2022-01-14 10:28:30 +00:00
|
|
|
|
rendered_body += "\n"
|
2022-01-10 15:35:37 +00:00
|
|
|
|
if element.string:
|
|
|
|
|
rendered_body += element.string
|
2022-01-14 10:28:30 +00:00
|
|
|
|
else:
|
|
|
|
|
for child in element.children:
|
|
|
|
|
rendered_body += recursive_render(child,indent=indent)
|
|
|
|
|
rendered_body += "\n"
|
2022-01-10 15:35:37 +00:00
|
|
|
|
elif element.name == "li":
|
2022-01-13 09:47:39 +00:00
|
|
|
|
line = ""
|
2022-01-10 15:35:37 +00:00
|
|
|
|
for child in element.children:
|
2022-01-13 09:47:39 +00:00
|
|
|
|
line += recursive_render(child,indent=indent).strip("\n")
|
|
|
|
|
#print("in li: ***%s***"%line)
|
|
|
|
|
rendered_body += " * " + line.strip() + "\n"
|
|
|
|
|
elif element.name in ["code","em","b","i"]:
|
|
|
|
|
# we don’t do anything with those markup right now. Maybe later?
|
|
|
|
|
for child in element.children:
|
|
|
|
|
rendered_body += recursive_render(child,indent=indent).strip("\n")
|
2022-01-10 14:30:23 +00:00
|
|
|
|
elif element.name == "p":
|
2022-01-10 15:35:37 +00:00
|
|
|
|
temp_str = ""
|
2022-01-10 14:30:23 +00:00
|
|
|
|
if element.string:
|
2022-01-10 15:35:37 +00:00
|
|
|
|
temp_str += element.string.strip()
|
2022-01-10 14:30:23 +00:00
|
|
|
|
#print("p string : ",element.string)
|
|
|
|
|
else:
|
|
|
|
|
#print("p no string : ",element.contents)
|
|
|
|
|
for child in element.children:
|
2022-01-12 12:23:22 +00:00
|
|
|
|
temp_str += recursive_render(child,indent=indent)
|
2022-01-11 21:26:57 +00:00
|
|
|
|
rendered_body = temp_str + "\n\n"
|
2022-01-10 14:30:23 +00:00
|
|
|
|
elif element.name == "a":
|
2022-01-10 15:35:37 +00:00
|
|
|
|
text = element.get_text().strip()
|
2022-01-11 08:16:34 +00:00
|
|
|
|
link = element.get('href')
|
|
|
|
|
if link:
|
|
|
|
|
line = "=> " + link + " " +text
|
|
|
|
|
link_id = " [%s] "%(len(self.index)+1)
|
|
|
|
|
temp_gi = GeminiItem.from_map_line(line, gi)
|
2022-01-12 11:21:11 +00:00
|
|
|
|
if temp_gi:
|
|
|
|
|
self.index.append(temp_gi)
|
|
|
|
|
rendered_body = "\x1b[34m\x1b[2m " + text + link_id + "\x1b[0m"
|
|
|
|
|
else:
|
|
|
|
|
#No real link found
|
|
|
|
|
rendered_body = text
|
2022-01-11 21:26:57 +00:00
|
|
|
|
elif element.name == "br":
|
|
|
|
|
rendered_body = "\n"
|
2022-01-10 14:30:23 +00:00
|
|
|
|
elif element.string:
|
2022-01-10 11:49:24 +00:00
|
|
|
|
#print("tag without children:",element.name)
|
2022-01-10 14:30:23 +00:00
|
|
|
|
#print("string : **%s** "%element.string.strip())
|
|
|
|
|
#print("########")
|
2022-01-13 09:47:39 +00:00
|
|
|
|
rendered_body = element.string.strip("\n").strip("\t")
|
2022-01-10 11:49:24 +00:00
|
|
|
|
else:
|
|
|
|
|
#print("tag children:",element.name)
|
|
|
|
|
for child in element.children:
|
2022-01-12 12:23:22 +00:00
|
|
|
|
rendered_body += recursive_render(child,indent=indent)
|
2022-01-10 14:30:23 +00:00
|
|
|
|
#print("body for element %s: %s"%(element.name,rendered_body))
|
2022-01-12 12:23:22 +00:00
|
|
|
|
return indent + rendered_body
|
2022-01-10 14:30:23 +00:00
|
|
|
|
|
|
|
|
|
# the real _handle_html method
|
|
|
|
|
self.index = []
|
2022-01-10 11:49:24 +00:00
|
|
|
|
if self.idx_filename:
|
|
|
|
|
os.unlink(self.idx_filename)
|
|
|
|
|
tmpf = tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False)
|
|
|
|
|
self.idx_filename = tmpf.name
|
|
|
|
|
tmpf.write(self._make_terminal_title(gi))
|
|
|
|
|
body = gi.get_body()
|
|
|
|
|
title = Document(body).title()
|
|
|
|
|
tmpf.write("\x1b[1;34m\x1b[4m" + title + "\x1b[0m""\n")
|
|
|
|
|
summary = Document(body).summary()
|
|
|
|
|
soup = BeautifulSoup(summary, 'html.parser')
|
|
|
|
|
rendered_body = ""
|
2022-01-12 08:23:27 +00:00
|
|
|
|
if soup and soup.body :
|
|
|
|
|
for el in soup.body.contents:
|
|
|
|
|
rendered_body += recursive_render(el)
|
|
|
|
|
paragraphs = rendered_body.split("\n\n")
|
|
|
|
|
for par in paragraphs:
|
|
|
|
|
lines = par.splitlines()
|
|
|
|
|
for line in lines:
|
2022-01-12 12:23:22 +00:00
|
|
|
|
if line.startswith("\t"):
|
2022-01-13 09:47:39 +00:00
|
|
|
|
i_indent = " "
|
|
|
|
|
s_indent = i_indent
|
2022-01-12 12:23:22 +00:00
|
|
|
|
line = line.strip("\t")
|
2022-01-13 09:47:39 +00:00
|
|
|
|
elif line.startswith(" * "):
|
|
|
|
|
i_indent = "" # we keep the initial bullet)
|
|
|
|
|
s_indent = " "
|
2022-01-12 12:23:22 +00:00
|
|
|
|
else:
|
2022-01-13 09:47:39 +00:00
|
|
|
|
i_indent = ""
|
|
|
|
|
s_indent = i_indent
|
2022-01-12 08:23:27 +00:00
|
|
|
|
if line.strip() != "":
|
2022-01-12 12:23:22 +00:00
|
|
|
|
wrapped = textwrap.fill(line,self.options["width"],\
|
2022-01-13 09:47:39 +00:00
|
|
|
|
initial_indent=i_indent,subsequent_indent=s_indent)
|
2022-01-12 08:23:27 +00:00
|
|
|
|
wrapped += "\n"
|
|
|
|
|
else:
|
|
|
|
|
wrapped = ""
|
|
|
|
|
tmpf.write(wrapped)
|
|
|
|
|
tmpf.write("\n")
|
2022-01-10 11:49:24 +00:00
|
|
|
|
tmpf.close()
|
2022-01-10 14:30:23 +00:00
|
|
|
|
self.lookup = self.index
|
|
|
|
|
self.page_index = 0
|
|
|
|
|
self.index_index = -1
|
2022-01-10 11:49:24 +00:00
|
|
|
|
if display:
|
|
|
|
|
cmd_str = self._get_handler_cmd("text/gemini")
|
|
|
|
|
subprocess.call(shlex.split(cmd_str % self.idx_filename))
|
|
|
|
|
|
2022-01-08 13:16:55 +00:00
|
|
|
|
# Gemtext Rendering Engine
|
|
|
|
|
# this method renders the original Gemtext then call the handler to display it.
|
2022-01-08 20:32:25 +00:00
|
|
|
|
def _handle_gemtext(self, menu_gi, display=True):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.index = []
|
2020-03-07 20:11:49 +00:00
|
|
|
|
preformatted = False
|
2019-06-22 12:58:21 +00:00
|
|
|
|
if self.idx_filename:
|
|
|
|
|
os.unlink(self.idx_filename)
|
2021-12-02 15:27:14 +00:00
|
|
|
|
# this tempfile will contains a parsed version of the gemtext
|
|
|
|
|
# to display it. This is the output, not native gemtext.
|
2019-06-22 12:58:21 +00:00
|
|
|
|
tmpf = tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False)
|
|
|
|
|
self.idx_filename = tmpf.name
|
2022-01-10 10:19:29 +00:00
|
|
|
|
tmpf.write(self._make_terminal_title(menu_gi))
|
2022-01-12 15:23:55 +00:00
|
|
|
|
#This local method takes a line and apply the ansi code given as "color"
|
|
|
|
|
#The whole line is then wrapped and ansi code are ended.
|
2022-01-12 11:21:11 +00:00
|
|
|
|
def wrap_line(line,color=None,i_indent="",s_indent=""):
|
|
|
|
|
wrapped = textwrap.wrap(line,self.options["width"],\
|
|
|
|
|
initial_indent=i_indent,subsequent_indent=s_indent)
|
2022-01-12 09:41:21 +00:00
|
|
|
|
final = ""
|
|
|
|
|
for l in wrapped:
|
|
|
|
|
if color:
|
|
|
|
|
l = color + l + "\x1b[0m"
|
|
|
|
|
if l.strip() != "":
|
|
|
|
|
final += l + "\n"
|
|
|
|
|
return final
|
|
|
|
|
|
2022-01-08 20:32:25 +00:00
|
|
|
|
for line in menu_gi.get_body().splitlines():
|
2020-03-07 20:11:49 +00:00
|
|
|
|
if line.startswith("```"):
|
|
|
|
|
preformatted = not preformatted
|
|
|
|
|
elif preformatted:
|
|
|
|
|
tmpf.write(line + "\n")
|
|
|
|
|
elif line.startswith("=>"):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
try:
|
2019-08-13 16:56:15 +00:00
|
|
|
|
gi = GeminiItem.from_map_line(line, menu_gi)
|
2022-01-12 11:21:11 +00:00
|
|
|
|
if gi:
|
|
|
|
|
self.index.append(gi)
|
2022-01-12 14:39:03 +00:00
|
|
|
|
#tmpf.write(self._format_geminiitem(len(self.index), gi) + "\n")
|
2022-01-12 11:21:11 +00:00
|
|
|
|
#tentative to wrapp long links. Not sure it worth the trouble
|
2022-01-12 14:39:03 +00:00
|
|
|
|
link = self._format_geminiitem(len(self.index), gi)
|
|
|
|
|
startpos = link.find("] ") + 2
|
|
|
|
|
wrapped = wrap_line(link,s_indent=startpos*" ")
|
|
|
|
|
tmpf.write(wrapped)
|
2022-01-12 11:21:11 +00:00
|
|
|
|
else:
|
|
|
|
|
self._debug("Skipping possible link: %s" % line)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
except:
|
|
|
|
|
self._debug("Skipping possible link: %s" % line)
|
2020-06-07 17:07:30 +00:00
|
|
|
|
elif line.startswith("* "):
|
2020-06-07 17:06:39 +00:00
|
|
|
|
line = line[1:].lstrip("\t ")
|
2020-03-07 20:30:34 +00:00
|
|
|
|
tmpf.write(textwrap.fill(line, self.options["width"],
|
|
|
|
|
initial_indent = "• ", subsequent_indent=" ") + "\n")
|
2020-06-07 17:09:53 +00:00
|
|
|
|
elif line.startswith(">"):
|
|
|
|
|
line = line[1:].lstrip("\t ")
|
|
|
|
|
tmpf.write(textwrap.fill(line, self.options["width"],
|
|
|
|
|
initial_indent = "> ", subsequent_indent="> ") + "\n")
|
2020-03-07 20:30:34 +00:00
|
|
|
|
elif line.startswith("###"):
|
2020-06-07 17:06:39 +00:00
|
|
|
|
line = line[3:].lstrip("\t ")
|
2021-12-23 20:47:44 +00:00
|
|
|
|
#tmpf.write("\x1b[4m" + line + "\x1b[0m""\n")
|
2022-01-12 09:41:21 +00:00
|
|
|
|
tmpf.write(wrap_line(line, color="\x1b[34m\x1b[2m"))
|
2020-03-07 20:30:34 +00:00
|
|
|
|
elif line.startswith("##"):
|
2020-06-07 17:06:39 +00:00
|
|
|
|
line = line[2:].lstrip("\t ")
|
2022-01-12 09:41:21 +00:00
|
|
|
|
tmpf.write(wrap_line(line, color="\x1b[34m"))
|
2020-03-07 20:30:34 +00:00
|
|
|
|
elif line.startswith("#"):
|
2020-06-07 17:06:39 +00:00
|
|
|
|
line = line[1:].lstrip("\t ")
|
2022-01-12 09:41:21 +00:00
|
|
|
|
tmpf.write(wrap_line(line,color="\x1b[1;34m\x1b[4m"))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
2022-01-12 09:41:21 +00:00
|
|
|
|
tmpf.write(wrap_line(line).rstrip() + "\n")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
tmpf.close()
|
|
|
|
|
|
|
|
|
|
self.lookup = self.index
|
|
|
|
|
self.page_index = 0
|
|
|
|
|
self.index_index = -1
|
|
|
|
|
|
2020-03-24 19:41:37 +00:00
|
|
|
|
if display:
|
2020-08-18 19:13:26 +00:00
|
|
|
|
cmd_str = self._get_handler_cmd("text/gemini")
|
2020-03-24 19:41:37 +00:00
|
|
|
|
subprocess.call(shlex.split(cmd_str % self.idx_filename))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def _format_geminiitem(self, index, gi, url=False):
|
2020-08-30 15:23:36 +00:00
|
|
|
|
protocol = "" if gi.scheme == "gemini" else " %s" % gi.scheme
|
|
|
|
|
line = "[%d%s] %s" % (index, protocol, gi.name or gi.url)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
if gi.name and url:
|
2019-08-13 16:56:15 +00:00
|
|
|
|
line += " (%s)" % gi.url
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return line
|
|
|
|
|
|
|
|
|
|
def _show_lookup(self, offset=0, end=None, url=False):
|
|
|
|
|
for n, gi in enumerate(self.lookup[offset:end]):
|
|
|
|
|
print(self._format_geminiitem(n+offset+1, gi, url))
|
|
|
|
|
|
|
|
|
|
def _update_history(self, gi):
|
|
|
|
|
# Don't duplicate
|
|
|
|
|
if self.history and self.history[self.hist_index] == gi:
|
|
|
|
|
return
|
|
|
|
|
self.history = self.history[0:self.hist_index+1]
|
|
|
|
|
self.history.append(gi)
|
|
|
|
|
self.hist_index = len(self.history) - 1
|
|
|
|
|
|
|
|
|
|
def _log_visit(self, gi, address, size):
|
|
|
|
|
if not address:
|
|
|
|
|
return
|
|
|
|
|
self.log["requests"] += 1
|
|
|
|
|
self.log["bytes_recvd"] += size
|
|
|
|
|
self.visited_hosts.add(address)
|
|
|
|
|
if address[0] == socket.AF_INET:
|
|
|
|
|
self.log["ipv4_requests"] += 1
|
|
|
|
|
self.log["ipv4_bytes_recvd"] += size
|
|
|
|
|
elif address[0] == socket.AF_INET6:
|
|
|
|
|
self.log["ipv6_requests"] += 1
|
|
|
|
|
self.log["ipv6_bytes_recvd"] += size
|
|
|
|
|
|
|
|
|
|
def _get_active_tmpfile(self):
|
2022-01-10 14:53:41 +00:00
|
|
|
|
if self.gi.get_mime() in ["text/gemini","text/html"]:
|
2019-06-22 14:36:03 +00:00
|
|
|
|
return self.idx_filename
|
|
|
|
|
else:
|
|
|
|
|
return self.tmp_filename
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def _debug(self, debug_text):
|
|
|
|
|
if not self.options["debug"]:
|
|
|
|
|
return
|
|
|
|
|
debug_text = "\x1b[0;32m[DEBUG] " + debug_text + "\x1b[0m"
|
|
|
|
|
print(debug_text)
|
|
|
|
|
|
2020-05-10 14:09:54 +00:00
|
|
|
|
def _load_client_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Interactively load a TLS client certificate from the filesystem in PEM
|
|
|
|
|
format.
|
|
|
|
|
"""
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Loading client certificate file, in PEM format (blank line to cancel)")
|
|
|
|
|
certfile = input("Certfile path: ").strip()
|
|
|
|
|
if not certfile:
|
|
|
|
|
print("Aborting.")
|
|
|
|
|
return
|
2020-08-30 14:50:52 +00:00
|
|
|
|
certfile = os.path.expanduser(certfile)
|
|
|
|
|
if not os.path.isfile(certfile):
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Certificate file {} does not exist.".format(certfile))
|
|
|
|
|
return
|
|
|
|
|
print("Loading private key file, in PEM format (blank line to cancel)")
|
|
|
|
|
keyfile = input("Keyfile path: ").strip()
|
|
|
|
|
if not keyfile:
|
|
|
|
|
print("Aborting.")
|
|
|
|
|
return
|
2020-08-30 14:50:52 +00:00
|
|
|
|
keyfile = os.path.expanduser(keyfile)
|
|
|
|
|
if not os.path.isfile(keyfile):
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Private key file {} does not exist.".format(keyfile))
|
|
|
|
|
return
|
|
|
|
|
self._activate_client_cert(certfile, keyfile)
|
|
|
|
|
|
2020-05-11 20:22:24 +00:00
|
|
|
|
def _generate_transient_cert_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Use `openssl` command to generate a new transient client certificate
|
|
|
|
|
with 24 hours of validity.
|
|
|
|
|
"""
|
2020-05-11 20:22:24 +00:00
|
|
|
|
certdir = os.path.join(self.config_dir, "transient_certs")
|
|
|
|
|
name = str(uuid.uuid4())
|
2020-05-17 10:18:09 +00:00
|
|
|
|
self._generate_client_cert(certdir, name, transient=True)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
self.active_is_transient = True
|
|
|
|
|
self.transient_certs_created.append(name)
|
|
|
|
|
|
|
|
|
|
def _generate_persistent_client_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Interactively use `openssl` command to generate a new persistent client
|
|
|
|
|
certificate with one year of validity.
|
|
|
|
|
"""
|
2020-08-14 20:29:21 +00:00
|
|
|
|
certdir = os.path.join(self.config_dir, "client_certs")
|
2020-05-10 15:25:03 +00:00
|
|
|
|
print("What do you want to name this new certificate?")
|
2020-08-14 20:29:21 +00:00
|
|
|
|
print("Answering `mycert` will create `{0}/mycert.crt` and `{0}/mycert.key`".format(certdir))
|
|
|
|
|
name = input("> ")
|
2020-05-10 15:25:03 +00:00
|
|
|
|
if not name.strip():
|
|
|
|
|
print("Aborting.")
|
|
|
|
|
return
|
2020-05-27 06:57:44 +00:00
|
|
|
|
self._generate_client_cert(certdir, name)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
|
2020-05-17 10:18:09 +00:00
|
|
|
|
def _generate_client_cert(self, certdir, basename, transient=False):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Use `openssl` binary to generate a client certificate (which may be
|
|
|
|
|
transient or persistent) and save the certificate and private key to the
|
|
|
|
|
specified directory with the specified basename.
|
|
|
|
|
"""
|
2020-05-10 15:25:03 +00:00
|
|
|
|
if not os.path.exists(certdir):
|
|
|
|
|
os.makedirs(certdir)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
certfile = os.path.join(certdir, basename+".crt")
|
|
|
|
|
keyfile = os.path.join(certdir, basename+".key")
|
2020-05-17 10:18:09 +00:00
|
|
|
|
cmd = "openssl req -x509 -newkey rsa:2048 -days {} -nodes -keyout {} -out {}".format(1 if transient else 365, keyfile, certfile)
|
|
|
|
|
if transient:
|
2020-05-31 08:58:45 +00:00
|
|
|
|
cmd += " -subj '/CN={}'".format(basename)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
os.system(cmd)
|
2020-05-10 15:25:03 +00:00
|
|
|
|
self._activate_client_cert(certfile, keyfile)
|
|
|
|
|
|
2020-05-11 21:27:48 +00:00
|
|
|
|
def _choose_client_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Interactively select a previously generated client certificate and
|
|
|
|
|
activate it.
|
|
|
|
|
"""
|
2020-05-17 16:35:35 +00:00
|
|
|
|
certdir = os.path.join(self.config_dir, "client_certs")
|
2020-05-11 21:27:48 +00:00
|
|
|
|
certs = glob.glob(os.path.join(certdir, "*.crt"))
|
2020-08-14 20:29:21 +00:00
|
|
|
|
if len(certs) == 0:
|
|
|
|
|
print("There are no previously generated certificates.")
|
|
|
|
|
return
|
2020-05-11 21:27:48 +00:00
|
|
|
|
certdir = {}
|
|
|
|
|
for n, cert in enumerate(certs):
|
|
|
|
|
certdir[str(n+1)] = (cert, os.path.splitext(cert)[0] + ".key")
|
|
|
|
|
print("{}. {}".format(n+1, os.path.splitext(os.path.basename(cert))[0]))
|
|
|
|
|
choice = input("> ").strip()
|
|
|
|
|
if choice in certdir:
|
|
|
|
|
certfile, keyfile = certdir[choice]
|
|
|
|
|
self._activate_client_cert(certfile, keyfile)
|
|
|
|
|
else:
|
|
|
|
|
print("What?")
|
|
|
|
|
|
2020-05-10 12:17:35 +00:00
|
|
|
|
def _activate_client_cert(self, certfile, keyfile):
|
|
|
|
|
self.client_certs["active"] = (certfile, keyfile)
|
|
|
|
|
self.active_cert_domains = []
|
2021-02-03 07:06:27 +00:00
|
|
|
|
self.prompt = self.cert_prompt + "+" + os.path.basename(certfile).replace('.crt','') + "> " + "\x1b[0m"
|
2020-05-10 12:17:35 +00:00
|
|
|
|
self._debug("Using ID {} / {}.".format(*self.client_certs["active"]))
|
|
|
|
|
|
2020-05-10 11:48:25 +00:00
|
|
|
|
def _deactivate_client_cert(self):
|
2020-05-11 20:22:24 +00:00
|
|
|
|
if self.active_is_transient:
|
|
|
|
|
for filename in self.client_certs["active"]:
|
|
|
|
|
os.remove(filename)
|
|
|
|
|
for domain in self.active_cert_domains:
|
|
|
|
|
self.client_certs.pop(domain)
|
2020-05-10 11:48:25 +00:00
|
|
|
|
self.client_certs["active"] = None
|
|
|
|
|
self.active_cert_domains = []
|
|
|
|
|
self.prompt = self.no_cert_prompt
|
2020-05-11 20:22:24 +00:00
|
|
|
|
self.active_is_transient = False
|
2020-05-10 11:48:25 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Cmd implementation follows
|
|
|
|
|
|
|
|
|
|
def default(self, line):
|
|
|
|
|
if line.strip() == "EOF":
|
|
|
|
|
return self.onecmd("quit")
|
|
|
|
|
elif line.strip() == "..":
|
|
|
|
|
return self.do_up()
|
|
|
|
|
elif line.startswith("/"):
|
|
|
|
|
return self.do_search(line[1:])
|
2021-12-22 12:20:10 +00:00
|
|
|
|
elif looks_like_url(line):
|
2021-12-16 15:09:54 +00:00
|
|
|
|
return self.do_go(line)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
# Expand abbreviated commands
|
|
|
|
|
first_word = line.split()[0].strip()
|
|
|
|
|
if first_word in _ABBREVS:
|
|
|
|
|
full_cmd = _ABBREVS[first_word]
|
|
|
|
|
expanded = line.replace(first_word, full_cmd, 1)
|
|
|
|
|
return self.onecmd(expanded)
|
|
|
|
|
|
|
|
|
|
# Try to parse numerical index for lookup table
|
|
|
|
|
try:
|
|
|
|
|
n = int(line.strip())
|
|
|
|
|
except ValueError:
|
|
|
|
|
print("What?")
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
gi = self.lookup[n-1]
|
|
|
|
|
except IndexError:
|
|
|
|
|
print ("Index too high!")
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
self.index_index = n
|
|
|
|
|
self._go_to_gi(gi)
|
|
|
|
|
|
|
|
|
|
### Settings
|
2020-04-07 20:46:05 +00:00
|
|
|
|
@restricted
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def do_set(self, line):
|
|
|
|
|
"""View or set various options."""
|
|
|
|
|
if not line.strip():
|
|
|
|
|
# Show all current settings
|
|
|
|
|
for option in sorted(self.options.keys()):
|
|
|
|
|
print("%s %s" % (option, self.options[option]))
|
|
|
|
|
elif len(line.split()) == 1:
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# Show current value of one specific setting
|
2019-06-22 12:58:21 +00:00
|
|
|
|
option = line.strip()
|
|
|
|
|
if option in self.options:
|
|
|
|
|
print("%s %s" % (option, self.options[option]))
|
|
|
|
|
else:
|
|
|
|
|
print("Unrecognised option %s" % option)
|
|
|
|
|
else:
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# Set value of one specific setting
|
2019-06-22 12:58:21 +00:00
|
|
|
|
option, value = line.split(" ", 1)
|
|
|
|
|
if option not in self.options:
|
|
|
|
|
print("Unrecognised option %s" % option)
|
|
|
|
|
return
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# Validate / convert values
|
|
|
|
|
if option == "gopher_proxy":
|
|
|
|
|
if ":" not in value:
|
|
|
|
|
value += ":1965"
|
|
|
|
|
else:
|
|
|
|
|
host, port = value.rsplit(":",1)
|
|
|
|
|
if not port.isnumeric():
|
|
|
|
|
print("Invalid proxy port %s" % port)
|
|
|
|
|
return
|
2020-05-19 21:14:09 +00:00
|
|
|
|
elif option == "tls_mode":
|
|
|
|
|
if value.lower() not in ("ca", "tofu"):
|
|
|
|
|
print("TLS mode must be `ca` or `tofu`!")
|
|
|
|
|
return
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif value.isnumeric():
|
|
|
|
|
value = int(value)
|
|
|
|
|
elif value.lower() == "false":
|
|
|
|
|
value = False
|
|
|
|
|
elif value.lower() == "true":
|
|
|
|
|
value = True
|
|
|
|
|
else:
|
|
|
|
|
try:
|
|
|
|
|
value = float(value)
|
|
|
|
|
except ValueError:
|
|
|
|
|
pass
|
|
|
|
|
self.options[option] = value
|
|
|
|
|
|
2020-05-10 10:35:46 +00:00
|
|
|
|
@restricted
|
|
|
|
|
def do_cert(self, line):
|
2020-05-23 11:35:13 +00:00
|
|
|
|
"""Manage client certificates"""
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Managing client certificates")
|
2020-05-10 10:59:26 +00:00
|
|
|
|
if self.client_certs["active"]:
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Active certificate: {}".format(self.client_certs["active"][0]))
|
|
|
|
|
print("1. Deactivate client certificate.")
|
2020-05-11 21:27:48 +00:00
|
|
|
|
print("2. Generate new certificate.")
|
|
|
|
|
print("3. Load previously generated certificate.")
|
2020-05-23 11:35:13 +00:00
|
|
|
|
print("4. Load externally created client certificate from file.")
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Enter blank line to exit certificate manager.")
|
|
|
|
|
choice = input("> ").strip()
|
|
|
|
|
if choice == "1":
|
2020-05-10 10:35:46 +00:00
|
|
|
|
print("Deactivating client certificate.")
|
2020-05-10 11:48:25 +00:00
|
|
|
|
self._deactivate_client_cert()
|
2020-05-10 14:09:54 +00:00
|
|
|
|
elif choice == "2":
|
2020-05-11 21:27:48 +00:00
|
|
|
|
self._generate_persistent_client_cert()
|
2020-05-10 14:09:54 +00:00
|
|
|
|
elif choice == "3":
|
2020-05-11 21:27:48 +00:00
|
|
|
|
self._choose_client_cert()
|
|
|
|
|
elif choice == "4":
|
|
|
|
|
self._load_client_cert()
|
2020-05-10 10:35:46 +00:00
|
|
|
|
else:
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Aborting.")
|
2020-05-10 10:35:46 +00:00
|
|
|
|
|
2020-04-07 20:46:05 +00:00
|
|
|
|
@restricted
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def do_handler(self, line):
|
|
|
|
|
"""View or set handler commands for different MIME types."""
|
|
|
|
|
if not line.strip():
|
|
|
|
|
# Show all current handlers
|
|
|
|
|
for mime in sorted(_MIME_HANDLERS.keys()):
|
|
|
|
|
print("%s %s" % (mime, _MIME_HANDLERS[mime]))
|
|
|
|
|
elif len(line.split()) == 1:
|
|
|
|
|
mime = line.strip()
|
|
|
|
|
if mime in _MIME_HANDLERS:
|
|
|
|
|
print("%s %s" % (mime, _MIME_HANDLERS[mime]))
|
|
|
|
|
else:
|
|
|
|
|
print("No handler set for MIME type %s" % mime)
|
|
|
|
|
else:
|
|
|
|
|
mime, handler = line.split(" ", 1)
|
|
|
|
|
_MIME_HANDLERS[mime] = handler
|
|
|
|
|
if "%s" not in handler:
|
|
|
|
|
print("Are you sure you don't want to pass the filename to the handler?")
|
|
|
|
|
|
2020-05-27 13:16:22 +00:00
|
|
|
|
def do_abbrevs(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Print all Offpunk command abbreviations."""
|
2020-06-13 10:39:18 +00:00
|
|
|
|
header = "Command Abbreviations:"
|
|
|
|
|
self.stdout.write("\n{}\n".format(str(header)))
|
2020-05-27 13:16:22 +00:00
|
|
|
|
if self.ruler:
|
|
|
|
|
self.stdout.write("{}\n".format(str(self.ruler * len(header))))
|
|
|
|
|
for k, v in _ABBREVS.items():
|
|
|
|
|
self.stdout.write("{:<7} {}\n".format(k, v))
|
2020-06-13 10:39:18 +00:00
|
|
|
|
self.stdout.write("\n")
|
2020-05-27 13:16:22 +00:00
|
|
|
|
|
2021-12-06 15:43:16 +00:00
|
|
|
|
def do_offline(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Use Offpunk offline by only accessing cached content"""
|
2021-12-06 15:43:16 +00:00
|
|
|
|
if self.offline_only:
|
2021-12-16 09:43:25 +00:00
|
|
|
|
print("Offline and undisturbed.")
|
2021-12-06 15:43:16 +00:00
|
|
|
|
else:
|
|
|
|
|
self.offline_only = True
|
2021-12-10 14:24:26 +00:00
|
|
|
|
self.prompt = self.offline_prompt
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Offpunk is now offline and will only access cached content")
|
2021-12-16 09:43:25 +00:00
|
|
|
|
|
|
|
|
|
def do_online(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Use Offpunk online with a direct connection"""
|
2021-12-16 09:43:25 +00:00
|
|
|
|
if self.offline_only:
|
|
|
|
|
self.offline_only = False
|
|
|
|
|
self.prompt = self.no_cert_prompt
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Offpunk is online and will access the network")
|
2021-12-16 09:43:25 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Already online. Try offline.")
|
2021-12-06 15:43:16 +00:00
|
|
|
|
|
2022-01-05 20:12:59 +00:00
|
|
|
|
def do_copy(self, *args):
|
|
|
|
|
"""Copy the content of the last visited page as gemtext in the clipboard.
|
|
|
|
|
Use with "url" as argument to only copy the adress.
|
|
|
|
|
Use with "raw" to copy the content as seen in your terminal (not gemtext)"""
|
|
|
|
|
if self.gi:
|
|
|
|
|
if shutil.which('xsel'):
|
|
|
|
|
if args and args[0] == "url":
|
|
|
|
|
subprocess.call(("echo %s |xsel -b -i" % self.gi.url), shell=True)
|
|
|
|
|
elif args and args[0] == "raw":
|
|
|
|
|
subprocess.call(("cat %s |xsel -b -i" % self._get_active_tmpfile()), shell=True)
|
|
|
|
|
else:
|
2022-01-08 20:46:57 +00:00
|
|
|
|
subprocess.call(("cat %s |xsel -b -i" % self.gi.get_body(as_file=True)), shell=True)
|
2022-01-05 20:12:59 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Please install xsel to use copy")
|
|
|
|
|
else:
|
|
|
|
|
print("No content to copy, visit a page first")
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
### Stuff for getting around
|
|
|
|
|
def do_go(self, line):
|
|
|
|
|
"""Go to a gemini URL or marked item."""
|
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line:
|
2021-12-22 12:20:10 +00:00
|
|
|
|
if shutil.which('xsel'):
|
|
|
|
|
clipboards = []
|
|
|
|
|
urls = []
|
|
|
|
|
clipboards.append(subprocess.check_output(['xsel','-p'],text=True))
|
|
|
|
|
clipboards.append(subprocess.check_output(['xsel','-s'],text=True))
|
|
|
|
|
clipboards.append(subprocess.check_output(['xsel','-b'],text=True))
|
|
|
|
|
for u in clipboards:
|
|
|
|
|
if looks_like_url(u) :
|
|
|
|
|
urls.append(u)
|
|
|
|
|
if len(urls) > 1:
|
|
|
|
|
self.lookup = []
|
|
|
|
|
for u in urls:
|
|
|
|
|
self.lookup.append(GeminiItem(u))
|
|
|
|
|
print("Where do you want to go today?")
|
|
|
|
|
self._show_lookup()
|
|
|
|
|
elif len(urls) == 1:
|
|
|
|
|
self.do_go(urls[0])
|
|
|
|
|
else:
|
|
|
|
|
print("Go where? (hint: simply copy an URL)")
|
|
|
|
|
else:
|
|
|
|
|
print("Go where? (hint: install xsel to go to copied URLs)")
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# First, check for possible marks
|
|
|
|
|
elif line in self.marks:
|
|
|
|
|
gi = self.marks[line]
|
|
|
|
|
self._go_to_gi(gi)
|
|
|
|
|
# or a local file
|
|
|
|
|
elif os.path.exists(os.path.expanduser(line)):
|
2021-11-18 11:02:00 +00:00
|
|
|
|
self._go_to_gi(GeminiItem(line))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# If this isn't a mark, treat it as a URL
|
2022-01-11 13:04:20 +00:00
|
|
|
|
elif looks_like_url(line):
|
2019-08-13 16:56:15 +00:00
|
|
|
|
self._go_to_gi(GeminiItem(line))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_reload(self, *args):
|
|
|
|
|
"""Reload the current URL."""
|
2021-12-14 15:33:17 +00:00
|
|
|
|
if self.offline_only:
|
|
|
|
|
with open(self.syncfile,mode='a') as sf:
|
|
|
|
|
line = self.gi.url + '\n'
|
|
|
|
|
sf.write(line)
|
|
|
|
|
sf.close()
|
|
|
|
|
else:
|
|
|
|
|
self._go_to_gi(self.gi, check_cache=False)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_up(self, *args):
|
|
|
|
|
"""Go up one directory in the path."""
|
2019-08-13 17:39:55 +00:00
|
|
|
|
self._go_to_gi(self.gi.up())
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def do_back(self, *args):
|
|
|
|
|
"""Go back to the previous gemini item."""
|
|
|
|
|
if not self.history or self.hist_index == 0:
|
|
|
|
|
return
|
|
|
|
|
self.hist_index -= 1
|
|
|
|
|
gi = self.history[self.hist_index]
|
|
|
|
|
self._go_to_gi(gi, update_hist=False)
|
|
|
|
|
|
|
|
|
|
def do_forward(self, *args):
|
|
|
|
|
"""Go forward to the next gemini item."""
|
|
|
|
|
if not self.history or self.hist_index == len(self.history) - 1:
|
|
|
|
|
return
|
|
|
|
|
self.hist_index += 1
|
|
|
|
|
gi = self.history[self.hist_index]
|
|
|
|
|
self._go_to_gi(gi, update_hist=False)
|
|
|
|
|
|
|
|
|
|
def do_next(self, *args):
|
|
|
|
|
"""Go to next item after current in index."""
|
|
|
|
|
return self.onecmd(str(self.index_index+1))
|
|
|
|
|
|
|
|
|
|
def do_previous(self, *args):
|
|
|
|
|
"""Go to previous item before current in index."""
|
|
|
|
|
self.lookup = self.index
|
|
|
|
|
return self.onecmd(str(self.index_index-1))
|
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_root(self, *args):
|
|
|
|
|
"""Go to root selector of the server hosting current item."""
|
2019-08-13 17:39:55 +00:00
|
|
|
|
self._go_to_gi(self.gi.root())
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def do_tour(self, line):
|
|
|
|
|
"""Add index items as waypoints on a tour, which is basically a FIFO
|
|
|
|
|
queue of gemini items.
|
|
|
|
|
|
|
|
|
|
Items can be added with `tour 1 2 3 4` or ranges like `tour 1-4`.
|
|
|
|
|
All items in current menu can be added with `tour *`.
|
2022-01-03 15:40:52 +00:00
|
|
|
|
Current item can be added back to the end of the tour with `tour .`.
|
2019-06-22 12:58:21 +00:00
|
|
|
|
Current tour can be listed with `tour ls` and scrubbed with `tour clear`."""
|
2021-12-22 09:21:32 +00:00
|
|
|
|
def add_to_tourfile(url):
|
|
|
|
|
with open(self.tourfile,'a') as f:
|
|
|
|
|
l = url.strip()+"\n"
|
|
|
|
|
f.write(l)
|
|
|
|
|
f.close
|
|
|
|
|
def pop_from_tourfile():
|
|
|
|
|
l = None
|
|
|
|
|
lines = []
|
|
|
|
|
with open(self.tourfile,'r') as f:
|
|
|
|
|
l = f.readline()
|
|
|
|
|
lines = f.readlines()
|
|
|
|
|
f.close()
|
|
|
|
|
with open(self.tourfile,'w') as f:
|
|
|
|
|
if len(lines) > 0:
|
|
|
|
|
f.writelines(lines)
|
|
|
|
|
f.close()
|
|
|
|
|
return l
|
|
|
|
|
def list_tourfile():
|
|
|
|
|
lines = []
|
|
|
|
|
with open(self.tourfile,'r') as f:
|
|
|
|
|
lines = f.readlines()
|
|
|
|
|
f.close()
|
|
|
|
|
return lines
|
|
|
|
|
def clear_tourfile():
|
|
|
|
|
with open(self.tourfile,'w') as f:
|
|
|
|
|
f.close()
|
2019-06-22 12:58:21 +00:00
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line:
|
|
|
|
|
# Fly to next waypoint on tour
|
2021-12-22 09:21:32 +00:00
|
|
|
|
nexttour = pop_from_tourfile()
|
|
|
|
|
if not nexttour:
|
2019-06-22 12:58:21 +00:00
|
|
|
|
print("End of tour.")
|
|
|
|
|
else:
|
2021-12-22 09:21:32 +00:00
|
|
|
|
self._go_to_gi(GeminiItem(nexttour))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif line == "ls":
|
|
|
|
|
old_lookup = self.lookup
|
2021-12-22 09:21:32 +00:00
|
|
|
|
self.lookup = []
|
|
|
|
|
for u in list_tourfile():
|
|
|
|
|
self.lookup.append(GeminiItem(u))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self._show_lookup()
|
|
|
|
|
self.lookup = old_lookup
|
|
|
|
|
elif line == "clear":
|
2021-12-22 09:21:32 +00:00
|
|
|
|
clear_tourfile()
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif line == "*":
|
2021-12-22 09:21:32 +00:00
|
|
|
|
for l in self.lookup:
|
|
|
|
|
add_to_tourfile(l.url)
|
2022-01-03 15:40:52 +00:00
|
|
|
|
elif line == ".":
|
|
|
|
|
add_to_tourfile(self.gi.url)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif looks_like_url(line):
|
2021-12-22 09:21:32 +00:00
|
|
|
|
add_to_tourfile(line)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
for index in line.split():
|
|
|
|
|
try:
|
|
|
|
|
pair = index.split('-')
|
|
|
|
|
if len(pair) == 1:
|
|
|
|
|
# Just a single index
|
|
|
|
|
n = int(index)
|
|
|
|
|
gi = self.lookup[n-1]
|
2021-12-22 09:21:32 +00:00
|
|
|
|
add_to_tourfile(gi.url)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif len(pair) == 2:
|
|
|
|
|
# Two endpoints for a range of indices
|
2021-08-25 06:01:16 +00:00
|
|
|
|
if int(pair[0]) < int(pair[1]):
|
|
|
|
|
for n in range(int(pair[0]), int(pair[1]) + 1):
|
|
|
|
|
gi = self.lookup[n-1]
|
2021-12-22 09:21:32 +00:00
|
|
|
|
add_to_tourfile(gi.url)
|
2021-08-25 06:01:16 +00:00
|
|
|
|
else:
|
|
|
|
|
for n in range(int(pair[0]), int(pair[1]) - 1, -1):
|
|
|
|
|
gi = self.lookup[n-1]
|
2021-12-22 09:21:32 +00:00
|
|
|
|
add_to_tourfile(gi.url)
|
2021-08-25 06:01:16 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
# Syntax error
|
|
|
|
|
print("Invalid use of range syntax %s, skipping" % index)
|
|
|
|
|
except ValueError:
|
|
|
|
|
print("Non-numeric index %s, skipping." % index)
|
|
|
|
|
except IndexError:
|
|
|
|
|
print("Invalid index %d, skipping." % n)
|
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_mark(self, line):
|
|
|
|
|
"""Mark the current item with a single letter. This letter can then
|
|
|
|
|
be passed to the 'go' command to return to the current item later.
|
|
|
|
|
Think of it like marks in vi: 'mark a'='ma' and 'go a'=''a'."""
|
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line:
|
|
|
|
|
for mark, gi in self.marks.items():
|
2019-08-13 16:56:15 +00:00
|
|
|
|
print("[%s] %s (%s)" % (mark, gi.name, gi.url))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif line.isalpha() and len(line) == 1:
|
|
|
|
|
self.marks[line] = self.gi
|
|
|
|
|
else:
|
|
|
|
|
print("Invalid mark, must be one letter")
|
|
|
|
|
|
2020-05-10 12:34:48 +00:00
|
|
|
|
def do_version(self, line):
|
|
|
|
|
"""Display version information."""
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Offpunk " + _VERSION)
|
2020-05-10 12:34:48 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
### Stuff that modifies the lookup table
|
|
|
|
|
def do_ls(self, line):
|
|
|
|
|
"""List contents of current index.
|
|
|
|
|
Use 'ls -l' to see URLs."""
|
|
|
|
|
self.lookup = self.index
|
|
|
|
|
self._show_lookup(url = "-l" in line)
|
|
|
|
|
self.page_index = 0
|
|
|
|
|
|
2020-05-15 11:38:51 +00:00
|
|
|
|
def do_gus(self, line):
|
2021-04-27 07:55:12 +00:00
|
|
|
|
"""Submit a search query to the geminispace.info search engine."""
|
|
|
|
|
gus = GeminiItem("gemini://geminispace.info/search")
|
2020-05-15 11:38:51 +00:00
|
|
|
|
self._go_to_gi(gus.query(line))
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def do_history(self, *args):
|
|
|
|
|
"""Display history."""
|
|
|
|
|
self.lookup = self.history
|
|
|
|
|
self._show_lookup(url=True)
|
|
|
|
|
self.page_index = 0
|
|
|
|
|
|
|
|
|
|
def do_search(self, searchterm):
|
|
|
|
|
"""Search index (case insensitive)."""
|
|
|
|
|
results = [
|
|
|
|
|
gi for gi in self.lookup if searchterm.lower() in gi.name.lower()]
|
|
|
|
|
if results:
|
|
|
|
|
self.lookup = results
|
|
|
|
|
self._show_lookup()
|
|
|
|
|
self.page_index = 0
|
|
|
|
|
else:
|
|
|
|
|
print("No results found.")
|
|
|
|
|
|
|
|
|
|
def emptyline(self):
|
|
|
|
|
"""Page through index ten lines at a time."""
|
|
|
|
|
i = self.page_index
|
|
|
|
|
if i > len(self.lookup):
|
|
|
|
|
return
|
|
|
|
|
self._show_lookup(offset=i, end=i+10)
|
|
|
|
|
self.page_index += 10
|
|
|
|
|
|
|
|
|
|
### Stuff that does something to most recently viewed item
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_cat(self, *args):
|
|
|
|
|
"""Run most recently visited item through "cat" command."""
|
|
|
|
|
subprocess.call(shlex.split("cat %s" % self._get_active_tmpfile()))
|
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_less(self, *args):
|
|
|
|
|
"""Run most recently visited item through "less" command."""
|
2022-01-08 20:32:25 +00:00
|
|
|
|
cmd_str = self._get_handler_cmd(self.gi.get_mime())
|
2019-06-22 12:58:21 +00:00
|
|
|
|
cmd_str = cmd_str % self._get_active_tmpfile()
|
2022-01-10 20:29:19 +00:00
|
|
|
|
subprocess.call("%s | less -RM" % cmd_str, shell=True)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_fold(self, *args):
|
|
|
|
|
"""Run most recently visited item through "fold" command."""
|
2022-01-08 20:32:25 +00:00
|
|
|
|
cmd_str = self._get_handler_cmd(self.gi.get_mime())
|
2019-06-22 12:58:21 +00:00
|
|
|
|
cmd_str = cmd_str % self._get_active_tmpfile()
|
|
|
|
|
subprocess.call("%s | fold -w 70 -s" % cmd_str, shell=True)
|
|
|
|
|
|
2020-04-07 20:46:05 +00:00
|
|
|
|
@restricted
|
2019-06-22 12:58:21 +00:00
|
|
|
|
@needs_gi
|
|
|
|
|
def do_shell(self, line):
|
|
|
|
|
"""'cat' most recently visited item through a shell pipeline."""
|
|
|
|
|
subprocess.call(("cat %s |" % self._get_active_tmpfile()) + line, shell=True)
|
|
|
|
|
|
2020-04-07 20:46:05 +00:00
|
|
|
|
@restricted
|
2019-06-22 12:58:21 +00:00
|
|
|
|
@needs_gi
|
|
|
|
|
def do_save(self, line):
|
|
|
|
|
"""Save an item to the filesystem.
|
|
|
|
|
'save n filename' saves menu item n to the specified filename.
|
|
|
|
|
'save filename' saves the last viewed item to the specified filename.
|
|
|
|
|
'save n' saves menu item n to an automagic filename."""
|
|
|
|
|
args = line.strip().split()
|
|
|
|
|
|
|
|
|
|
# First things first, figure out what our arguments are
|
|
|
|
|
if len(args) == 0:
|
|
|
|
|
# No arguments given at all
|
|
|
|
|
# Save current item, if there is one, to a file whose name is
|
|
|
|
|
# inferred from the gemini path
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if not self.gi.is_cache_valid():
|
2022-01-01 21:05:02 +00:00
|
|
|
|
print("You cannot save if not cached!")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return
|
|
|
|
|
else:
|
|
|
|
|
index = None
|
|
|
|
|
filename = None
|
|
|
|
|
elif len(args) == 1:
|
|
|
|
|
# One argument given
|
|
|
|
|
# If it's numeric, treat it as an index, and infer the filename
|
|
|
|
|
try:
|
|
|
|
|
index = int(args[0])
|
|
|
|
|
filename = None
|
|
|
|
|
# If it's not numeric, treat it as a filename and
|
|
|
|
|
# save the current item
|
|
|
|
|
except ValueError:
|
|
|
|
|
index = None
|
|
|
|
|
filename = os.path.expanduser(args[0])
|
|
|
|
|
elif len(args) == 2:
|
|
|
|
|
# Two arguments given
|
|
|
|
|
# Treat first as an index and second as filename
|
|
|
|
|
index, filename = args
|
|
|
|
|
try:
|
|
|
|
|
index = int(index)
|
|
|
|
|
except ValueError:
|
|
|
|
|
print("First argument is not a valid item index!")
|
|
|
|
|
return
|
|
|
|
|
filename = os.path.expanduser(filename)
|
|
|
|
|
else:
|
|
|
|
|
print("You must provide an index, a filename, or both.")
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
# Next, fetch the item to save, if it's not the current one.
|
|
|
|
|
if index:
|
|
|
|
|
last_gi = self.gi
|
|
|
|
|
try:
|
|
|
|
|
gi = self.lookup[index-1]
|
|
|
|
|
self._go_to_gi(gi, update_hist = False, handle = False)
|
|
|
|
|
except IndexError:
|
|
|
|
|
print ("Index too high!")
|
|
|
|
|
self.gi = last_gi
|
|
|
|
|
return
|
|
|
|
|
else:
|
|
|
|
|
gi = self.gi
|
|
|
|
|
|
|
|
|
|
# Derive filename from current GI's path, if one hasn't been set
|
|
|
|
|
if not filename:
|
2022-01-08 20:46:57 +00:00
|
|
|
|
filename = gi.get_filename()
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Check for filename collisions and actually do the save if safe
|
|
|
|
|
if os.path.exists(filename):
|
|
|
|
|
print("File %s already exists!" % filename)
|
|
|
|
|
else:
|
|
|
|
|
# Don't use _get_active_tmpfile() here, because we want to save the
|
2021-12-30 15:03:08 +00:00
|
|
|
|
# "source code" of menus, not the rendered view - this way Offpunk
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# can navigate to it later.
|
|
|
|
|
print("Saved to %s" % filename)
|
2022-01-08 20:46:57 +00:00
|
|
|
|
shutil.copyfile(gi.get_body(as_file=True), filename)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
# Restore gi if necessary
|
|
|
|
|
if index != None:
|
|
|
|
|
self._go_to_gi(last_gi, handle=False)
|
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_url(self, *args):
|
|
|
|
|
"""Print URL of most recently visited item."""
|
2019-08-13 16:56:15 +00:00
|
|
|
|
print(self.gi.url)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
### Bookmarking stuff
|
2020-04-07 20:46:05 +00:00
|
|
|
|
@restricted
|
2019-06-22 12:58:21 +00:00
|
|
|
|
@needs_gi
|
|
|
|
|
def do_add(self, line):
|
|
|
|
|
"""Add the current URL to the bookmarks menu.
|
|
|
|
|
Optionally, specify the new name for the bookmark."""
|
2020-05-11 20:22:24 +00:00
|
|
|
|
with open(os.path.join(self.config_dir, "bookmarks.gmi"), "a") as fp:
|
2019-08-13 16:56:15 +00:00
|
|
|
|
fp.write(self.gi.to_map_line(line))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2020-03-24 19:41:37 +00:00
|
|
|
|
def do_bookmarks(self, line):
|
|
|
|
|
"""Show or access the bookmarks menu.
|
|
|
|
|
'bookmarks' shows all bookmarks.
|
|
|
|
|
'bookmarks n' navigates immediately to item n in the bookmark menu.
|
2020-05-10 20:51:33 +00:00
|
|
|
|
Bookmarks are stored using the 'add' command."""
|
2020-05-11 20:22:24 +00:00
|
|
|
|
bm_file = os.path.join(self.config_dir, "bookmarks.gmi")
|
2019-08-18 19:32:34 +00:00
|
|
|
|
if not os.path.exists(bm_file):
|
|
|
|
|
print("You need to 'add' some bookmarks, first!")
|
2020-03-24 19:41:37 +00:00
|
|
|
|
return
|
|
|
|
|
args = line.strip()
|
|
|
|
|
if len(args.split()) > 1 or (args and not args.isnumeric()):
|
|
|
|
|
print("bookmarks command takes a single integer argument!")
|
|
|
|
|
return
|
|
|
|
|
with open(bm_file, "r") as fp:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
gi = GeminiItem("localhost:/" + bm_file,"Bookmarks")
|
|
|
|
|
gi.body = fp.read()
|
2021-12-09 16:23:50 +00:00
|
|
|
|
# We don’t display bookmarks if accessing directly one
|
|
|
|
|
# or if in sync_only
|
|
|
|
|
display = not ( args or self.sync_only)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self._handle_gemtext(gi, display = display)
|
2020-03-24 19:41:37 +00:00
|
|
|
|
if args:
|
|
|
|
|
# Use argument as a numeric index
|
|
|
|
|
self.default(line)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
### Help
|
|
|
|
|
def do_help(self, arg):
|
|
|
|
|
"""ALARM! Recursion detected! ALARM! Prepare to eject!"""
|
|
|
|
|
if arg == "!":
|
|
|
|
|
print("! is an alias for 'shell'")
|
|
|
|
|
elif arg == "?":
|
|
|
|
|
print("? is an alias for 'help'")
|
2022-01-05 20:12:59 +00:00
|
|
|
|
elif arg in _ABBREVS:
|
|
|
|
|
full_cmd = _ABBREVS[arg]
|
|
|
|
|
print("%s is aan alias for '%s'" %(arg,full_cmd))
|
|
|
|
|
print("See the list of aliases with 'abbrevs'")
|
|
|
|
|
print("'help %s':"%full_cmd)
|
|
|
|
|
cmd.Cmd.do_help(self, full_cmd)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
cmd.Cmd.do_help(self, arg)
|
|
|
|
|
|
|
|
|
|
### Flight recorder
|
|
|
|
|
def do_blackbox(self, *args):
|
|
|
|
|
"""Display contents of flight recorder, showing statistics for the
|
|
|
|
|
current gemini browsing session."""
|
|
|
|
|
lines = []
|
|
|
|
|
# Compute flight time
|
|
|
|
|
now = time.time()
|
|
|
|
|
delta = now - self.log["start_time"]
|
2020-06-14 10:28:34 +00:00
|
|
|
|
hours, remainder = divmod(delta, 3600)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
minutes, seconds = divmod(remainder, 60)
|
|
|
|
|
# Count hosts
|
|
|
|
|
ipv4_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET])
|
|
|
|
|
ipv6_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET6])
|
|
|
|
|
# Assemble lines
|
2020-05-10 16:47:07 +00:00
|
|
|
|
lines.append(("Patrol duration", "%02d:%02d:%02d" % (hours, minutes, seconds)))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
lines.append(("Requests sent:", self.log["requests"]))
|
|
|
|
|
lines.append((" IPv4 requests:", self.log["ipv4_requests"]))
|
|
|
|
|
lines.append((" IPv6 requests:", self.log["ipv6_requests"]))
|
|
|
|
|
lines.append(("Bytes received:", self.log["bytes_recvd"]))
|
|
|
|
|
lines.append((" IPv4 bytes:", self.log["ipv4_bytes_recvd"]))
|
|
|
|
|
lines.append((" IPv6 bytes:", self.log["ipv6_bytes_recvd"]))
|
|
|
|
|
lines.append(("Unique hosts visited:", len(self.visited_hosts)))
|
|
|
|
|
lines.append((" IPv4 hosts:", ipv4_hosts))
|
|
|
|
|
lines.append((" IPv6 hosts:", ipv6_hosts))
|
|
|
|
|
lines.append(("DNS failures:", self.log["dns_failures"]))
|
|
|
|
|
lines.append(("Timeouts:", self.log["timeouts"]))
|
|
|
|
|
lines.append(("Refused connections:", self.log["refused_connections"]))
|
|
|
|
|
lines.append(("Reset connections:", self.log["reset_connections"]))
|
2020-09-01 19:14:17 +00:00
|
|
|
|
lines.append(("Cache hits:", self.log["cache_hits"]))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Print
|
|
|
|
|
for key, value in lines:
|
|
|
|
|
print(key.ljust(24) + str(value).rjust(8))
|
|
|
|
|
|
|
|
|
|
### The end!
|
|
|
|
|
def do_quit(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Exit Offpunk."""
|
2020-05-16 16:58:53 +00:00
|
|
|
|
# Close TOFU DB
|
|
|
|
|
self.db_conn.commit()
|
|
|
|
|
self.db_conn.close()
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Clean up after ourself
|
2020-06-08 19:52:28 +00:00
|
|
|
|
if self.tmp_filename and os.path.exists(self.tmp_filename):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
os.unlink(self.tmp_filename)
|
2020-06-08 19:52:28 +00:00
|
|
|
|
if self.idx_filename and os.path.exists(self.idx_filename):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
os.unlink(self.idx_filename)
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2020-05-11 20:22:24 +00:00
|
|
|
|
for cert in self.transient_certs_created:
|
|
|
|
|
for ext in (".crt", ".key"):
|
|
|
|
|
certfile = os.path.join(self.config_dir, "transient_certs", cert+ext)
|
|
|
|
|
if os.path.exists(certfile):
|
|
|
|
|
os.remove(certfile)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
print()
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("You can close your screen!")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
sys.exit()
|
|
|
|
|
|
|
|
|
|
do_exit = do_quit
|
|
|
|
|
|
|
|
|
|
# Main function
|
|
|
|
|
def main():
|
|
|
|
|
|
|
|
|
|
# Parse args
|
|
|
|
|
parser = argparse.ArgumentParser(description='A command line gemini client.')
|
|
|
|
|
parser.add_argument('--bookmarks', action='store_true',
|
|
|
|
|
help='start with your list of bookmarks')
|
2020-03-23 02:12:00 +00:00
|
|
|
|
parser.add_argument('--tls-cert', metavar='FILE', help='TLS client certificate file')
|
|
|
|
|
parser.add_argument('--tls-key', metavar='FILE', help='TLS client certificate private key file')
|
2020-04-07 20:46:05 +00:00
|
|
|
|
parser.add_argument('--restricted', action="store_true", help='Disallow shell, add, and save commands')
|
2021-12-14 13:06:07 +00:00
|
|
|
|
parser.add_argument('--sync', action='store_true',
|
|
|
|
|
help='run non-interactively to build cache by exploring bookmarks')
|
2021-12-16 14:58:05 +00:00
|
|
|
|
parser.add_argument('--cache-validity',
|
|
|
|
|
help='duration for which a cache is valid before sync (seconds)')
|
2020-05-10 12:34:48 +00:00
|
|
|
|
parser.add_argument('--version', action='store_true',
|
|
|
|
|
help='display version information and quit')
|
2019-06-22 12:58:21 +00:00
|
|
|
|
parser.add_argument('url', metavar='URL', nargs='*',
|
|
|
|
|
help='start with this URL')
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
2020-05-10 12:34:48 +00:00
|
|
|
|
# Handle --version
|
|
|
|
|
if args.version:
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Offpunk " + _VERSION)
|
2020-05-10 12:34:48 +00:00
|
|
|
|
sys.exit()
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Instantiate client
|
2021-12-14 13:06:07 +00:00
|
|
|
|
gc = GeminiClient(restricted=args.restricted,synconly=args.sync)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
# Process config file
|
2021-12-30 15:03:08 +00:00
|
|
|
|
rcfile = os.path.join(gc.config_dir, "offpunkrc")
|
2020-05-10 20:51:33 +00:00
|
|
|
|
if os.path.exists(rcfile):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
print("Using config %s" % rcfile)
|
|
|
|
|
with open(rcfile, "r") as fp:
|
|
|
|
|
for line in fp:
|
|
|
|
|
line = line.strip()
|
|
|
|
|
if ((args.bookmarks or args.url) and
|
|
|
|
|
any((line.startswith(x) for x in ("go", "g", "tour", "t")))
|
|
|
|
|
):
|
|
|
|
|
if args.bookmarks:
|
|
|
|
|
print("Skipping rc command \"%s\" due to --bookmarks option." % line)
|
|
|
|
|
else:
|
|
|
|
|
print("Skipping rc command \"%s\" due to provided URLs." % line)
|
|
|
|
|
continue
|
|
|
|
|
gc.cmdqueue.append(line)
|
|
|
|
|
|
|
|
|
|
# Say hi
|
2021-12-14 13:06:07 +00:00
|
|
|
|
if not args.sync:
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Welcome to Offpunk!")
|
2021-12-09 16:23:50 +00:00
|
|
|
|
if args.restricted:
|
|
|
|
|
print("Restricted mode engaged!")
|
|
|
|
|
print("Enjoy your patrol through Geminispace...")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
# Act on args
|
2020-03-23 02:12:00 +00:00
|
|
|
|
if args.tls_cert:
|
|
|
|
|
# If tls_key is None, python will attempt to load the key from tls_cert.
|
2020-05-27 07:00:42 +00:00
|
|
|
|
gc._activate_client_cert(args.tls_cert, args.tls_key)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
if args.bookmarks:
|
|
|
|
|
gc.cmdqueue.append("bookmarks")
|
|
|
|
|
elif args.url:
|
|
|
|
|
if len(args.url) == 1:
|
|
|
|
|
gc.cmdqueue.append("go %s" % args.url[0])
|
|
|
|
|
else:
|
|
|
|
|
for url in args.url:
|
|
|
|
|
if not url.startswith("gemini://"):
|
|
|
|
|
url = "gemini://" + url
|
|
|
|
|
gc.cmdqueue.append("tour %s" % url)
|
|
|
|
|
gc.cmdqueue.append("tour")
|
|
|
|
|
|
|
|
|
|
# Endless interpret loop
|
2021-12-14 13:06:07 +00:00
|
|
|
|
if args.sync:
|
2021-12-18 09:16:19 +00:00
|
|
|
|
# fetch_cache is the core of the sync algorithm.
|
|
|
|
|
# It takes as input :
|
2022-01-09 20:21:09 +00:00
|
|
|
|
# - a list of GeminiItems to be fetched
|
2021-12-18 09:16:19 +00:00
|
|
|
|
# - depth : the degree of recursion to build the cache (0 means no recursion)
|
|
|
|
|
# - validity : the age, in seconds, existing caches need to have before
|
|
|
|
|
# being refreshed (0 = never refreshed if it already exists)
|
|
|
|
|
# - savetotour : if True, newly cached items are added to tour
|
2021-12-19 14:51:59 +00:00
|
|
|
|
def add_to_tour(gitem):
|
|
|
|
|
if gitem.is_cache_valid():
|
2021-12-20 14:32:19 +00:00
|
|
|
|
print(" -> adding to tour: %s" %gitem.url)
|
2021-12-19 14:51:59 +00:00
|
|
|
|
with open(gc.tourfile,mode='a') as tf:
|
|
|
|
|
line = gitem.url.strip() + "\n"
|
|
|
|
|
tf.write(line)
|
|
|
|
|
tf.close()
|
|
|
|
|
|
|
|
|
|
def fetch_cache(gitem,depth=0,validity=0,savetotour=False,count=[0,0],strin=""):
|
|
|
|
|
#savetotour = True will save to tour newly cached content
|
2021-12-17 11:35:46 +00:00
|
|
|
|
# else, do not save to tour
|
2021-12-17 11:08:27 +00:00
|
|
|
|
#regardless of valitidy
|
|
|
|
|
if not gitem.is_cache_valid(validity=validity):
|
|
|
|
|
if strin != "":
|
|
|
|
|
endline = '\r'
|
|
|
|
|
else:
|
2021-12-17 14:50:20 +00:00
|
|
|
|
endline = None
|
2021-12-19 14:51:59 +00:00
|
|
|
|
#Did we already had a cache (even an old one) ?
|
|
|
|
|
isnew = not gitem.is_cache_valid()
|
2021-12-17 11:08:27 +00:00
|
|
|
|
print("%s [%s/%s] Fetch "%(strin,count[0],count[1]),gitem.url,end=endline)
|
|
|
|
|
gc.onecmd("go %s" %gitem.url)
|
2021-12-19 14:51:59 +00:00
|
|
|
|
if savetotour and isnew and gitem.is_cache_valid():
|
2021-12-17 11:08:27 +00:00
|
|
|
|
#we add to the next tour only if we managed to cache
|
|
|
|
|
#the ressource
|
2021-12-19 14:51:59 +00:00
|
|
|
|
add_to_tour(gitem)
|
2021-12-17 11:08:27 +00:00
|
|
|
|
if depth > 0:
|
|
|
|
|
d = depth - 1
|
|
|
|
|
temp_lookup = set(gc.lookup)
|
|
|
|
|
subcount = [0,len(temp_lookup)]
|
|
|
|
|
for k in temp_lookup:
|
|
|
|
|
#recursive call
|
|
|
|
|
substri = strin + " -->"
|
2021-12-17 17:09:55 +00:00
|
|
|
|
subcount[0] += 1
|
2021-12-19 14:51:59 +00:00
|
|
|
|
fetch_cache(k,depth=d,validity=0,savetotour=savetotour,\
|
2021-12-17 11:35:46 +00:00
|
|
|
|
count=subcount,strin=substri)
|
2021-12-17 11:08:27 +00:00
|
|
|
|
|
2021-12-16 14:58:05 +00:00
|
|
|
|
if args.cache_validity:
|
|
|
|
|
refresh_time = int(args.cache_validity)
|
|
|
|
|
else:
|
2022-01-09 20:21:09 +00:00
|
|
|
|
# if no refresh time, a default of 0 is used (which means "infinite")
|
2021-12-18 09:16:19 +00:00
|
|
|
|
refresh_time = 0
|
2021-12-14 13:06:07 +00:00
|
|
|
|
gc.sync_only = True
|
2021-12-18 09:16:19 +00:00
|
|
|
|
# We start by syncing the bookmarks
|
|
|
|
|
gc.onecmd("bm")
|
|
|
|
|
original_lookup = gc.lookup
|
|
|
|
|
end = len(original_lookup)
|
|
|
|
|
counter = 0
|
|
|
|
|
print(" * * * %s bookmarks to fetch * * *" %end)
|
|
|
|
|
for j in original_lookup:
|
|
|
|
|
#refreshing the bookmarks
|
|
|
|
|
counter += 1
|
2021-12-19 14:51:59 +00:00
|
|
|
|
fetch_cache(j,depth=1,validity=refresh_time,savetotour=True,count=[counter,end])
|
2021-12-18 09:16:19 +00:00
|
|
|
|
## Second we get ressources in the tour
|
2021-12-14 15:07:02 +00:00
|
|
|
|
lines_lookup = []
|
2021-12-17 11:35:46 +00:00
|
|
|
|
if os.path.exists(gc.tourfile):
|
|
|
|
|
with open(gc.tourfile,mode='r') as tf:
|
|
|
|
|
lines_lookup += tf.readlines()
|
|
|
|
|
tf.close
|
2021-12-17 11:08:27 +00:00
|
|
|
|
tot = len(set(lines_lookup))
|
|
|
|
|
counter = 0
|
2021-12-17 11:35:46 +00:00
|
|
|
|
if tot > 0:
|
|
|
|
|
print(" * * * %s to fetch from your tour browsing * * *" %tot)
|
2021-12-17 11:08:27 +00:00
|
|
|
|
for l in set(lines_lookup):
|
|
|
|
|
#always get to_fetch and tour, regarless of refreshtime
|
2021-12-17 11:35:46 +00:00
|
|
|
|
#we don’t save to tour (it’s already there)
|
2021-12-17 11:08:27 +00:00
|
|
|
|
counter += 1
|
2022-01-09 20:21:09 +00:00
|
|
|
|
if l.startswith("gemini://") or l.startswith("http"):
|
2021-12-20 14:32:19 +00:00
|
|
|
|
fetch_cache(GeminiItem(l.strip()),depth=1,validity=refresh_time,\
|
2021-12-19 14:51:59 +00:00
|
|
|
|
savetotour=False,count=[counter,tot])
|
2021-12-17 11:35:46 +00:00
|
|
|
|
# Then we get ressources from syncfile
|
2021-12-17 11:08:27 +00:00
|
|
|
|
lines_lookup = []
|
2021-12-17 11:35:46 +00:00
|
|
|
|
if os.path.exists(gc.syncfile):
|
|
|
|
|
with open(gc.syncfile,mode='r') as sf:
|
|
|
|
|
lines_lookup += sf.readlines()
|
|
|
|
|
sf.close()
|
2021-12-17 11:08:27 +00:00
|
|
|
|
tot = len(set(lines_lookup))
|
|
|
|
|
counter = 0
|
2021-12-17 11:35:46 +00:00
|
|
|
|
if tot > 0:
|
|
|
|
|
print(" * * * %s to fetch from your offline browsing * * *" %tot)
|
2021-12-17 11:08:27 +00:00
|
|
|
|
for l in set(lines_lookup):
|
2022-01-09 20:21:09 +00:00
|
|
|
|
#always fetch the cache (we allows only a 3 minutes time
|
|
|
|
|
# to avoid multiple fetch in the same sync run)
|
2021-12-19 14:51:59 +00:00
|
|
|
|
#then add to tour
|
2021-12-17 11:08:27 +00:00
|
|
|
|
counter += 1
|
2021-12-20 14:32:19 +00:00
|
|
|
|
gitem = GeminiItem(l.strip())
|
2022-01-09 20:21:09 +00:00
|
|
|
|
if l.startswith("gemini://") or l.startswith("http"):
|
2021-12-19 14:51:59 +00:00
|
|
|
|
fetch_cache(gitem,depth=1,validity=180,\
|
|
|
|
|
savetotour=False,count=[counter,tot])
|
|
|
|
|
add_to_tour(gitem)
|
|
|
|
|
|
2022-01-11 08:39:09 +00:00
|
|
|
|
# let’s empty the sync file once the fetch is successful
|
|
|
|
|
if os.path.exists(gc.syncfile):
|
|
|
|
|
open(gc.syncfile,mode='w').close()
|
2021-12-09 16:23:50 +00:00
|
|
|
|
gc.onecmd("blackbox")
|
2021-12-09 14:12:32 +00:00
|
|
|
|
else:
|
|
|
|
|
while True:
|
|
|
|
|
try:
|
|
|
|
|
gc.cmdloop()
|
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
|
print("")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
main()
|