forked from solderpunk/AV-98
2014 lines
81 KiB
Python
Executable File
2014 lines
81 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# Offpunk Offline Gemini client
|
||
# Derived from AV-98 by Solderpunk,
|
||
# (C) From 2021: Ploum <offpunk@ploum.eu>
|
||
# (C) 2019, 2020: Solderpunk <solderpunk@sdf.org>
|
||
# With contributions from:
|
||
# - danceka <hannu.hartikainen@gmail.com>
|
||
# - <jprjr@tilde.club>
|
||
# - <vee@vnsf.xyz>
|
||
# - Klaus Alexander Seistrup <klaus@seistrup.dk>
|
||
# - govynnus <govynnus@sdf.org>
|
||
# - Björn Wärmedal <bjorn.warmedal@gmail.com>
|
||
# - <jake@rmgr.dev>
|
||
# - Maeve Sproule <code@sprock.dev>
|
||
|
||
"""
|
||
Offline-First Gemini/Web/Gopher/RSS reader and browser
|
||
"""
|
||
|
||
__version__ = "1.9.2"
|
||
|
||
import argparse
|
||
import cmd
|
||
import datetime
|
||
import fnmatch
|
||
import io
|
||
import os
|
||
import os.path
|
||
import filecmp
|
||
import random
|
||
import shlex
|
||
import shutil
|
||
import socket
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
import uuid
|
||
import webbrowser
|
||
import base64
|
||
import subprocess
|
||
import ansirenderer
|
||
import netcache
|
||
from offutils import run,term_width
|
||
try:
|
||
import setproctitle
|
||
setproctitle.setproctitle("offpunk")
|
||
_HAS_SETPROCTITLE = True
|
||
except ModuleNotFoundError:
|
||
_HAS_SETPROCTITLE = False
|
||
|
||
|
||
_HAS_XSEL = shutil.which('xsel')
|
||
_HAS_XDGOPEN = shutil.which('xdg-open')
|
||
|
||
try:
|
||
import requests
|
||
_DO_HTTP = True
|
||
except ModuleNotFoundError:
|
||
_DO_HTTP = False
|
||
|
||
|
||
|
||
## Config directories
|
||
## We implement our own python-xdg to avoid conflict with existing libraries.
|
||
_home = os.path.expanduser('~')
|
||
data_home = os.environ.get('XDG_DATA_HOME') or \
|
||
os.path.join(_home,'.local','share')
|
||
config_home = os.environ.get('XDG_CONFIG_HOME') or \
|
||
os.path.join(_home,'.config')
|
||
_CONFIG_DIR = os.path.join(config_home,"offpunk/")
|
||
_DATA_DIR = os.path.join(data_home,"offpunk/")
|
||
_old_config = os.path.expanduser("~/.offpunk/")
|
||
## Look for pre-existing config directory, if any
|
||
if os.path.exists(_old_config):
|
||
_CONFIG_DIR = _old_config
|
||
#if no XDG .local/share and not XDG .config, we use the old config
|
||
if not os.path.exists(data_home) and os.path.exists(_old_config):
|
||
_DATA_DIR = _CONFIG_DIR
|
||
_MAX_CACHE_SIZE = 10
|
||
_MAX_CACHE_AGE_SECS = 180
|
||
|
||
_GREP = "grep --color=auto"
|
||
|
||
#TODO: remove this debug code
|
||
_CACHE_PATH = "/home/ploum/dev/netcache/"
|
||
_DATA_DIR = "/home/ploum/dev/netcache/"
|
||
_CONFIG_DIR = "/home/ploum/dev/netcache/"
|
||
# Command abbreviations
|
||
_ABBREVS = {
|
||
"..": "up",
|
||
"a": "add",
|
||
"b": "back",
|
||
"bb": "blackbox",
|
||
"bm": "bookmarks",
|
||
"book": "bookmarks",
|
||
"cp": "copy",
|
||
"f": "forward",
|
||
"g": "go",
|
||
"h": "history",
|
||
"hist": "history",
|
||
"l": "view",
|
||
"less": "view",
|
||
"man": "help",
|
||
"mv": "move",
|
||
"n": "next",
|
||
"off": "offline",
|
||
"on": "online",
|
||
"p": "previous",
|
||
"prev": "previous",
|
||
"q": "quit",
|
||
"r": "reload",
|
||
"s": "save",
|
||
"se": "search",
|
||
"/": "find",
|
||
"t": "tour",
|
||
"u": "up",
|
||
"v": "view",
|
||
"w": "wikipedia",
|
||
"wen": "wikipedia en",
|
||
"wfr": "wikipedia fr",
|
||
"wes": "wikipedia es",
|
||
}
|
||
|
||
_MIME_HANDLERS = {
|
||
}
|
||
|
||
#An IPV6 URL should be put between []
|
||
#We try to detect them has location with more than 2 ":"
|
||
def fix_ipv6_url(url):
|
||
if not url or url.startswith("mailto"):
|
||
return url
|
||
if "://" in url:
|
||
schema, schemaless = url.split("://",maxsplit=1)
|
||
else:
|
||
schema, schemaless = None, url
|
||
if "/" in schemaless:
|
||
netloc, rest = schemaless.split("/",1)
|
||
if netloc.count(":") > 2 and "[" not in netloc and "]" not in netloc:
|
||
schemaless = "[" + netloc + "]" + "/" + rest
|
||
elif schemaless.count(":") > 2:
|
||
schemaless = "[" + schemaless + "]/"
|
||
if schema:
|
||
return schema + "://" + schemaless
|
||
return schemaless
|
||
|
||
# Cheap and cheerful URL detector
|
||
def looks_like_url(word):
|
||
try:
|
||
if not word.strip():
|
||
return False
|
||
url = fix_ipv6_url(word).strip()
|
||
parsed = urllib.parse.urlparse(url)
|
||
#sometimes, urllib crashed only when requesting the port
|
||
port = parsed.port
|
||
scheme = word.split("://")[0]
|
||
mailto = word.startswith("mailto:")
|
||
start = scheme in netcache.standard_ports
|
||
local = scheme in ["file","list"]
|
||
if mailto:
|
||
return "@" in word
|
||
elif not local:
|
||
return start and ("." in word or "localhost" in word)
|
||
else:
|
||
return "/" in word
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
# GeminiClient Decorators
|
||
def needs_gi(inner):
|
||
def outer(self, *args, **kwargs):
|
||
if not self.current_url:
|
||
print("You need to 'go' somewhere, first")
|
||
return None
|
||
else:
|
||
return inner(self, *args, **kwargs)
|
||
outer.__doc__ = inner.__doc__
|
||
return outer
|
||
|
||
class GeminiClient(cmd.Cmd):
|
||
|
||
def __init__(self, completekey="tab", synconly=False):
|
||
cmd.Cmd.__init__(self)
|
||
|
||
# Set umask so that nothing we create can be read by anybody else.
|
||
# The certificate cache and TOFU database contain "browser history"
|
||
# type sensitivie information.
|
||
os.umask(0o077)
|
||
|
||
# This dictionary contains an url -> ansirenderer mapping. This allows
|
||
# to reuse a renderer when visiting several times the same URL during
|
||
# the same session
|
||
self.rendererdic = {}
|
||
self.no_cert_prompt = "\001\x1b[38;5;76m\002" + "ON" + "\001\x1b[38;5;255m\002" + "> " + "\001\x1b[0m\002"
|
||
self.cert_prompt = "\001\x1b[38;5;202m\002" + "ON" + "\001\x1b[38;5;255m\002"
|
||
self.offline_prompt = "\001\x1b[38;5;76m\002" + "OFF" + "\001\x1b[38;5;255m\002" + "> " + "\001\x1b[0m\001"
|
||
self.prompt = self.no_cert_prompt
|
||
self.current_url = None
|
||
self.hist_index = 0
|
||
self.marks = {}
|
||
self.page_index = 0
|
||
self.permanent_redirects = {}
|
||
# Sync-only mode is restriced by design
|
||
self.visited_hosts = set()
|
||
self.offline_only = False
|
||
self.sync_only = False
|
||
self.support_http = _DO_HTTP
|
||
self.automatic_choice = "n"
|
||
|
||
self.client_certs = {
|
||
"active": None
|
||
}
|
||
self.active_cert_domains = []
|
||
self.active_is_transient = False
|
||
self.transient_certs_created = []
|
||
|
||
self.options = {
|
||
"debug" : False,
|
||
"beta" : False,
|
||
"timeout" : 600,
|
||
"short_timeout" : 5,
|
||
"width" : 72,
|
||
"auto_follow_redirects" : True,
|
||
"tls_mode" : "tofu",
|
||
"archives_size" : 200,
|
||
"history_size" : 200,
|
||
"max_size_download" : 10,
|
||
"editor" : None,
|
||
"download_images_first" : True,
|
||
"redirects" : True,
|
||
# the wikipedia entry needs two %s, one for lang, other for search
|
||
"wikipedia" : "gemini://vault.transjovian.org:1965/search/%s/%s",
|
||
"search" : "gemini://kennedy.gemi.dev/search?%s",
|
||
"accept_bad_ssl_certificates" : False,
|
||
}
|
||
|
||
self.redirects = {
|
||
"twitter.com" : "nitter.42l.fr",
|
||
"facebook.com" : "blocked",
|
||
"doubleclick.net": "blocked",
|
||
"google-analytics.com" : "blocked",
|
||
"youtube.com" : "yewtu.be",
|
||
"reddit.com" : "teddit.net",
|
||
"old.reddit.com": "teddit.net",
|
||
"medium.com" : "scribe.rip",
|
||
|
||
}
|
||
term_width(new_width=self.options["width"])
|
||
self.log = {
|
||
"start_time": time.time(),
|
||
"requests": 0,
|
||
"ipv4_requests": 0,
|
||
"ipv6_requests": 0,
|
||
"bytes_recvd": 0,
|
||
"ipv4_bytes_recvd": 0,
|
||
"ipv6_bytes_recvd": 0,
|
||
"dns_failures": 0,
|
||
"refused_connections": 0,
|
||
"reset_connections": 0,
|
||
"timeouts": 0,
|
||
"cache_hits": 0,
|
||
}
|
||
|
||
|
||
def complete_list(self,text,line,begidx,endidx):
|
||
allowed = []
|
||
cmds = ["create","edit","subscribe","freeze","normal","delete","help"]
|
||
lists = self.list_lists()
|
||
words = len(line.split())
|
||
# We need to autocomplete listname for the first or second argument
|
||
# If the first one is a cmds
|
||
if words <= 1:
|
||
allowed = lists + cmds
|
||
elif words == 2:
|
||
# if text, the completing word is the second
|
||
cond = bool(text)
|
||
if text:
|
||
allowed = lists + cmds
|
||
else:
|
||
current_cmd = line.split()[1]
|
||
if current_cmd in ["help", "create"]:
|
||
allowed = []
|
||
elif current_cmd in cmds:
|
||
allowed = lists
|
||
elif words == 3 and text != "":
|
||
current_cmd = line.split()[1]
|
||
if current_cmd in ["help", "create"]:
|
||
allowed = []
|
||
elif current_cmd in cmds:
|
||
allowed = lists
|
||
return [i+" " for i in allowed if i.startswith(text)]
|
||
|
||
def complete_add(self,text,line,begidx,endidx):
|
||
if len(line.split()) == 2 and text != "":
|
||
allowed = self.list_lists()
|
||
elif len(line.split()) == 1:
|
||
allowed = self.list_lists()
|
||
else:
|
||
allowed = []
|
||
return [i+" " for i in allowed if i.startswith(text)]
|
||
def complete_move(self,text,line,begidx,endidx):
|
||
return self.complete_add(text,line,begidx,endidx)
|
||
|
||
def get_renderer(self,url=None):
|
||
# If launched without argument, we return the renderer for the current URL
|
||
mode = None
|
||
if not url: url = self.current_url
|
||
findmode = url.split("##offpunk_mode=")
|
||
if len(findmode) > 1:
|
||
url = findmode[0]
|
||
if findmode[1] in ["full"] or findmode[1].isnumeric():
|
||
mode = findmode[1]
|
||
# reuse existing renderer if any
|
||
if url in self.rendererdic.keys():
|
||
renderer = self.rendererdic[url]
|
||
else:
|
||
cache_path = netcache.get_cache_path(url)
|
||
if cache_path:
|
||
renderer = ansirenderer.renderer_from_file(cache_path,url)
|
||
self.rendererdic[url] = renderer
|
||
else:
|
||
print("WARNING: no cache for requested renderer for %s" %url)
|
||
return None
|
||
if mode:
|
||
renderer.set_mode(mode)
|
||
return renderer
|
||
|
||
def _go_to_url(self, url, update_hist=True, check_cache=True, handle=True,\
|
||
name=None, mode=None,limit_size=False):
|
||
"""This method might be considered "the heart of Offpunk".
|
||
Everything involved in fetching a gemini resource happens here:
|
||
sending the request over the network, parsing the response,
|
||
storing the response in a temporary file, choosing
|
||
and calling a handler program, and updating the history.
|
||
Nothing is returned."""
|
||
if not url:
|
||
return
|
||
#TODO: we don’t handle the name anymore !
|
||
if name:
|
||
print("We don’t handle name of URL: %s"%name)
|
||
# Don't try to speak to servers running other protocols
|
||
#TODO: support for mailto and unsupported protocols
|
||
# elif gi.scheme == "mailto":
|
||
# if handle and not self.sync_only:
|
||
# resp = input("Send an email to %s Y/N? " %gi.path)
|
||
# self.gi = gi
|
||
# if resp.strip().lower() in ("y", "yes"):
|
||
# if _HAS_XDGOPEN :
|
||
# run("xdg-open mailto:%s", parameter=gi.path ,direct_output=True)
|
||
# else:
|
||
# print("Cannot find a mail client to send mail to %s" %gi.path)
|
||
# print("Please install xdg-open (usually from xdg-util package)")
|
||
# return
|
||
# elif gi.scheme not in ["file","list"] and gi.scheme not in netcache.standard_ports \
|
||
# and not self.sync_only:
|
||
# print("Sorry, no support for {} links.".format(gi.scheme))
|
||
# return
|
||
# Obey permanent redirects
|
||
if url in self.permanent_redirects:
|
||
self._go_to_url(self.permanent_redirects[url],update_hist=update_hist,\
|
||
check_cache=check_cache, handle=handle, name=name,mode=mode,\
|
||
limit_size=limit_size)
|
||
return
|
||
|
||
# check if local file exists.
|
||
#TODO
|
||
#if gi.local and not os.path.exists(gi.path):
|
||
# print("Local file %s does not exist!" %gi.path)
|
||
# return
|
||
|
||
#elif not gi.local:
|
||
if True:
|
||
params = {}
|
||
params["timeout"] = self.options["short_timeout"]
|
||
params["max_size"] = int(self.options["max_size_download"])*1000000
|
||
params["print_error"] = not self.sync_only
|
||
params["offline"] = self.offline_only
|
||
cachepath = netcache.fetch(url,**params)
|
||
renderer = self.get_renderer(url)
|
||
if not renderer:
|
||
print("no renderer for %s"%url)
|
||
# Use cache or mark as to_fetch if resource is not cached
|
||
if not cachepath:
|
||
self.get_list("to_fetch")
|
||
r = self.list_add_line("to_fetch",url=url,verbose=False)
|
||
if r:
|
||
print("%s not available, marked for syncing"%url)
|
||
else:
|
||
print("%s already marked for syncing"%url)
|
||
return
|
||
#Ok, we have a cached version
|
||
else:
|
||
# Pass file to handler, unless we were asked not to
|
||
display = handle and not self.sync_only
|
||
#TODO: take into account _RENDER_IMAGE
|
||
if display and self.options["download_images_first"] \
|
||
and not self.offline_only:
|
||
|
||
# We download images first
|
||
#TODO: this should go into netcache
|
||
print("url: ## %s ##"%url)
|
||
for image in renderer.get_images(mode=mode):
|
||
if image and image.startswith("http"):
|
||
if not netcache.is_cache_valid(image):
|
||
width = term_width() - 1
|
||
toprint = "Downloading %s" %image
|
||
toprint = toprint[:width]
|
||
toprint += " "*(width-len(toprint))
|
||
print(toprint,end="\r")
|
||
self._go_to_url(image, update_hist=False, check_cache=True, \
|
||
handle=False,limit_size=True)
|
||
is_rendered = False
|
||
if display and netcache.is_cache_valid(url):
|
||
title = renderer.get_url_title()
|
||
nbr = len(renderer.get_links(mode=mode))
|
||
if renderer.is_local():
|
||
title += " (%s items)"%nbr
|
||
str_last = "local file"
|
||
else:
|
||
str_last = "last accessed on %s"\
|
||
%time.ctime(netcache.cache_last_modified(url))
|
||
title += " (%s links)"%nbr
|
||
is_rendered = renderer.display(mode=mode,\
|
||
window_title=title,window_info=str_last)
|
||
if display and is_rendered:
|
||
self.page_index = 0
|
||
# Update state (external files are not added to history)
|
||
if mode and mode != "readable":
|
||
url += "##offpunk_mode=" + mode
|
||
self.current_url = url
|
||
if update_hist and not self.sync_only:
|
||
self._update_history(url)
|
||
elif display and not is_rendered :
|
||
cmd_str = self._get_handler_cmd(ansirenderer.get_mime(url))
|
||
try:
|
||
run(cmd_str, netcache.get_cache_path(url), direct_output=True)
|
||
except FileNotFoundError:
|
||
print("Handler program %s not found!" % shlex.split(cmd_str)[0])
|
||
print("You can use the ! command to specify another handler program or pipeline.")
|
||
|
||
|
||
|
||
|
||
def _get_handler_cmd(self, mimetype):
|
||
# Now look for a handler for this mimetype
|
||
# Consider exact matches before wildcard matches
|
||
exact_matches = []
|
||
wildcard_matches = []
|
||
for handled_mime, cmd_str in _MIME_HANDLERS.items():
|
||
if "*" in handled_mime:
|
||
wildcard_matches.append((handled_mime, cmd_str))
|
||
else:
|
||
exact_matches.append((handled_mime, cmd_str))
|
||
for handled_mime, cmd_str in exact_matches + wildcard_matches:
|
||
if fnmatch.fnmatch(mimetype, handled_mime):
|
||
break
|
||
else:
|
||
# Use "xdg-open" as a last resort.
|
||
if _HAS_XDGOPEN:
|
||
cmd_str = "xdg-open %s"
|
||
else:
|
||
cmd_str = "echo \"Can’t find how to open \"%s"
|
||
print("Please install xdg-open (usually from xdg-util package)")
|
||
return cmd_str
|
||
|
||
@needs_gi
|
||
def _show_lookup(self, offset=0, end=None, show_url=False):
|
||
for n, u in enumerate(self.get_renderer().get_links()[offset:end]):
|
||
index = n+offset+1
|
||
line = "[%s] %s" %(index,u)
|
||
#TODO: implement proper listing of url (with protocol and show_url)
|
||
# protocol = "" if gi.scheme == "gemini" else " %s" % gi.scheme
|
||
# line = "[%d%s] %s" % (index, protocol, gi.name or gi.url)
|
||
# line += " (%s)" % gi.url
|
||
print(line)
|
||
|
||
def _update_history(self, url):
|
||
# We never update while in sync_only
|
||
if self.sync_only:
|
||
return
|
||
# We don’t add lists to history
|
||
#if not gi or os.path.join(_DATA_DIR,"lists") in gi.url:
|
||
# return
|
||
histlist = self.get_list("history")
|
||
links = self.list_get_links("history")
|
||
# avoid duplicate
|
||
length = len(links)
|
||
if length > self.options["history_size"]:
|
||
length = self.options["history_size"]
|
||
if length > 0 and links[self.hist_index] == url:
|
||
return
|
||
self.list_add_top("history",limit=self.options["history_size"],truncate_lines=self.hist_index)
|
||
self.hist_index = 0
|
||
|
||
|
||
def _log_visit(self, gi, address, size):
|
||
if not address:
|
||
return
|
||
self.log["requests"] += 1
|
||
self.log["bytes_recvd"] += size
|
||
self.visited_hosts.add(address)
|
||
if address[0] == socket.AF_INET:
|
||
self.log["ipv4_requests"] += 1
|
||
self.log["ipv4_bytes_recvd"] += size
|
||
elif address[0] == socket.AF_INET6:
|
||
self.log["ipv6_requests"] += 1
|
||
self.log["ipv6_bytes_recvd"] += size
|
||
|
||
def _debug(self, debug_text):
|
||
if not self.options["debug"]:
|
||
return
|
||
debug_text = "\x1b[0;32m[DEBUG] " + debug_text + "\x1b[0m"
|
||
print(debug_text)
|
||
|
||
|
||
# Cmd implementation follows
|
||
|
||
def default(self, line):
|
||
if line.strip() == "EOF":
|
||
return self.onecmd("quit")
|
||
elif line.startswith("/"):
|
||
return self.do_find(line[1:])
|
||
# Expand abbreviated commands
|
||
first_word = line.split()[0].strip()
|
||
if first_word in _ABBREVS:
|
||
full_cmd = _ABBREVS[first_word]
|
||
expanded = line.replace(first_word, full_cmd, 1)
|
||
return self.onecmd(expanded)
|
||
# Try to access it like an URL
|
||
if looks_like_url(line):
|
||
return self.do_go(line)
|
||
# Try to parse numerical index for lookup table
|
||
try:
|
||
n = int(line.strip())
|
||
except ValueError:
|
||
print("What?")
|
||
return
|
||
# if we have no url, there's nothing to do
|
||
if self.current_url is None:
|
||
print("No links to index")
|
||
return
|
||
else:
|
||
r = self.get_renderer()
|
||
if r:
|
||
url = r.get_link(n)
|
||
self._go_to_url(url)
|
||
else:
|
||
print("No page with links")
|
||
return
|
||
|
||
|
||
### Settings
|
||
def do_redirect(self,line):
|
||
"""Display and manage the list of redirected URLs. This features is mostly useful to use privacy-friendly frontends for popular websites."""
|
||
if len(line.split()) == 1:
|
||
if line in self.redirects:
|
||
print("%s is redirected to %s" %(line,self.redirects[line]))
|
||
else:
|
||
print("Please add a destination to redirect %s" %line)
|
||
elif len(line.split()) >= 2:
|
||
orig, dest = line.split(" ",1)
|
||
if dest.lower() == "none":
|
||
if orig in self.redirects:
|
||
self.redirects.pop(orig)
|
||
print("Redirection for %s has been removed"%orig)
|
||
else:
|
||
print("%s was not redirected. Nothing has changed."%orig)
|
||
elif dest.lower() == "block":
|
||
self.redirects[orig] = "blocked"
|
||
print("%s will now be blocked"%orig)
|
||
else:
|
||
self.redirects[orig] = dest
|
||
print("%s will now be redirected to %s" %(orig,dest))
|
||
else:
|
||
toprint="Current redirections:\n"
|
||
toprint+="--------------------\n"
|
||
for r in self.redirects:
|
||
toprint += ("%s\t->\t%s\n" %(r,self.redirects[r]))
|
||
toprint +="\nTo add new, use \"redirect origine.com destination.org\""
|
||
toprint +="\nTo remove a redirect, use \"redirect origine.com NONE\""
|
||
toprint +="\nTo completely block a website, use \"redirect origine.com BLOCK\""
|
||
print(toprint)
|
||
|
||
def do_set(self, line):
|
||
"""View or set various options."""
|
||
if not line.strip():
|
||
# Show all current settings
|
||
for option in sorted(self.options.keys()):
|
||
print("%s %s" % (option, self.options[option]))
|
||
elif len(line.split()) == 1 :
|
||
# Show current value of one specific setting
|
||
option = line.strip()
|
||
if option in self.options:
|
||
print("%s %s" % (option, self.options[option]))
|
||
else:
|
||
print("Unrecognised option %s" % option)
|
||
else:
|
||
# Set value of one specific setting
|
||
option, value = line.split(" ", 1)
|
||
if option not in self.options:
|
||
print("Unrecognised option %s" % option)
|
||
return
|
||
# Validate / convert values
|
||
elif option == "tls_mode":
|
||
if value.lower() not in ("ca", "tofu"):
|
||
print("TLS mode must be `ca` or `tofu`!")
|
||
return
|
||
elif option == "accept_bad_ssl_certificates":
|
||
if value.lower() == "false":
|
||
print("Only high security certificates are now accepted")
|
||
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:@SECLEVEL=2'
|
||
elif value.lower() == "true":
|
||
print("Low security SSL certificates are now accepted")
|
||
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:@SECLEVEL=1'
|
||
else:
|
||
print("accept_bad_ssl_certificates should be True or False")
|
||
return
|
||
elif option == "width":
|
||
if value.isnumeric():
|
||
value = int(value)
|
||
print("changing width to ",value)
|
||
term_width(new_width=value)
|
||
else:
|
||
print("%s is not a valid width (integer required)"%value)
|
||
elif value.isnumeric():
|
||
value = int(value)
|
||
elif value.lower() == "false":
|
||
value = False
|
||
elif value.lower() == "true":
|
||
value = True
|
||
else:
|
||
try:
|
||
value = float(value)
|
||
except ValueError:
|
||
pass
|
||
self.options[option] = value
|
||
|
||
def do_cert(self, line):
|
||
"""Manage client certificates"""
|
||
print("Managing client certificates")
|
||
if self.client_certs["active"]:
|
||
print("Active certificate: {}".format(self.client_certs["active"][0]))
|
||
print("1. Deactivate client certificate.")
|
||
print("2. Generate new certificate.")
|
||
print("3. Load previously generated certificate.")
|
||
print("4. Load externally created client certificate from file.")
|
||
print("Enter blank line to exit certificate manager.")
|
||
choice = input("> ").strip()
|
||
if choice == "1":
|
||
print("Deactivating client certificate.")
|
||
netcache._deactivate_client_cert()
|
||
elif choice == "2":
|
||
netcache._generate_persistent_client_cert()
|
||
elif choice == "3":
|
||
netcache._choose_client_cert()
|
||
elif choice == "4":
|
||
netcache._load_client_cert()
|
||
else:
|
||
print("Aborting.")
|
||
|
||
def do_handler(self, line):
|
||
"""View or set handler commands for different MIME types."""
|
||
if not line.strip():
|
||
# Show all current handlers
|
||
for mime in sorted(_MIME_HANDLERS.keys()):
|
||
print("%s %s" % (mime, _MIME_HANDLERS[mime]))
|
||
elif len(line.split()) == 1:
|
||
mime = line.strip()
|
||
if mime in _MIME_HANDLERS:
|
||
print("%s %s" % (mime, _MIME_HANDLERS[mime]))
|
||
else:
|
||
print("No handler set for MIME type %s" % mime)
|
||
else:
|
||
mime, handler = line.split(" ", 1)
|
||
_MIME_HANDLERS[mime] = handler
|
||
if "%s" not in handler:
|
||
print("Are you sure you don't want to pass the filename to the handler?")
|
||
|
||
def do_abbrevs(self, *args):
|
||
"""Print all Offpunk command abbreviations."""
|
||
header = "Command Abbreviations:"
|
||
self.stdout.write("\n{}\n".format(str(header)))
|
||
if self.ruler:
|
||
self.stdout.write("{}\n".format(str(self.ruler * len(header))))
|
||
for k, v in _ABBREVS.items():
|
||
self.stdout.write("{:<7} {}\n".format(k, v))
|
||
self.stdout.write("\n")
|
||
|
||
def do_offline(self, *args):
|
||
"""Use Offpunk offline by only accessing cached content"""
|
||
if self.offline_only:
|
||
print("Offline and undisturbed.")
|
||
else:
|
||
self.offline_only = True
|
||
self.prompt = self.offline_prompt
|
||
print("Offpunk is now offline and will only access cached content")
|
||
|
||
def do_online(self, *args):
|
||
"""Use Offpunk online with a direct connection"""
|
||
if self.offline_only:
|
||
self.offline_only = False
|
||
self.prompt = self.no_cert_prompt
|
||
print("Offpunk is online and will access the network")
|
||
else:
|
||
print("Already online. Try offline.")
|
||
|
||
def do_copy(self, arg):
|
||
"""Copy the content of the last visited page as gemtext in the clipboard.
|
||
Use with "url" as argument to only copy the adress.
|
||
Use with "raw" to copy ANSI content as seen in your terminal (not gemtext).
|
||
Use with "cache" to copy the path of the cached content."""
|
||
if self.current_url:
|
||
if _HAS_XSEL:
|
||
args = arg.split()
|
||
if args and args[0] == "url":
|
||
if len(args) > 1 and args[1].isdecimal():
|
||
url = self.get_renderer().get_link(int(args[1])-1)
|
||
else:
|
||
url = self.current_url
|
||
run("xsel -b -i", input=url, direct_output=True)
|
||
elif args and args[0] == "raw":
|
||
run("xsel -b -i", input=open(self.get_renderer().get_temp_filename(), "rb"),\
|
||
direct_output=True)
|
||
elif args and args[0] == "cache":
|
||
run("xsel -b -i", input=netcache.get_cache_path(self.current_url),\
|
||
direct_output=True)
|
||
else:
|
||
run("xsel -b -i", input=open(netcache.get_cache_path(self.current_url), "rb"),\
|
||
direct_output=True)
|
||
else:
|
||
print("Please install xsel to use copy")
|
||
else:
|
||
print("No content to copy, visit a page first")
|
||
|
||
### Stuff for getting around
|
||
def do_go(self, line):
|
||
"""Go to a gemini URL or marked item."""
|
||
line = line.strip()
|
||
if not line:
|
||
if shutil.which('xsel'):
|
||
clipboards = []
|
||
urls = []
|
||
for selec in ["-p","-s","-b"]:
|
||
try:
|
||
clipboards.append(run("xsel "+selec))
|
||
except Exception as err:
|
||
#print("Skippink clipboard %s because %s"%(selec,err))
|
||
pass
|
||
for u in clipboards:
|
||
if "://" in u and looks_like_url(u) and u not in urls :
|
||
urls.append(u)
|
||
if len(urls) > 1:
|
||
stri = "URLs in your clipboard\n"
|
||
counter = 0
|
||
for u in urls:
|
||
counter += 1
|
||
stri += "[%s] %s\n"%(counter,u)
|
||
stri += "Where do you want to go today ?> "
|
||
ans = input(stri)
|
||
if ans.isdigit() and 0 < int(ans) <= len(urls):
|
||
self.do_go(urls[int(ans)-1])
|
||
elif len(urls) == 1:
|
||
self.do_go(urls[0])
|
||
else:
|
||
print("Go where? (hint: simply copy an URL in your clipboard)")
|
||
else:
|
||
print("Go where? (hint: install xsel to go to copied URLs)")
|
||
|
||
# First, check for possible marks
|
||
elif line in self.marks:
|
||
url = self.marks[line]
|
||
self._go_to_url(url)
|
||
# or a local file
|
||
elif os.path.exists(os.path.expanduser(line)):
|
||
self._go_to_url(line)
|
||
# If this isn't a mark, treat it as a URL
|
||
elif looks_like_url(line):
|
||
self._go_to_url(line)
|
||
else:
|
||
print("%s is not a valid URL to go"%line)
|
||
|
||
@needs_gi
|
||
def do_reload(self, *args):
|
||
"""Reload the current URL."""
|
||
if self.offline_only:
|
||
self.get_list("to_fetch")
|
||
r = self.list_add_line("to_fetch",url=self.current_url,verbose=False)
|
||
if r:
|
||
print("%s marked for syncing" %self.current_url)
|
||
else:
|
||
print("%s already marked for syncing" %self.current_url)
|
||
else:
|
||
self._go_to_url(self.current_url, check_cache=False)
|
||
|
||
@needs_gi
|
||
def do_up(self, *args):
|
||
"""Go up one directory in the path.
|
||
Take an integer as argument to go up multiple times."""
|
||
level = 1
|
||
if args[0].isnumeric():
|
||
level = int(args[0])
|
||
elif args[0] != "":
|
||
print("Up only take integer as arguments")
|
||
#TODO : implement up, this code is copy/pasted from GeminiItem
|
||
path = self.path.rstrip('/')
|
||
count = 0
|
||
while count < level:
|
||
pathbits = list(os.path.split(path))
|
||
# Don't try to go higher than root or in config
|
||
if self.local or len(pathbits) == 1 :
|
||
return self
|
||
# Get rid of bottom component
|
||
if len(pathbits) > 1:
|
||
pathbits.pop()
|
||
path = os.path.join(*pathbits)
|
||
count += 1
|
||
if self.scheme == "gopher":
|
||
path = "/1" + path
|
||
return self._derive_url(path)
|
||
|
||
def do_back(self, *args):
|
||
"""Go back to the previous gemini item."""
|
||
histfile = self.get_list("history")
|
||
links = self.list_get_links("history")
|
||
if self.hist_index >= len(links) -1:
|
||
return
|
||
self.hist_index += 1
|
||
url = links[self.hist_index]
|
||
self._go_to_url(url, update_hist=False)
|
||
|
||
def do_forward(self, *args):
|
||
"""Go forward to the next gemini item."""
|
||
histfile = self.get_list("history")
|
||
links = self.list_get_links("history")
|
||
if self.hist_index <= 0:
|
||
return
|
||
self.hist_index -= 1
|
||
url = links[self.hist_index]
|
||
self._go_to_url(url, update_hist=False)
|
||
|
||
@needs_gi
|
||
def do_root(self, *args):
|
||
"""Go to root selector of the server hosting current item."""
|
||
parse = urllib.parse.urlparse(self.url)
|
||
self._go_to_url(urllib.parse.urlunparse(parse.scheme,parse.netloc,"/","","",""))
|
||
|
||
def do_tour(self, line):
|
||
"""Add index items as waypoints on a tour, which is basically a FIFO
|
||
queue of gemini items.
|
||
|
||
`tour` or `t` alone brings you to the next item in your tour.
|
||
Items can be added with `tour 1 2 3 4` or ranges like `tour 1-4`.
|
||
All items in current menu can be added with `tour *`.
|
||
All items in $LIST can be added with `tour $LIST`.
|
||
Current item can be added back to the end of the tour with `tour .`.
|
||
Current tour can be listed with `tour ls` and scrubbed with `tour clear`."""
|
||
# Creating the tour list if needed
|
||
self.get_list("tour")
|
||
line = line.strip()
|
||
if not line:
|
||
# Fly to next waypoint on tour
|
||
if len(self.list_get_links("tour")) < 1:
|
||
print("End of tour.")
|
||
else:
|
||
url = self.list_go_to_line("1","tour")
|
||
if url:
|
||
self.list_rm_url(url,"tour")
|
||
elif line == "ls":
|
||
self.list_show("tour")
|
||
elif line == "clear":
|
||
for l in self.list_get_links("tour"):
|
||
self.list_rm_url(l,"tour")
|
||
elif line == "*":
|
||
for l in self.get_renderer().get_links():
|
||
self.list_add_line("tour",url=l,verbose=False)
|
||
elif line == ".":
|
||
self.list_add_line("tour",verbose=False)
|
||
elif looks_like_url(line):
|
||
self.list_add_line("tour",url=line)
|
||
elif line in self.list_lists():
|
||
list_path = self.list_path(line)
|
||
if not list_path:
|
||
print("List %s does not exist. Cannot add it to tour"%(list))
|
||
else:
|
||
url = "list:///%s"%line
|
||
display = not self.sync_only
|
||
for l in self.get_renderer(url).get_links():
|
||
self.list_add_line("tour",url=l,verbose=False)
|
||
else:
|
||
for index in line.split():
|
||
try:
|
||
pair = index.split('-')
|
||
if len(pair) == 1:
|
||
# Just a single index
|
||
n = int(index)
|
||
url = self.get_renderer().get_link(n)
|
||
self.list_add_line("tour",url=url,verbose=False)
|
||
elif len(pair) == 2:
|
||
# Two endpoints for a range of indices
|
||
if int(pair[0]) < int(pair[1]):
|
||
for n in range(int(pair[0]), int(pair[1]) + 1):
|
||
url = self.get_renderer().get_link(n)
|
||
self.list_add_line("tour",url=url,verbose=False)
|
||
else:
|
||
for n in range(int(pair[0]), int(pair[1]) - 1, -1):
|
||
url = self.get_renderer().get_link(n)
|
||
self.list_add_line("tour",url=url,verbose=False)
|
||
|
||
else:
|
||
# Syntax error
|
||
print("Invalid use of range syntax %s, skipping" % index)
|
||
except ValueError:
|
||
print("Non-numeric index %s, skipping." % index)
|
||
except IndexError:
|
||
print("Invalid index %d, skipping." % n)
|
||
|
||
@needs_gi
|
||
def do_mark(self, line):
|
||
"""Mark the current item with a single letter. This letter can then
|
||
be passed to the 'go' command to return to the current item later.
|
||
Think of it like marks in vi: 'mark a'='ma' and 'go a'=''a'.
|
||
Marks are temporary until shutdown (not saved to disk)."""
|
||
line = line.strip()
|
||
if not line:
|
||
for mark, gi in self.marks.items():
|
||
print("[%s] %s (%s)" % (mark, gi.name, gi.url))
|
||
elif line.isalpha() and len(line) == 1:
|
||
self.marks[line] = self.current_url
|
||
else:
|
||
print("Invalid mark, must be one letter")
|
||
|
||
@needs_gi
|
||
def do_info(self,line):
|
||
"""Display information about current page."""
|
||
print("current url: %s" %self.current_url)
|
||
print("current renderer: %s" %self.rendererdic)
|
||
renderer = self.get_renderer()
|
||
url = self.current_url
|
||
out = renderer.get_page_title() + "\n\n"
|
||
out += "URL : " + url + "\n"
|
||
out += "Mime : " + renderer.get_mime() + "\n"
|
||
out += "Cache : " + netcache.get_cache_path(url) + "\n"
|
||
tmp = self.get_renderer().get_temp_filename()
|
||
if tmp != netcache.get_cache_path(self.current_url):
|
||
out += "Tempfile : " + self.get_renderer().get_temp_filename() + "\n"
|
||
if self.get_renderer() :
|
||
rend = str(self.get_renderer().__class__)
|
||
rend = rend.lstrip("<class '__main__.").rstrip("'>")
|
||
else:
|
||
rend = "None"
|
||
out += "Renderer : " + rend + "\n\n"
|
||
lists = []
|
||
for l in self.list_lists():
|
||
if self.list_has_url(self.current_url,l):
|
||
lists.append(l)
|
||
if len(lists) > 0:
|
||
out += "Page appeard in following lists :\n"
|
||
for l in lists:
|
||
if not self.list_is_system(l):
|
||
status = "normal list"
|
||
if self.list_is_subscribed(l):
|
||
status = "subscription"
|
||
elif self.list_is_frozen(l):
|
||
status = "frozen list"
|
||
out += " • %s\t(%s)\n" %(l,status)
|
||
for l in lists:
|
||
if self.list_is_system(l):
|
||
out += " • %s\n" %l
|
||
else:
|
||
out += "Page is not save in any list"
|
||
print(out)
|
||
|
||
def do_version(self, line):
|
||
"""Display version and system information."""
|
||
def has(value):
|
||
if value:
|
||
return "\t\x1b[1;32mInstalled\x1b[0m\n"
|
||
else:
|
||
return "\t\x1b[1;31mNot Installed\x1b[0m\n"
|
||
output = "Offpunk " + __version__ + "\n"
|
||
output += "===========\n"
|
||
output += "Highly recommended:\n"
|
||
output += " - python-cryptography : " + has(netcache._HAS_CRYPTOGRAPHY)
|
||
output += " - xdg-open : " + has(_HAS_XDGOPEN)
|
||
output += "\nWeb browsing:\n"
|
||
output += " - python-requests : " + has(_DO_HTTP)
|
||
output += " - python-feedparser : " + has(_DO_FEED)
|
||
output += " - python-bs4 : " + has(_HAS_SOUP)
|
||
output += " - python-readability : " + has(_HAS_READABILITY)
|
||
output += " - timg 1.3.2+ : " + has(_NEW_TIMG)
|
||
if _NEW_CHAFA:
|
||
output += " - chafa 1.10+ : " + has(_HAS_CHAFA)
|
||
else:
|
||
output += " - chafa : " + has(_HAS_CHAFA)
|
||
output += " - python-pil : " + has(_HAS_PIL)
|
||
output += "\nNice to have:\n"
|
||
output += " - python-setproctitle : " + has(_HAS_SETPROCTITLE)
|
||
output += " - xsel : " + has(_HAS_XSEL)
|
||
|
||
output += "\nFeatures :\n"
|
||
if _NEW_CHAFA:
|
||
output += " - Render images (chafa or timg) : " + has(_RENDER_IMAGE)
|
||
else:
|
||
output += " - Render images (python-pil, chafa or timg) : " + has(_RENDER_IMAGE)
|
||
output += " - Render HTML (bs4, readability) : " + has(_DO_HTML)
|
||
output += " - Render Atom/RSS feeds (feedparser) : " + has(_DO_FEED)
|
||
output += " - Connect to http/https (requests) : " + has(_DO_HTTP)
|
||
output += " - Detect text encoding (python-chardet) : " + has(_HAS_CHARDET)
|
||
output += " - copy to/from clipboard (xsel) : " + has(_HAS_XSEL)
|
||
output += " - restore last position (less 572+) : " + has(_LESS_RESTORE_POSITION)
|
||
output += "\n"
|
||
output += "Config directory : " + _CONFIG_DIR + "\n"
|
||
output += "User Data directory : " + _DATA_DIR + "\n"
|
||
output += "Cache directoy : " + _CACHE_PATH
|
||
|
||
print(output)
|
||
|
||
### Stuff that modifies the lookup table
|
||
def do_ls(self, line):
|
||
"""List contents of current index.
|
||
Use 'ls -l' to see URLs."""
|
||
self._show_lookup(show_url = "-l" in line)
|
||
self.page_index = 0
|
||
|
||
def do_search(self,line):
|
||
"""Search on Gemini using the engine configured (by default kennedy.gemi.dev)
|
||
You can configure it using "set search URL".
|
||
URL should contains one "%s" that will be replaced by the search term."""
|
||
search = urllib.parse.quote(line)
|
||
url = self.options["search"]%search
|
||
self._go_to_url(url)
|
||
|
||
def do_wikipedia(self,line):
|
||
"""Search on wikipedia using the configured Gemini interface.
|
||
The first word should be the two letters code for the language.
|
||
Exemple : "wikipedia en Gemini protocol"
|
||
But you can also use abbreviations to go faster:
|
||
"wen Gemini protocol". (your abbreviation might be missing, report the bug)
|
||
The interface used can be modified with the command:
|
||
"set wikipedia URL" where URL should contains two "%s", the first
|
||
one used for the language, the second for the search string."""
|
||
words = line.split(" ",maxsplit=1)
|
||
if len(words[0]) == 2:
|
||
lang = words[0]
|
||
search = urllib.parse.quote(words[1])
|
||
else:
|
||
lang = "en"
|
||
search = urllib.parse.quote(line)
|
||
url = self.options["wikipedia"]%(lang,search)
|
||
self._go_to_url(url)
|
||
|
||
def do_gus(self, line):
|
||
"""Submit a search query to the geminispace.info search engine."""
|
||
self._go_to_url(urllib.parse.urlunparse("gemini","geminispace.info","/search","",line,""))
|
||
|
||
def do_history(self, *args):
|
||
"""Display history."""
|
||
self.list_show("history")
|
||
|
||
@needs_gi
|
||
def do_find(self, searchterm):
|
||
"""Find in current page by displaying only relevant lines (grep)."""
|
||
self.get_renderer().display(grep=searchterm)
|
||
|
||
def emptyline(self):
|
||
"""Page through index ten lines at a time."""
|
||
i = self.page_index
|
||
if not self.current_url or i > len(self.get_renderer().get_links()):
|
||
return
|
||
self._show_lookup(offset=i, end=i+10)
|
||
self.page_index += 10
|
||
|
||
### Stuff that does something to most recently viewed item
|
||
@needs_gi
|
||
def do_cat(self, *args):
|
||
"""Run most recently visited item through "cat" command."""
|
||
run("cat", input=open(self.get_renderer().get_temp_filename(), "rb"), direct_output=True)
|
||
|
||
@needs_gi
|
||
def do_view(self, *args):
|
||
"""Run most recently visited item through "less" command, restoring \
|
||
previous position.
|
||
Use "view normal" to see the default article view on html page.
|
||
Use "view full" to see a complete html page instead of the article view.
|
||
Use "view feed" to see the the linked feed of the page (in any).
|
||
Use "view feeds" to see available feeds on this page.
|
||
(full, feed, feeds have no effect on non-html content)."""
|
||
if self.current_url and args and args[0] != "":
|
||
if args[0] in ["full","debug"]:
|
||
self._go_to_url(self.current_url,mode=args[0])
|
||
elif args[0] in ["normal","readable"]:
|
||
self._go_to_url(self.current_url,mode="readable")
|
||
elif args[0] == "feed":
|
||
subs = self.get_renderer().get_subscribe_links()
|
||
if len(subs) > 1:
|
||
self.do_go(subs[1][0])
|
||
elif "rss" in subs[0][1] or "atom" in subs[0][1]:
|
||
print("%s is already a feed" %self.current_url)
|
||
else:
|
||
print("No other feed found on %s"%self.current_url)
|
||
elif args[0] == "feeds":
|
||
subs = self.get_renderer().get_subscribe_links()
|
||
stri = "Available views :\n"
|
||
counter = 0
|
||
for s in subs:
|
||
counter += 1
|
||
stri += "[%s] %s [%s]\n"%(counter,s[0],s[1])
|
||
stri += "Which view do you want to see ? >"
|
||
ans = input(stri)
|
||
if ans.isdigit() and 0 < int(ans) <= len(subs):
|
||
self.do_go(subs[int(ans)-1][0])
|
||
else:
|
||
print("Valid argument for view are : normal, full, feed, feeds")
|
||
else:
|
||
self._go_to_url(self.current_url)
|
||
|
||
@needs_gi
|
||
def do_open(self, *args):
|
||
"""Open current item with the configured handler or xdg-open.
|
||
Uses "open url" to open current URL in a browser.
|
||
see "handler" command to set your handler."""
|
||
if args[0] == "url":
|
||
run("xdg-open %s", parameter=self.current_url, direct_output=True)
|
||
else:
|
||
cmd_str = self._get_handler_cmd(ansirenderer.get_mime(self.current_url))
|
||
run(cmd_str, parameter=netcache.get_cache_path(self.current_url), direct_output=True)
|
||
|
||
@needs_gi
|
||
def do_shell(self, line):
|
||
"""'cat' most recently visited item through a shell pipeline.
|
||
'!' is an useful shortcut."""
|
||
run(line, input=open(self.get_renderer().get_temp_filename(), "rb"), direct_output=True)
|
||
|
||
@needs_gi
|
||
def do_save(self, line):
|
||
"""Save an item to the filesystem.
|
||
'save n filename' saves menu item n to the specified filename.
|
||
'save filename' saves the last viewed item to the specified filename.
|
||
'save n' saves menu item n to an automagic filename."""
|
||
args = line.strip().split()
|
||
|
||
# First things first, figure out what our arguments are
|
||
if len(args) == 0:
|
||
# No arguments given at all
|
||
# Save current item, if there is one, to a file whose name is
|
||
# inferred from the gemini path
|
||
if not netcache.is_cache_valid(self.current_url):
|
||
print("You cannot save if not cached!")
|
||
return
|
||
else:
|
||
index = None
|
||
filename = None
|
||
elif len(args) == 1:
|
||
# One argument given
|
||
# If it's numeric, treat it as an index, and infer the filename
|
||
try:
|
||
index = int(args[0])
|
||
filename = None
|
||
# If it's not numeric, treat it as a filename and
|
||
# save the current item
|
||
except ValueError:
|
||
index = None
|
||
filename = os.path.expanduser(args[0])
|
||
elif len(args) == 2:
|
||
# Two arguments given
|
||
# Treat first as an index and second as filename
|
||
index, filename = args
|
||
try:
|
||
index = int(index)
|
||
except ValueError:
|
||
print("First argument is not a valid item index!")
|
||
return
|
||
filename = os.path.expanduser(filename)
|
||
else:
|
||
print("You must provide an index, a filename, or both.")
|
||
return
|
||
|
||
# Next, fetch the item to save, if it's not the current one.
|
||
if index:
|
||
last_url = self.current_url
|
||
try:
|
||
url = self.get_renderer().get_link(index)
|
||
self._go_to_url(url, update_hist = False, handle = False)
|
||
except IndexError:
|
||
print ("Index too high!")
|
||
self.current_url = last_url
|
||
return
|
||
else:
|
||
url = current_url
|
||
|
||
# Derive filename from current GI's path, if one hasn't been set
|
||
if not filename:
|
||
filename = os.path.basename(netcache.get_cache_path(self.url))
|
||
# Check for filename collisions and actually do the save if safe
|
||
if os.path.exists(filename):
|
||
print("File %s already exists!" % filename)
|
||
else:
|
||
# Don't use _get_active_tmpfile() here, because we want to save the
|
||
# "source code" of menus, not the rendered view - this way Offpunk
|
||
# can navigate to it later.
|
||
path = netcache.get_cache_path(url)
|
||
if os.path.isdir(path):
|
||
print("Can’t save %s because it’s a folder, not a file"%path)
|
||
else:
|
||
print("Saved to %s" % filename)
|
||
shutil.copyfile(path, filename)
|
||
|
||
# Restore gi if necessary
|
||
if index != None:
|
||
self._go_to_url(last_url, handle=False)
|
||
|
||
@needs_gi
|
||
def do_url(self, *args):
|
||
"""Print URL of most recently visited item."""
|
||
print(self.current_url)
|
||
|
||
### Bookmarking stuff
|
||
@needs_gi
|
||
def do_add(self, line):
|
||
"""Add the current URL to the list specied as argument.
|
||
If no argument given, URL is added to Bookmarks."""
|
||
args = line.split()
|
||
if len(args) < 1 :
|
||
list = "bookmarks"
|
||
if not self.list_path(list):
|
||
self.list_create(list)
|
||
self.list_add_line(list)
|
||
else:
|
||
self.list_add_line(args[0])
|
||
|
||
# Get the list file name, creating or migrating it if needed.
|
||
# Migrate bookmarks/tour/to_fetch from XDG_CONFIG to XDG_DATA
|
||
# We migrate only if the file exists in XDG_CONFIG and not XDG_DATA
|
||
def get_list(self,list):
|
||
list_path = self.list_path(list)
|
||
if not list_path:
|
||
old_file_gmi = os.path.join(_CONFIG_DIR,list + ".gmi")
|
||
old_file_nogmi = os.path.join(_CONFIG_DIR,list)
|
||
target = os.path.join(_DATA_DIR,"lists")
|
||
if os.path.exists(old_file_gmi):
|
||
shutil.move(old_file_gmi,target)
|
||
elif os.path.exists(old_file_nogmi):
|
||
targetgmi = os.path.join(target,list+".gmi")
|
||
shutil.move(old_file_nogmi,targetgmi)
|
||
else:
|
||
if list == "subscribed":
|
||
title = "Subscriptions #subscribed (new links in those pages will be added to tour)"
|
||
elif list == "to_fetch":
|
||
title = "Links requested and to be fetched during the next --sync"
|
||
else:
|
||
title = None
|
||
self.list_create(list, title=title,quite=True)
|
||
list_path = self.list_path(list)
|
||
return list_path
|
||
|
||
@needs_gi
|
||
def do_subscribe(self,line):
|
||
"""Subscribe to current page by saving it in the "subscribed" list.
|
||
If a new link is found in the page during a --sync, the new link is automatically
|
||
fetched and added to your next tour.
|
||
To unsubscribe, remove the page from the "subscribed" list."""
|
||
subs = self.get_renderer().get_subscribe_links()
|
||
if len(subs) > 1:
|
||
stri = "Multiple feeds have been found :\n"
|
||
elif "rss" in subs[0][1] or "atom" in subs[0][1] :
|
||
stri = "This page is already a feed:\n"
|
||
else:
|
||
stri = "No feed detected. You can still watch the page :\n"
|
||
counter = 0
|
||
for l in subs:
|
||
link = l[0]
|
||
already = []
|
||
for li in self.list_lists():
|
||
if self.list_is_subscribed(li):
|
||
if self.list_has_url(link,li):
|
||
already.append(li)
|
||
stri += "[%s] %s [%s]\n"%(counter+1,link,l[1])
|
||
if len(already) > 0:
|
||
stri += "\t -> (already subscribed through lists %s)\n"%(str(already))
|
||
counter += 1
|
||
stri += "\n"
|
||
stri += "Which feed do you want to subscribe ? > "
|
||
ans = input(stri)
|
||
if ans.isdigit() and 0 < int(ans) <= len(subs):
|
||
sublink,mime,title = subs[int(ans)-1]
|
||
else:
|
||
sublink,title = None,None
|
||
if sublink:
|
||
list_path = self.get_list("subscribed")
|
||
added = self.list_add_line("subscribed",url=sublink,verbose=False)
|
||
if added :
|
||
print("Subscribed to %s" %sublink)
|
||
else:
|
||
print("You are already subscribed to %s"%sublink)
|
||
else:
|
||
print("No subscription registered")
|
||
|
||
def do_bookmarks(self, line):
|
||
"""Show or access the bookmarks menu.
|
||
'bookmarks' shows all bookmarks.
|
||
'bookmarks n' navigates immediately to item n in the bookmark menu.
|
||
Bookmarks are stored using the 'add' command."""
|
||
list_path = self.get_list("bookmarks")
|
||
args = line.strip()
|
||
if len(args.split()) > 1 or (args and not args.isnumeric()):
|
||
print("bookmarks command takes a single integer argument!")
|
||
elif args:
|
||
self.list_go_to_line(args,"bookmarks")
|
||
else:
|
||
self.list_show("bookmarks")
|
||
|
||
@needs_gi
|
||
def do_archive(self,args):
|
||
"""Archive current page by removing it from every list and adding it to
|
||
archives, which is a special historical list limited in size. It is similar to `move archives`."""
|
||
for li in self.list_lists():
|
||
if li not in ["archives", "history"]:
|
||
deleted = self.list_rm_url(self.current_url,li)
|
||
if deleted:
|
||
print("Removed from %s"%li)
|
||
self.list_add_top("archives",limit=self.options["archives_size"])
|
||
print("Archiving: %s"%self.get_renderer().get_page_title())
|
||
print("\x1b[2;34mCurrent maximum size of archives : %s\x1b[0m" %self.options["archives_size"])
|
||
|
||
#what is the line to add to a list for this url ?
|
||
def to_map_line(self):
|
||
return "=> {} {}\n".format(self.current_url, self.get_renderer().get_page_title())
|
||
|
||
def list_add_line(self,list,url=None,verbose=True):
|
||
list_path = self.list_path(list)
|
||
if not list_path and self.list_is_system(list):
|
||
self.list_create(list,quite=True)
|
||
list_path = self.list_path(list)
|
||
if not list_path:
|
||
print("List %s does not exist. Create it with ""list create %s"""%(list,list))
|
||
return False
|
||
else:
|
||
if not url:
|
||
url = self.current_url
|
||
# first we check if url already exists in the file
|
||
with open(list_path,"r") as l_file:
|
||
lines = l_file.readlines()
|
||
l_file.close()
|
||
for l in lines:
|
||
sp = l.split()
|
||
if self.current_url in sp:
|
||
if verbose:
|
||
print("%s already in %s."%(url,list))
|
||
return False
|
||
with open(list_path,"a") as l_file:
|
||
l_file.write(self.to_map_line())
|
||
l_file.close()
|
||
if verbose:
|
||
print("%s added to %s" %(url,list))
|
||
return True
|
||
|
||
@needs_gi
|
||
def list_add_top(self,list,limit=0,truncate_lines=0):
|
||
stri = self.to_map_line().strip("\n")
|
||
if list == "archives":
|
||
stri += ", archived on "
|
||
elif list == "history":
|
||
stri += ", visited on "
|
||
else:
|
||
stri += ", added to %s on "%list
|
||
stri += time.ctime() + "\n"
|
||
list_path = self.get_list(list)
|
||
with open(list_path,"r") as l_file:
|
||
lines = l_file.readlines()
|
||
l_file.close()
|
||
with open(list_path,"w") as l_file:
|
||
l_file.write("#%s\n"%list)
|
||
l_file.write(stri)
|
||
counter = 0
|
||
# Truncating is useful in case we open a new branch
|
||
# after a few back in history
|
||
to_truncate = truncate_lines
|
||
for l in lines:
|
||
if not l.startswith("#"):
|
||
if to_truncate > 0:
|
||
to_truncate -= 1
|
||
elif limit == 0 or counter < limit:
|
||
l_file.write(l)
|
||
counter += 1
|
||
l_file.close()
|
||
|
||
|
||
# remove an url from a list.
|
||
# return True if the URL was removed
|
||
# return False if the URL was not found
|
||
def list_rm_url(self,url,list):
|
||
return self.list_has_url(url,list,deletion=True)
|
||
|
||
# deletion and has_url are so similar, I made them the same method
|
||
def list_has_url(self,url,list,deletion=False):
|
||
list_path = self.list_path(list)
|
||
if list_path:
|
||
to_return = False
|
||
with open(list_path,"r") as lf:
|
||
lines = lf.readlines()
|
||
lf.close()
|
||
to_write = []
|
||
# let’s remove the mode
|
||
url = url.split("##offpunk_mode=")[0]
|
||
for l in lines:
|
||
# we separate components of the line
|
||
# to ensure we identify a complete URL, not a part of it
|
||
splitted = l.split()
|
||
if url not in splitted and len(splitted) > 1:
|
||
current = splitted[1].split("##offpunk_mode=")[0]
|
||
#sometimes, we must remove the ending "/"
|
||
if url == current:
|
||
to_return = True
|
||
elif url.endswith("/") and url[:-1] == current:
|
||
to_return = True
|
||
else:
|
||
to_write.append(l)
|
||
else:
|
||
to_return = True
|
||
if deletion :
|
||
with open(list_path,"w") as lf:
|
||
for l in to_write:
|
||
lf.write(l)
|
||
lf.close()
|
||
return to_return
|
||
else:
|
||
return False
|
||
|
||
def list_get_links(self,list):
|
||
list_path = self.list_path(list)
|
||
if os.path.exists(list_path):
|
||
print("path is %s and exists %s"%(list_path,os.path.exists(list_path)))
|
||
return self.get_renderer("list:///%s"%list).get_links()
|
||
else:
|
||
return []
|
||
|
||
def list_go_to_line(self,line,list):
|
||
list_path = self.list_path(list)
|
||
if not list_path:
|
||
print("List %s does not exist. Create it with ""list create %s"""%(list,list))
|
||
elif not line.isnumeric():
|
||
print("go_to_line requires a number as parameter")
|
||
else:
|
||
r = self.get_renderer("list:///%s"%list)
|
||
url = r.get_lin(int(line))
|
||
display = not self.sync_only
|
||
if url:
|
||
self._go_to_url(url,handle=display)
|
||
#TODO : why returning the mode??
|
||
return self.renderer(url).last_mode
|
||
|
||
def list_show(self,list):
|
||
list_path = self.list_path(list)
|
||
if not list_path:
|
||
print("List %s does not exist. Create it with ""list create %s"""%(list,list))
|
||
else:
|
||
url = "list:///%s"%list
|
||
display = not self.sync_only
|
||
self._go_to_url(url,handle=display)
|
||
|
||
#return the path of the list file if list exists.
|
||
#return None if the list doesn’t exist.
|
||
def list_path(self,list):
|
||
listdir = os.path.join(_DATA_DIR,"lists")
|
||
list_path = os.path.join(listdir, "%s.gmi"%list)
|
||
if os.path.exists(list_path):
|
||
return list_path
|
||
else:
|
||
return None
|
||
|
||
def list_create(self,list,title=None,quite=False):
|
||
list_path = self.list_path(list)
|
||
if list in ["create","edit","delete","help"]:
|
||
print("%s is not allowed as a name for a list"%list)
|
||
elif not list_path:
|
||
listdir = os.path.join(_DATA_DIR,"lists")
|
||
os.makedirs(listdir,exist_ok=True)
|
||
list_path = os.path.join(listdir, "%s.gmi"%list)
|
||
with open(list_path,"a") as lfile:
|
||
if title:
|
||
lfile.write("# %s\n"%title)
|
||
else:
|
||
lfile.write("# %s\n"%list)
|
||
lfile.close()
|
||
if not quite:
|
||
print("list created. Display with `list %s`"%list)
|
||
else:
|
||
print("list %s already exists" %list)
|
||
|
||
def do_move(self,arg):
|
||
"""move LIST will add the current page to the list LIST.
|
||
With a major twist: current page will be removed from all other lists.
|
||
If current page was not in a list, this command is similar to `add LIST`."""
|
||
if not arg:
|
||
print("LIST argument is required as the target for your move")
|
||
elif arg[0] == "archives":
|
||
self.do_archive()
|
||
else:
|
||
args = arg.split()
|
||
list_path = self.list_path(args[0])
|
||
if not list_path:
|
||
print("%s is not a list, aborting the move" %args[0])
|
||
else:
|
||
lists = self.list_lists()
|
||
for l in lists:
|
||
if l != args[0] and l not in ["archives", "history"]:
|
||
isremoved = self.list_rm_url(self.current_url,l)
|
||
if isremoved:
|
||
print("Removed from %s"%l)
|
||
self.list_add_line(args[0])
|
||
|
||
def list_lists(self):
|
||
listdir = os.path.join(_DATA_DIR,"lists")
|
||
to_return = []
|
||
if os.path.exists(listdir):
|
||
lists = os.listdir(listdir)
|
||
if len(lists) > 0:
|
||
for l in lists:
|
||
#removing the .gmi at the end of the name
|
||
to_return.append(l[:-4])
|
||
return to_return
|
||
|
||
def list_has_status(self,list,status):
|
||
path = self.list_path(list)
|
||
toreturn = False
|
||
if path:
|
||
with open(path) as f:
|
||
line = f.readline().strip()
|
||
f.close()
|
||
if line.startswith("#") and status in line:
|
||
toreturn = True
|
||
return toreturn
|
||
|
||
def list_is_subscribed(self,list):
|
||
return self.list_has_status(list,"#subscribed")
|
||
def list_is_frozen(self,list):
|
||
return self.list_has_status(list,"#frozen")
|
||
def list_is_system(self,list):
|
||
return list in ["history","to_fetch","archives","tour"]
|
||
|
||
# This modify the status of a list to one of :
|
||
# normal, frozen, subscribed
|
||
# action is either #frozen, #subscribed or None
|
||
def list_modify(self,list,action=None):
|
||
path = self.list_path(list)
|
||
with open(path) as f:
|
||
lines = f.readlines()
|
||
f.close()
|
||
if lines[0].strip().startswith("#"):
|
||
first_line = lines.pop(0).strip("\n")
|
||
else:
|
||
first_line = "# %s "%list
|
||
first_line = first_line.replace("#subscribed","").replace("#frozen","")
|
||
if action:
|
||
first_line += " " + action
|
||
print("List %s has been marked as %s"%(list,action))
|
||
else:
|
||
print("List %s is now a normal list" %list)
|
||
first_line += "\n"
|
||
lines.insert(0,first_line)
|
||
with open(path,"w") as f:
|
||
for line in lines:
|
||
f.write(line)
|
||
f.close()
|
||
def do_list(self,arg):
|
||
"""Manage list of bookmarked pages.
|
||
- list : display available lists
|
||
- list $LIST : display pages in $LIST
|
||
- list create $NEWLIST : create a new list
|
||
- list edit $LIST : edit the list
|
||
- list subscribe $LIST : during sync, add new links found in listed pages to tour
|
||
- list freeze $LIST : don’t update pages in list during sync if a cache already exists
|
||
- list normal $LIST : update pages in list during sync but don’t add anything to tour
|
||
- list delete $LIST : delete a list permanently (a confirmation is required)
|
||
- list help : print this help
|
||
See also :
|
||
- add $LIST (to add current page to $LIST or, by default, to bookmarks)
|
||
- move $LIST (to add current page to list while removing from all others)
|
||
- archive (to remove current page from all lists while adding to archives)
|
||
|
||
There’s no "delete" on purpose. The use of "archive" is recommended.
|
||
|
||
The following lists cannot be removed or frozen but can be edited with "list edit"
|
||
- list archives : contains last 200 archived URLs
|
||
- history : contains last 200 visisted URLs
|
||
- to_fetch : contains URLs that will be fetch during the next sync
|
||
- tour : contains the next URLs to visit during a tour (see "help tour")
|
||
|
||
"""
|
||
listdir = os.path.join(_DATA_DIR,"lists")
|
||
os.makedirs(listdir,exist_ok=True)
|
||
if not arg:
|
||
lists = self.list_lists()
|
||
if len(lists) > 0:
|
||
lurl = "list:///"
|
||
self._go_to_url(lurl)
|
||
else:
|
||
print("No lists yet. Use `list create`")
|
||
else:
|
||
args = arg.split()
|
||
if args[0] == "create":
|
||
if len(args) > 2:
|
||
name = " ".join(args[2:])
|
||
self.list_create(args[1].lower(),title=name)
|
||
elif len(args) == 2:
|
||
self.list_create(args[1].lower())
|
||
else:
|
||
print("A name is required to create a new list. Use `list create NAME`")
|
||
elif args[0] == "edit":
|
||
editor = None
|
||
if "editor" in self.options and self.options["editor"]:
|
||
editor = self.options["editor"]
|
||
elif os.environ.get("VISUAL"):
|
||
editor = os.environ.get("VISUAL")
|
||
elif os.environ.get("EDITOR"):
|
||
editor = os.environ.get("EDITOR")
|
||
if editor:
|
||
if len(args) > 1 and args[1] in self.list_lists():
|
||
path = os.path.join(listdir,args[1]+".gmi")
|
||
try:
|
||
# Note that we intentionally don't quote the editor.
|
||
# In the unlikely case `editor` includes a percent
|
||
# sign, we also escape it for the %-formatting.
|
||
cmd = editor.replace("%", "%%") + " %s"
|
||
run(cmd, parameter=path, direct_output=True)
|
||
except Exception as err:
|
||
print(err)
|
||
print("Please set a valid editor with \"set editor\"")
|
||
else:
|
||
print("A valid list name is required to edit a list")
|
||
else:
|
||
print("No valid editor has been found.")
|
||
print("You can use the following command to set your favourite editor:")
|
||
print("set editor EDITOR")
|
||
print("or use the $VISUAL or $EDITOR environment variables.")
|
||
elif args[0] == "delete":
|
||
if len(args) > 1:
|
||
if self.list_is_system(args[1]):
|
||
print("%s is a system list which cannot be deleted"%args[1])
|
||
elif args[1] in self.list_lists():
|
||
size = len(self.list_get_links(args[1]))
|
||
stri = "Are you sure you want to delete %s ?\n"%args[1]
|
||
confirm = "YES"
|
||
if size > 0:
|
||
stri += "! %s items in the list will be lost !\n"%size
|
||
confirm = "YES DELETE %s" %size
|
||
else :
|
||
stri += "The list is empty, it should be safe to delete it.\n"
|
||
stri += "Type \"%s\" (in capital, without quotes) to confirm :"%confirm
|
||
answer = input(stri)
|
||
if answer == confirm:
|
||
path = os.path.join(listdir,args[1]+".gmi")
|
||
os.remove(path)
|
||
print("* * * %s has been deleted" %args[1])
|
||
else:
|
||
print("A valid list name is required to be deleted")
|
||
else:
|
||
print("A valid list name is required to be deleted")
|
||
elif args[0] in ["subscribe","freeze","normal"]:
|
||
if len(args) > 1:
|
||
if self.list_is_system(args[1]):
|
||
print("You cannot modify %s which is a system list"%args[1])
|
||
elif args[1] in self.list_lists():
|
||
if args[0] == "subscribe":
|
||
action = "#subscribed"
|
||
elif args[0] == "freeze":
|
||
action = "#frozen"
|
||
else:
|
||
action = None
|
||
self.list_modify(args[1],action=action)
|
||
else:
|
||
print("A valid list name is required after %s" %args[0])
|
||
elif args[0] == "help":
|
||
self.onecmd("help list")
|
||
elif len(args) == 1:
|
||
self.list_show(args[0].lower())
|
||
else:
|
||
self.list_go_to_line(args[1],args[0].lower())
|
||
|
||
def do_help(self, arg):
|
||
"""ALARM! Recursion detected! ALARM! Prepare to eject!"""
|
||
if arg == "!":
|
||
print("! is an alias for 'shell'")
|
||
elif arg == "?":
|
||
print("? is an alias for 'help'")
|
||
elif arg in _ABBREVS:
|
||
full_cmd = _ABBREVS[arg]
|
||
print("%s is an alias for '%s'" %(arg,full_cmd))
|
||
print("See the list of aliases with 'abbrevs'")
|
||
print("'help %s':"%full_cmd)
|
||
cmd.Cmd.do_help(self, full_cmd)
|
||
else:
|
||
cmd.Cmd.do_help(self, arg)
|
||
|
||
### Flight recorder
|
||
def do_blackbox(self, *args):
|
||
"""Display contents of flight recorder, showing statistics for the
|
||
current gemini browsing session."""
|
||
lines = []
|
||
# Compute flight time
|
||
now = time.time()
|
||
delta = now - self.log["start_time"]
|
||
hours, remainder = divmod(delta, 3600)
|
||
minutes, seconds = divmod(remainder, 60)
|
||
# Count hosts
|
||
ipv4_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET])
|
||
ipv6_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET6])
|
||
# Assemble lines
|
||
lines.append(("Patrol duration", "%02d:%02d:%02d" % (hours, minutes, seconds)))
|
||
lines.append(("Requests sent:", self.log["requests"]))
|
||
lines.append((" IPv4 requests:", self.log["ipv4_requests"]))
|
||
lines.append((" IPv6 requests:", self.log["ipv6_requests"]))
|
||
lines.append(("Bytes received:", self.log["bytes_recvd"]))
|
||
lines.append((" IPv4 bytes:", self.log["ipv4_bytes_recvd"]))
|
||
lines.append((" IPv6 bytes:", self.log["ipv6_bytes_recvd"]))
|
||
lines.append(("Unique hosts visited:", len(self.visited_hosts)))
|
||
lines.append((" IPv4 hosts:", ipv4_hosts))
|
||
lines.append((" IPv6 hosts:", ipv6_hosts))
|
||
lines.append(("DNS failures:", self.log["dns_failures"]))
|
||
lines.append(("Timeouts:", self.log["timeouts"]))
|
||
lines.append(("Refused connections:", self.log["refused_connections"]))
|
||
lines.append(("Reset connections:", self.log["reset_connections"]))
|
||
lines.append(("Cache hits:", self.log["cache_hits"]))
|
||
# Print
|
||
for key, value in lines:
|
||
print(key.ljust(24) + str(value).rjust(8))
|
||
|
||
|
||
def do_sync(self, line):
|
||
"""Synchronize all bookmarks lists and URLs from the to_fetch list.
|
||
- New elements in pages in subscribed lists will be added to tour
|
||
- Elements in list to_fetch will be retrieved and added to tour
|
||
- Normal lists will be synchronized and updated
|
||
- Frozen lists will be fetched only if not present.
|
||
|
||
Before a sync, you can edit the list of URLs that will be fetched with the
|
||
following command: "list edit to_fetch"
|
||
|
||
Argument : duration of cache validity (in seconds)."""
|
||
if self.offline_only:
|
||
print("Sync can only be achieved online. Change status with `online`.")
|
||
return
|
||
args = line.split()
|
||
if len(args) > 0:
|
||
if not args[0].isdigit():
|
||
print("sync argument should be the cache validity expressed in seconds")
|
||
return
|
||
else:
|
||
validity = int(args[0])
|
||
else:
|
||
validity = 0
|
||
self.call_sync(refresh_time=validity)
|
||
|
||
def call_sync(self,refresh_time=0,depth=1):
|
||
# fetch_url is the core of the sync algorithm.
|
||
# It takes as input :
|
||
# - an URL to be fetched
|
||
# - depth : the degree of recursion to build the cache (0 means no recursion)
|
||
# - validity : the age, in seconds, existing caches need to have before
|
||
# being refreshed (0 = never refreshed if it already exists)
|
||
# - savetotour : if True, newly cached items are added to tour
|
||
def add_to_tour(url):
|
||
if url and netcache.is_cache_valid(url):
|
||
toprint = " -> adding to tour: %s" %url
|
||
width = term_width() - 1
|
||
toprint = toprint[:width]
|
||
toprint += " "*(width-len(toprint))
|
||
print(toprint)
|
||
self.list_add_line("tour",url=url,verbose=False)
|
||
return True
|
||
else:
|
||
return False
|
||
def fetch_url(url,depth=0,validity=0,savetotour=False,count=[0,0],strin=""):
|
||
#savetotour = True will save to tour newly cached content
|
||
# else, do not save to tour
|
||
#regardless of valitidy
|
||
if not url: return
|
||
if not netcache.is_cache_valid(url,validity=validity):
|
||
if strin != "":
|
||
endline = '\r'
|
||
else:
|
||
endline = None
|
||
#Did we already had a cache (even an old one) ?
|
||
isnew = not netcache.is_cache_valid(url)
|
||
toprint = "%s [%s/%s] Fetch "%(strin,count[0],count[1]) + url
|
||
width = term_width() - 1
|
||
toprint = toprint[:width]
|
||
toprint += " "*(width-len(toprint))
|
||
print(toprint,end=endline)
|
||
#If not saving to tour, then we should limit download size
|
||
limit = not savetotour
|
||
self._go_to_url(url,update_hist=False,limit_size=limit)
|
||
if savetotour and isnew and netcache.is_cache_valid(url):
|
||
#we add to the next tour only if we managed to cache
|
||
#the ressource
|
||
add_to_tour(url)
|
||
#Now, recursive call, even if we didn’t refresh the cache
|
||
# This recursive call is impacting performances a lot but is needed
|
||
# For the case when you add a address to a list to read later
|
||
# You then expect the links to be loaded during next refresh, even
|
||
# if the link itself is fresh enough
|
||
# see fetch_list()
|
||
if depth > 0:
|
||
#we should only savetotour at the first level of recursion
|
||
# The code for this was removed so, currently, we savetotour
|
||
# at every level of recursion.
|
||
links = self.get_renderer(url).get_links(mode="links_only")
|
||
subcount = [0,len(links)]
|
||
d = depth - 1
|
||
for k in links:
|
||
#recursive call (validity is always 0 in recursion)
|
||
substri = strin + " -->"
|
||
subcount[0] += 1
|
||
fetch_url(k,depth=d,validity=0,savetotour=savetotour,\
|
||
count=subcount,strin=substri)
|
||
def fetch_list(list,validity=0,depth=1,tourandremove=False,tourchildren=False):
|
||
links = self.list_get_links(list)
|
||
end = len(links)
|
||
counter = 0
|
||
print(" * * * %s to fetch in %s * * *" %(end,list))
|
||
for l in links:
|
||
counter += 1
|
||
# If cache for a link is newer than the list
|
||
fetch_url(l,depth=depth,validity=validity,savetotour=tourchildren,count=[counter,end])
|
||
if tourandremove:
|
||
if add_to_tour(l):
|
||
self.list_rm_url(l,list)
|
||
|
||
self.sync_only = True
|
||
lists = self.list_lists()
|
||
# We will fetch all the lists except "archives" and "history"
|
||
# We keep tour for the last round
|
||
subscriptions = []
|
||
normal_lists = []
|
||
fridge = []
|
||
for l in lists:
|
||
if not self.list_is_system(l):
|
||
if self.list_is_frozen(l):
|
||
fridge.append(l)
|
||
elif self.list_is_subscribed(l):
|
||
subscriptions.append(l)
|
||
else:
|
||
normal_lists.append(l)
|
||
# We start with the "subscribed" as we need to find new items
|
||
starttime = int(time.time())
|
||
for l in subscriptions:
|
||
fetch_list(l,validity=refresh_time,depth=depth,tourchildren=True)
|
||
#Then the fetch list (item are removed from the list after fetch)
|
||
# We fetch regarless of the refresh_time
|
||
if "to_fetch" in lists:
|
||
nowtime = int(time.time())
|
||
short_valid = nowtime - starttime
|
||
fetch_list("to_fetch",validity=short_valid,depth=depth,tourandremove=True)
|
||
#then we fetch all the rest (including bookmarks and tour)
|
||
for l in normal_lists:
|
||
fetch_list(l,validity=refresh_time,depth=depth)
|
||
for l in fridge:
|
||
fetch_list(l,validity=0,depth=depth)
|
||
#tour should be the last one as item my be added to it by others
|
||
fetch_list("tour",validity=refresh_time,depth=depth)
|
||
print("End of sync")
|
||
self.sync_only = False
|
||
|
||
### The end!
|
||
def do_quit(self, *args):
|
||
"""Exit Offpunk."""
|
||
def unlink(filename):
|
||
if filename and os.path.exists(filename):
|
||
os.unlink(filename)
|
||
|
||
for cert in self.transient_certs_created:
|
||
for ext in (".crt", ".key"):
|
||
certfile = os.path.join(_CONFIG_DIR, "transient_certs", cert+ext)
|
||
if os.path.exists(certfile):
|
||
os.remove(certfile)
|
||
print("You can close your screen!")
|
||
sys.exit()
|
||
|
||
do_exit = do_quit
|
||
|
||
|
||
|
||
# Main function
|
||
def main():
|
||
|
||
# Parse args
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--bookmarks', action='store_true',
|
||
help='start with your list of bookmarks')
|
||
parser.add_argument('--tls-cert', metavar='FILE', help='TLS client certificate file')
|
||
parser.add_argument('--tls-key', metavar='FILE', help='TLS client certificate private key file')
|
||
parser.add_argument('--sync', action='store_true',
|
||
help='run non-interactively to build cache by exploring bookmarks')
|
||
parser.add_argument('--assume-yes', action='store_true',
|
||
help='assume-yes when asked questions about certificates/redirections during sync (lower security)')
|
||
parser.add_argument('--disable-http',action='store_true',
|
||
help='do not try to get http(s) links (but already cached will be displayed)')
|
||
parser.add_argument('--fetch-later', action='store_true',
|
||
help='run non-interactively with an URL as argument to fetch it later')
|
||
parser.add_argument('--depth',
|
||
help='depth of the cache to build. Default is 1. More is crazy. Use at your own risks!')
|
||
parser.add_argument('--cache-validity',
|
||
help='duration for which a cache is valid before sync (seconds)')
|
||
parser.add_argument('--version', action='store_true',
|
||
help='display version information and quit')
|
||
parser.add_argument('--features', action='store_true',
|
||
help='display available features and dependancies then quit')
|
||
parser.add_argument('url', metavar='URL', nargs='*',
|
||
help='start with this URL')
|
||
args = parser.parse_args()
|
||
|
||
# Handle --version
|
||
if args.version:
|
||
print("Offpunk " + __version__)
|
||
sys.exit()
|
||
elif args.features:
|
||
GeminiClient.do_version(None,None)
|
||
sys.exit()
|
||
else:
|
||
for f in [_CONFIG_DIR, _DATA_DIR]:
|
||
if not os.path.exists(f):
|
||
print("Creating config directory {}".format(f))
|
||
os.makedirs(f)
|
||
|
||
# Instantiate client
|
||
gc = GeminiClient(synconly=args.sync)
|
||
torun_queue = []
|
||
|
||
# Interactive if offpunk started normally
|
||
# False if started with --sync
|
||
# Queue is a list of command (potentially empty)
|
||
def read_config(queue,interactive=True):
|
||
rcfile = os.path.join(_CONFIG_DIR, "offpunkrc")
|
||
if os.path.exists(rcfile):
|
||
print("Using config %s" % rcfile)
|
||
with open(rcfile, "r") as fp:
|
||
for line in fp:
|
||
line = line.strip()
|
||
if ((args.bookmarks or args.url) and
|
||
any((line.startswith(x) for x in ("go", "g", "tour", "t")))
|
||
):
|
||
if args.bookmarks:
|
||
print("Skipping rc command \"%s\" due to --bookmarks option." % line)
|
||
else:
|
||
print("Skipping rc command \"%s\" due to provided URLs." % line)
|
||
continue
|
||
# We always consider redirect
|
||
# for the rest, we need to be interactive
|
||
if line.startswith("redirect") or interactive:
|
||
queue.append(line)
|
||
return queue
|
||
# Act on args
|
||
if args.tls_cert:
|
||
# If tls_key is None, python will attempt to load the key from tls_cert.
|
||
netcache._activate_client_cert(args.tls_cert, args.tls_key)
|
||
if args.bookmarks:
|
||
torun_queue.append("bookmarks")
|
||
elif args.url:
|
||
if len(args.url) == 1:
|
||
torun_queue.append("go %s" % args.url[0])
|
||
else:
|
||
for url in args.url:
|
||
torun_queue.append("tour %s" % url)
|
||
torun_queue.append("tour")
|
||
|
||
if args.disable_http:
|
||
gc.support_http = False
|
||
|
||
# Endless interpret loop (except while --sync or --fetch-later)
|
||
if args.fetch_later:
|
||
if args.url:
|
||
gc.sync_only = True
|
||
for u in args.url:
|
||
if netcache.is_cache_valid(u):
|
||
gc.list_add_line("tour",u)
|
||
else:
|
||
gc.list_add_line("to_fetch",u)
|
||
else:
|
||
print("--fetch-later requires an URL (or a list of URLS) as argument")
|
||
elif args.sync:
|
||
if args.assume_yes:
|
||
gc.automatic_choice = "y"
|
||
gc.onecmd("set accept_bad_ssl_certificates True")
|
||
if args.cache_validity:
|
||
refresh_time = int(args.cache_validity)
|
||
else:
|
||
# if no refresh time, a default of 0 is used (which means "infinite")
|
||
refresh_time = 0
|
||
if args.depth:
|
||
depth = int(args.depth)
|
||
else:
|
||
depth = 1
|
||
read_config(torun_queue, interactive=False)
|
||
for line in torun_queue:
|
||
gc.onecmd(line)
|
||
gc.call_sync(refresh_time=refresh_time,depth=depth)
|
||
gc.onecmd("blackbox")
|
||
else:
|
||
# We are in the normal mode. First process config file
|
||
torun_queue = read_config(torun_queue,interactive=True)
|
||
print("Welcome to Offpunk!")
|
||
print("Type `help` to get the list of available command.")
|
||
for line in torun_queue:
|
||
gc.onecmd(line)
|
||
|
||
while True:
|
||
try:
|
||
gc.cmdloop()
|
||
except KeyboardInterrupt:
|
||
print("")
|
||
|
||
if __name__ == '__main__':
|
||
main()
|