2019-06-22 12:58:21 +00:00
|
|
|
|
#!/usr/bin/env python3
|
2021-12-30 15:03:08 +00:00
|
|
|
|
# Offpunk Offline Gemini client
|
|
|
|
|
# Derived from AV-98 by Solderpunk,
|
2022-01-08 21:18:54 +00:00
|
|
|
|
# (C) 2021, 2022 Ploum <offpunk@ploum.eu>
|
2020-05-17 15:57:34 +00:00
|
|
|
|
# (C) 2019, 2020 Solderpunk <solderpunk@sdf.org>
|
|
|
|
|
# With contributions from:
|
2020-06-02 20:57:48 +00:00
|
|
|
|
# - danceka <hannu.hartikainen@gmail.com>
|
2020-05-17 15:57:34 +00:00
|
|
|
|
# - <jprjr@tilde.club>
|
2020-06-02 20:57:48 +00:00
|
|
|
|
# - <vee@vnsf.xyz>
|
2020-06-07 17:06:39 +00:00
|
|
|
|
# - Klaus Alexander Seistrup <klaus@seistrup.dk>
|
2020-08-14 20:29:21 +00:00
|
|
|
|
# - govynnus <govynnus@sdf.org>
|
2021-08-25 06:07:18 +00:00
|
|
|
|
# - Björn Wärmedal <bjorn.warmedal@gmail.com>
|
|
|
|
|
# - <jake@rmgr.dev>
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
# - Maeve Sproule <code@sprock.dev>
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2022-11-15 16:01:13 +00:00
|
|
|
|
_VERSION = "1.7.1"
|
2022-02-06 13:08:25 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import argparse
|
|
|
|
|
import cmd
|
|
|
|
|
import codecs
|
2020-05-16 16:58:53 +00:00
|
|
|
|
import datetime
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import fnmatch
|
2020-06-07 17:13:00 +00:00
|
|
|
|
import getpass
|
2020-05-11 21:27:48 +00:00
|
|
|
|
import glob
|
2020-05-16 16:58:53 +00:00
|
|
|
|
import hashlib
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import io
|
|
|
|
|
import mimetypes
|
2020-05-10 15:25:03 +00:00
|
|
|
|
import os
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import os.path
|
2021-12-13 12:49:12 +00:00
|
|
|
|
import filecmp
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import random
|
|
|
|
|
import shlex
|
|
|
|
|
import shutil
|
|
|
|
|
import socket
|
2020-05-16 16:58:53 +00:00
|
|
|
|
import sqlite3
|
2020-05-11 20:22:24 +00:00
|
|
|
|
import ssl
|
2020-05-17 18:38:06 +00:00
|
|
|
|
from ssl import CertificateError
|
2019-06-22 12:58:21 +00:00
|
|
|
|
import sys
|
|
|
|
|
import tempfile
|
|
|
|
|
import time
|
2020-05-11 20:22:24 +00:00
|
|
|
|
import urllib.parse
|
|
|
|
|
import uuid
|
2019-11-05 17:58:04 +00:00
|
|
|
|
import webbrowser
|
2022-03-14 23:04:11 +00:00
|
|
|
|
import html
|
2022-10-07 09:19:23 +00:00
|
|
|
|
import base64
|
2022-03-30 17:23:44 +00:00
|
|
|
|
import subprocess
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
|
|
|
|
|
# In terms of arguments, this can take an input file/string to be passed to
|
|
|
|
|
# stdin, a parameter to do (well-escaped) "%" replacement on the command, a
|
|
|
|
|
# flag requesting that the output go directly to the stdout, and a list of
|
|
|
|
|
# additional environment variables to set.
|
|
|
|
|
def run(cmd, *, input=None, parameter=None, direct_output=False, env={}):
|
2022-11-01 16:22:26 +00:00
|
|
|
|
#print("running %s"%cmd)
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
if parameter:
|
|
|
|
|
cmd = cmd % shlex.quote(parameter)
|
|
|
|
|
env = dict(os.environ) | env
|
|
|
|
|
if isinstance(input, io.IOBase):
|
|
|
|
|
stdin = input
|
|
|
|
|
input = None
|
|
|
|
|
else:
|
|
|
|
|
if input:
|
|
|
|
|
input = input.encode()
|
|
|
|
|
stdin = None
|
2022-03-30 17:23:44 +00:00
|
|
|
|
if not direct_output:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
# subprocess.check_output() wouldn't allow us to pass stdin.
|
|
|
|
|
result = subprocess.run(cmd, check=True, env=env, input=input,
|
|
|
|
|
shell=True, stdin=stdin, stdout=subprocess.PIPE,
|
|
|
|
|
stderr=subprocess.STDOUT)
|
|
|
|
|
return result.stdout.decode()
|
2022-03-30 17:23:44 +00:00
|
|
|
|
else:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
subprocess.run(cmd, env=env, input=input, shell=True, stdin=stdin)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2022-02-18 22:50:08 +00:00
|
|
|
|
try:
|
|
|
|
|
import setproctitle
|
|
|
|
|
setproctitle.setproctitle("offpunk")
|
|
|
|
|
_HAS_SETPROCTITLE = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_HAS_SETPROCTITLE = False
|
|
|
|
|
|
2022-03-19 19:58:18 +00:00
|
|
|
|
import textwrap
|
2022-02-11 10:03:19 +00:00
|
|
|
|
|
2022-03-07 17:23:48 +00:00
|
|
|
|
global TERM_WIDTH
|
2022-03-24 10:06:35 +00:00
|
|
|
|
TERM_WIDTH = 80
|
2022-03-07 17:23:48 +00:00
|
|
|
|
|
|
|
|
|
def term_width():
|
|
|
|
|
width = TERM_WIDTH
|
|
|
|
|
cur = shutil.get_terminal_size()[0]
|
|
|
|
|
if cur < width:
|
|
|
|
|
width = cur
|
|
|
|
|
return width
|
|
|
|
|
|
2022-03-17 09:18:00 +00:00
|
|
|
|
try:
|
|
|
|
|
from PIL import Image
|
|
|
|
|
_HAS_PIL = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_HAS_PIL = False
|
|
|
|
|
_HAS_TIMG = shutil.which('timg')
|
2022-02-14 22:02:53 +00:00
|
|
|
|
_HAS_CHAFA = shutil.which('chafa')
|
2022-03-17 09:18:00 +00:00
|
|
|
|
_NEW_CHAFA = False
|
2022-10-06 09:59:03 +00:00
|
|
|
|
_NEW_TIMG = False
|
2022-10-08 21:20:30 +00:00
|
|
|
|
_RENDER_IMAGE = False
|
2022-03-17 09:18:00 +00:00
|
|
|
|
|
|
|
|
|
# All this code to know if we render image inline or not
|
|
|
|
|
if _HAS_CHAFA:
|
|
|
|
|
# starting with 1.10, chafa can return only one frame
|
|
|
|
|
# which allows us to drop dependancy for PIL
|
2022-03-30 17:23:44 +00:00
|
|
|
|
output = run("chafa --version")
|
2022-12-09 01:29:10 +00:00
|
|
|
|
# output is "Chafa version M.m.p"
|
|
|
|
|
# check for m < 1.10
|
|
|
|
|
try:
|
|
|
|
|
chafa_major, chafa_minor, _ = output.split("\n")[0].split(" ")[-1].split(".")
|
|
|
|
|
if int(chafa_major) >= 1 and int(chafa_minor) >= 10:
|
|
|
|
|
_NEW_CHAFA = True
|
|
|
|
|
except:
|
|
|
|
|
pass
|
2022-03-21 21:43:33 +00:00
|
|
|
|
if _NEW_CHAFA :
|
2022-03-17 09:18:00 +00:00
|
|
|
|
_RENDER_IMAGE = True
|
2022-10-08 21:20:30 +00:00
|
|
|
|
if _HAS_TIMG :
|
2022-10-07 22:42:46 +00:00
|
|
|
|
try:
|
|
|
|
|
output = run("timg --version")
|
2022-10-08 21:20:30 +00:00
|
|
|
|
except subprocess.CalledProcessError:
|
2022-10-07 22:42:46 +00:00
|
|
|
|
output = False
|
2022-10-06 09:59:03 +00:00
|
|
|
|
# We don’t deal with timg before 1.3.2 (looping options)
|
2022-10-07 22:42:46 +00:00
|
|
|
|
if output and output[5:10] > "1.3.2":
|
2022-10-06 09:59:03 +00:00
|
|
|
|
_NEW_TIMG = True
|
|
|
|
|
_RENDER_IMAGE = True
|
2022-03-21 21:43:33 +00:00
|
|
|
|
elif _HAS_CHAFA and _HAS_PIL:
|
2022-03-17 09:18:00 +00:00
|
|
|
|
_RENDER_IMAGE = True
|
2022-10-08 21:20:30 +00:00
|
|
|
|
if not _RENDER_IMAGE:
|
2022-03-21 21:43:33 +00:00
|
|
|
|
print("To render images inline, you need either chafa or timg.")
|
2022-10-08 21:20:30 +00:00
|
|
|
|
if not _NEW_CHAFA and not _NEW_TIMG:
|
2022-03-17 09:18:00 +00:00
|
|
|
|
print("Before Chafa 1.10, you also need python-pil")
|
|
|
|
|
|
|
|
|
|
#return ANSI text that can be show by less
|
|
|
|
|
def inline_image(img_file,width):
|
|
|
|
|
#Chafa is faster than timg inline. Let use that one by default
|
|
|
|
|
inline = None
|
|
|
|
|
ansi_img = ""
|
2022-04-09 15:15:56 +00:00
|
|
|
|
#We avoid errors by not trying to render non-image files
|
|
|
|
|
if shutil.which("file"):
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
mime = run("file -b --mime-type %s", parameter=img_file).strip()
|
2022-04-09 15:15:56 +00:00
|
|
|
|
if not "image" in mime:
|
|
|
|
|
return ansi_img
|
2022-03-21 21:43:33 +00:00
|
|
|
|
if _HAS_CHAFA:
|
2022-03-17 09:18:00 +00:00
|
|
|
|
if _HAS_PIL and not _NEW_CHAFA:
|
|
|
|
|
# this code is a hack to remove frames from animated gif
|
|
|
|
|
img_obj = Image.open(img_file)
|
|
|
|
|
if hasattr(img_obj,"n_frames") and img_obj.n_frames > 1:
|
|
|
|
|
# we remove all frames but the first one
|
2022-03-18 12:25:26 +00:00
|
|
|
|
img_obj.save(img_file,format="gif",save_all=False)
|
2022-03-21 15:21:02 +00:00
|
|
|
|
inline = "chafa --bg white -s %s -f symbols"
|
2022-03-17 09:18:00 +00:00
|
|
|
|
elif _NEW_CHAFA:
|
2022-04-18 19:24:26 +00:00
|
|
|
|
inline = "chafa --bg white -t 1 -s %s -f symbols --animate=off"
|
2022-10-08 21:20:30 +00:00
|
|
|
|
if not inline and _NEW_TIMG:
|
2022-03-17 09:18:00 +00:00
|
|
|
|
inline = "timg --frames=1 -p q -g %sx1000"
|
|
|
|
|
if inline:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
cmd = inline%width + " %s"
|
2022-03-16 22:58:10 +00:00
|
|
|
|
try:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
ansi_img = run(cmd, parameter=img_file)
|
2022-03-17 09:18:00 +00:00
|
|
|
|
except Exception as err:
|
|
|
|
|
ansi_img = "***image failed : %s***\n" %err
|
|
|
|
|
return ansi_img
|
|
|
|
|
|
|
|
|
|
def terminal_image(img_file):
|
|
|
|
|
#Render by timg is better than old chafa.
|
|
|
|
|
# it is also centered
|
|
|
|
|
cmd = None
|
2022-10-08 21:20:30 +00:00
|
|
|
|
if _NEW_TIMG:
|
2022-03-30 12:35:37 +00:00
|
|
|
|
cmd = "timg --loops=1 -C"
|
2022-03-17 09:18:00 +00:00
|
|
|
|
elif _HAS_CHAFA:
|
2022-04-18 19:24:26 +00:00
|
|
|
|
cmd = "chafa -d 0 --bg white -t 1 -w 1"
|
2022-03-17 09:18:00 +00:00
|
|
|
|
if cmd:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
cmd = cmd + " %s"
|
|
|
|
|
run(cmd, parameter=img_file, direct_output=True)
|
2022-02-11 10:03:19 +00:00
|
|
|
|
|
2022-11-29 23:11:44 +00:00
|
|
|
|
def parse_mime(mime):
|
|
|
|
|
options = {}
|
|
|
|
|
if mime:
|
|
|
|
|
if ";" in mime:
|
|
|
|
|
splited = mime.split(";",maxsplit=1)
|
|
|
|
|
mime = splited[0]
|
|
|
|
|
if len(splited) >= 1:
|
|
|
|
|
options_list = splited[1].split()
|
|
|
|
|
for o in options_list:
|
|
|
|
|
spl = o.split("=",maxsplit=1)
|
|
|
|
|
if len(spl) > 0:
|
|
|
|
|
options[spl[0]] = spl[1]
|
|
|
|
|
return mime, options
|
2020-05-10 13:02:24 +00:00
|
|
|
|
|
2022-03-12 17:41:38 +00:00
|
|
|
|
_HAS_XSEL = shutil.which('xsel')
|
|
|
|
|
_HAS_XDGOPEN = shutil.which('xdg-open')
|
2020-05-17 18:38:06 +00:00
|
|
|
|
try:
|
|
|
|
|
from cryptography import x509
|
|
|
|
|
from cryptography.hazmat.backends import default_backend
|
|
|
|
|
_HAS_CRYPTOGRAPHY = True
|
|
|
|
|
_BACKEND = default_backend()
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_HAS_CRYPTOGRAPHY = False
|
|
|
|
|
|
2022-01-10 10:19:29 +00:00
|
|
|
|
try:
|
|
|
|
|
import requests
|
|
|
|
|
_DO_HTTP = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_DO_HTTP = False
|
|
|
|
|
|
2022-01-10 11:49:24 +00:00
|
|
|
|
try:
|
|
|
|
|
from readability import Document
|
2022-02-14 22:02:53 +00:00
|
|
|
|
_HAS_READABILITY = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_HAS_READABILITY = False
|
|
|
|
|
|
|
|
|
|
try:
|
2022-01-10 11:49:24 +00:00
|
|
|
|
from bs4 import BeautifulSoup
|
2022-03-15 19:24:06 +00:00
|
|
|
|
from bs4 import Comment
|
2022-02-14 22:02:53 +00:00
|
|
|
|
_HAS_SOUP = True
|
2022-01-10 11:49:24 +00:00
|
|
|
|
except ModuleNotFoundError:
|
2022-02-14 22:02:53 +00:00
|
|
|
|
_HAS_SOUP = False
|
|
|
|
|
|
2022-04-02 16:12:46 +00:00
|
|
|
|
_DO_HTML = _HAS_SOUP #and _HAS_READABILITY
|
|
|
|
|
if _DO_HTML and not _HAS_READABILITY:
|
|
|
|
|
print("To improve your web experience (less cruft in webpages),")
|
2022-11-15 10:58:31 +00:00
|
|
|
|
print("please install python3-readability or readability-lxml")
|
2022-02-06 13:08:25 +00:00
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
import feedparser
|
|
|
|
|
_DO_FEED = True
|
|
|
|
|
except ModuleNotFoundError:
|
|
|
|
|
_DO_FEED = False
|
2020-05-10 12:34:48 +00:00
|
|
|
|
|
2022-01-18 17:04:29 +00:00
|
|
|
|
## Config directories
|
2022-03-08 09:14:36 +00:00
|
|
|
|
## We implement our own python-xdg to avoid conflict with existing libraries.
|
|
|
|
|
_home = os.path.expanduser('~')
|
|
|
|
|
data_home = os.environ.get('XDG_DATA_HOME') or \
|
|
|
|
|
os.path.join(_home,'.local','share')
|
|
|
|
|
config_home = os.environ.get('XDG_CONFIG_HOME') or \
|
|
|
|
|
os.path.join(_home,'.config')
|
|
|
|
|
cache_home = os.environ.get('XDG_CACHE_HOME') or\
|
|
|
|
|
os.path.join(_home,'.cache')
|
|
|
|
|
_CACHE_PATH = os.path.join(cache_home,"offpunk/")
|
|
|
|
|
_CONFIG_DIR = os.path.join(config_home,"offpunk/")
|
|
|
|
|
_DATA_DIR = os.path.join(data_home,"offpunk/")
|
|
|
|
|
_old_config = os.path.expanduser("~/.offpunk/")
|
|
|
|
|
## Look for pre-existing config directory, if any
|
|
|
|
|
if os.path.exists(_old_config):
|
|
|
|
|
_CONFIG_DIR = _old_config
|
|
|
|
|
#if no XDG .local/share and not XDG .config, we use the old config
|
|
|
|
|
if not os.path.exists(data_home) and os.path.exists(_old_config):
|
2022-01-19 09:46:53 +00:00
|
|
|
|
_DATA_DIR = _CONFIG_DIR
|
2019-10-13 17:42:04 +00:00
|
|
|
|
_MAX_REDIRECTS = 5
|
2020-08-30 21:17:21 +00:00
|
|
|
|
_MAX_CACHE_SIZE = 10
|
|
|
|
|
_MAX_CACHE_AGE_SECS = 180
|
2022-02-17 22:16:16 +00:00
|
|
|
|
|
2022-04-26 11:15:20 +00:00
|
|
|
|
_GREP = "grep --color=auto"
|
2022-02-17 22:16:16 +00:00
|
|
|
|
less_version = 0
|
2022-03-23 15:35:10 +00:00
|
|
|
|
if not shutil.which("less"):
|
|
|
|
|
print("Please install the pager \"less\" to run Offpunk.")
|
|
|
|
|
print("If you wish to use another pager, send your request to offpunk@ploum.eu.")
|
|
|
|
|
print("(I’m really curious to hear about people not having \"less\" on their system.)")
|
|
|
|
|
sys.exit()
|
2022-03-30 17:23:44 +00:00
|
|
|
|
output = run("less --version")
|
2022-02-17 22:16:16 +00:00
|
|
|
|
# We get less Version (which is the only integer on the first line)
|
|
|
|
|
words = output.split("\n")[0].split()
|
2022-03-23 15:35:10 +00:00
|
|
|
|
less_version = 0
|
2022-02-17 22:16:16 +00:00
|
|
|
|
for w in words:
|
|
|
|
|
if w.isdigit():
|
|
|
|
|
less_version = int(w)
|
|
|
|
|
# restoring position only works for version of less > 572
|
|
|
|
|
if less_version >= 572:
|
|
|
|
|
_LESS_RESTORE_POSITION = True
|
|
|
|
|
else:
|
|
|
|
|
_LESS_RESTORE_POSITION = False
|
2022-01-06 12:44:02 +00:00
|
|
|
|
#_DEFAULT_LESS = "less -EXFRfM -PMurl\ lines\ \%lt-\%lb/\%L\ \%Pb\%$ %s"
|
2022-02-12 18:07:55 +00:00
|
|
|
|
# -E : quit when reaching end of file (to behave like "cat")
|
|
|
|
|
# -F : quit if content fits the screen (behave like "cat")
|
|
|
|
|
# -X : does not clear the screen
|
|
|
|
|
# -R : interpret ANSI colors correctly
|
|
|
|
|
# -f : suppress warning for some contents
|
|
|
|
|
# -M : long prompt (to have info about where you are in the file)
|
2022-03-25 12:37:23 +00:00
|
|
|
|
# -W : hilite the new first line after a page skip (space)
|
2022-02-12 18:07:55 +00:00
|
|
|
|
# -i : ignore case in search
|
2022-03-20 18:25:04 +00:00
|
|
|
|
# -S : do not wrap long lines. Wrapping is done by offpunk, longlines
|
|
|
|
|
# are there on purpose (surch in asciiart)
|
2022-02-17 22:16:16 +00:00
|
|
|
|
#--incsearch : incremental search starting rev581
|
|
|
|
|
if less_version >= 581:
|
2022-03-25 12:37:23 +00:00
|
|
|
|
less_base = "less --incsearch --save-marks -~ -XRfMWiS"
|
2022-03-29 08:26:53 +00:00
|
|
|
|
elif less_version >= 572:
|
2022-03-25 12:37:23 +00:00
|
|
|
|
less_base = "less --save-marks -XRfMWiS"
|
2022-03-29 08:26:53 +00:00
|
|
|
|
else:
|
|
|
|
|
less_base = "less -XRfMWiS"
|
2022-02-17 22:16:16 +00:00
|
|
|
|
_DEFAULT_LESS = less_base + " \"+''\" %s"
|
|
|
|
|
_DEFAULT_CAT = less_base + " -EF %s"
|
2022-03-25 12:37:23 +00:00
|
|
|
|
def less_cmd(file, histfile=None,cat=False,grep=None):
|
2022-02-17 16:00:24 +00:00
|
|
|
|
if histfile:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
env = {"LESSHISTFILE": histfile}
|
2022-02-17 16:00:24 +00:00
|
|
|
|
else:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
env = {}
|
2022-02-17 16:00:24 +00:00
|
|
|
|
if cat:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
cmd_str = _DEFAULT_CAT
|
2022-03-25 12:37:23 +00:00
|
|
|
|
elif grep:
|
|
|
|
|
grep_cmd = _GREP
|
|
|
|
|
#case insensitive for lowercase search
|
|
|
|
|
if grep.islower():
|
|
|
|
|
grep_cmd += " -i"
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
cmd_str = _DEFAULT_CAT + "|" + grep_cmd + " %s"%grep
|
2022-02-17 16:00:24 +00:00
|
|
|
|
else:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
cmd_str = _DEFAULT_LESS
|
|
|
|
|
run(cmd_str, parameter=file, direct_output=True, env=env)
|
2022-02-17 14:57:04 +00:00
|
|
|
|
|
2019-10-13 17:42:04 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Command abbreviations
|
|
|
|
|
_ABBREVS = {
|
|
|
|
|
"a": "add",
|
|
|
|
|
"b": "back",
|
|
|
|
|
"bb": "blackbox",
|
|
|
|
|
"bm": "bookmarks",
|
|
|
|
|
"book": "bookmarks",
|
2022-01-05 20:12:59 +00:00
|
|
|
|
"cp": "copy",
|
2022-03-07 22:16:17 +00:00
|
|
|
|
"f": "forward",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"g": "go",
|
|
|
|
|
"h": "history",
|
|
|
|
|
"hist": "history",
|
2022-03-07 22:16:17 +00:00
|
|
|
|
"l": "view",
|
|
|
|
|
"less": "view",
|
2022-03-09 09:15:24 +00:00
|
|
|
|
"man": "help",
|
2022-02-14 22:02:53 +00:00
|
|
|
|
"mv": "move",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"n": "next",
|
2022-01-05 20:12:59 +00:00
|
|
|
|
"off": "offline",
|
|
|
|
|
"on": "online",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"p": "previous",
|
|
|
|
|
"prev": "previous",
|
|
|
|
|
"q": "quit",
|
|
|
|
|
"r": "reload",
|
|
|
|
|
"s": "save",
|
|
|
|
|
"se": "search",
|
2022-11-14 20:33:40 +00:00
|
|
|
|
"/": "find",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"t": "tour",
|
|
|
|
|
"u": "up",
|
2022-03-07 22:16:17 +00:00
|
|
|
|
"v": "view",
|
2022-11-14 20:33:40 +00:00
|
|
|
|
"w": "wikipedia",
|
|
|
|
|
"wen": "wikipedia en",
|
|
|
|
|
"wfr": "wikipedia fr",
|
|
|
|
|
"wes": "wikipedia es",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
_MIME_HANDLERS = {
|
2021-12-19 14:51:59 +00:00
|
|
|
|
"application/pdf": "zathura %s",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
"audio/mpeg": "mpg123 %s",
|
|
|
|
|
"audio/ogg": "ogg123 %s",
|
2022-01-08 20:32:25 +00:00
|
|
|
|
"image/*": "feh -. %s",
|
2019-06-22 12:58:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-06-01 07:21:55 +00:00
|
|
|
|
# monkey-patch Gemini support in urllib.parse
|
|
|
|
|
# see https://github.com/python/cpython/blob/master/Lib/urllib/parse.py
|
|
|
|
|
urllib.parse.uses_relative.append("gemini")
|
|
|
|
|
urllib.parse.uses_netloc.append("gemini")
|
2022-03-04 22:56:57 +00:00
|
|
|
|
urllib.parse.uses_relative.append("spartan")
|
|
|
|
|
urllib.parse.uses_netloc.append("spartan")
|
2020-06-01 07:21:55 +00:00
|
|
|
|
|
2022-03-16 18:26:46 +00:00
|
|
|
|
#An IPV6 URL should be put between []
|
|
|
|
|
#We try to detect them has location with more than 2 ":"
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def fix_ipv6_url(url):
|
2022-03-16 18:26:46 +00:00
|
|
|
|
if not url or url.startswith("mailto"):
|
2019-08-13 16:56:15 +00:00
|
|
|
|
return url
|
2019-06-22 12:58:21 +00:00
|
|
|
|
if "://" in url:
|
2021-12-20 15:32:54 +00:00
|
|
|
|
schema, schemaless = url.split("://",maxsplit=1)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
schema, schemaless = None, url
|
|
|
|
|
if "/" in schemaless:
|
|
|
|
|
netloc, rest = schemaless.split("/",1)
|
2022-03-16 18:26:46 +00:00
|
|
|
|
if netloc.count(":") > 2 and "[" not in netloc and "]" not in netloc:
|
|
|
|
|
schemaless = "[" + netloc + "]" + "/" + rest
|
|
|
|
|
elif schemaless.count(":") > 2:
|
|
|
|
|
schemaless = "[" + schemaless + "]/"
|
2019-06-22 12:58:21 +00:00
|
|
|
|
if schema:
|
|
|
|
|
return schema + "://" + schemaless
|
|
|
|
|
return schemaless
|
|
|
|
|
|
2022-02-12 23:37:31 +00:00
|
|
|
|
# This list is also used as a list of supported protocols
|
2019-08-13 16:56:15 +00:00
|
|
|
|
standard_ports = {
|
2022-03-04 22:38:35 +00:00
|
|
|
|
"gemini" : 1965,
|
|
|
|
|
"gopher" : 70,
|
2022-11-24 11:24:03 +00:00
|
|
|
|
"finger" : 79,
|
2022-03-04 22:38:35 +00:00
|
|
|
|
"http" : 80,
|
|
|
|
|
"https" : 443,
|
|
|
|
|
"spartan": 300,
|
2019-08-13 16:56:15 +00:00
|
|
|
|
}
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2022-02-12 23:37:31 +00:00
|
|
|
|
# First, we define the different content->text renderers, outside of the rest
|
2022-01-17 22:37:12 +00:00
|
|
|
|
# (They could later be factorized in other files or replaced)
|
2022-02-12 23:37:31 +00:00
|
|
|
|
class AbstractRenderer():
|
2022-03-18 22:17:59 +00:00
|
|
|
|
def __init__(self,content,url,center=True):
|
2022-02-09 20:46:29 +00:00
|
|
|
|
self.url = url
|
2022-01-29 14:35:28 +00:00
|
|
|
|
self.body = content
|
2022-03-16 09:03:29 +00:00
|
|
|
|
#there’s one rendered text and one links table per mode
|
|
|
|
|
self.rendered_text = {}
|
|
|
|
|
self.links = {}
|
2022-03-30 10:04:07 +00:00
|
|
|
|
self.images = {}
|
2022-01-30 18:11:23 +00:00
|
|
|
|
self.title = None
|
2022-02-12 23:37:31 +00:00
|
|
|
|
self.validity = True
|
2022-03-15 23:13:45 +00:00
|
|
|
|
self.temp_file = {}
|
|
|
|
|
self.less_histfile = {}
|
2022-03-18 22:02:21 +00:00
|
|
|
|
self.center = center
|
2022-02-18 13:11:09 +00:00
|
|
|
|
|
2022-03-21 10:36:01 +00:00
|
|
|
|
#This class hold an internal representation of the HTML text
|
|
|
|
|
class representation:
|
2022-03-21 15:21:02 +00:00
|
|
|
|
def __init__(self,width,title=None,center=True):
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.title=title
|
2022-03-21 15:21:02 +00:00
|
|
|
|
self.center = center
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.final_text = ""
|
|
|
|
|
self.opened = []
|
|
|
|
|
self.width = width
|
|
|
|
|
self.last_line = ""
|
|
|
|
|
self.last_line_colors = {}
|
|
|
|
|
self.last_line_center = False
|
|
|
|
|
self.new_paragraph = True
|
|
|
|
|
self.i_indent = ""
|
|
|
|
|
self.s_indent = ""
|
|
|
|
|
self.r_indent = ""
|
|
|
|
|
self.current_indent = ""
|
|
|
|
|
self.disabled_indents = None
|
|
|
|
|
# each color is an [open,close] pair code
|
2022-03-21 21:43:33 +00:00
|
|
|
|
self.colors = {
|
2022-03-21 10:36:01 +00:00
|
|
|
|
"bold" : ["1","22"],
|
|
|
|
|
"faint" : ["2","22"],
|
2022-03-21 21:43:33 +00:00
|
|
|
|
"italic" : ["3","23"],
|
|
|
|
|
"underline": ["4","24"],
|
|
|
|
|
"red" : ["31","39"],
|
2022-03-21 10:36:01 +00:00
|
|
|
|
"yellow" : ["33","39"],
|
2022-03-21 21:43:33 +00:00
|
|
|
|
"blue" : ["34","39"],
|
2022-03-21 10:36:01 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def _insert(self,color,open=True):
|
|
|
|
|
if open: o = 0
|
|
|
|
|
else: o = 1
|
|
|
|
|
pos = len(self.last_line)
|
|
|
|
|
#we remember the position where to insert color codes
|
|
|
|
|
if not pos in self.last_line_colors:
|
|
|
|
|
self.last_line_colors[pos] = []
|
|
|
|
|
#Two inverse code cancel each other
|
|
|
|
|
if [color,int(not o)] in self.last_line_colors[pos]:
|
|
|
|
|
self.last_line_colors[pos].remove([color,int(not o)])
|
|
|
|
|
else:
|
|
|
|
|
self.last_line_colors[pos].append([color,o])#+color+str(o))
|
|
|
|
|
|
|
|
|
|
# Take self.last line and add ANSI codes to it before adding it to
|
|
|
|
|
# self.final_text.
|
2022-03-21 20:44:34 +00:00
|
|
|
|
def _endline(self):
|
2022-03-21 10:36:01 +00:00
|
|
|
|
if len(self.last_line.strip()) > 0:
|
|
|
|
|
for c in self.opened:
|
|
|
|
|
self._insert(c,open=False)
|
2022-03-21 15:21:02 +00:00
|
|
|
|
nextline = ""
|
2022-03-21 10:36:01 +00:00
|
|
|
|
added_char = 0
|
|
|
|
|
#we insert the color code at the saved positions
|
|
|
|
|
while len (self.last_line_colors) > 0:
|
|
|
|
|
pos,colors = self.last_line_colors.popitem()
|
|
|
|
|
#popitem itterates LIFO.
|
|
|
|
|
#So we go, backward, to the pos (starting at the end of last_line)
|
2022-03-21 15:21:02 +00:00
|
|
|
|
nextline = self.last_line[pos:] + nextline
|
2022-03-21 10:36:01 +00:00
|
|
|
|
ansicol = "\x1b["
|
|
|
|
|
for c,o in colors:
|
|
|
|
|
ansicol += self.colors[c][o] + ";"
|
|
|
|
|
ansicol = ansicol[:-1]+"m"
|
2022-03-21 15:21:02 +00:00
|
|
|
|
nextline = ansicol + nextline
|
2022-03-21 10:36:01 +00:00
|
|
|
|
added_char += len(ansicol)
|
|
|
|
|
self.last_line = self.last_line[:pos]
|
2022-03-21 15:21:02 +00:00
|
|
|
|
nextline = self.last_line + nextline
|
2022-03-21 10:36:01 +00:00
|
|
|
|
if self.last_line_center:
|
|
|
|
|
#we have to care about the ansi char while centering
|
|
|
|
|
width = term_width() + added_char
|
2022-03-21 15:21:02 +00:00
|
|
|
|
nextline = nextline.strip().center(width)
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.last_line_center = False
|
|
|
|
|
else:
|
2022-03-21 20:44:34 +00:00
|
|
|
|
#should we lstrip the nextline in the addition ?
|
2022-03-22 17:21:31 +00:00
|
|
|
|
nextline = self.current_indent + nextline.lstrip() + self.r_indent
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.current_indent = self.s_indent
|
2022-03-21 15:21:02 +00:00
|
|
|
|
self.final_text += nextline
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.last_line = ""
|
2022-03-21 20:44:34 +00:00
|
|
|
|
self.final_text += "\n"
|
|
|
|
|
for c in self.opened:
|
|
|
|
|
self._insert(c,open=True)
|
2022-03-21 10:36:01 +00:00
|
|
|
|
else:
|
|
|
|
|
self.last_line = ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def center_line(self):
|
|
|
|
|
self.last_line_center = True
|
|
|
|
|
|
|
|
|
|
def open_color(self,color):
|
|
|
|
|
if color in self.colors and color not in self.opened:
|
|
|
|
|
self._insert(color,open=True)
|
|
|
|
|
self.opened.append(color)
|
|
|
|
|
def close_color(self,color):
|
|
|
|
|
if color in self.colors and color in self.opened:
|
|
|
|
|
self._insert(color,open=False)
|
|
|
|
|
self.opened.remove(color)
|
|
|
|
|
def close_all(self):
|
|
|
|
|
if len(self.colors) > 0:
|
|
|
|
|
self.last_line += "\x1b[0m"
|
|
|
|
|
self.opened.clear()
|
|
|
|
|
|
|
|
|
|
def startindent(self,indent,sub=None,reverse=None):
|
|
|
|
|
self._endline()
|
|
|
|
|
self.i_indent = indent
|
|
|
|
|
self.current_indent = indent
|
|
|
|
|
if sub:
|
|
|
|
|
self.s_indent = sub
|
|
|
|
|
else:
|
|
|
|
|
self.s_indent = indent
|
|
|
|
|
if reverse:
|
|
|
|
|
self.r_indent = reverse
|
|
|
|
|
else:
|
|
|
|
|
self.r_indent = ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def endindent(self):
|
2022-03-21 16:49:37 +00:00
|
|
|
|
self._endline()
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.i_indent = ""
|
|
|
|
|
self.s_indent = ""
|
|
|
|
|
self.r_indent = ""
|
|
|
|
|
self.current_indent = ""
|
|
|
|
|
|
|
|
|
|
def _disable_indents(self):
|
|
|
|
|
self.disabled_indents = []
|
|
|
|
|
self.disabled_indents.append(self.current_indent)
|
|
|
|
|
self.disabled_indents.append(self.i_indent)
|
|
|
|
|
self.disabled_indents.append(self.s_indent)
|
|
|
|
|
self.disabled_indents.append(self.r_indent)
|
|
|
|
|
self.endindent()
|
|
|
|
|
|
|
|
|
|
def _enable_indents(self):
|
|
|
|
|
if self.disabled_indents:
|
|
|
|
|
self.current_indent = self.disabled_indents[0]
|
|
|
|
|
self.i_indent = self.disabled_indents[1]
|
|
|
|
|
self.s_indent = self.disabled_indents[2]
|
|
|
|
|
self.r_indent = self.disabled_indents[3]
|
|
|
|
|
self.disabled_indents = None
|
|
|
|
|
|
|
|
|
|
def newline(self):
|
|
|
|
|
self._endline()
|
|
|
|
|
|
2022-03-22 10:11:35 +00:00
|
|
|
|
#A new paragraph implies 2 newlines (1 blank line between paragraphs)
|
|
|
|
|
#But it is only used if didn’t already started one to avoid plenty
|
|
|
|
|
#of blank lines. force=True allows to bypass that limit.
|
2022-03-21 10:36:01 +00:00
|
|
|
|
#new_paragraph becomes false as soon as text is entered into it
|
2022-03-21 20:44:34 +00:00
|
|
|
|
def newparagraph(self,force=False):
|
|
|
|
|
if force or not self.new_paragraph:
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self._endline()
|
|
|
|
|
self.final_text += "\n"
|
|
|
|
|
self.new_paragraph = True
|
|
|
|
|
|
2022-03-25 13:05:45 +00:00
|
|
|
|
def add_space(self):
|
|
|
|
|
if len(self.last_line) > 0 and self.last_line[-1] != " ":
|
|
|
|
|
self.last_line += " "
|
|
|
|
|
|
2022-03-21 10:36:01 +00:00
|
|
|
|
def _title_first(self,intext=None):
|
|
|
|
|
if self.title:
|
|
|
|
|
if not self.title == intext:
|
|
|
|
|
self._disable_indents()
|
|
|
|
|
self.open_color("blue")
|
|
|
|
|
self.open_color("bold")
|
|
|
|
|
self.open_color("underline")
|
|
|
|
|
self.add_text(self.title)
|
|
|
|
|
self.close_all()
|
|
|
|
|
self.newparagraph()
|
|
|
|
|
self._enable_indents()
|
|
|
|
|
self.title = None
|
|
|
|
|
|
|
|
|
|
# Beware, blocks are not wrapped nor indented and left untouched!
|
2022-03-22 10:11:35 +00:00
|
|
|
|
# They are mostly useful for pictures and preformatted text.
|
2022-03-21 20:44:34 +00:00
|
|
|
|
def add_block(self,intext):
|
2022-03-22 10:11:35 +00:00
|
|
|
|
# If necessary, we add the title before a block
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self._title_first()
|
|
|
|
|
# we don’t want to indent blocks
|
2022-03-21 20:44:34 +00:00
|
|
|
|
self._endline()
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self._disable_indents()
|
2022-03-22 10:11:35 +00:00
|
|
|
|
self.final_text += self.current_indent + intext
|
|
|
|
|
self.new_paragraph = False
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self._endline()
|
|
|
|
|
self._enable_indents()
|
|
|
|
|
|
|
|
|
|
def add_text(self,intext):
|
|
|
|
|
self._title_first(intext=intext)
|
|
|
|
|
lines = []
|
2022-03-31 22:20:34 +00:00
|
|
|
|
last = (self.last_line + intext)
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.last_line = ""
|
2022-03-27 14:15:34 +00:00
|
|
|
|
# With the following, we basically cancel adding only spaces
|
|
|
|
|
# on an empty line
|
|
|
|
|
if len(last.strip()) > 0:
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self.new_paragraph = False
|
2022-03-27 14:15:34 +00:00
|
|
|
|
else:
|
|
|
|
|
last = last.strip()
|
2022-03-21 10:36:01 +00:00
|
|
|
|
if len(last) > self.width:
|
|
|
|
|
width = self.width - len(self.current_indent) - len(self.r_indent)
|
2022-03-21 20:44:34 +00:00
|
|
|
|
spaces_left = len(last) - len(last.lstrip())
|
|
|
|
|
spaces_right = len(last) - len(last.rstrip())
|
|
|
|
|
lines = textwrap.wrap(last,width,drop_whitespace=True)
|
|
|
|
|
self.last_line += spaces_left*" "
|
2022-03-21 10:36:01 +00:00
|
|
|
|
while len(lines) > 1:
|
|
|
|
|
l = lines.pop(0)
|
2022-03-22 10:11:35 +00:00
|
|
|
|
self.last_line += l
|
2022-03-21 10:36:01 +00:00
|
|
|
|
self._endline()
|
|
|
|
|
if len(lines) == 1:
|
|
|
|
|
li = lines[0]
|
2022-03-21 20:44:34 +00:00
|
|
|
|
self.last_line += li + spaces_right*" "
|
2022-03-21 10:36:01 +00:00
|
|
|
|
else:
|
2022-03-22 10:11:35 +00:00
|
|
|
|
self.last_line = last
|
2022-03-21 10:36:01 +00:00
|
|
|
|
|
|
|
|
|
def get_final(self):
|
|
|
|
|
self.close_all()
|
|
|
|
|
self._endline()
|
|
|
|
|
#if no content, we still add the title
|
|
|
|
|
self._title_first()
|
|
|
|
|
lines = self.final_text.splitlines()
|
|
|
|
|
lines2 = []
|
|
|
|
|
termspace = shutil.get_terminal_size()[0]
|
|
|
|
|
#Following code instert blanck spaces to center the content
|
2022-03-21 15:21:02 +00:00
|
|
|
|
if self.center and termspace > term_width():
|
2022-03-21 10:36:01 +00:00
|
|
|
|
margin = int((termspace - term_width())//2)
|
|
|
|
|
else:
|
|
|
|
|
margin = 0
|
|
|
|
|
for l in lines :
|
|
|
|
|
lines2.append(margin*" "+l)
|
|
|
|
|
return "\n".join(lines2)
|
|
|
|
|
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_subscribe_links(self):
|
|
|
|
|
return [[self.url,self.get_mime(),self.get_title()]]
|
2022-02-06 23:29:26 +00:00
|
|
|
|
def is_valid(self):
|
2022-02-12 23:37:31 +00:00
|
|
|
|
return self.validity
|
2022-03-16 08:47:53 +00:00
|
|
|
|
def get_links(self,mode="links_only"):
|
2022-03-16 09:03:29 +00:00
|
|
|
|
if mode not in self.links :
|
2022-03-16 08:47:53 +00:00
|
|
|
|
prepared_body = self.prepare(self.body,mode=mode)
|
|
|
|
|
results = self.render(prepared_body,mode=mode)
|
2022-02-20 22:22:25 +00:00
|
|
|
|
if results:
|
2022-03-16 09:03:29 +00:00
|
|
|
|
self.links[mode] = results[1]
|
2022-10-06 20:30:09 +00:00
|
|
|
|
for l in self.get_subscribe_links()[1:]:
|
|
|
|
|
self.links[mode].append(l[0])
|
2022-03-16 09:03:29 +00:00
|
|
|
|
return self.links[mode]
|
2022-02-13 00:12:55 +00:00
|
|
|
|
def get_title(self):
|
|
|
|
|
return "Abstract title"
|
2022-03-30 10:04:07 +00:00
|
|
|
|
|
|
|
|
|
# This function return a list of URL which should be downloaded
|
|
|
|
|
# before displaying the page (images in HTML pages, typically)
|
|
|
|
|
def get_images(self,mode="readable"):
|
|
|
|
|
if not mode in self.images:
|
|
|
|
|
self.get_body(mode=mode)
|
|
|
|
|
# we also invalidate the body that was done without images
|
|
|
|
|
self.rendered_text.pop(mode)
|
|
|
|
|
if mode in self.images:
|
|
|
|
|
return self.images[mode]
|
|
|
|
|
else:
|
|
|
|
|
return []
|
2022-02-13 21:59:16 +00:00
|
|
|
|
#This function will give gemtext to the gemtext renderer
|
2022-02-14 10:05:39 +00:00
|
|
|
|
def prepare(self,body,mode=None):
|
2022-02-13 21:59:16 +00:00
|
|
|
|
return body
|
|
|
|
|
|
2022-03-15 19:14:21 +00:00
|
|
|
|
def get_body(self,width=None,mode="readable"):
|
2022-02-13 16:50:15 +00:00
|
|
|
|
if not width:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
width = term_width()
|
2022-03-16 09:03:29 +00:00
|
|
|
|
if mode not in self.rendered_text:
|
2022-02-14 10:05:39 +00:00
|
|
|
|
prepared_body = self.prepare(self.body,mode=mode)
|
2022-02-20 22:22:25 +00:00
|
|
|
|
result = self.render(prepared_body,width=width,mode=mode)
|
|
|
|
|
if result:
|
2022-03-16 09:03:29 +00:00
|
|
|
|
self.rendered_text[mode] = result[0]
|
|
|
|
|
self.links[mode] = result[1]
|
|
|
|
|
return self.rendered_text[mode]
|
2022-03-15 15:23:44 +00:00
|
|
|
|
|
2022-03-21 21:43:33 +00:00
|
|
|
|
def _window_title(self,title,info=None):
|
|
|
|
|
title_r = self.representation(term_width())
|
|
|
|
|
title_r.open_color("red")
|
|
|
|
|
title_r.open_color("bold")
|
|
|
|
|
title_r.add_text(title)
|
|
|
|
|
title_r.close_color("bold")
|
|
|
|
|
if info:
|
|
|
|
|
title_r.add_text(" (%s)"%info)
|
|
|
|
|
title_r.close_color("red")
|
|
|
|
|
return title_r.get_final()
|
|
|
|
|
|
2022-03-25 12:37:23 +00:00
|
|
|
|
def display(self,mode="readable",window_title="",window_info=None,grep=None):
|
2022-11-27 00:15:37 +00:00
|
|
|
|
if not mode: mode = "readable"
|
2022-03-21 21:43:33 +00:00
|
|
|
|
wtitle = self._window_title(window_title,info=window_info)
|
|
|
|
|
body = wtitle + "\n" + self.get_body(mode=mode)
|
2022-03-15 22:00:05 +00:00
|
|
|
|
if not body:
|
|
|
|
|
return False
|
|
|
|
|
# We actually put the body in a tmpfile before giving it to less
|
2022-03-15 23:13:45 +00:00
|
|
|
|
if mode not in self.temp_file:
|
2022-03-15 22:00:05 +00:00
|
|
|
|
tmpf = tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False)
|
2022-03-15 23:13:45 +00:00
|
|
|
|
self.temp_file[mode] = tmpf.name
|
2022-03-15 22:00:05 +00:00
|
|
|
|
tmpf.write(body)
|
|
|
|
|
tmpf.close()
|
2022-03-15 23:13:45 +00:00
|
|
|
|
if mode not in self.less_histfile:
|
2022-03-15 22:00:05 +00:00
|
|
|
|
firsttime = True
|
|
|
|
|
tmpf = tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False)
|
2022-03-15 23:13:45 +00:00
|
|
|
|
self.less_histfile[mode] = tmpf.name
|
2022-03-15 22:00:05 +00:00
|
|
|
|
else:
|
|
|
|
|
firsttime = False
|
2022-03-25 12:37:23 +00:00
|
|
|
|
less_cmd(self.temp_file[mode], histfile=self.less_histfile[mode],cat=firsttime,grep=grep)
|
2022-03-15 22:00:05 +00:00
|
|
|
|
return True
|
|
|
|
|
|
2022-03-15 23:13:45 +00:00
|
|
|
|
def get_temp_file(self,mode="readable"):
|
|
|
|
|
if mode in self.temp_file:
|
|
|
|
|
return self.temp_file[mode]
|
|
|
|
|
else:
|
|
|
|
|
return None
|
2022-03-15 22:00:05 +00:00
|
|
|
|
|
2022-02-13 16:50:15 +00:00
|
|
|
|
# An instance of AbstractRenderer should have a self.render(body,width,mode) method.
|
|
|
|
|
# 3 modes are used : readable (by default), full and links_only (the fastest, when
|
|
|
|
|
# rendered content is not used, only the links are needed)
|
2022-02-13 21:59:16 +00:00
|
|
|
|
# The prepare() function is called before the rendering. It is useful if
|
|
|
|
|
# your renderer output in a format suitable for another existing renderer (such as gemtext)
|
2022-01-17 22:37:12 +00:00
|
|
|
|
|
2022-02-12 23:37:31 +00:00
|
|
|
|
# Gemtext Rendering Engine
|
|
|
|
|
class GemtextRenderer(AbstractRenderer):
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_mime(self):
|
|
|
|
|
return "text/gemini"
|
2022-01-30 18:11:23 +00:00
|
|
|
|
def get_title(self):
|
|
|
|
|
if self.title:
|
|
|
|
|
return self.title
|
2022-03-27 14:24:19 +00:00
|
|
|
|
elif self.body:
|
2022-01-30 18:11:23 +00:00
|
|
|
|
lines = self.body.splitlines()
|
|
|
|
|
for line in lines:
|
|
|
|
|
if line.startswith("#"):
|
|
|
|
|
self.title = line.strip("#").strip()
|
|
|
|
|
return self.title
|
|
|
|
|
if len(lines) > 0:
|
2022-02-12 12:12:37 +00:00
|
|
|
|
# If not title found, we take the first 50 char
|
2022-01-30 18:11:23 +00:00
|
|
|
|
# of the first line
|
|
|
|
|
title_line = lines[0].strip()
|
2022-02-12 12:12:37 +00:00
|
|
|
|
if len(title_line) > 50:
|
|
|
|
|
title_line = title_line[:49] + "…"
|
2022-01-30 18:11:23 +00:00
|
|
|
|
self.title = title_line
|
|
|
|
|
return self.title
|
|
|
|
|
else:
|
|
|
|
|
self.title = "Empty Page"
|
|
|
|
|
return self.title
|
2022-03-27 14:24:19 +00:00
|
|
|
|
else:
|
|
|
|
|
return "Unknown Gopher Page"
|
2022-02-13 16:50:15 +00:00
|
|
|
|
|
|
|
|
|
#render_gemtext
|
|
|
|
|
def render(self,gemtext, width=None,mode=None):
|
2022-02-12 12:12:37 +00:00
|
|
|
|
if not width:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
width = term_width()
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r = self.representation(width)
|
2022-01-29 14:35:28 +00:00
|
|
|
|
links = []
|
2022-11-10 22:50:16 +00:00
|
|
|
|
hidden_links = []
|
2022-01-29 14:35:28 +00:00
|
|
|
|
preformatted = False
|
|
|
|
|
def format_link(url,index,name=None):
|
|
|
|
|
if "://" in url:
|
|
|
|
|
protocol,adress = url.split("://",maxsplit=1)
|
|
|
|
|
protocol = " %s" %protocol
|
2022-01-17 22:37:12 +00:00
|
|
|
|
else:
|
2022-01-29 14:35:28 +00:00
|
|
|
|
adress = url
|
|
|
|
|
protocol = ""
|
2022-03-22 22:17:47 +00:00
|
|
|
|
if "gemini" in protocol or "list" in protocol:
|
2022-01-29 14:35:28 +00:00
|
|
|
|
protocol = ""
|
|
|
|
|
if not name:
|
|
|
|
|
name = adress
|
|
|
|
|
line = "[%d%s] %s" % (index, protocol, name)
|
|
|
|
|
return line
|
|
|
|
|
for line in gemtext.splitlines():
|
2022-03-21 20:44:34 +00:00
|
|
|
|
r.newline()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
if line.startswith("```"):
|
|
|
|
|
preformatted = not preformatted
|
|
|
|
|
elif preformatted:
|
2022-03-07 17:23:48 +00:00
|
|
|
|
# infinite line to not wrap preformated
|
2022-03-21 20:44:34 +00:00
|
|
|
|
r.add_block(line+"\n")
|
2022-03-21 16:49:37 +00:00
|
|
|
|
elif len(line.strip()) == 0:
|
2022-03-21 20:44:34 +00:00
|
|
|
|
r.newparagraph(force=True)
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif line.startswith("=>"):
|
|
|
|
|
strippedline = line[2:].strip()
|
|
|
|
|
if strippedline:
|
|
|
|
|
links.append(strippedline)
|
|
|
|
|
splitted = strippedline.split(maxsplit=1)
|
|
|
|
|
url = splitted[0]
|
|
|
|
|
name = None
|
|
|
|
|
if len(splitted) > 1:
|
|
|
|
|
name = splitted[1]
|
|
|
|
|
link = format_link(url,len(links),name=name)
|
2022-03-21 15:21:02 +00:00
|
|
|
|
#r.open_color("blue")
|
|
|
|
|
#r.open_color("faint")
|
|
|
|
|
#r.open_color("underline")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
startpos = link.find("] ") + 2
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r.startindent("",sub=startpos*" ")
|
|
|
|
|
r.add_text(link)
|
|
|
|
|
r.endindent()
|
|
|
|
|
#r.close_all()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif line.startswith("* "):
|
|
|
|
|
line = line[1:].lstrip("\t ")
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r.startindent("• ",sub=" ")
|
|
|
|
|
r.add_text(line)
|
|
|
|
|
r.endindent()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif line.startswith(">"):
|
|
|
|
|
line = line[1:].lstrip("\t ")
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r.startindent("> ")
|
|
|
|
|
r.add_text(line)
|
|
|
|
|
r.endindent()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif line.startswith("###"):
|
|
|
|
|
line = line[3:].lstrip("\t ")
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r.open_color("blue")
|
|
|
|
|
r.add_text(line)
|
|
|
|
|
r.close_color("blue")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif line.startswith("##"):
|
|
|
|
|
line = line[2:].lstrip("\t ")
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r.open_color("blue")
|
|
|
|
|
r.add_text(line)
|
|
|
|
|
r.close_color("blue")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif line.startswith("#"):
|
|
|
|
|
line = line[1:].lstrip("\t ")
|
2022-01-30 18:11:23 +00:00
|
|
|
|
if not self.title:
|
|
|
|
|
self.title = line
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r.open_color("bold")
|
|
|
|
|
r.open_color("blue")
|
|
|
|
|
r.open_color("underline")
|
|
|
|
|
r.add_text(line)
|
|
|
|
|
r.close_color("underline")
|
|
|
|
|
r.close_color("bold")
|
|
|
|
|
r.close_color("blue")
|
2022-01-17 22:37:12 +00:00
|
|
|
|
else:
|
2022-11-10 22:50:16 +00:00
|
|
|
|
if "://" in line:
|
|
|
|
|
words = line.split()
|
|
|
|
|
for w in words:
|
|
|
|
|
if "://" in w:
|
|
|
|
|
hidden_links.append(w)
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r.add_text(line.rstrip())
|
2022-11-10 22:50:16 +00:00
|
|
|
|
links += hidden_links
|
2022-03-21 20:44:34 +00:00
|
|
|
|
return r.get_final(), links
|
2022-01-29 14:35:28 +00:00
|
|
|
|
|
2022-02-12 23:37:31 +00:00
|
|
|
|
class GopherRenderer(AbstractRenderer):
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_mime(self):
|
|
|
|
|
return "text/gopher"
|
2022-02-10 22:41:10 +00:00
|
|
|
|
def get_title(self):
|
2022-03-21 21:43:33 +00:00
|
|
|
|
if not self.title:
|
2022-03-24 09:51:36 +00:00
|
|
|
|
self.title = ""
|
2022-03-21 21:43:33 +00:00
|
|
|
|
if self.body:
|
|
|
|
|
firstline = self.body.splitlines()[0]
|
|
|
|
|
firstline = firstline.split("\t")[0]
|
|
|
|
|
if firstline.startswith("i"):
|
|
|
|
|
firstline = firstline[1:]
|
|
|
|
|
self.title = firstline
|
|
|
|
|
return self.title
|
2022-02-10 22:41:10 +00:00
|
|
|
|
|
2022-02-13 16:50:15 +00:00
|
|
|
|
#menu_or_text
|
|
|
|
|
def render(self,body,width=None,mode=None):
|
2022-02-12 12:12:37 +00:00
|
|
|
|
if not width:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
width = term_width()
|
2022-02-10 22:41:10 +00:00
|
|
|
|
try:
|
2022-03-21 21:43:33 +00:00
|
|
|
|
render,links = self._render_goph(body,width=width,mode=mode)
|
2022-02-10 22:41:10 +00:00
|
|
|
|
except Exception as err:
|
2022-03-21 21:43:33 +00:00
|
|
|
|
print("Error rendering Gopher ",err)
|
|
|
|
|
r = self.representation(width)
|
|
|
|
|
r.add_block(body)
|
|
|
|
|
render = r.get_final()
|
2022-02-10 22:41:10 +00:00
|
|
|
|
links = []
|
|
|
|
|
return render,links
|
|
|
|
|
|
2022-03-21 21:43:33 +00:00
|
|
|
|
def _render_goph(self,body,width=None,mode=None):
|
2022-02-12 12:12:37 +00:00
|
|
|
|
if not width:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
width = term_width()
|
2022-03-21 21:43:33 +00:00
|
|
|
|
# This was copied straight from Agena (then later adapted)
|
2022-02-10 22:41:10 +00:00
|
|
|
|
links = []
|
2022-03-21 21:43:33 +00:00
|
|
|
|
r = self.representation(width)
|
2022-02-10 22:41:10 +00:00
|
|
|
|
for line in self.body.split("\n"):
|
2022-03-21 21:43:33 +00:00
|
|
|
|
r.newline()
|
2022-02-10 22:41:10 +00:00
|
|
|
|
if line.startswith("i"):
|
2022-03-21 21:43:33 +00:00
|
|
|
|
towrap = line[1:].split("\t")[0]
|
|
|
|
|
if len(towrap.strip()) > 0:
|
|
|
|
|
r.add_text(towrap)
|
|
|
|
|
else:
|
|
|
|
|
r.newparagraph()
|
2022-02-10 22:41:10 +00:00
|
|
|
|
elif not line.strip() in [".",""]:
|
|
|
|
|
parts = line.split("\t")
|
|
|
|
|
parts[-1] = parts[-1].strip()
|
|
|
|
|
if parts[-1] == "+":
|
|
|
|
|
parts = parts[:-1]
|
|
|
|
|
if len(parts) == 4:
|
|
|
|
|
name,path,host,port = parts
|
|
|
|
|
itemtype = name[0]
|
|
|
|
|
name = name[1:]
|
|
|
|
|
if port == "70":
|
|
|
|
|
port = ""
|
|
|
|
|
else:
|
|
|
|
|
port = ":%s"%port
|
|
|
|
|
if itemtype == "h" and path.startswith("URL:"):
|
|
|
|
|
url = path[4:]
|
|
|
|
|
else:
|
2022-02-16 08:45:38 +00:00
|
|
|
|
if not path.startswith("/"):
|
|
|
|
|
path = "/"+path
|
2022-02-10 22:41:10 +00:00
|
|
|
|
url = "gopher://%s%s/%s%s" %(host,port,itemtype,path)
|
2022-03-31 09:00:55 +00:00
|
|
|
|
url = url.replace(" ","%20")
|
2022-02-10 22:41:10 +00:00
|
|
|
|
linkline = url + " " + name
|
|
|
|
|
links.append(linkline)
|
2022-03-21 21:43:33 +00:00
|
|
|
|
towrap = "[%s] "%len(links)+ name
|
|
|
|
|
r.add_text(towrap)
|
2022-02-10 22:41:10 +00:00
|
|
|
|
else:
|
2022-03-21 21:43:33 +00:00
|
|
|
|
r.add_text(line)
|
|
|
|
|
return r.get_final(),links
|
2022-02-10 22:41:10 +00:00
|
|
|
|
|
|
|
|
|
|
2022-02-13 21:59:16 +00:00
|
|
|
|
class FolderRenderer(GemtextRenderer):
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_mime(self):
|
|
|
|
|
return "Directory"
|
2022-02-14 10:28:12 +00:00
|
|
|
|
def prepare(self,body,mode=None):
|
2022-02-15 13:56:35 +00:00
|
|
|
|
def get_first_line(l):
|
2022-02-13 00:12:55 +00:00
|
|
|
|
path = os.path.join(listdir,l+".gmi")
|
2022-02-15 13:56:35 +00:00
|
|
|
|
with open(path) as f:
|
|
|
|
|
first_line = f.readline().strip()
|
|
|
|
|
f.close()
|
|
|
|
|
if first_line.startswith("#"):
|
|
|
|
|
return first_line
|
|
|
|
|
else:
|
|
|
|
|
return None
|
|
|
|
|
def write_list(l):
|
|
|
|
|
body = ""
|
|
|
|
|
for li in l:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
path = "list:///%s"%li
|
|
|
|
|
gi = GeminiItem(path)
|
2022-02-15 13:56:35 +00:00
|
|
|
|
size = len(gi.get_links())
|
|
|
|
|
body += "=> %s %s (%s items)\n" %(str(path),li,size)
|
|
|
|
|
return body
|
2022-02-13 00:12:55 +00:00
|
|
|
|
listdir = os.path.join(_DATA_DIR,"lists")
|
|
|
|
|
if self.url != listdir:
|
|
|
|
|
return "This is folder %s" %self.url
|
|
|
|
|
else:
|
2022-02-14 22:02:53 +00:00
|
|
|
|
self.title = "My lists"
|
2022-02-13 00:12:55 +00:00
|
|
|
|
lists = []
|
|
|
|
|
if os.path.exists(listdir):
|
|
|
|
|
listfiles = os.listdir(listdir)
|
|
|
|
|
if len(listfiles) > 0:
|
|
|
|
|
for l in listfiles:
|
|
|
|
|
#removing the .gmi at the end of the name
|
|
|
|
|
lists.append(l[:-4])
|
|
|
|
|
if len(lists) > 0:
|
|
|
|
|
body = ""
|
|
|
|
|
my_lists = []
|
|
|
|
|
system_lists = []
|
|
|
|
|
subscriptions = []
|
2022-02-15 13:56:35 +00:00
|
|
|
|
frozen = []
|
2022-02-13 00:12:55 +00:00
|
|
|
|
lists.sort()
|
|
|
|
|
for l in lists:
|
|
|
|
|
if l in ["history","to_fetch","archives","tour"]:
|
|
|
|
|
system_lists.append(l)
|
|
|
|
|
else:
|
2022-02-15 13:56:35 +00:00
|
|
|
|
first_line = get_first_line(l)
|
|
|
|
|
if first_line and "#subscribed" in first_line:
|
|
|
|
|
subscriptions.append(l)
|
|
|
|
|
elif first_line and "#frozen" in first_line:
|
|
|
|
|
frozen.append(l)
|
|
|
|
|
else:
|
|
|
|
|
my_lists.append(l)
|
2022-02-13 00:12:55 +00:00
|
|
|
|
if len(my_lists) > 0:
|
2022-02-13 09:59:10 +00:00
|
|
|
|
body+= "\n## Bookmarks Lists (updated during sync)\n"
|
2022-02-15 13:56:35 +00:00
|
|
|
|
body += write_list(my_lists)
|
2022-02-13 00:12:55 +00:00
|
|
|
|
if len(subscriptions) > 0:
|
2022-02-13 09:59:10 +00:00
|
|
|
|
body +="\n## Subscriptions (new links in those are added to tour)\n"
|
2022-02-15 13:56:35 +00:00
|
|
|
|
body += write_list(subscriptions)
|
|
|
|
|
if len(frozen) > 0:
|
2022-02-15 15:50:33 +00:00
|
|
|
|
body +="\n## Frozen (fetched but never updated)\n"
|
2022-02-15 13:56:35 +00:00
|
|
|
|
body += write_list(frozen)
|
2022-02-13 00:12:55 +00:00
|
|
|
|
if len(system_lists) > 0:
|
|
|
|
|
body +="\n## System Lists\n"
|
2022-02-15 13:56:35 +00:00
|
|
|
|
body += write_list(system_lists)
|
2022-02-13 21:59:16 +00:00
|
|
|
|
return body
|
2022-02-13 00:12:55 +00:00
|
|
|
|
|
2022-02-14 10:05:39 +00:00
|
|
|
|
class FeedRenderer(GemtextRenderer):
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_mime(self):
|
|
|
|
|
return "application/rss+xml"
|
2022-02-06 23:29:26 +00:00
|
|
|
|
def is_valid(self):
|
2022-02-13 16:50:15 +00:00
|
|
|
|
if _DO_FEED:
|
|
|
|
|
parsed = feedparser.parse(self.body)
|
2022-02-06 13:08:25 +00:00
|
|
|
|
else:
|
2022-02-13 16:50:15 +00:00
|
|
|
|
return False
|
|
|
|
|
if parsed.bozo:
|
|
|
|
|
return False
|
|
|
|
|
else:
|
2022-02-25 21:13:40 +00:00
|
|
|
|
#If no content, then fallback to HTML
|
|
|
|
|
return len(parsed.entries) > 0
|
2022-02-06 13:08:25 +00:00
|
|
|
|
|
|
|
|
|
def get_title(self):
|
2022-02-06 14:42:41 +00:00
|
|
|
|
if not self.title:
|
2022-03-24 09:51:36 +00:00
|
|
|
|
self.get_body()
|
2022-02-06 14:42:41 +00:00
|
|
|
|
return self.title
|
2022-02-06 13:08:25 +00:00
|
|
|
|
|
2022-02-14 10:05:39 +00:00
|
|
|
|
def prepare(self,content,mode="readable",width=None):
|
2022-02-12 12:12:37 +00:00
|
|
|
|
if not width:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
width = term_width()
|
2022-02-06 14:42:41 +00:00
|
|
|
|
self.title = "RSS/Atom feed"
|
|
|
|
|
page = ""
|
2022-02-06 13:08:25 +00:00
|
|
|
|
if _DO_FEED:
|
|
|
|
|
parsed = feedparser.parse(content)
|
|
|
|
|
else:
|
2022-02-06 14:42:41 +00:00
|
|
|
|
page += "Please install python-feedparser to handle RSS/Atom feeds\n"
|
2022-02-06 23:29:26 +00:00
|
|
|
|
self.validity = False
|
2022-02-06 13:08:25 +00:00
|
|
|
|
return page
|
|
|
|
|
if parsed.bozo:
|
|
|
|
|
page += "Invalid RSS feed\n\n"
|
2022-02-09 20:46:29 +00:00
|
|
|
|
page += str(parsed.bozo_exception)
|
2022-02-06 23:29:26 +00:00
|
|
|
|
self.validity = False
|
2022-02-06 13:08:25 +00:00
|
|
|
|
else:
|
2022-02-06 23:29:26 +00:00
|
|
|
|
if "title" in parsed.feed:
|
2022-02-06 14:42:41 +00:00
|
|
|
|
t = parsed.feed.title
|
|
|
|
|
else:
|
|
|
|
|
t = "Unknown"
|
2022-02-14 09:25:55 +00:00
|
|
|
|
self.title = "%s (XML feed)" %t
|
2022-02-14 10:05:39 +00:00
|
|
|
|
title = "# %s"%self.title
|
|
|
|
|
page += title + "\n"
|
|
|
|
|
if "updated" in parsed.feed:
|
|
|
|
|
page += "Last updated on %s\n\n" %parsed.feed.updated
|
2022-02-06 14:42:41 +00:00
|
|
|
|
if "subtitle" in parsed.feed:
|
2022-02-14 10:05:39 +00:00
|
|
|
|
page += parsed.feed.subtitle + "\n"
|
2022-02-06 23:29:26 +00:00
|
|
|
|
if "link" in parsed.feed:
|
2022-02-14 10:05:39 +00:00
|
|
|
|
page += "=> %s\n" %parsed.feed.link
|
|
|
|
|
page += "\n## Entries\n"
|
2022-02-06 23:29:26 +00:00
|
|
|
|
if len(parsed.entries) < 1:
|
|
|
|
|
self.validity = False
|
2022-02-06 13:08:25 +00:00
|
|
|
|
for i in parsed.entries:
|
2022-02-14 10:05:39 +00:00
|
|
|
|
line = "=> %s " %i.link
|
2022-02-06 14:42:41 +00:00
|
|
|
|
if "published" in i:
|
2022-02-14 10:05:39 +00:00
|
|
|
|
pub_date = time.strftime("%Y-%m-%d",i.published_parsed)
|
|
|
|
|
line += pub_date + " : "
|
|
|
|
|
line += "%s" %(i.title)
|
|
|
|
|
if "author" in i:
|
|
|
|
|
line += " (by %s)"%i.author
|
|
|
|
|
page += line + "\n"
|
2022-02-13 16:50:15 +00:00
|
|
|
|
if mode == "full":
|
2022-02-06 14:42:41 +00:00
|
|
|
|
if "summary" in i:
|
2022-03-18 22:02:21 +00:00
|
|
|
|
html = HtmlRenderer(i.summary,self.url,center=False)
|
|
|
|
|
rendered = html.get_body(width=None,mode="full")
|
2022-02-13 21:59:16 +00:00
|
|
|
|
page += rendered
|
|
|
|
|
page += "\n"
|
2022-02-14 10:05:39 +00:00
|
|
|
|
return page
|
2022-02-06 13:08:25 +00:00
|
|
|
|
|
2022-02-12 23:37:31 +00:00
|
|
|
|
class ImageRenderer(AbstractRenderer):
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_mime(self):
|
|
|
|
|
return "image/*"
|
2022-02-12 13:24:43 +00:00
|
|
|
|
def is_valid(self):
|
|
|
|
|
if _RENDER_IMAGE:
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
return False
|
2022-03-16 09:08:29 +00:00
|
|
|
|
def get_links(self,mode=None):
|
2022-02-12 13:24:43 +00:00
|
|
|
|
return []
|
|
|
|
|
def get_title(self):
|
2022-02-12 23:37:31 +00:00
|
|
|
|
return "Picture file"
|
2022-02-13 16:50:15 +00:00
|
|
|
|
def render(self,img,width=None,mode=None):
|
2022-03-15 15:23:44 +00:00
|
|
|
|
#with inline, we use symbols to be rendered with less.
|
|
|
|
|
#else we use the best possible renderer.
|
2022-02-13 16:50:15 +00:00
|
|
|
|
if mode == "links_only":
|
|
|
|
|
return "", []
|
2022-02-12 13:24:43 +00:00
|
|
|
|
if not width:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
width = term_width()
|
2022-03-07 20:54:43 +00:00
|
|
|
|
spaces = 0
|
|
|
|
|
else:
|
|
|
|
|
spaces = int((term_width() - width)//2)
|
2022-03-17 09:18:00 +00:00
|
|
|
|
ansi_img = inline_image(img,width)
|
2022-03-07 20:54:43 +00:00
|
|
|
|
#Now centering the image
|
|
|
|
|
lines = ansi_img.splitlines()
|
|
|
|
|
new_img = ""
|
|
|
|
|
for l in lines:
|
|
|
|
|
new_img += spaces*" " + l + "\n"
|
|
|
|
|
return new_img, []
|
2022-03-25 12:37:23 +00:00
|
|
|
|
def display(self,mode=None,window_title=None,window_info=None,grep=None):
|
2022-03-21 21:43:33 +00:00
|
|
|
|
if window_title:
|
|
|
|
|
print(self._window_title(window_title,info=window_info))
|
2022-03-17 09:18:00 +00:00
|
|
|
|
terminal_image(self.body)
|
2022-03-15 22:00:05 +00:00
|
|
|
|
return True
|
2022-02-06 13:08:25 +00:00
|
|
|
|
|
2022-02-12 23:37:31 +00:00
|
|
|
|
class HtmlRenderer(AbstractRenderer):
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_mime(self):
|
|
|
|
|
return "text/html"
|
2022-02-20 22:22:25 +00:00
|
|
|
|
def is_valid(self):
|
|
|
|
|
if not _DO_HTML:
|
|
|
|
|
print("HTML document detected. Please install python-bs4 and python-readability.")
|
|
|
|
|
return _DO_HTML and self.validity
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_subscribe_links(self):
|
|
|
|
|
subs = [[self.url,self.get_mime(),self.get_title()]]
|
|
|
|
|
soup = BeautifulSoup(self.body, 'html.parser')
|
|
|
|
|
links = soup.find_all("link",rel="alternate",recursive=True)
|
|
|
|
|
for l in links:
|
|
|
|
|
ty = l.get("type")
|
2022-02-25 09:59:54 +00:00
|
|
|
|
if ty :
|
|
|
|
|
if "rss" in ty or "atom" in ty or "feed" in ty:
|
|
|
|
|
subs.append([l.get("href"),ty,l.get("title")])
|
2022-02-18 13:11:09 +00:00
|
|
|
|
return subs
|
|
|
|
|
|
2022-01-30 18:11:23 +00:00
|
|
|
|
def get_title(self):
|
|
|
|
|
if self.title:
|
|
|
|
|
return self.title
|
2022-04-02 15:55:45 +00:00
|
|
|
|
elif self.body:
|
2022-04-02 16:12:46 +00:00
|
|
|
|
if _HAS_READABILITY:
|
|
|
|
|
try:
|
|
|
|
|
readable = Document(self.body)
|
|
|
|
|
self.title = readable.short_title()
|
|
|
|
|
return self.title
|
|
|
|
|
except Exception as err:
|
|
|
|
|
pass
|
|
|
|
|
soup = BeautifulSoup(self.body,"html.parser")
|
|
|
|
|
self.title = str(soup.title.string)
|
2022-01-30 18:11:23 +00:00
|
|
|
|
else:
|
2022-04-02 15:55:45 +00:00
|
|
|
|
return ""
|
2022-03-18 10:25:47 +00:00
|
|
|
|
|
2022-01-29 14:35:28 +00:00
|
|
|
|
# Our own HTML engine (crazy, isn’t it?)
|
|
|
|
|
# Return [rendered_body, list_of_links]
|
2022-02-13 16:50:15 +00:00
|
|
|
|
# mode is either links_only, readable or full
|
2022-03-18 22:02:21 +00:00
|
|
|
|
def render(self,body,mode="readable",width=None,add_title=True):
|
2022-02-12 12:12:37 +00:00
|
|
|
|
if not width:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
width = term_width()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
if not _DO_HTML:
|
|
|
|
|
print("HTML document detected. Please install python-bs4 and python-readability.")
|
|
|
|
|
return
|
|
|
|
|
# This method recursively parse the HTML
|
2022-03-21 15:21:02 +00:00
|
|
|
|
r = self.representation(width,title=self.get_title(),center=self.center)
|
2022-01-29 14:35:28 +00:00
|
|
|
|
links = []
|
2022-02-04 23:26:51 +00:00
|
|
|
|
# You know how bad html is when you realize that space sometimes meaningful, somtimes not.
|
|
|
|
|
# CR are not meaniningful. Except that, somethimes, they should be interpreted as spaces.
|
|
|
|
|
# HTML is real crap. At least the one people are generating.
|
2022-10-07 11:56:55 +00:00
|
|
|
|
|
|
|
|
|
def render_image(src,width=40,mode=None):
|
|
|
|
|
ansi_img = ""
|
2022-10-07 13:51:26 +00:00
|
|
|
|
imgurl,imgdata = looks_like_base64(src,self.url)
|
2022-10-07 11:56:55 +00:00
|
|
|
|
if _RENDER_IMAGE and mode != "links_only" and imgurl:
|
2022-02-14 23:04:30 +00:00
|
|
|
|
try:
|
2022-03-31 22:20:34 +00:00
|
|
|
|
#4 followings line are there to translate the URL into cache path
|
2022-10-07 11:56:55 +00:00
|
|
|
|
g = GeminiItem(imgurl)
|
2022-10-07 09:19:23 +00:00
|
|
|
|
img = g.get_cache_path()
|
|
|
|
|
if imgdata:
|
|
|
|
|
with open(img,"wb") as cached:
|
|
|
|
|
cached.write(base64.b64decode(imgdata))
|
|
|
|
|
cached.close()
|
2022-02-14 23:04:30 +00:00
|
|
|
|
if g.is_cache_valid():
|
2022-10-07 11:56:55 +00:00
|
|
|
|
renderer = ImageRenderer(img,imgurl)
|
2022-02-14 23:04:30 +00:00
|
|
|
|
# Image are 40px wide except if terminal is smaller
|
|
|
|
|
if width > 40:
|
|
|
|
|
size = 40
|
|
|
|
|
else:
|
|
|
|
|
size = width
|
2022-03-15 15:23:44 +00:00
|
|
|
|
ansi_img = "\n" + renderer.get_body(width=size,mode="inline")
|
2022-02-14 23:04:30 +00:00
|
|
|
|
except Exception as err:
|
|
|
|
|
#we sometimes encounter really bad formatted files or URL
|
2022-10-07 09:19:23 +00:00
|
|
|
|
ansi_img = textwrap.fill("[BAD IMG] %s - %s"%(err,src),width) + "\n"
|
2022-02-14 23:04:30 +00:00
|
|
|
|
return ansi_img
|
2022-03-18 10:25:47 +00:00
|
|
|
|
def sanitize_string(string):
|
2022-03-22 17:21:31 +00:00
|
|
|
|
#never start with a "\n"
|
2022-04-06 09:41:56 +00:00
|
|
|
|
#string = string.lstrip("\n")
|
2022-03-22 17:21:31 +00:00
|
|
|
|
string = string.replace("\r","").replace("\n", " ").replace("\t"," ")
|
2022-02-14 23:04:30 +00:00
|
|
|
|
endspace = string.endswith(" ") or string.endswith("\xa0")
|
|
|
|
|
startspace = string.startswith(" ") or string.startswith("\xa0")
|
2022-02-04 23:26:51 +00:00
|
|
|
|
toreturn = string.replace("\n", " ").replace("\t"," ").strip()
|
2022-02-04 00:14:22 +00:00
|
|
|
|
while " " in toreturn:
|
|
|
|
|
toreturn = toreturn.replace(" "," ")
|
2022-03-14 23:04:11 +00:00
|
|
|
|
toreturn = html.unescape(toreturn)
|
2022-02-17 08:58:42 +00:00
|
|
|
|
if endspace and not toreturn.endswith(" ") and not toreturn.endswith("\xa0"):
|
2022-02-04 23:26:51 +00:00
|
|
|
|
toreturn += " "
|
2022-02-17 08:58:42 +00:00
|
|
|
|
if startspace and not toreturn.startswith(" ") and not toreturn.startswith("\xa0"):
|
2022-02-04 23:26:51 +00:00
|
|
|
|
toreturn = " " + toreturn
|
2022-02-04 00:14:22 +00:00
|
|
|
|
return toreturn
|
2022-03-14 17:46:14 +00:00
|
|
|
|
def recursive_render(element,indent="",preformatted=False):
|
2022-01-29 14:35:28 +00:00
|
|
|
|
if element.name == "blockquote":
|
2022-03-20 18:25:04 +00:00
|
|
|
|
r.newparagraph()
|
|
|
|
|
r.startindent(" ",reverse=" ")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
for child in element.children:
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.open_color("italic")
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent="\t")
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.close_color("italic")
|
2022-03-20 18:25:04 +00:00
|
|
|
|
r.endindent()
|
2022-02-04 23:26:51 +00:00
|
|
|
|
elif element.name in ["div","p"]:
|
2022-03-19 16:44:28 +00:00
|
|
|
|
r.newparagraph()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent=indent)
|
2022-03-25 13:05:45 +00:00
|
|
|
|
r.newparagraph()
|
|
|
|
|
elif element.name in ["span"]:
|
|
|
|
|
r.add_space()
|
|
|
|
|
for child in element.children:
|
|
|
|
|
recursive_render(child,indent=indent)
|
|
|
|
|
r.add_space()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif element.name in ["h1","h2","h3","h4","h5","h6"]:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
r.open_color("blue")
|
|
|
|
|
if element.name in ["h1"]:
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.open_color("bold")
|
|
|
|
|
r.open_color("underline")
|
2022-03-21 10:57:35 +00:00
|
|
|
|
elif element.name in ["h2"]:
|
|
|
|
|
r.open_color("bold")
|
|
|
|
|
elif element.name in ["h5","h6"]:
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.open_color("faint")
|
2022-03-17 14:34:43 +00:00
|
|
|
|
for child in element.children:
|
2022-03-19 16:44:28 +00:00
|
|
|
|
r.newparagraph()
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child)
|
2022-03-22 22:17:47 +00:00
|
|
|
|
r.newparagraph()
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.close_all()
|
2022-03-20 18:25:04 +00:00
|
|
|
|
elif element.name in ["code","tt"]:
|
2022-01-29 14:35:28 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent=indent,preformatted=True)
|
2022-03-20 18:25:04 +00:00
|
|
|
|
elif element.name in ["pre"]:
|
2022-03-25 13:05:45 +00:00
|
|
|
|
r.newparagraph()
|
2022-03-20 18:25:04 +00:00
|
|
|
|
r.add_block(element.text)
|
2022-03-25 13:05:45 +00:00
|
|
|
|
r.newparagraph()
|
2022-03-20 18:25:04 +00:00
|
|
|
|
elif element.name in ["li"]:
|
2022-03-19 19:58:18 +00:00
|
|
|
|
r.startindent(" • ",sub=" ")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent=indent)
|
2022-03-19 16:44:28 +00:00
|
|
|
|
r.endindent()
|
2022-03-20 18:25:04 +00:00
|
|
|
|
elif element.name in ["tr"]:
|
|
|
|
|
r.startindent("|",reverse="|")
|
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent=indent)
|
2022-03-20 18:25:04 +00:00
|
|
|
|
r.endindent()
|
|
|
|
|
elif element.name in ["td","th"]:
|
|
|
|
|
r.add_text("| ")
|
2022-02-04 23:26:51 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child)
|
2022-03-20 18:25:04 +00:00
|
|
|
|
r.add_text(" |")
|
2022-02-04 23:26:51 +00:00
|
|
|
|
# italics
|
2022-03-14 17:46:14 +00:00
|
|
|
|
elif element.name in ["em","i"]:
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.open_color("italic")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent=indent,preformatted=preformatted)
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.close_color("italic")
|
2022-02-04 23:26:51 +00:00
|
|
|
|
#bold
|
2022-02-17 08:58:42 +00:00
|
|
|
|
elif element.name in ["b","strong"]:
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.open_color("bold")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent=indent,preformatted=preformatted)
|
2022-03-18 10:25:47 +00:00
|
|
|
|
r.close_color("bold")
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif element.name == "a":
|
2022-03-18 10:25:47 +00:00
|
|
|
|
link = element.get('href')
|
2022-02-10 16:19:20 +00:00
|
|
|
|
# support for images nested in links
|
2022-01-29 14:35:28 +00:00
|
|
|
|
if link:
|
2022-03-19 21:03:31 +00:00
|
|
|
|
text = ""
|
|
|
|
|
imgtext = ""
|
|
|
|
|
#we display images first in a link
|
2022-03-18 21:11:58 +00:00
|
|
|
|
for child in element.children:
|
|
|
|
|
if child.name == "img":
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child)
|
2022-03-19 21:03:31 +00:00
|
|
|
|
imgtext = "[IMG LINK %s]"
|
|
|
|
|
links.append(link+" "+text)
|
|
|
|
|
link_id = str(len(links))
|
|
|
|
|
r.open_color("blue")
|
|
|
|
|
r.open_color("faint")
|
|
|
|
|
for child in element.children:
|
|
|
|
|
if child.name != "img":
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,preformatted=preformatted)
|
|
|
|
|
if imgtext != "":
|
2022-03-19 21:03:31 +00:00
|
|
|
|
r.center_line()
|
|
|
|
|
r.add_text(imgtext%link_id)
|
|
|
|
|
else:
|
|
|
|
|
r.add_text(" [%s]"%link_id)
|
|
|
|
|
r.close_color("blue")
|
|
|
|
|
r.close_color("faint")
|
2022-01-17 22:37:12 +00:00
|
|
|
|
else:
|
2022-01-29 14:35:28 +00:00
|
|
|
|
#No real link found
|
2022-03-18 21:30:40 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,preformatted=preformatted)
|
2022-02-04 00:14:22 +00:00
|
|
|
|
elif element.name == "img":
|
2022-02-09 20:46:29 +00:00
|
|
|
|
src = element.get("src")
|
|
|
|
|
text = ""
|
2022-02-14 23:04:30 +00:00
|
|
|
|
ansi_img = render_image(src,width=width,mode=mode)
|
2022-02-04 00:14:22 +00:00
|
|
|
|
alt = element.get("alt")
|
|
|
|
|
if alt:
|
|
|
|
|
alt = sanitize_string(alt)
|
2022-02-09 20:46:29 +00:00
|
|
|
|
text += "[IMG] %s"%alt
|
2022-02-04 00:14:22 +00:00
|
|
|
|
else:
|
2022-02-09 20:46:29 +00:00
|
|
|
|
text += "[IMG]"
|
2022-02-04 00:14:22 +00:00
|
|
|
|
if src:
|
|
|
|
|
links.append(src+" "+text)
|
2022-03-30 10:04:07 +00:00
|
|
|
|
if not mode in self.images:
|
|
|
|
|
self.images[mode] = []
|
|
|
|
|
abs_url = urllib.parse.urljoin(self.url, src)
|
|
|
|
|
self.images[mode].append(abs_url)
|
2022-02-04 00:14:22 +00:00
|
|
|
|
link_id = " [%s]"%(len(links))
|
2022-03-18 16:35:14 +00:00
|
|
|
|
r.add_block(ansi_img)
|
|
|
|
|
r.open_color("faint")
|
|
|
|
|
r.open_color("yellow")
|
2022-03-18 17:10:38 +00:00
|
|
|
|
r.center_line()
|
2022-03-21 10:57:35 +00:00
|
|
|
|
r.add_text(text + link_id)
|
2022-03-18 16:35:14 +00:00
|
|
|
|
r.close_color("faint")
|
|
|
|
|
r.close_color("yellow")
|
2022-03-19 23:00:55 +00:00
|
|
|
|
r.newline()
|
2022-01-29 14:35:28 +00:00
|
|
|
|
elif element.name == "br":
|
2022-03-19 16:44:28 +00:00
|
|
|
|
r.newline()
|
2022-03-15 19:24:06 +00:00
|
|
|
|
elif element.name not in ["script","style","template"] and type(element) != Comment:
|
|
|
|
|
if element.string:
|
|
|
|
|
if preformatted :
|
2022-03-19 21:51:56 +00:00
|
|
|
|
r.open_color("faint")
|
|
|
|
|
r.add_text(element.string)
|
|
|
|
|
r.close_color("faint")
|
2022-03-15 19:24:06 +00:00
|
|
|
|
else:
|
2022-03-18 10:25:47 +00:00
|
|
|
|
s = sanitize_string(element.string)
|
2022-03-19 16:44:28 +00:00
|
|
|
|
if len(s.strip()) > 0:
|
|
|
|
|
r.add_text(s)
|
2022-03-14 17:46:14 +00:00
|
|
|
|
else:
|
2022-03-15 19:24:06 +00:00
|
|
|
|
for child in element.children:
|
2022-03-21 10:57:35 +00:00
|
|
|
|
recursive_render(child,indent=indent)
|
2022-01-29 14:35:28 +00:00
|
|
|
|
# the real render_html hearth
|
2022-02-11 11:32:59 +00:00
|
|
|
|
if mode == "full":
|
|
|
|
|
summary = body
|
2022-04-02 16:12:46 +00:00
|
|
|
|
elif _HAS_READABILITY:
|
2022-04-02 15:55:45 +00:00
|
|
|
|
try:
|
|
|
|
|
readable = Document(body)
|
|
|
|
|
summary = readable.summary()
|
|
|
|
|
except Exception as err:
|
|
|
|
|
summary = body
|
2022-04-02 16:12:46 +00:00
|
|
|
|
else:
|
|
|
|
|
summary = body
|
2022-01-29 14:35:28 +00:00
|
|
|
|
soup = BeautifulSoup(summary, 'html.parser')
|
2022-03-01 21:03:42 +00:00
|
|
|
|
#soup = BeautifulSoup(summary, 'html5lib')
|
2022-02-13 21:59:16 +00:00
|
|
|
|
if soup :
|
|
|
|
|
if soup.body :
|
2022-03-21 15:21:02 +00:00
|
|
|
|
recursive_render(soup.body)
|
2022-02-13 21:59:16 +00:00
|
|
|
|
else:
|
2022-03-21 15:21:02 +00:00
|
|
|
|
recursive_render(soup)
|
2022-03-21 10:36:01 +00:00
|
|
|
|
return r.get_final(),links
|
2022-01-17 22:37:12 +00:00
|
|
|
|
|
2022-01-19 08:41:04 +00:00
|
|
|
|
# Mapping mimetypes with renderers
|
2022-02-12 23:37:31 +00:00
|
|
|
|
# (any content with a mimetype text/* not listed here will be rendered with as GemText)
|
2022-01-18 21:19:43 +00:00
|
|
|
|
_FORMAT_RENDERERS = {
|
2022-01-29 14:35:28 +00:00
|
|
|
|
"text/gemini": GemtextRenderer,
|
|
|
|
|
"text/html" : HtmlRenderer,
|
2022-02-10 22:41:10 +00:00
|
|
|
|
"text/xml" : FeedRenderer,
|
2022-02-14 10:54:51 +00:00
|
|
|
|
"application/xml" : FeedRenderer,
|
|
|
|
|
"application/rss+xml" : FeedRenderer,
|
|
|
|
|
"application/atom+xml" : FeedRenderer,
|
2022-02-10 22:41:10 +00:00
|
|
|
|
"text/gopher": GopherRenderer,
|
2022-02-12 13:24:43 +00:00
|
|
|
|
"image/*": ImageRenderer
|
2022-01-18 21:19:43 +00:00
|
|
|
|
}
|
2022-01-19 08:41:04 +00:00
|
|
|
|
# Offpunk is organized as follow:
|
|
|
|
|
# - a GeminiClient instance which handles the browsing of GeminiItems (= pages).
|
2022-01-17 22:37:12 +00:00
|
|
|
|
# - There’s only one GeminiClient. Each page is a GeminiItem (name is historical, as
|
|
|
|
|
# it could be non-gemini content)
|
|
|
|
|
# - A GeminiItem is created with an URL from which it will derives content.
|
2022-03-31 22:20:34 +00:00
|
|
|
|
# - Content include : a title, a body (raw source) and a renderer. The renderer will provide
|
|
|
|
|
# ANSI rendered version of the content and a list of links
|
2022-01-19 08:41:04 +00:00
|
|
|
|
# - Each GeminiItem generates a "cache_path" in which it maintains a cached version of its content.
|
2022-01-17 22:37:12 +00:00
|
|
|
|
|
2019-08-13 16:56:15 +00:00
|
|
|
|
class GeminiItem():
|
|
|
|
|
|
|
|
|
|
def __init__(self, url, name=""):
|
2021-11-18 11:02:00 +00:00
|
|
|
|
if "://" not in url and ("./" not in url and url[0] != "/"):
|
2022-01-23 16:40:28 +00:00
|
|
|
|
if not url.startswith("mailto:"):
|
|
|
|
|
url = "gemini://" + url
|
2022-03-23 13:28:19 +00:00
|
|
|
|
self.last_mode = None
|
|
|
|
|
findmode = url.split("##offpunk_mode=")
|
|
|
|
|
if len(findmode) > 1:
|
|
|
|
|
self.url = findmode[0]
|
|
|
|
|
if findmode[1] in ["full"] or findmode[1].isnumeric():
|
|
|
|
|
self.last_mode = findmode[1]
|
|
|
|
|
else:
|
|
|
|
|
self.url = url
|
|
|
|
|
self.url = fix_ipv6_url(self.url).strip()
|
2022-02-17 09:38:41 +00:00
|
|
|
|
self._cache_path = None
|
2019-08-13 16:56:15 +00:00
|
|
|
|
self.name = name
|
2022-01-08 13:16:55 +00:00
|
|
|
|
self.mime = None
|
2022-01-17 22:37:12 +00:00
|
|
|
|
self.renderer = None
|
2022-02-12 23:37:31 +00:00
|
|
|
|
self.body = None
|
2019-08-13 16:56:15 +00:00
|
|
|
|
parsed = urllib.parse.urlparse(self.url)
|
2022-03-22 22:17:47 +00:00
|
|
|
|
if url[0] == "/" or url.startswith("./"):
|
2022-01-25 11:20:11 +00:00
|
|
|
|
self.scheme = "file"
|
2022-01-08 21:18:54 +00:00
|
|
|
|
else:
|
|
|
|
|
self.scheme = parsed.scheme
|
2022-03-22 22:17:47 +00:00
|
|
|
|
if self.scheme in ["file","mailto","list"]:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self.local = True
|
2022-01-28 12:25:55 +00:00
|
|
|
|
self.host = ""
|
|
|
|
|
self.port = None
|
2022-02-17 11:48:39 +00:00
|
|
|
|
# file:// is 7 char
|
|
|
|
|
if self.url.startswith("file://"):
|
|
|
|
|
self.path = self.url[7:]
|
|
|
|
|
elif self.scheme == "mailto":
|
|
|
|
|
self.path = parsed.path
|
2022-03-22 22:17:47 +00:00
|
|
|
|
elif self.url.startswith("list://"):
|
|
|
|
|
listdir = os.path.join(_DATA_DIR,"lists")
|
|
|
|
|
listname = self.url[7:].lstrip("/")
|
|
|
|
|
if listname in [""]:
|
|
|
|
|
self.name = "My Lists"
|
|
|
|
|
self.path = listdir
|
|
|
|
|
else:
|
|
|
|
|
self.name = listname
|
|
|
|
|
self.path = os.path.join(listdir, "%s.gmi"%listname)
|
2022-02-17 11:48:39 +00:00
|
|
|
|
else:
|
|
|
|
|
self.path = self.url
|
2022-01-08 20:32:25 +00:00
|
|
|
|
else:
|
2022-02-10 22:41:10 +00:00
|
|
|
|
self.local = False
|
2022-11-24 15:26:36 +00:00
|
|
|
|
# Convert unicode hostname to punycode using idna RFC3490
|
|
|
|
|
self.host = parsed.hostname #.encode("idna").decode()
|
2022-02-17 09:38:41 +00:00
|
|
|
|
self.port = parsed.port or standard_ports.get(self.scheme, 0)
|
2022-02-17 11:48:39 +00:00
|
|
|
|
# special gopher selector case
|
|
|
|
|
if self.scheme == "gopher":
|
|
|
|
|
if parsed.path and parsed.path[0] == "/" and len(parsed.path) > 1:
|
|
|
|
|
splitted = parsed.path.split("/")
|
|
|
|
|
# We check if we have well a gopher type
|
|
|
|
|
if len(splitted[1]) == 1:
|
2022-02-12 12:12:37 +00:00
|
|
|
|
itemtype = parsed.path[1]
|
2022-02-17 11:48:39 +00:00
|
|
|
|
selector = parsed.path[2:]
|
2022-02-12 12:12:37 +00:00
|
|
|
|
else:
|
|
|
|
|
itemtype = "1"
|
2022-02-17 11:48:39 +00:00
|
|
|
|
selector = parsed.path
|
2022-02-19 21:09:57 +00:00
|
|
|
|
self.path = selector
|
2022-02-10 22:41:10 +00:00
|
|
|
|
else:
|
2022-02-17 11:48:39 +00:00
|
|
|
|
itemtype = "1"
|
2022-02-10 22:41:10 +00:00
|
|
|
|
self.path = parsed.path
|
2022-02-17 11:48:39 +00:00
|
|
|
|
if itemtype == "0":
|
|
|
|
|
self.mime = "text/gemini"
|
|
|
|
|
elif itemtype == "1":
|
|
|
|
|
self.mime = "text/gopher"
|
|
|
|
|
elif itemtype == "h":
|
|
|
|
|
self.mime = "text/html"
|
|
|
|
|
elif itemtype in ("9","g","I","s"):
|
2022-02-24 09:29:14 +00:00
|
|
|
|
self.mime = "binary"
|
2022-02-12 12:12:37 +00:00
|
|
|
|
else:
|
2022-02-17 11:48:39 +00:00
|
|
|
|
self.mime = "text/gopher"
|
|
|
|
|
else:
|
|
|
|
|
self.path = parsed.path
|
|
|
|
|
if parsed.query:
|
|
|
|
|
# we don’t add the query if path is too long because path above 260 char
|
|
|
|
|
# are not supported and crash python.
|
|
|
|
|
# Also, very long query are usually useless stuff
|
|
|
|
|
if len(self.path+parsed.query) < 258:
|
|
|
|
|
self.path += "/" + parsed.query
|
2022-02-17 16:00:24 +00:00
|
|
|
|
|
2022-02-17 11:48:39 +00:00
|
|
|
|
def get_cache_path(self):
|
|
|
|
|
if self._cache_path and not os.path.isdir(self._cache_path):
|
|
|
|
|
return self._cache_path
|
|
|
|
|
elif self.local:
|
|
|
|
|
self._cache_path = self.path
|
|
|
|
|
#if not local, we create a local cache path.
|
|
|
|
|
else:
|
|
|
|
|
self._cache_path = os.path.expanduser(_CACHE_PATH + self.scheme +\
|
|
|
|
|
"/" + self.host + self.path)
|
|
|
|
|
#There’s an OS limitation of 260 characters per path.
|
|
|
|
|
#We will thus cut the path enough to add the index afterward
|
|
|
|
|
self._cache_path = self._cache_path[:249]
|
|
|
|
|
# FIXME : this is a gross hack to give a name to
|
|
|
|
|
# index files. This will break if the index is not
|
|
|
|
|
# index.gmi. I don’t know how to know the real name
|
|
|
|
|
# of the file. But first, we need to ensure that the domain name
|
|
|
|
|
# finish by "/". Else, the cache will create a file, not a folder.
|
|
|
|
|
if self.scheme.startswith("http"):
|
|
|
|
|
index = "index.html"
|
2022-11-24 11:24:03 +00:00
|
|
|
|
elif self.scheme in ["gopher", "finger"]:
|
2022-02-17 11:48:39 +00:00
|
|
|
|
index = "index.txt"
|
|
|
|
|
else:
|
|
|
|
|
index = "index.gmi"
|
|
|
|
|
if self.path == "" or os.path.isdir(self._cache_path):
|
|
|
|
|
if not self._cache_path.endswith("/"):
|
|
|
|
|
self._cache_path += "/"
|
|
|
|
|
if not self.url.endswith("/"):
|
|
|
|
|
self.url += "/"
|
|
|
|
|
if self._cache_path.endswith("/"):
|
|
|
|
|
self._cache_path += index
|
2022-03-24 20:03:03 +00:00
|
|
|
|
#sometimes, the index itself is a dir
|
|
|
|
|
#like when folder/index.gmi?param has been created
|
|
|
|
|
#and we try to access folder
|
|
|
|
|
if os.path.isdir(self._cache_path):
|
|
|
|
|
self._cache_path += "/" + index
|
2022-02-17 09:38:41 +00:00
|
|
|
|
return self._cache_path
|
2022-01-10 10:19:29 +00:00
|
|
|
|
|
2022-01-30 18:11:23 +00:00
|
|
|
|
def get_capsule_title(self):
|
2022-01-08 20:32:25 +00:00
|
|
|
|
#small intelligence to try to find a good name for a capsule
|
|
|
|
|
#we try to find eithe ~username or /users/username
|
|
|
|
|
#else we fallback to hostname
|
2022-02-17 11:35:50 +00:00
|
|
|
|
if self.local:
|
2022-01-10 10:19:29 +00:00
|
|
|
|
if self.name != "":
|
2022-02-14 09:25:55 +00:00
|
|
|
|
red_title = self.name
|
2022-01-10 10:19:29 +00:00
|
|
|
|
else:
|
2022-02-14 09:25:55 +00:00
|
|
|
|
red_title = self.path
|
2022-01-10 10:19:29 +00:00
|
|
|
|
else:
|
2022-02-14 09:25:55 +00:00
|
|
|
|
red_title = self.host
|
2022-01-10 10:19:29 +00:00
|
|
|
|
if "user" in self.path:
|
|
|
|
|
i = 0
|
|
|
|
|
splitted = self.path.split("/")
|
|
|
|
|
while i < (len(splitted)-1):
|
|
|
|
|
if splitted[i].startswith("user"):
|
2022-02-14 09:25:55 +00:00
|
|
|
|
red_title = splitted[i+1]
|
2022-01-10 10:19:29 +00:00
|
|
|
|
i += 1
|
|
|
|
|
if "~" in self.path:
|
|
|
|
|
for pp in self.path.split("/"):
|
|
|
|
|
if pp.startswith("~"):
|
2022-02-14 09:25:55 +00:00
|
|
|
|
red_title = pp[1:]
|
|
|
|
|
return red_title
|
2022-03-22 22:17:47 +00:00
|
|
|
|
|
|
|
|
|
def get_page_title(self):
|
2022-03-23 07:48:45 +00:00
|
|
|
|
title = ""
|
2022-03-22 22:17:47 +00:00
|
|
|
|
if not self.renderer:
|
|
|
|
|
self._set_renderer()
|
2022-03-23 07:48:45 +00:00
|
|
|
|
if self.renderer:
|
|
|
|
|
title = self.renderer.get_title()
|
2022-04-02 15:55:45 +00:00
|
|
|
|
if not title or len(title) == 0:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
title = self.get_capsule_title()
|
|
|
|
|
else:
|
|
|
|
|
title += " (%s)" %self.get_capsule_title()
|
|
|
|
|
return title
|
|
|
|
|
|
2021-12-18 09:16:19 +00:00
|
|
|
|
def is_cache_valid(self,validity=0):
|
2021-12-16 12:10:55 +00:00
|
|
|
|
# Validity is the acceptable time for
|
|
|
|
|
# a cache to be valid (in seconds)
|
2021-12-18 09:16:19 +00:00
|
|
|
|
# If 0, then any cache is considered as valid
|
|
|
|
|
# (use validity = 1 if you want to refresh everything)
|
2022-02-17 09:38:41 +00:00
|
|
|
|
cache = self.get_cache_path()
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if self.local:
|
2022-02-23 11:06:53 +00:00
|
|
|
|
return os.path.exists(cache)
|
2022-02-17 09:38:41 +00:00
|
|
|
|
elif cache :
|
2022-02-04 12:00:29 +00:00
|
|
|
|
# If path is too long, we always return True to avoid
|
|
|
|
|
# fetching it.
|
2022-02-17 09:38:41 +00:00
|
|
|
|
if len(cache) > 259:
|
2022-02-13 15:35:30 +00:00
|
|
|
|
print("We return False because path is too long")
|
|
|
|
|
return False
|
2022-02-17 09:38:41 +00:00
|
|
|
|
if os.path.exists(cache) and not os.path.isdir(cache):
|
2021-12-18 09:16:19 +00:00
|
|
|
|
if validity > 0 :
|
2022-01-08 20:32:25 +00:00
|
|
|
|
last_modification = self.cache_last_modified()
|
2021-12-16 12:10:55 +00:00
|
|
|
|
now = time.time()
|
2021-12-17 11:08:27 +00:00
|
|
|
|
age = now - last_modification
|
2021-12-16 12:10:55 +00:00
|
|
|
|
return age < validity
|
|
|
|
|
else:
|
|
|
|
|
return True
|
2021-12-13 12:49:12 +00:00
|
|
|
|
else:
|
|
|
|
|
#Cache has not been build
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
#There’s not even a cache!
|
|
|
|
|
return False
|
|
|
|
|
|
2022-01-08 20:32:25 +00:00
|
|
|
|
def cache_last_modified(self):
|
2022-02-17 09:38:41 +00:00
|
|
|
|
path = self.get_cache_path()
|
|
|
|
|
if path:
|
|
|
|
|
return os.path.getmtime(path)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
elif self.local:
|
|
|
|
|
return 0
|
|
|
|
|
else:
|
|
|
|
|
print("ERROR : NO CACHE in cache_last_modified")
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
def get_body(self,as_file=False):
|
2022-02-12 23:37:31 +00:00
|
|
|
|
if self.body and not as_file:
|
|
|
|
|
return self.body
|
2022-02-17 11:35:50 +00:00
|
|
|
|
if self.is_cache_valid():
|
2022-02-17 09:38:41 +00:00
|
|
|
|
path = self.get_cache_path()
|
2022-01-08 20:32:25 +00:00
|
|
|
|
else:
|
|
|
|
|
path = None
|
|
|
|
|
if path:
|
2022-02-04 12:00:29 +00:00
|
|
|
|
# There’s on OS limit on path length
|
|
|
|
|
if len(path) > 259:
|
|
|
|
|
toreturn = "Path is too long. This is an OS limitation.\n\n"
|
|
|
|
|
toreturn += self.url
|
|
|
|
|
return toreturn
|
|
|
|
|
elif as_file:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
return path
|
|
|
|
|
else:
|
|
|
|
|
with open(path) as f:
|
|
|
|
|
body = f.read()
|
|
|
|
|
f.close()
|
2022-01-10 11:49:24 +00:00
|
|
|
|
return body
|
2022-01-08 20:32:25 +00:00
|
|
|
|
else:
|
2022-03-27 14:24:19 +00:00
|
|
|
|
#print("ERROR: NO CACHE for %s" %self._cache_path)
|
|
|
|
|
return None
|
2022-03-30 10:04:07 +00:00
|
|
|
|
|
|
|
|
|
def get_images(self,mode=None):
|
|
|
|
|
if not self.renderer:
|
|
|
|
|
self._set_renderer()
|
|
|
|
|
if self.renderer:
|
|
|
|
|
return self.renderer.get_images(mode=mode)
|
|
|
|
|
else:
|
|
|
|
|
return []
|
|
|
|
|
|
2022-01-18 21:19:43 +00:00
|
|
|
|
# This method is used to load once the list of links in a gi
|
2022-01-19 10:02:49 +00:00
|
|
|
|
# Links can be followed, after a space, by a description/title
|
2022-03-23 13:28:19 +00:00
|
|
|
|
def get_links(self,mode=None):
|
2022-03-16 08:47:53 +00:00
|
|
|
|
links = []
|
|
|
|
|
toreturn = []
|
|
|
|
|
if not self.renderer:
|
|
|
|
|
self._set_renderer()
|
|
|
|
|
if self.renderer:
|
2022-03-16 09:03:29 +00:00
|
|
|
|
if not mode:
|
|
|
|
|
mode = self.last_mode
|
|
|
|
|
links = self.renderer.get_links(mode=mode)
|
2022-01-17 22:37:12 +00:00
|
|
|
|
for l in links:
|
|
|
|
|
#split between link and potential name
|
2022-10-15 16:25:15 +00:00
|
|
|
|
# check that l is non-empty
|
|
|
|
|
url = None
|
|
|
|
|
if l:
|
|
|
|
|
splitted = l.split(maxsplit=1)
|
|
|
|
|
url = self.absolutise_url(splitted[0])
|
|
|
|
|
if url and looks_like_url(url):
|
2022-01-17 22:37:12 +00:00
|
|
|
|
if len(splitted) > 1:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
#We add a name only for Gopher items
|
|
|
|
|
if url.startswith("gopher://"):
|
|
|
|
|
newgi = GeminiItem(url,name=splitted[1])
|
|
|
|
|
else:
|
|
|
|
|
newgi = GeminiItem(url)
|
2022-01-17 22:37:12 +00:00
|
|
|
|
else:
|
|
|
|
|
newgi = GeminiItem(url)
|
2022-03-16 08:47:53 +00:00
|
|
|
|
toreturn.append(newgi)
|
2022-10-15 16:25:15 +00:00
|
|
|
|
elif url and mode != "links_only" and url.startswith("data:image/"):
|
2022-10-07 13:51:26 +00:00
|
|
|
|
imgurl,imgdata = looks_like_base64(url,self.url)
|
2022-12-02 11:49:37 +00:00
|
|
|
|
if imgurl:
|
|
|
|
|
toreturn.append(GeminiItem(imgurl))
|
|
|
|
|
else:
|
|
|
|
|
toreturn.append(None)
|
2022-10-07 13:51:26 +00:00
|
|
|
|
else:
|
|
|
|
|
# We must include a None item to keep the link count valid
|
|
|
|
|
toreturn.append(None)
|
2022-03-16 08:47:53 +00:00
|
|
|
|
return toreturn
|
2022-01-17 22:37:12 +00:00
|
|
|
|
|
2022-01-18 21:19:43 +00:00
|
|
|
|
def get_link(self,nb):
|
2022-01-25 10:49:31 +00:00
|
|
|
|
# == None allows to return False, even if the list is empty
|
2022-03-16 08:47:53 +00:00
|
|
|
|
links = self.get_links()
|
|
|
|
|
if len(links) < nb:
|
2022-01-18 21:19:43 +00:00
|
|
|
|
print("Index too high! No link %s for %s" %(nb,self.url))
|
|
|
|
|
return None
|
|
|
|
|
else:
|
2022-03-16 08:47:53 +00:00
|
|
|
|
return links[nb-1]
|
2022-01-18 21:19:43 +00:00
|
|
|
|
|
2022-02-18 13:11:09 +00:00
|
|
|
|
def get_subscribe_links(self):
|
|
|
|
|
if not self.renderer:
|
|
|
|
|
self._set_renderer()
|
|
|
|
|
if self.renderer:
|
2022-03-07 22:16:17 +00:00
|
|
|
|
subs = self.renderer.get_subscribe_links()
|
|
|
|
|
abssubs = []
|
|
|
|
|
# some rss links are relatives
|
|
|
|
|
for s in subs:
|
|
|
|
|
s[0] = self.absolutise_url(s[0])
|
|
|
|
|
abssubs.append(s)
|
|
|
|
|
return abssubs
|
2022-02-18 13:11:09 +00:00
|
|
|
|
else:
|
|
|
|
|
return []
|
|
|
|
|
|
2022-02-10 22:41:10 +00:00
|
|
|
|
def _set_renderer(self,mime=None):
|
2022-02-17 11:35:50 +00:00
|
|
|
|
if self.local and os.path.isdir(self.get_cache_path()):
|
|
|
|
|
self.renderer = FolderRenderer("",self.get_cache_path())
|
2022-02-13 00:12:55 +00:00
|
|
|
|
return
|
2022-02-10 22:41:10 +00:00
|
|
|
|
if not mime:
|
|
|
|
|
mime = self.get_mime()
|
2022-02-18 11:14:42 +00:00
|
|
|
|
#we don’t even have a mime (so probably we don’t have a cache)
|
|
|
|
|
if not mime:
|
|
|
|
|
return
|
2022-02-12 13:24:43 +00:00
|
|
|
|
mime_to_use = []
|
|
|
|
|
for m in _FORMAT_RENDERERS:
|
|
|
|
|
if fnmatch.fnmatch(mime, m):
|
|
|
|
|
mime_to_use.append(m)
|
|
|
|
|
if len(mime_to_use) > 0:
|
|
|
|
|
current_mime = mime_to_use[0]
|
|
|
|
|
func = _FORMAT_RENDERERS[current_mime]
|
|
|
|
|
if current_mime.startswith("text"):
|
2022-02-10 22:41:10 +00:00
|
|
|
|
self.renderer = func(self.get_body(),self.url)
|
2022-02-12 13:24:43 +00:00
|
|
|
|
# We double check if the renderer is correct.
|
|
|
|
|
# If not, we fallback to html
|
|
|
|
|
# (this is currently only for XHTML, often being
|
|
|
|
|
# mislabelled as xml thus RSS feeds)
|
|
|
|
|
if not self.renderer.is_valid():
|
|
|
|
|
func = _FORMAT_RENDERERS["text/html"]
|
|
|
|
|
#print("Set (fallback)RENDERER to html instead of %s"%mime)
|
|
|
|
|
self.renderer = func(self.get_body(),self.url)
|
|
|
|
|
else:
|
|
|
|
|
#we don’t parse text, we give the file to the renderer
|
2022-02-17 09:38:41 +00:00
|
|
|
|
self.renderer = func(self.get_cache_path(),self.url)
|
2022-02-12 13:24:43 +00:00
|
|
|
|
if not self.renderer.is_valid():
|
|
|
|
|
self.renderer = None
|
2022-02-10 22:41:10 +00:00
|
|
|
|
|
2022-03-25 12:37:23 +00:00
|
|
|
|
def display(self,mode=None,grep=None):
|
2022-01-17 22:37:12 +00:00
|
|
|
|
if not self.renderer:
|
2022-02-10 22:41:10 +00:00
|
|
|
|
self._set_renderer()
|
2022-02-20 22:22:25 +00:00
|
|
|
|
if self.renderer and self.renderer.is_valid():
|
2022-03-23 13:28:19 +00:00
|
|
|
|
if not mode:
|
|
|
|
|
mode = self.last_mode
|
|
|
|
|
else:
|
|
|
|
|
self.last_mode = mode
|
2022-03-21 21:43:33 +00:00
|
|
|
|
title = self.get_capsule_title()
|
|
|
|
|
if self.is_cache_valid(): #and self.offline_only and not self.local:
|
2022-03-23 13:28:19 +00:00
|
|
|
|
nbr = len(self.get_links(mode=mode))
|
2022-03-21 21:43:33 +00:00
|
|
|
|
if self.local:
|
|
|
|
|
title += " (%s items)"%nbr
|
2022-03-22 22:17:47 +00:00
|
|
|
|
str_last = "local file"
|
2022-03-21 21:43:33 +00:00
|
|
|
|
else:
|
|
|
|
|
str_last = "last accessed on %s" %time.ctime(self.cache_last_modified())
|
|
|
|
|
title += " (%s links)"%nbr
|
2022-04-02 15:55:45 +00:00
|
|
|
|
return self.renderer.display(mode=mode,window_title=title,window_info=str_last,grep=grep)
|
|
|
|
|
else:
|
|
|
|
|
return False
|
2022-01-17 22:37:12 +00:00
|
|
|
|
else:
|
2022-03-15 22:00:05 +00:00
|
|
|
|
return False
|
|
|
|
|
|
2022-02-09 20:46:29 +00:00
|
|
|
|
def get_filename(self):
|
|
|
|
|
filename = os.path.basename(self.get_cache_path())
|
2022-01-08 20:46:57 +00:00
|
|
|
|
return filename
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
2022-03-15 22:00:05 +00:00
|
|
|
|
def get_temp_filename(self):
|
2022-03-15 23:13:45 +00:00
|
|
|
|
tmpf = None
|
2022-03-15 22:00:05 +00:00
|
|
|
|
if not self.renderer:
|
|
|
|
|
self._set_renderer()
|
|
|
|
|
if self.renderer and self.renderer.is_valid():
|
2022-03-15 23:13:45 +00:00
|
|
|
|
tmpf = self.renderer.get_temp_file()
|
|
|
|
|
if not tmpf:
|
|
|
|
|
tmpf = self.get_cache_path()
|
|
|
|
|
return tmpf
|
2022-03-15 22:00:05 +00:00
|
|
|
|
|
2022-02-12 23:37:31 +00:00
|
|
|
|
def write_body(self,body,mime=None):
|
2022-01-08 20:32:25 +00:00
|
|
|
|
## body is a copy of the raw gemtext
|
2022-01-09 10:35:09 +00:00
|
|
|
|
## Write_body() also create the cache !
|
2022-01-09 14:53:14 +00:00
|
|
|
|
# DEFAULT GEMINI MIME
|
2022-02-12 23:37:31 +00:00
|
|
|
|
self.body = body
|
2022-11-29 23:11:44 +00:00
|
|
|
|
self.mime, options = parse_mime(mime)
|
2022-02-12 23:37:31 +00:00
|
|
|
|
if not self.local:
|
|
|
|
|
if self.mime and self.mime.startswith("text/"):
|
|
|
|
|
mode = "w"
|
|
|
|
|
else:
|
|
|
|
|
mode = "wb"
|
2022-02-17 09:38:41 +00:00
|
|
|
|
cache_dir = os.path.dirname(self.get_cache_path())
|
2022-02-12 23:37:31 +00:00
|
|
|
|
# If the subdirectory already exists as a file (not a folder)
|
|
|
|
|
# We remove it (happens when accessing URL/subfolder before
|
|
|
|
|
# URL/subfolder/file.gmi.
|
|
|
|
|
# This causes loss of data in the cache
|
|
|
|
|
# proper solution would be to save "sufolder" as "sufolder/index.gmi"
|
|
|
|
|
# If the subdirectory doesn’t exist, we recursively try to find one
|
|
|
|
|
# until it exists to avoid a file blocking the creation of folders
|
|
|
|
|
root_dir = cache_dir
|
|
|
|
|
while not os.path.exists(root_dir):
|
|
|
|
|
root_dir = os.path.dirname(root_dir)
|
|
|
|
|
if os.path.isfile(root_dir):
|
|
|
|
|
os.remove(root_dir)
|
|
|
|
|
os.makedirs(cache_dir,exist_ok=True)
|
2022-02-17 09:38:41 +00:00
|
|
|
|
with open(self.get_cache_path(), mode=mode) as f:
|
2022-02-12 23:37:31 +00:00
|
|
|
|
f.write(body)
|
|
|
|
|
f.close()
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
|
|
|
|
def get_mime(self):
|
2022-03-29 20:08:30 +00:00
|
|
|
|
#Beware, this one is really a shaddy ad-hoc function
|
2022-02-10 22:41:10 +00:00
|
|
|
|
if self.mime:
|
|
|
|
|
return self.mime
|
|
|
|
|
elif self.is_cache_valid():
|
2022-02-17 11:35:50 +00:00
|
|
|
|
path = self.get_cache_path()
|
2022-02-22 12:24:36 +00:00
|
|
|
|
if self.scheme == "mailto":
|
|
|
|
|
mime = "mailto"
|
|
|
|
|
elif os.path.isdir(path):
|
2022-02-19 21:09:57 +00:00
|
|
|
|
mime = "Local Folder"
|
|
|
|
|
elif path.endswith(".gmi"):
|
2022-02-06 13:08:25 +00:00
|
|
|
|
mime = "text/gemini"
|
2022-03-29 20:08:30 +00:00
|
|
|
|
elif shutil.which("file") :
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
mime = run("file -b --mime-type %s", parameter=path).strip()
|
2022-02-13 21:59:16 +00:00
|
|
|
|
mime2,encoding = mimetypes.guess_type(path,strict=False)
|
|
|
|
|
#If we hesitate between html and xml, takes the xml one
|
|
|
|
|
#because the FeedRendered fallback to HtmlRenderer
|
2022-02-13 22:18:42 +00:00
|
|
|
|
if mime2 and mime != mime2 and "html" in mime and "xml" in mime2:
|
|
|
|
|
mime = "text/xml"
|
2022-10-10 09:00:58 +00:00
|
|
|
|
# If it’s a xml file, consider it as such, regardless of what file thinks
|
|
|
|
|
elif path.endswith(".xml"):
|
|
|
|
|
mime = "text/xml"
|
2022-02-13 22:18:42 +00:00
|
|
|
|
#Some xml/html document are considered as octet-stream
|
|
|
|
|
if mime == "application/octet-stream":
|
2022-02-13 21:59:16 +00:00
|
|
|
|
mime = "text/xml"
|
2022-02-06 11:55:54 +00:00
|
|
|
|
else:
|
|
|
|
|
mime,encoding = mimetypes.guess_type(path,strict=False)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
#gmi Mimetype is not recognized yet
|
2022-03-29 20:08:30 +00:00
|
|
|
|
if not mime and not shutil.which("file") :
|
|
|
|
|
print("Cannot guess the mime type of the file. Please install \"file\".")
|
|
|
|
|
print("(and send me an email, I’m curious of systems without \"file\" installed!")
|
2022-01-18 21:19:43 +00:00
|
|
|
|
if mime.startswith("text") and mime not in _FORMAT_RENDERERS:
|
2022-03-06 17:11:59 +00:00
|
|
|
|
if mime2 and mime2 in _FORMAT_RENDERERS:
|
|
|
|
|
mime = mime2
|
|
|
|
|
else:
|
|
|
|
|
#by default, we consider it’s gemini except for html
|
|
|
|
|
mime = "text/gemini"
|
2022-01-08 20:32:25 +00:00
|
|
|
|
self.mime = mime
|
|
|
|
|
return self.mime
|
|
|
|
|
|
|
|
|
|
def set_error(self,err):
|
|
|
|
|
# If we get an error, we want to keep an existing cache
|
|
|
|
|
# but we need to touch it or to create an empty one
|
|
|
|
|
# to avoid hitting the error at each refresh
|
2022-02-17 09:38:41 +00:00
|
|
|
|
cache = self.get_cache_path()
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if self.is_cache_valid():
|
2022-02-17 09:38:41 +00:00
|
|
|
|
os.utime(cache)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
else:
|
2022-02-17 09:38:41 +00:00
|
|
|
|
cache_dir = os.path.dirname(cache)
|
2022-02-04 12:00:29 +00:00
|
|
|
|
root_dir = cache_dir
|
|
|
|
|
while not os.path.exists(root_dir):
|
|
|
|
|
root_dir = os.path.dirname(root_dir)
|
|
|
|
|
if os.path.isfile(root_dir):
|
|
|
|
|
os.remove(root_dir)
|
2022-02-03 12:38:07 +00:00
|
|
|
|
os.makedirs(cache_dir,exist_ok=True)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if os.path.isdir(cache_dir):
|
2022-02-17 09:38:41 +00:00
|
|
|
|
with open(cache, "w") as cache:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
cache.write(str(datetime.datetime.now())+"\n")
|
2022-01-24 18:23:03 +00:00
|
|
|
|
cache.write("ERROR while caching %s\n\n" %self.url)
|
|
|
|
|
cache.write("*****\n\n")
|
2022-02-09 20:46:29 +00:00
|
|
|
|
cache.write(str(type(err)) + " = " + str(err))
|
2022-03-04 22:38:35 +00:00
|
|
|
|
#cache.write("\n" + str(err.with_traceback(None)))
|
2022-02-09 20:46:29 +00:00
|
|
|
|
cache.write("\n*****\n\n")
|
2022-01-24 18:23:03 +00:00
|
|
|
|
cache.write("If you believe this error was temporary, type ""reload"".\n")
|
2022-01-17 22:37:12 +00:00
|
|
|
|
cache.write("The ressource will be tentatively fetched during next sync.\n")
|
2022-01-08 20:32:25 +00:00
|
|
|
|
cache.close()
|
|
|
|
|
|
|
|
|
|
|
2019-08-13 17:36:58 +00:00
|
|
|
|
def root(self):
|
|
|
|
|
return GeminiItem(self._derive_url("/"))
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
2022-03-08 21:21:44 +00:00
|
|
|
|
def up(self,level=1):
|
|
|
|
|
path = self.path.rstrip('/')
|
|
|
|
|
count = 0
|
|
|
|
|
while count < level:
|
|
|
|
|
pathbits = list(os.path.split(path))
|
|
|
|
|
# Don't try to go higher than root or in config
|
|
|
|
|
if self.local or len(pathbits) == 1 :
|
|
|
|
|
return self
|
|
|
|
|
# Get rid of bottom component
|
|
|
|
|
if len(pathbits) > 1:
|
|
|
|
|
pathbits.pop()
|
|
|
|
|
path = os.path.join(*pathbits)
|
|
|
|
|
count += 1
|
2022-02-19 21:09:57 +00:00
|
|
|
|
if self.scheme == "gopher":
|
2022-03-08 21:21:44 +00:00
|
|
|
|
path = "/1" + path
|
|
|
|
|
return GeminiItem(self._derive_url(path))
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
2019-08-13 17:36:58 +00:00
|
|
|
|
def query(self, query):
|
2020-05-12 20:00:17 +00:00
|
|
|
|
query = urllib.parse.quote(query)
|
2019-08-13 17:24:00 +00:00
|
|
|
|
return GeminiItem(self._derive_url(query=query))
|
|
|
|
|
|
|
|
|
|
def _derive_url(self, path="", query=""):
|
|
|
|
|
"""
|
|
|
|
|
A thin wrapper around urlunparse which avoids inserting standard ports
|
|
|
|
|
into URLs just to keep things clean.
|
|
|
|
|
"""
|
2022-01-28 12:25:55 +00:00
|
|
|
|
if not self.port or self.port == standard_ports[self.scheme] :
|
|
|
|
|
host = self.host
|
|
|
|
|
else:
|
|
|
|
|
host = self.host + ":" + str(self.port)
|
|
|
|
|
return urllib.parse.urlunparse((self.scheme,host,path or self.path, "", query, ""))
|
2019-08-13 17:17:40 +00:00
|
|
|
|
|
2019-08-13 16:56:15 +00:00
|
|
|
|
def absolutise_url(self, relative_url):
|
|
|
|
|
"""
|
|
|
|
|
Convert a relative URL to an absolute URL by using the URL of this
|
|
|
|
|
GeminiItem as a base.
|
|
|
|
|
"""
|
2021-12-15 10:05:38 +00:00
|
|
|
|
abs_url = urllib.parse.urljoin(self.url, relative_url)
|
|
|
|
|
return abs_url
|
2022-03-23 13:28:19 +00:00
|
|
|
|
|
|
|
|
|
def url_mode(self):
|
|
|
|
|
url = self.url
|
|
|
|
|
if self.last_mode and self.last_mode != "readable":
|
|
|
|
|
url += "##offpunk_mode=" + self.last_mode
|
|
|
|
|
return url
|
|
|
|
|
|
2022-02-04 23:26:51 +00:00
|
|
|
|
def to_map_line(self):
|
2022-03-23 13:28:19 +00:00
|
|
|
|
return "=> {} {}\n".format(self.url_mode(), self.get_page_title())
|
2019-08-13 16:56:15 +00:00
|
|
|
|
|
2022-01-19 08:41:04 +00:00
|
|
|
|
CRLF = '\r\n'
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
# Cheap and cheerful URL detector
|
|
|
|
|
def looks_like_url(word):
|
2022-01-31 13:33:05 +00:00
|
|
|
|
try:
|
|
|
|
|
if not word.strip():
|
|
|
|
|
return False
|
2022-01-11 13:04:20 +00:00
|
|
|
|
url = fix_ipv6_url(word).strip()
|
2022-01-12 08:19:08 +00:00
|
|
|
|
parsed = urllib.parse.urlparse(url)
|
|
|
|
|
#sometimes, urllib crashed only when requesting the port
|
|
|
|
|
port = parsed.port
|
2022-01-30 16:47:03 +00:00
|
|
|
|
mailto = word.startswith("mailto:")
|
2022-03-04 22:56:57 +00:00
|
|
|
|
scheme = word.split("://")[0]
|
|
|
|
|
start = scheme in standard_ports
|
2022-03-22 22:17:47 +00:00
|
|
|
|
local = scheme in ["file","list"]
|
|
|
|
|
if not start and not local and not mailto:
|
2022-01-14 11:14:46 +00:00
|
|
|
|
return looks_like_url("gemini://"+word)
|
2022-01-30 16:47:03 +00:00
|
|
|
|
elif mailto:
|
|
|
|
|
return "@" in word
|
2022-03-22 22:17:47 +00:00
|
|
|
|
elif not local:
|
2022-11-16 20:45:51 +00:00
|
|
|
|
return "." in word or "localhost" in word
|
2022-03-22 22:17:47 +00:00
|
|
|
|
else:
|
|
|
|
|
return "/" in word
|
2022-01-11 13:04:20 +00:00
|
|
|
|
except ValueError:
|
|
|
|
|
return False
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2022-10-07 13:51:26 +00:00
|
|
|
|
# This method return the image URL or invent it if it’s a base64 inline image
|
|
|
|
|
# It returns [url,image_data] where image_data is None for normal image
|
|
|
|
|
def looks_like_base64(src,baseurl):
|
|
|
|
|
imgdata = None
|
|
|
|
|
imgname = src
|
2022-12-02 11:49:37 +00:00
|
|
|
|
if src and src.startswith("data:image/"):
|
|
|
|
|
if ";base64," in src:
|
|
|
|
|
splitted = src.split(";base64,")
|
|
|
|
|
extension = splitted[0].strip("data:image/")[:3]
|
|
|
|
|
imgdata = splitted[1]
|
|
|
|
|
imgname = imgdata[:20] + "." + extension
|
|
|
|
|
imgurl = urllib.parse.urljoin(baseurl, imgname)
|
|
|
|
|
else:
|
|
|
|
|
#We can’t handle other data:image such as svg for now
|
|
|
|
|
imgurl = None
|
|
|
|
|
else:
|
|
|
|
|
imgurl = urllib.parse.urljoin(baseurl, imgname)
|
2022-10-07 13:51:26 +00:00
|
|
|
|
return imgurl,imgdata
|
|
|
|
|
|
2020-08-31 19:18:15 +00:00
|
|
|
|
class UserAbortException(Exception):
|
|
|
|
|
pass
|
|
|
|
|
|
2019-08-13 16:56:15 +00:00
|
|
|
|
# GeminiClient Decorators
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def needs_gi(inner):
|
|
|
|
|
def outer(self, *args, **kwargs):
|
|
|
|
|
if not self.gi:
|
|
|
|
|
print("You need to 'go' somewhere, first")
|
|
|
|
|
return None
|
|
|
|
|
else:
|
|
|
|
|
return inner(self, *args, **kwargs)
|
|
|
|
|
outer.__doc__ = inner.__doc__
|
|
|
|
|
return outer
|
|
|
|
|
|
|
|
|
|
class GeminiClient(cmd.Cmd):
|
|
|
|
|
|
2022-11-30 19:57:23 +00:00
|
|
|
|
def __init__(self, completekey="tab", synconly=False):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
cmd.Cmd.__init__(self)
|
2020-05-10 20:51:33 +00:00
|
|
|
|
|
2020-05-23 11:17:12 +00:00
|
|
|
|
# Set umask so that nothing we create can be read by anybody else.
|
|
|
|
|
# The certificate cache and TOFU database contain "browser history"
|
|
|
|
|
# type sensitivie information.
|
2020-05-23 11:24:39 +00:00
|
|
|
|
os.umask(0o077)
|
2020-05-23 11:17:12 +00:00
|
|
|
|
|
2020-05-10 20:51:33 +00:00
|
|
|
|
|
2021-12-30 15:03:08 +00:00
|
|
|
|
self.no_cert_prompt = "\x1b[38;5;76m" + "ON" + "\x1b[38;5;255m" + "> " + "\x1b[0m"
|
|
|
|
|
self.cert_prompt = "\x1b[38;5;202m" + "ON" + "\x1b[38;5;255m"
|
|
|
|
|
self.offline_prompt = "\x1b[38;5;76m" + "OFF" + "\x1b[38;5;255m" + "> " + "\x1b[0m"
|
2020-05-10 10:35:46 +00:00
|
|
|
|
self.prompt = self.no_cert_prompt
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.gi = None
|
|
|
|
|
self.hist_index = 0
|
|
|
|
|
self.index = []
|
|
|
|
|
self.index_index = -1
|
|
|
|
|
self.marks = {}
|
|
|
|
|
self.page_index = 0
|
2019-10-15 19:12:32 +00:00
|
|
|
|
self.permanent_redirects = {}
|
2019-10-13 17:42:04 +00:00
|
|
|
|
self.previous_redirectors = set()
|
2021-12-09 14:12:32 +00:00
|
|
|
|
# Sync-only mode is restriced by design
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.visited_hosts = set()
|
2021-12-06 15:43:16 +00:00
|
|
|
|
self.offline_only = False
|
2021-12-09 14:12:32 +00:00
|
|
|
|
self.sync_only = False
|
2022-02-12 13:41:05 +00:00
|
|
|
|
self.support_http = _DO_HTTP
|
2022-01-22 14:33:39 +00:00
|
|
|
|
self.automatic_choice = "n"
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2020-05-10 10:59:26 +00:00
|
|
|
|
self.client_certs = {
|
|
|
|
|
"active": None
|
|
|
|
|
}
|
2020-05-10 11:44:40 +00:00
|
|
|
|
self.active_cert_domains = []
|
2020-05-11 20:22:24 +00:00
|
|
|
|
self.active_is_transient = False
|
|
|
|
|
self.transient_certs_created = []
|
2020-05-10 10:59:26 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.options = {
|
|
|
|
|
"debug" : False,
|
2022-03-19 19:58:18 +00:00
|
|
|
|
"beta" : False,
|
2019-08-13 10:04:39 +00:00
|
|
|
|
"ipv6" : True,
|
2021-04-27 07:53:58 +00:00
|
|
|
|
"timeout" : 600,
|
2021-12-10 10:27:48 +00:00
|
|
|
|
"short_timeout" : 5,
|
2022-03-19 16:44:28 +00:00
|
|
|
|
"width" : 72,
|
2019-10-14 16:47:02 +00:00
|
|
|
|
"auto_follow_redirects" : True,
|
2020-05-19 21:14:09 +00:00
|
|
|
|
"tls_mode" : "tofu",
|
2022-03-05 15:36:39 +00:00
|
|
|
|
"archives_size" : 200,
|
|
|
|
|
"history_size" : 200,
|
2022-03-24 21:00:04 +00:00
|
|
|
|
"max_size_download" : 10,
|
2022-03-29 11:08:35 +00:00
|
|
|
|
"editor" : None,
|
2022-03-30 10:04:07 +00:00
|
|
|
|
"download_images_first" : True,
|
2022-03-30 13:46:25 +00:00
|
|
|
|
"redirects" : True,
|
2022-11-14 20:33:40 +00:00
|
|
|
|
# the wikipedia entry needs two %s, one for lang, other for search
|
|
|
|
|
"wikipedia" : "gemini://vault.transjovian.org:1965/search/%s/%s",
|
|
|
|
|
"search" : "gemini://kennedy.gemi.dev/search?%s",
|
2022-11-16 13:44:47 +00:00
|
|
|
|
"accept_bad_ssl_certificates" : False,
|
2022-03-30 13:46:25 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self.redirects = {
|
|
|
|
|
"twitter.com" : "nitter.42l.fr",
|
2022-07-24 08:17:33 +00:00
|
|
|
|
"facebook.com" : "blocked",
|
|
|
|
|
"google-analytics.com" : "blocked",
|
2022-03-30 13:46:25 +00:00
|
|
|
|
"youtube.com" : "yewtu.be",
|
2022-10-07 22:16:28 +00:00
|
|
|
|
"reddit.com" : "teddit.net",
|
|
|
|
|
"old.reddit.com": "teddit.net",
|
2022-03-30 13:46:25 +00:00
|
|
|
|
"medium.com" : "scribe.rip",
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
}
|
2022-02-12 12:12:37 +00:00
|
|
|
|
global TERM_WIDTH
|
|
|
|
|
TERM_WIDTH = self.options["width"]
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self.log = {
|
|
|
|
|
"start_time": time.time(),
|
|
|
|
|
"requests": 0,
|
|
|
|
|
"ipv4_requests": 0,
|
|
|
|
|
"ipv6_requests": 0,
|
|
|
|
|
"bytes_recvd": 0,
|
|
|
|
|
"ipv4_bytes_recvd": 0,
|
|
|
|
|
"ipv6_bytes_recvd": 0,
|
|
|
|
|
"dns_failures": 0,
|
|
|
|
|
"refused_connections": 0,
|
|
|
|
|
"reset_connections": 0,
|
|
|
|
|
"timeouts": 0,
|
2020-09-01 19:14:17 +00:00
|
|
|
|
"cache_hits": 0,
|
2019-06-22 12:58:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
self._connect_to_tofu_db()
|
|
|
|
|
|
2022-11-01 23:58:41 +00:00
|
|
|
|
def complete_list(self,text,line,begidx,endidx):
|
2022-11-02 14:47:57 +00:00
|
|
|
|
allowed = []
|
2022-11-01 23:58:41 +00:00
|
|
|
|
cmds = ["create","edit","subscribe","freeze","normal","delete","help"]
|
2022-11-02 14:47:57 +00:00
|
|
|
|
lists = self.list_lists()
|
|
|
|
|
words = len(line.split())
|
|
|
|
|
# We need to autocomplete listname for the first or second argument
|
|
|
|
|
# If the first one is a cmds
|
|
|
|
|
if words <= 1:
|
|
|
|
|
allowed = lists + cmds
|
|
|
|
|
elif words == 2:
|
|
|
|
|
# if text, the completing word is the second
|
|
|
|
|
cond = bool(text)
|
|
|
|
|
if text:
|
|
|
|
|
allowed = lists + cmds
|
|
|
|
|
else:
|
|
|
|
|
current_cmd = line.split()[1]
|
|
|
|
|
if current_cmd in ["help", "create"]:
|
|
|
|
|
allowed = []
|
|
|
|
|
elif current_cmd in cmds:
|
|
|
|
|
allowed = lists
|
|
|
|
|
elif words == 3 and text != "":
|
|
|
|
|
current_cmd = line.split()[1]
|
|
|
|
|
if current_cmd in ["help", "create"]:
|
|
|
|
|
allowed = []
|
|
|
|
|
elif current_cmd in cmds:
|
|
|
|
|
allowed = lists
|
2022-12-07 22:23:31 +00:00
|
|
|
|
return [i+" " for i in allowed if i.startswith(text)]
|
2022-11-01 23:58:41 +00:00
|
|
|
|
|
|
|
|
|
def complete_add(self,text,line,begidx,endidx):
|
2022-11-02 14:47:57 +00:00
|
|
|
|
if len(line.split()) == 2 and text != "":
|
|
|
|
|
allowed = self.list_lists()
|
|
|
|
|
elif len(line.split()) == 1:
|
|
|
|
|
allowed = self.list_lists()
|
|
|
|
|
else:
|
|
|
|
|
allowed = []
|
2022-12-07 22:23:31 +00:00
|
|
|
|
return [i+" " for i in allowed if i.startswith(text)]
|
2022-11-01 23:58:41 +00:00
|
|
|
|
def complete_move(self,text,line,begidx,endidx):
|
2022-11-02 14:47:57 +00:00
|
|
|
|
return self.complete_add(text,line,begidx,endidx)
|
2022-11-01 23:58:41 +00:00
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
def _connect_to_tofu_db(self):
|
|
|
|
|
|
2022-01-18 17:04:29 +00:00
|
|
|
|
db_path = os.path.join(_CONFIG_DIR, "tofu.db")
|
2020-05-16 16:58:53 +00:00
|
|
|
|
self.db_conn = sqlite3.connect(db_path)
|
|
|
|
|
self.db_cur = self.db_conn.cursor()
|
|
|
|
|
|
|
|
|
|
self.db_cur.execute("""CREATE TABLE IF NOT EXISTS cert_cache
|
|
|
|
|
(hostname text, address text, fingerprint text,
|
|
|
|
|
first_seen date, last_seen date, count integer)""")
|
|
|
|
|
|
2022-03-03 15:16:32 +00:00
|
|
|
|
def _go_to_gi(self, gi, update_hist=True, check_cache=True, handle=True,\
|
2022-03-23 13:28:19 +00:00
|
|
|
|
mode=None,limit_size=False):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""This method might be considered "the heart of Offpunk".
|
2019-06-22 12:58:21 +00:00
|
|
|
|
Everything involved in fetching a gemini resource happens here:
|
2022-01-08 13:16:55 +00:00
|
|
|
|
sending the request over the network, parsing the response,
|
|
|
|
|
storing the response in a temporary file, choosing
|
|
|
|
|
and calling a handler program, and updating the history.
|
|
|
|
|
Nothing is returned."""
|
2022-01-18 21:19:43 +00:00
|
|
|
|
if not gi:
|
|
|
|
|
return
|
2019-08-14 18:16:58 +00:00
|
|
|
|
# Don't try to speak to servers running other protocols
|
2022-01-24 15:50:22 +00:00
|
|
|
|
elif gi.scheme == "mailto":
|
2022-01-23 16:40:28 +00:00
|
|
|
|
if handle and not self.sync_only:
|
|
|
|
|
resp = input("Send an email to %s Y/N? " %gi.path)
|
|
|
|
|
self.gi = gi
|
|
|
|
|
if resp.strip().lower() in ("y", "yes"):
|
2022-02-20 22:22:25 +00:00
|
|
|
|
if _HAS_XDGOPEN :
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run("xdg-open mailto:%s", parameter=gi.path ,direct_output=True)
|
2022-02-20 22:22:25 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Cannot find a mail client to send mail to %s" %gi.path)
|
|
|
|
|
print("Please install xdg-open (usually from xdg-util package)")
|
2022-01-23 16:40:28 +00:00
|
|
|
|
return
|
2022-03-22 22:17:47 +00:00
|
|
|
|
elif gi.scheme not in ["file","list"] and gi.scheme not in standard_ports \
|
|
|
|
|
and not self.sync_only:
|
2022-01-18 09:53:56 +00:00
|
|
|
|
print("Sorry, no support for {} links.".format(gi.scheme))
|
|
|
|
|
return
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2022-12-02 12:11:26 +00:00
|
|
|
|
if not mode:
|
|
|
|
|
mode = gi.last_mode
|
2019-10-15 19:12:32 +00:00
|
|
|
|
# Obey permanent redirects
|
|
|
|
|
if gi.url in self.permanent_redirects:
|
|
|
|
|
new_gi = GeminiItem(self.permanent_redirects[gi.url], name=gi.name)
|
2022-12-01 16:39:04 +00:00
|
|
|
|
self._go_to_gi(new_gi,mode=mode)
|
2019-10-15 19:12:32 +00:00
|
|
|
|
return
|
2022-01-24 15:50:22 +00:00
|
|
|
|
|
2022-01-08 20:32:25 +00:00
|
|
|
|
# Use cache or mark as to_fetch if resource is not cached
|
|
|
|
|
# Why is this code useful ? It set the mimetype !
|
|
|
|
|
if self.offline_only:
|
|
|
|
|
if not gi.is_cache_valid():
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.get_list("to_fetch")
|
2022-02-15 13:56:35 +00:00
|
|
|
|
r = self.list_add_line("to_fetch",gi=gi,verbose=False)
|
|
|
|
|
if r:
|
|
|
|
|
print("%s not available, marked for syncing"%gi.url)
|
|
|
|
|
else:
|
|
|
|
|
print("%s already marked for syncing"%gi.url)
|
2021-12-06 16:03:22 +00:00
|
|
|
|
return
|
2022-01-25 10:49:31 +00:00
|
|
|
|
# check if local file exists.
|
2022-02-12 23:37:31 +00:00
|
|
|
|
if gi.local and not os.path.exists(gi.path):
|
2022-01-25 10:49:31 +00:00
|
|
|
|
print("Local file %s does not exist!" %gi.path)
|
|
|
|
|
return
|
2021-12-06 16:03:22 +00:00
|
|
|
|
|
2022-01-18 09:53:56 +00:00
|
|
|
|
elif not self.offline_only and not gi.local:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
try:
|
2022-01-09 15:12:07 +00:00
|
|
|
|
if gi.scheme in ("http", "https"):
|
2022-02-12 13:41:05 +00:00
|
|
|
|
if self.support_http:
|
2022-03-03 15:16:32 +00:00
|
|
|
|
if limit_size:
|
2022-03-01 21:03:42 +00:00
|
|
|
|
# Let’s cap automatic downloads to 20Mo
|
2022-03-03 15:16:32 +00:00
|
|
|
|
max_download = int(self.options["max_size_download"])*1000000
|
2022-03-01 21:03:42 +00:00
|
|
|
|
else:
|
|
|
|
|
max_download = None
|
|
|
|
|
gi = self._fetch_http(gi,max_length=max_download)
|
2022-02-04 14:55:22 +00:00
|
|
|
|
elif handle and not self.sync_only:
|
|
|
|
|
if not _DO_HTTP:
|
|
|
|
|
print("Install python3-requests to handle http requests natively")
|
2022-01-10 10:19:29 +00:00
|
|
|
|
webbrowser.open_new_tab(gi.url)
|
|
|
|
|
return
|
2022-02-04 14:55:22 +00:00
|
|
|
|
else:
|
|
|
|
|
return
|
2022-02-10 22:41:10 +00:00
|
|
|
|
elif gi.scheme in ("gopher"):
|
2022-02-11 10:03:19 +00:00
|
|
|
|
gi = self._fetch_gopher(gi,timeout=self.options["short_timeout"])
|
2022-11-24 11:24:03 +00:00
|
|
|
|
elif gi.scheme in ("finger"):
|
|
|
|
|
gi = self._fetch_finger(gi,timeout=self.options["short_timeout"])
|
2022-03-04 22:38:35 +00:00
|
|
|
|
elif gi.scheme in ("spartan"):
|
|
|
|
|
gi = self._fetch_spartan(gi)
|
2022-01-09 15:12:07 +00:00
|
|
|
|
else:
|
|
|
|
|
gi = self._fetch_over_network(gi)
|
2020-08-31 19:18:15 +00:00
|
|
|
|
except UserAbortException:
|
|
|
|
|
return
|
2020-08-30 18:21:15 +00:00
|
|
|
|
except Exception as err:
|
2022-01-08 20:32:25 +00:00
|
|
|
|
gi.set_error(err)
|
2020-08-30 18:21:15 +00:00
|
|
|
|
# Print an error message
|
2022-01-08 13:16:55 +00:00
|
|
|
|
# we fail silently when sync_only
|
2021-12-14 12:29:09 +00:00
|
|
|
|
print_error = not self.sync_only
|
2020-08-30 18:21:15 +00:00
|
|
|
|
if isinstance(err, socket.gaierror):
|
|
|
|
|
self.log["dns_failures"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
|
|
|
|
print("ERROR: DNS error!")
|
2020-08-30 18:21:15 +00:00
|
|
|
|
elif isinstance(err, ConnectionRefusedError):
|
|
|
|
|
self.log["refused_connections"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
|
|
|
|
print("ERROR1: Connection refused!")
|
2020-08-30 18:21:15 +00:00
|
|
|
|
elif isinstance(err, ConnectionResetError):
|
|
|
|
|
self.log["reset_connections"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
|
|
|
|
print("ERROR2: Connection reset!")
|
2020-08-30 18:21:15 +00:00
|
|
|
|
elif isinstance(err, (TimeoutError, socket.timeout)):
|
|
|
|
|
self.log["timeouts"] += 1
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
2021-12-10 10:27:48 +00:00
|
|
|
|
print("""ERROR3: Connection timed out!
|
|
|
|
|
Slow internet connection? Use 'set timeout' to be more patient.""")
|
2021-12-16 09:43:25 +00:00
|
|
|
|
elif isinstance(err, FileExistsError):
|
|
|
|
|
print("""ERROR5: Trying to create a directory which already exists
|
|
|
|
|
in the cache : """)
|
|
|
|
|
print(err)
|
2022-11-16 13:44:47 +00:00
|
|
|
|
elif isinstance(err,requests.exceptions.SSLError):
|
|
|
|
|
print("""ERROR6: Bad SSL certificate:\n""")
|
|
|
|
|
print(err)
|
|
|
|
|
print("""\n If you know what you are doing, you can try to accept bad certificates with the following command:\n""")
|
|
|
|
|
print("""set accept_bad_ssl_certificates True""")
|
2020-08-30 18:21:15 +00:00
|
|
|
|
else:
|
2021-12-13 12:49:12 +00:00
|
|
|
|
if print_error:
|
2022-02-09 20:46:29 +00:00
|
|
|
|
print("ERROR4: " + str(type(err)) + " : " + str(err))
|
|
|
|
|
print("\n" + str(err.with_traceback(None)))
|
2020-08-30 18:21:15 +00:00
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
# Pass file to handler, unless we were asked not to
|
2022-01-17 22:37:12 +00:00
|
|
|
|
if gi :
|
|
|
|
|
display = handle and not self.sync_only
|
2022-03-30 10:09:08 +00:00
|
|
|
|
if display and _RENDER_IMAGE and self.options["download_images_first"] \
|
|
|
|
|
and not self.offline_only:
|
2022-03-30 10:04:07 +00:00
|
|
|
|
# We download images first
|
|
|
|
|
for image in gi.get_images(mode=mode):
|
2022-07-24 08:17:33 +00:00
|
|
|
|
if image and image.startswith("http"):
|
2022-03-30 10:04:07 +00:00
|
|
|
|
img_gi = GeminiItem(image)
|
|
|
|
|
if not img_gi.is_cache_valid():
|
|
|
|
|
width = term_width() - 1
|
|
|
|
|
toprint = "Downloading %s" %image
|
|
|
|
|
toprint = toprint[:width]
|
|
|
|
|
toprint += " "*(width-len(toprint))
|
|
|
|
|
print(toprint,end="\r")
|
|
|
|
|
self._go_to_gi(img_gi, update_hist=False, check_cache=True, \
|
|
|
|
|
handle=False,limit_size=True)
|
2022-03-15 22:00:05 +00:00
|
|
|
|
if display and gi.display(mode=mode):
|
2022-03-23 13:28:19 +00:00
|
|
|
|
self.index = gi.get_links()
|
2022-01-17 22:37:12 +00:00
|
|
|
|
self.page_index = 0
|
|
|
|
|
self.index_index = -1
|
2022-03-14 10:43:40 +00:00
|
|
|
|
# Update state (external files are not added to history)
|
|
|
|
|
self.gi = gi
|
|
|
|
|
if update_hist and not self.sync_only:
|
|
|
|
|
self._update_history(gi)
|
2022-01-17 22:37:12 +00:00
|
|
|
|
elif display :
|
2022-01-08 20:32:25 +00:00
|
|
|
|
cmd_str = self._get_handler_cmd(gi.get_mime())
|
2020-08-30 18:21:15 +00:00
|
|
|
|
try:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
# get body (tmpfile) from gi !
|
|
|
|
|
run(cmd_str, parameter=gi.get_body(as_file=True), direct_output=True)
|
2020-08-30 18:21:15 +00:00
|
|
|
|
except FileNotFoundError:
|
|
|
|
|
print("Handler program %s not found!" % shlex.split(cmd_str)[0])
|
|
|
|
|
print("You can use the ! command to specify another handler program or pipeline.")
|
|
|
|
|
|
2022-03-01 21:03:42 +00:00
|
|
|
|
def _fetch_http(self,gi,max_length=None):
|
2022-03-24 21:00:04 +00:00
|
|
|
|
def set_error(item,length,max_length):
|
|
|
|
|
err = "Size of %s is %s Mo\n"%(item.url,length)
|
|
|
|
|
err += "Offpunk only download automatically content under %s Mo\n" %(max_length/1000000)
|
|
|
|
|
err += "To retrieve this content anyway, type 'reload'."
|
|
|
|
|
item.set_error(err)
|
|
|
|
|
return item
|
2022-02-01 14:53:07 +00:00
|
|
|
|
header = {}
|
|
|
|
|
header["User-Agent"] = "Offpunk browser v%s"%_VERSION
|
2022-03-30 13:46:25 +00:00
|
|
|
|
parsed = urllib.parse.urlparse(gi.url)
|
|
|
|
|
# Code to translate URLs to better frontends (think twitter.com -> nitter)
|
|
|
|
|
if self.options["redirects"]:
|
|
|
|
|
netloc = parsed.netloc
|
|
|
|
|
if netloc.startswith("www."):
|
|
|
|
|
netloc = netloc[4:]
|
|
|
|
|
if netloc in self.redirects:
|
2022-07-21 15:02:36 +00:00
|
|
|
|
if self.redirects[netloc] == "blocked":
|
|
|
|
|
text = "This website has been blocked.\n"
|
|
|
|
|
text += "Use the redirect command to unblock it."
|
|
|
|
|
gi.write_body(text,"text/gemini")
|
|
|
|
|
return gi
|
|
|
|
|
else:
|
|
|
|
|
parsed = parsed._replace(netloc = self.redirects[netloc])
|
2022-03-30 13:46:25 +00:00
|
|
|
|
url = urllib.parse.urlunparse(parsed)
|
|
|
|
|
with requests.get(url,headers=header, stream=True,timeout=5) as response:
|
2022-03-24 21:00:04 +00:00
|
|
|
|
#print("This is header for %s"%gi.url)
|
|
|
|
|
#print(response.headers)
|
2022-03-15 22:23:29 +00:00
|
|
|
|
if "content-type" in response.headers:
|
|
|
|
|
mime = response.headers['content-type']
|
|
|
|
|
else:
|
|
|
|
|
mime = None
|
2022-03-01 21:03:42 +00:00
|
|
|
|
if "content-length" in response.headers:
|
|
|
|
|
length = int(response.headers['content-length'])
|
|
|
|
|
else:
|
|
|
|
|
length = 0
|
|
|
|
|
if max_length and length > max_length:
|
|
|
|
|
response.close()
|
2022-03-24 21:00:04 +00:00
|
|
|
|
return set_error(gi,str(length/1000000),max_length)
|
|
|
|
|
elif max_length and length == 0:
|
|
|
|
|
body = b''
|
2022-11-24 15:26:36 +00:00
|
|
|
|
downloaded = 0
|
2022-03-24 21:00:04 +00:00
|
|
|
|
for r in response.iter_content():
|
|
|
|
|
body += r
|
|
|
|
|
#We divide max_size for streamed content
|
|
|
|
|
#in order to catch them faster
|
2022-10-24 23:15:57 +00:00
|
|
|
|
size = sys.getsizeof(body)
|
2022-11-24 09:48:38 +00:00
|
|
|
|
max = max_length/2
|
2022-11-24 15:26:36 +00:00
|
|
|
|
current = round(size*100/max,0)
|
|
|
|
|
if current > downloaded:
|
|
|
|
|
downloaded = current
|
|
|
|
|
print(" -> Receiving stream: %s%% of allowed data"%downloaded,end='\r')
|
2022-11-14 18:17:20 +00:00
|
|
|
|
#print("size: %s (%s\% of maxlenght)"%(size,size/max_length))
|
2022-10-24 23:15:57 +00:00
|
|
|
|
if size > max_length/2:
|
2022-03-24 21:00:04 +00:00
|
|
|
|
response.close()
|
|
|
|
|
return set_error(gi,"streaming",max_length)
|
|
|
|
|
response.close()
|
2022-03-01 21:03:42 +00:00
|
|
|
|
else:
|
|
|
|
|
body = response.content
|
|
|
|
|
response.close()
|
2022-03-15 22:23:29 +00:00
|
|
|
|
if mime and "text/" in mime:
|
2022-03-25 19:53:45 +00:00
|
|
|
|
body = body.decode("UTF-8","replace")
|
2022-01-09 14:53:14 +00:00
|
|
|
|
gi.write_body(body,mime)
|
2022-01-09 15:12:07 +00:00
|
|
|
|
return gi
|
2022-01-09 14:53:14 +00:00
|
|
|
|
|
2022-02-11 10:03:19 +00:00
|
|
|
|
def _fetch_gopher(self,gi,timeout=10):
|
|
|
|
|
if not looks_like_url(gi.url):
|
|
|
|
|
print("%s is not a valide url" %gi.url)
|
2022-02-10 22:41:10 +00:00
|
|
|
|
parsed =urllib.parse.urlparse(gi.url)
|
|
|
|
|
host = parsed.hostname
|
|
|
|
|
port = parsed.port or 70
|
|
|
|
|
if parsed.path and parsed.path[0] == "/" and len(parsed.path) > 1:
|
|
|
|
|
splitted = parsed.path.split("/")
|
|
|
|
|
# We check if we have well a gopher type
|
|
|
|
|
if len(splitted[1]) == 1:
|
|
|
|
|
itemtype = parsed.path[1]
|
|
|
|
|
selector = parsed.path[2:]
|
|
|
|
|
else:
|
|
|
|
|
itemtype = "1"
|
|
|
|
|
selector = parsed.path
|
|
|
|
|
else:
|
|
|
|
|
itemtype = "1"
|
|
|
|
|
selector = parsed.path
|
2022-02-11 10:03:19 +00:00
|
|
|
|
addresses = socket.getaddrinfo(host, port, family=0,type=socket.SOCK_STREAM)
|
2022-02-10 22:41:10 +00:00
|
|
|
|
s = socket.create_connection((host,port))
|
2022-02-11 10:03:19 +00:00
|
|
|
|
for address in addresses:
|
|
|
|
|
self._debug("Connecting to: " + str(address[4]))
|
|
|
|
|
s = socket.socket(address[0], address[1])
|
|
|
|
|
s.settimeout(timeout)
|
|
|
|
|
try:
|
|
|
|
|
s.connect(address[4])
|
|
|
|
|
break
|
|
|
|
|
except OSError as e:
|
|
|
|
|
err = e
|
|
|
|
|
else:
|
|
|
|
|
# If we couldn't connect to *any* of the addresses, just
|
|
|
|
|
# bubble up the exception from the last attempt and deny
|
|
|
|
|
# knowledge of earlier failures.
|
|
|
|
|
raise err
|
2022-02-10 22:41:10 +00:00
|
|
|
|
if parsed.query:
|
|
|
|
|
request = selector + "\t" + parsed.query
|
|
|
|
|
else:
|
|
|
|
|
request = selector
|
|
|
|
|
request += "\r\n"
|
|
|
|
|
s.sendall(request.encode("UTF-8"))
|
|
|
|
|
response = s.makefile("rb").read()
|
|
|
|
|
# Transcode response into UTF-8
|
|
|
|
|
#if itemtype in ("0","1","h"):
|
|
|
|
|
if not itemtype in ("9","g","I","s"):
|
|
|
|
|
# Try most common encodings
|
|
|
|
|
for encoding in ("UTF-8", "ISO-8859-1"):
|
|
|
|
|
try:
|
|
|
|
|
response = response.decode("UTF-8")
|
|
|
|
|
break
|
|
|
|
|
except UnicodeDecodeError:
|
|
|
|
|
pass
|
|
|
|
|
else:
|
|
|
|
|
# try to find encoding
|
|
|
|
|
#if _HAS_CHARDET:
|
|
|
|
|
detected = chardet.detect(response)
|
|
|
|
|
response = response.decode(detected["encoding"])
|
|
|
|
|
#else:
|
|
|
|
|
#raise UnicodeDecodeError
|
|
|
|
|
if itemtype == "0":
|
|
|
|
|
mime = "text/gemini"
|
|
|
|
|
elif itemtype == "1":
|
|
|
|
|
mime = "text/gopher"
|
|
|
|
|
elif itemtype == "h":
|
|
|
|
|
mime = "text/html"
|
|
|
|
|
elif itemtype in ("9","g","I","s"):
|
|
|
|
|
mime = None
|
|
|
|
|
else:
|
|
|
|
|
# by default, we should consider Gopher
|
|
|
|
|
mime = "text/gopher"
|
|
|
|
|
gi.write_body(response,mime)
|
|
|
|
|
return gi
|
|
|
|
|
|
2022-11-24 11:24:03 +00:00
|
|
|
|
def _fetch_finger(self,gi,timeout=10):
|
|
|
|
|
if not looks_like_url(gi.url):
|
|
|
|
|
print("%s is not a valid url" %gi.url)
|
|
|
|
|
parsed = urllib.parse.urlparse(gi.url)
|
|
|
|
|
host = parsed.hostname
|
|
|
|
|
port = parsed.port or standard_ports["finger"]
|
|
|
|
|
query = parsed.path.lstrip("/") + "\r\n"
|
|
|
|
|
with socket.create_connection((host,port)) as sock:
|
|
|
|
|
sock.settimeout(timeout)
|
|
|
|
|
sock.send(query.encode())
|
|
|
|
|
response = sock.makefile("rb").read().decode("UTF-8")
|
|
|
|
|
gi.write_body(response,"text/plain")
|
|
|
|
|
return gi
|
|
|
|
|
|
2022-03-04 22:38:35 +00:00
|
|
|
|
# Copied from reference spartan client by Michael Lazar
|
|
|
|
|
def _fetch_spartan(self,gi):
|
|
|
|
|
url_parts = urllib.parse.urlparse(gi.url)
|
|
|
|
|
host = url_parts.hostname
|
|
|
|
|
port = url_parts.port or 300
|
|
|
|
|
path = url_parts.path or "/"
|
|
|
|
|
query = url_parts.query
|
|
|
|
|
|
|
|
|
|
redirect_url = None
|
|
|
|
|
|
|
|
|
|
with socket.create_connection((host,port)) as sock:
|
|
|
|
|
if query:
|
|
|
|
|
data = urllib.parse.unquote_to_bytes(query)
|
|
|
|
|
else:
|
|
|
|
|
data = b""
|
|
|
|
|
encoded_host = host.encode("idna")
|
|
|
|
|
ascii_path = urllib.parse.unquote_to_bytes(path)
|
|
|
|
|
encoded_path = urllib.parse.quote_from_bytes(ascii_path).encode("ascii")
|
|
|
|
|
sock.send(b"%s %s %d\r\n" % (encoded_host,encoded_path,len(data)))
|
|
|
|
|
fp = sock.makefile("rb")
|
|
|
|
|
response = fp.readline(4096).decode("ascii").strip("\r\n")
|
|
|
|
|
parts = response.split(" ",maxsplit=1)
|
|
|
|
|
code,meta = int(parts[0]),parts[1]
|
|
|
|
|
if code == 2:
|
|
|
|
|
body = fp.read()
|
|
|
|
|
if meta.startswith("text"):
|
|
|
|
|
body = body.decode("UTF-8")
|
|
|
|
|
gi.write_body(body,meta)
|
|
|
|
|
elif code == 3:
|
|
|
|
|
redirect_url = url_parts._replace(path=meta).geturl()
|
|
|
|
|
else:
|
|
|
|
|
gi.set_error("Spartan code %s: Error %s"%(code,meta))
|
|
|
|
|
if redirect_url:
|
|
|
|
|
gi = GeminiItem(redirect_url)
|
|
|
|
|
self._fetch_spartan(gi)
|
|
|
|
|
return gi
|
|
|
|
|
|
2022-01-09 19:41:47 +00:00
|
|
|
|
# fetch_over_network will modify with gi.write_body(body,mime)
|
2022-01-09 10:35:09 +00:00
|
|
|
|
# before returning the gi
|
2020-08-30 18:21:15 +00:00
|
|
|
|
def _fetch_over_network(self, gi):
|
2021-12-06 16:03:22 +00:00
|
|
|
|
|
2020-05-11 20:22:24 +00:00
|
|
|
|
# Be careful with client certificates!
|
|
|
|
|
# Are we crossing a domain boundary?
|
2020-05-10 11:44:40 +00:00
|
|
|
|
if self.active_cert_domains and gi.host not in self.active_cert_domains:
|
2020-05-10 20:51:33 +00:00
|
|
|
|
if self.active_is_transient:
|
|
|
|
|
print("Permanently delete currently active transient certificate?")
|
|
|
|
|
resp = input("Y/N? ")
|
|
|
|
|
if resp.strip().lower() in ("y", "yes"):
|
|
|
|
|
print("Destroying certificate.")
|
|
|
|
|
self._deactivate_client_cert()
|
|
|
|
|
else:
|
|
|
|
|
print("Staying here.")
|
2020-08-31 19:18:15 +00:00
|
|
|
|
raise UserAbortException()
|
2020-05-10 11:44:40 +00:00
|
|
|
|
else:
|
2020-05-10 20:51:33 +00:00
|
|
|
|
print("PRIVACY ALERT: Deactivate client cert before connecting to a new domain?")
|
|
|
|
|
resp = input("Y/N? ")
|
|
|
|
|
if resp.strip().lower() in ("n", "no"):
|
|
|
|
|
print("Keeping certificate active for {}".format(gi.host))
|
|
|
|
|
else:
|
|
|
|
|
print("Deactivating certificate.")
|
|
|
|
|
self._deactivate_client_cert()
|
|
|
|
|
|
2020-05-10 12:19:12 +00:00
|
|
|
|
# Suggest reactivating previous certs
|
|
|
|
|
if not self.client_certs["active"] and gi.host in self.client_certs:
|
|
|
|
|
print("PRIVACY ALERT: Reactivate previously used client cert for {}?".format(gi.host))
|
|
|
|
|
resp = input("Y/N? ")
|
|
|
|
|
if resp.strip().lower() in ("y", "yes"):
|
2020-05-10 15:00:30 +00:00
|
|
|
|
self._activate_client_cert(*self.client_certs[gi.host])
|
2020-05-10 12:19:12 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Remaining unidentified.")
|
|
|
|
|
self.client_certs.pop(gi.host)
|
|
|
|
|
|
2020-08-30 18:21:15 +00:00
|
|
|
|
# Is this a local file?
|
2021-11-18 11:02:00 +00:00
|
|
|
|
if gi.local:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
address, f = None, open(gi.path, "rb")
|
|
|
|
|
else:
|
|
|
|
|
address, f = self._send_request(gi)
|
|
|
|
|
|
|
|
|
|
# Spec dictates <META> should not exceed 1024 bytes,
|
|
|
|
|
# so maximum valid header length is 1027 bytes.
|
|
|
|
|
header = f.readline(1027)
|
2022-11-24 15:26:36 +00:00
|
|
|
|
header = urllib.parse.unquote(header.decode("UTF-8"))
|
2020-08-30 18:21:15 +00:00
|
|
|
|
if not header or header[-1] != '\n':
|
|
|
|
|
raise RuntimeError("Received invalid header from server!")
|
|
|
|
|
header = header.strip()
|
|
|
|
|
self._debug("Response header: %s." % header)
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Validate header
|
2019-08-13 17:00:15 +00:00
|
|
|
|
status, meta = header.split(maxsplit=1)
|
2020-05-16 13:59:05 +00:00
|
|
|
|
if len(meta) > 1024 or len(status) != 2 or not status.isnumeric():
|
2019-08-13 15:09:29 +00:00
|
|
|
|
f.close()
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Received invalid header from server!")
|
2019-08-13 15:09:29 +00:00
|
|
|
|
|
2019-10-13 17:42:04 +00:00
|
|
|
|
# Update redirect loop/maze escaping state
|
|
|
|
|
if not status.startswith("3"):
|
|
|
|
|
self.previous_redirectors = set()
|
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Handle non-SUCCESS headers, which don't have a response body
|
|
|
|
|
# Inputs
|
|
|
|
|
if status.startswith("1"):
|
2021-12-09 17:05:46 +00:00
|
|
|
|
if self.sync_only:
|
|
|
|
|
return None
|
2020-06-07 17:13:00 +00:00
|
|
|
|
else:
|
2021-12-09 17:05:46 +00:00
|
|
|
|
print(meta)
|
|
|
|
|
if status == "11":
|
|
|
|
|
user_input = getpass.getpass("> ")
|
|
|
|
|
else:
|
|
|
|
|
user_input = input("> ")
|
|
|
|
|
return self._fetch_over_network(gi.query(user_input))
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Redirects
|
2019-08-08 18:23:58 +00:00
|
|
|
|
elif status.startswith("3"):
|
2019-08-13 17:00:15 +00:00
|
|
|
|
new_gi = GeminiItem(gi.absolutise_url(meta))
|
2020-08-18 19:06:12 +00:00
|
|
|
|
if new_gi.url == gi.url:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("URL redirects to itself!")
|
2020-08-18 19:06:12 +00:00
|
|
|
|
elif new_gi.url in self.previous_redirectors:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Caught in redirect loop!")
|
2019-10-13 17:42:04 +00:00
|
|
|
|
elif len(self.previous_redirectors) == _MAX_REDIRECTS:
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Refusing to follow more than %d consecutive redirects!" % _MAX_REDIRECTS)
|
2022-01-22 14:33:39 +00:00
|
|
|
|
elif self.sync_only:
|
|
|
|
|
follow = self.automatic_choice
|
2020-05-23 10:53:20 +00:00
|
|
|
|
# Never follow cross-domain redirects without asking
|
|
|
|
|
elif new_gi.host != gi.host:
|
|
|
|
|
follow = input("Follow cross-domain redirect to %s? (y/n) " % new_gi.url)
|
2020-05-31 12:06:23 +00:00
|
|
|
|
# Never follow cross-protocol redirects without asking
|
|
|
|
|
elif new_gi.scheme != gi.scheme:
|
|
|
|
|
follow = input("Follow cross-protocol redirect to %s? (y/n) " % new_gi.url)
|
2020-05-31 12:23:30 +00:00
|
|
|
|
# Don't follow *any* redirect without asking if auto-follow is off
|
2019-10-15 19:12:32 +00:00
|
|
|
|
elif not self.options["auto_follow_redirects"]:
|
|
|
|
|
follow = input("Follow redirect to %s? (y/n) " % new_gi.url)
|
2020-05-31 12:23:30 +00:00
|
|
|
|
# Otherwise, follow away
|
|
|
|
|
else:
|
2020-05-31 12:24:23 +00:00
|
|
|
|
follow = "yes"
|
2020-05-31 12:06:23 +00:00
|
|
|
|
if follow.strip().lower() not in ("y", "yes"):
|
2020-08-31 19:18:15 +00:00
|
|
|
|
raise UserAbortException()
|
2020-05-31 12:06:23 +00:00
|
|
|
|
self._debug("Following redirect to %s." % new_gi.url)
|
|
|
|
|
self._debug("This is consecutive redirect number %d." % len(self.previous_redirectors))
|
|
|
|
|
self.previous_redirectors.add(gi.url)
|
|
|
|
|
if status == "31":
|
|
|
|
|
# Permanent redirect
|
|
|
|
|
self.permanent_redirects[gi.url] = new_gi.url
|
2020-08-30 18:21:15 +00:00
|
|
|
|
return self._fetch_over_network(new_gi)
|
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Errors
|
2019-08-08 18:23:58 +00:00
|
|
|
|
elif status.startswith("4") or status.startswith("5"):
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError(meta)
|
|
|
|
|
|
2019-08-11 19:26:30 +00:00
|
|
|
|
# Client cert
|
|
|
|
|
elif status.startswith("6"):
|
2020-08-31 19:18:15 +00:00
|
|
|
|
self._handle_cert_request(meta)
|
|
|
|
|
return self._fetch_over_network(gi)
|
2020-08-18 19:41:51 +00:00
|
|
|
|
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Invalid status
|
|
|
|
|
elif not status.startswith("2"):
|
2020-08-30 18:21:15 +00:00
|
|
|
|
raise RuntimeError("Server returned undefined status code %s!" % status)
|
2019-08-13 15:09:29 +00:00
|
|
|
|
|
|
|
|
|
# If we're here, this must be a success and there's a response body
|
|
|
|
|
assert status.startswith("2")
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
2019-08-13 17:00:15 +00:00
|
|
|
|
mime = meta
|
2019-08-13 15:09:29 +00:00
|
|
|
|
# Read the response body over the network
|
2022-01-09 15:12:07 +00:00
|
|
|
|
fbody = f.read()
|
|
|
|
|
# DEFAULT GEMINI MIME
|
|
|
|
|
if mime == "":
|
|
|
|
|
mime = "text/gemini; charset=utf-8"
|
2022-11-29 23:11:44 +00:00
|
|
|
|
shortmime, mime_options = parse_mime(mime)
|
2022-01-09 15:12:07 +00:00
|
|
|
|
if "charset" in mime_options:
|
|
|
|
|
try:
|
|
|
|
|
codecs.lookup(mime_options["charset"])
|
|
|
|
|
except LookupError:
|
|
|
|
|
raise RuntimeError("Header declared unknown encoding %s" % value)
|
|
|
|
|
if shortmime.startswith("text/"):
|
2022-01-11 21:26:57 +00:00
|
|
|
|
#Get the charset and default to UTF-8 in none
|
2022-01-09 15:12:07 +00:00
|
|
|
|
encoding = mime_options.get("charset", "UTF-8")
|
|
|
|
|
try:
|
|
|
|
|
body = fbody.decode(encoding)
|
|
|
|
|
except UnicodeError:
|
|
|
|
|
raise RuntimeError("Could not decode response body using %s\
|
|
|
|
|
encoding declared in header!" % encoding)
|
2022-01-13 09:06:10 +00:00
|
|
|
|
else:
|
|
|
|
|
body = fbody
|
2022-01-09 14:27:02 +00:00
|
|
|
|
gi.write_body(body,mime)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
return gi
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def _send_request(self, gi):
|
|
|
|
|
"""Send a selector to a given host and port.
|
|
|
|
|
Returns the resolved address and binary file with the reply."""
|
2022-03-04 22:38:35 +00:00
|
|
|
|
host, port = gi.host, gi.port
|
2022-11-24 15:26:36 +00:00
|
|
|
|
host = host.encode("idna").decode()
|
2020-05-10 10:35:46 +00:00
|
|
|
|
# Do DNS resolution
|
2019-08-13 10:04:07 +00:00
|
|
|
|
addresses = self._get_addresses(host, port)
|
2020-05-10 10:35:46 +00:00
|
|
|
|
|
|
|
|
|
# Prepare TLS context
|
2022-02-22 10:15:19 +00:00
|
|
|
|
protocol = ssl.PROTOCOL_TLS_CLIENT if sys.version_info.minor >=6 else ssl.PROTOCOL_TLSv1_2
|
2020-05-10 10:35:46 +00:00
|
|
|
|
context = ssl.SSLContext(protocol)
|
2020-05-19 21:14:09 +00:00
|
|
|
|
# Use CAs or TOFU
|
|
|
|
|
if self.options["tls_mode"] == "ca":
|
|
|
|
|
context.verify_mode = ssl.CERT_REQUIRED
|
|
|
|
|
context.check_hostname = True
|
|
|
|
|
context.load_default_certs()
|
|
|
|
|
else:
|
|
|
|
|
context.check_hostname = False
|
|
|
|
|
context.verify_mode = ssl.CERT_NONE
|
2020-05-10 10:35:46 +00:00
|
|
|
|
# Impose minimum TLS version
|
|
|
|
|
## In 3.7 and above, this is easy...
|
|
|
|
|
if sys.version_info.minor >= 7:
|
|
|
|
|
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
|
|
|
## Otherwise, it seems very hard...
|
|
|
|
|
## The below is less strict than it ought to be, but trying to disable
|
|
|
|
|
## TLS v1.1 here using ssl.OP_NO_TLSv1_1 produces unexpected failures
|
|
|
|
|
## with recent versions of OpenSSL. What a mess...
|
|
|
|
|
else:
|
|
|
|
|
context.options |= ssl.OP_NO_SSLv3
|
|
|
|
|
context.options |= ssl.OP_NO_SSLv2
|
|
|
|
|
# Try to enforce sensible ciphers
|
|
|
|
|
try:
|
2020-06-04 14:21:11 +00:00
|
|
|
|
context.set_ciphers("AESGCM+ECDHE:AESGCM+DHE:CHACHA20+ECDHE:CHACHA20+DHE:!DSS:!SHA1:!MD5:@STRENGTH")
|
2020-05-10 10:35:46 +00:00
|
|
|
|
except ssl.SSLError:
|
|
|
|
|
# Rely on the server to only support sensible things, I guess...
|
|
|
|
|
pass
|
|
|
|
|
# Load client certificate if needed
|
2020-05-10 10:59:26 +00:00
|
|
|
|
if self.client_certs["active"]:
|
|
|
|
|
certfile, keyfile = self.client_certs["active"]
|
|
|
|
|
context.load_cert_chain(certfile, keyfile)
|
2022-01-08 20:32:25 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Connect to remote host by any address possible
|
|
|
|
|
err = None
|
|
|
|
|
for address in addresses:
|
|
|
|
|
self._debug("Connecting to: " + str(address[4]))
|
|
|
|
|
s = socket.socket(address[0], address[1])
|
2021-12-10 10:27:48 +00:00
|
|
|
|
if self.sync_only:
|
|
|
|
|
timeout = self.options["short_timeout"]
|
|
|
|
|
else:
|
|
|
|
|
timeout = self.options["timeout"]
|
|
|
|
|
s.settimeout(timeout)
|
2022-11-24 15:26:36 +00:00
|
|
|
|
s = context.wrap_socket(s, server_hostname = host)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
try:
|
|
|
|
|
s.connect(address[4])
|
|
|
|
|
break
|
|
|
|
|
except OSError as e:
|
|
|
|
|
err = e
|
|
|
|
|
else:
|
|
|
|
|
# If we couldn't connect to *any* of the addresses, just
|
|
|
|
|
# bubble up the exception from the last attempt and deny
|
|
|
|
|
# knowledge of earlier failures.
|
|
|
|
|
raise err
|
2019-08-18 19:59:49 +00:00
|
|
|
|
|
2020-04-12 19:20:29 +00:00
|
|
|
|
if sys.version_info.minor >=5:
|
|
|
|
|
self._debug("Established {} connection.".format(s.version()))
|
2019-08-18 19:59:49 +00:00
|
|
|
|
self._debug("Cipher is: {}.".format(s.cipher()))
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
# Do TOFU
|
2020-05-19 21:14:09 +00:00
|
|
|
|
if self.options["tls_mode"] != "ca":
|
|
|
|
|
cert = s.getpeercert(binary_form=True)
|
|
|
|
|
self._validate_cert(address[4][0], host, cert)
|
2020-05-16 16:58:53 +00:00
|
|
|
|
|
2020-05-10 11:44:40 +00:00
|
|
|
|
# Remember that we showed the current cert to this domain...
|
|
|
|
|
if self.client_certs["active"]:
|
2022-11-24 15:26:36 +00:00
|
|
|
|
self.active_cert_domains.append(host)
|
|
|
|
|
self.client_certs[host] = self.client_certs["active"]
|
2020-05-10 11:44:40 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Send request and wrap response in a file descriptor
|
2019-08-13 16:56:15 +00:00
|
|
|
|
self._debug("Sending %s<CRLF>" % gi.url)
|
|
|
|
|
s.sendall((gi.url + CRLF).encode("UTF-8"))
|
2022-01-08 20:32:25 +00:00
|
|
|
|
mf= s.makefile(mode = "rb")
|
|
|
|
|
return address, mf
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2019-08-12 14:14:42 +00:00
|
|
|
|
def _get_addresses(self, host, port):
|
|
|
|
|
# DNS lookup - will get IPv4 and IPv6 records if IPv6 is enabled
|
|
|
|
|
if ":" in host:
|
|
|
|
|
# This is likely a literal IPv6 address, so we can *only* ask for
|
|
|
|
|
# IPv6 addresses or getaddrinfo will complain
|
|
|
|
|
family_mask = socket.AF_INET6
|
|
|
|
|
elif socket.has_ipv6 and self.options["ipv6"]:
|
|
|
|
|
# Accept either IPv4 or IPv6 addresses
|
|
|
|
|
family_mask = 0
|
|
|
|
|
else:
|
|
|
|
|
# IPv4 only
|
|
|
|
|
family_mask = socket.AF_INET
|
|
|
|
|
addresses = socket.getaddrinfo(host, port, family=family_mask,
|
|
|
|
|
type=socket.SOCK_STREAM)
|
|
|
|
|
# Sort addresses so IPv6 ones come first
|
|
|
|
|
addresses.sort(key=lambda add: add[0] == socket.AF_INET6, reverse=True)
|
|
|
|
|
|
|
|
|
|
return addresses
|
|
|
|
|
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2020-08-31 19:18:15 +00:00
|
|
|
|
def _handle_cert_request(self, meta):
|
|
|
|
|
print("SERVER SAYS: ", meta)
|
|
|
|
|
# Present different messages for different 6x statuses, but
|
|
|
|
|
# handle them the same.
|
|
|
|
|
if status in ("64", "65"):
|
|
|
|
|
print("The server rejected your certificate because it is either expired or not yet valid.")
|
|
|
|
|
elif status == "63":
|
|
|
|
|
print("The server did not accept your certificate.")
|
|
|
|
|
print("You may need to e.g. coordinate with the admin to get your certificate fingerprint whitelisted.")
|
|
|
|
|
else:
|
|
|
|
|
print("The site {} is requesting a client certificate.".format(gi.host))
|
|
|
|
|
print("This will allow the site to recognise you across requests.")
|
|
|
|
|
|
|
|
|
|
# Give the user choices
|
|
|
|
|
print("What do you want to do?")
|
|
|
|
|
print("1. Give up.")
|
|
|
|
|
print("2. Generate a new transient certificate.")
|
|
|
|
|
print("3. Generate a new persistent certificate.")
|
|
|
|
|
print("4. Load a previously generated persistent.")
|
|
|
|
|
print("5. Load certificate from an external file.")
|
2022-01-22 14:33:39 +00:00
|
|
|
|
if self.sync_only:
|
|
|
|
|
choice = 1
|
|
|
|
|
else:
|
|
|
|
|
choice = input("> ").strip()
|
2020-08-31 19:18:15 +00:00
|
|
|
|
if choice == "2":
|
|
|
|
|
self._generate_transient_cert_cert()
|
|
|
|
|
elif choice == "3":
|
|
|
|
|
self._generate_persistent_client_cert()
|
|
|
|
|
elif choice == "4":
|
|
|
|
|
self._choose_client_cert()
|
|
|
|
|
elif choice == "5":
|
|
|
|
|
self._load_client_cert()
|
|
|
|
|
else:
|
|
|
|
|
print("Giving up.")
|
|
|
|
|
raise UserAbortException()
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
def _validate_cert(self, address, host, cert):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Validate a TLS certificate in TOFU mode.
|
|
|
|
|
|
|
|
|
|
If the cryptography module is installed:
|
|
|
|
|
- Check the certificate Common Name or SAN matches `host`
|
|
|
|
|
- Check the certificate's not valid before date is in the past
|
|
|
|
|
- Check the certificate's not valid after date is in the future
|
2020-05-17 18:38:06 +00:00
|
|
|
|
|
2020-06-07 18:42:19 +00:00
|
|
|
|
Whether the cryptography module is installed or not, check the
|
|
|
|
|
certificate's fingerprint against the TOFU database to see if we've
|
|
|
|
|
previously encountered a different certificate for this IP address and
|
|
|
|
|
hostname.
|
|
|
|
|
"""
|
2020-05-28 19:01:04 +00:00
|
|
|
|
now = datetime.datetime.utcnow()
|
2020-05-17 18:38:06 +00:00
|
|
|
|
if _HAS_CRYPTOGRAPHY:
|
|
|
|
|
# Using the cryptography module we can get detailed access
|
|
|
|
|
# to the properties of even self-signed certs, unlike in
|
|
|
|
|
# the standard ssl library...
|
|
|
|
|
c = x509.load_der_x509_certificate(cert, _BACKEND)
|
2022-03-29 20:27:35 +00:00
|
|
|
|
# Check certificate validity dates
|
|
|
|
|
if c.not_valid_before >= now:
|
|
|
|
|
raise CertificateError("Certificate not valid until: {}!".format(c.not_valid_before))
|
|
|
|
|
elif c.not_valid_after <= now:
|
|
|
|
|
raise CertificateError("Certificate expired as of: {})!".format(c.not_valid_after))
|
|
|
|
|
|
|
|
|
|
# Check certificate hostnames
|
|
|
|
|
names = []
|
|
|
|
|
common_name = c.subject.get_attributes_for_oid(x509.oid.NameOID.COMMON_NAME)
|
|
|
|
|
if common_name:
|
|
|
|
|
names.append(common_name[0].value)
|
|
|
|
|
try:
|
|
|
|
|
names.extend([alt.value for alt in c.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME).value])
|
|
|
|
|
except x509.ExtensionNotFound:
|
|
|
|
|
pass
|
|
|
|
|
names = set(names)
|
|
|
|
|
for name in names:
|
|
|
|
|
try:
|
|
|
|
|
ssl._dnsname_match(name, host)
|
|
|
|
|
break
|
|
|
|
|
except CertificateError:
|
|
|
|
|
continue
|
|
|
|
|
else:
|
|
|
|
|
# If we didn't break out, none of the names were valid
|
|
|
|
|
raise CertificateError("Hostname does not match certificate common name or any alternative names.")
|
2020-05-17 18:38:06 +00:00
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
sha = hashlib.sha256()
|
|
|
|
|
sha.update(cert)
|
|
|
|
|
fingerprint = sha.hexdigest()
|
|
|
|
|
|
|
|
|
|
# Have we been here before?
|
|
|
|
|
self.db_cur.execute("""SELECT fingerprint, first_seen, last_seen, count
|
|
|
|
|
FROM cert_cache
|
|
|
|
|
WHERE hostname=? AND address=?""", (host, address))
|
|
|
|
|
cached_certs = self.db_cur.fetchall()
|
|
|
|
|
|
|
|
|
|
# If so, check for a match
|
|
|
|
|
if cached_certs:
|
|
|
|
|
max_count = 0
|
2020-05-23 10:53:02 +00:00
|
|
|
|
most_frequent_cert = None
|
2020-05-16 16:58:53 +00:00
|
|
|
|
for cached_fingerprint, first, last, count in cached_certs:
|
|
|
|
|
if count > max_count:
|
|
|
|
|
max_count = count
|
2020-05-23 10:53:02 +00:00
|
|
|
|
most_frequent_cert = cached_fingerprint
|
2020-05-16 16:58:53 +00:00
|
|
|
|
if fingerprint == cached_fingerprint:
|
|
|
|
|
# Matched!
|
|
|
|
|
self._debug("TOFU: Accepting previously seen ({} times) certificate {}".format(count, fingerprint))
|
|
|
|
|
self.db_cur.execute("""UPDATE cert_cache
|
|
|
|
|
SET last_seen=?, count=?
|
|
|
|
|
WHERE hostname=? AND address=? AND fingerprint=?""",
|
|
|
|
|
(now, count+1, host, address, fingerprint))
|
2020-05-17 12:02:36 +00:00
|
|
|
|
self.db_conn.commit()
|
2020-05-16 16:58:53 +00:00
|
|
|
|
break
|
|
|
|
|
else:
|
2022-11-16 09:06:35 +00:00
|
|
|
|
certdir = os.path.join(_CONFIG_DIR, "cert_cache")
|
|
|
|
|
with open(os.path.join(certdir, most_frequent_cert+".crt"), "rb") as fp:
|
|
|
|
|
previous_cert = fp.read()
|
2020-05-23 10:53:02 +00:00
|
|
|
|
if _HAS_CRYPTOGRAPHY:
|
|
|
|
|
# Load the most frequently seen certificate to see if it has
|
|
|
|
|
# expired
|
|
|
|
|
previous_cert = x509.load_der_x509_certificate(previous_cert, _BACKEND)
|
|
|
|
|
previous_ttl = previous_cert.not_valid_after - now
|
|
|
|
|
print(previous_ttl)
|
|
|
|
|
|
2020-05-16 16:58:53 +00:00
|
|
|
|
self._debug("TOFU: Unrecognised certificate {}! Raising the alarm...".format(fingerprint))
|
|
|
|
|
print("****************************************")
|
|
|
|
|
print("[SECURITY WARNING] Unrecognised certificate!")
|
|
|
|
|
print("The certificate presented for {} ({}) has never been seen before.".format(host, address))
|
|
|
|
|
print("This MIGHT be a Man-in-the-Middle attack.")
|
2020-05-23 10:53:02 +00:00
|
|
|
|
print("A different certificate has previously been seen {} times.".format(max_count))
|
|
|
|
|
if _HAS_CRYPTOGRAPHY:
|
|
|
|
|
if previous_ttl < datetime.timedelta():
|
|
|
|
|
print("That certificate has expired, which reduces suspicion somewhat.")
|
|
|
|
|
else:
|
|
|
|
|
print("That certificate is still valid for: {}".format(previous_ttl))
|
2020-05-16 16:58:53 +00:00
|
|
|
|
print("****************************************")
|
|
|
|
|
print("Attempt to verify the new certificate fingerprint out-of-band:")
|
|
|
|
|
print(fingerprint)
|
2022-01-22 14:33:39 +00:00
|
|
|
|
if self.sync_only:
|
|
|
|
|
choice = self.automatic_choice
|
|
|
|
|
else:
|
|
|
|
|
choice = input("Accept this new certificate? Y/N ").strip().lower()
|
2020-05-16 16:58:53 +00:00
|
|
|
|
if choice in ("y", "yes"):
|
|
|
|
|
self.db_cur.execute("""INSERT INTO cert_cache
|
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
|
|
|
(host, address, fingerprint, now, now, 1))
|
2020-05-17 12:02:36 +00:00
|
|
|
|
self.db_conn.commit()
|
2020-05-17 16:35:35 +00:00
|
|
|
|
with open(os.path.join(certdir, fingerprint+".crt"), "wb") as fp:
|
|
|
|
|
fp.write(cert)
|
2020-05-16 16:58:53 +00:00
|
|
|
|
else:
|
|
|
|
|
raise Exception("TOFU Failure!")
|
|
|
|
|
|
|
|
|
|
# If not, cache this cert
|
|
|
|
|
else:
|
|
|
|
|
self._debug("TOFU: Blindly trusting first ever certificate for this host!")
|
|
|
|
|
self.db_cur.execute("""INSERT INTO cert_cache
|
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
|
|
|
(host, address, fingerprint, now, now, 1))
|
2020-05-17 12:02:36 +00:00
|
|
|
|
self.db_conn.commit()
|
2022-01-18 17:04:29 +00:00
|
|
|
|
certdir = os.path.join(_CONFIG_DIR, "cert_cache")
|
2020-05-17 16:35:35 +00:00
|
|
|
|
if not os.path.exists(certdir):
|
|
|
|
|
os.makedirs(certdir)
|
|
|
|
|
with open(os.path.join(certdir, fingerprint+".crt"), "wb") as fp:
|
|
|
|
|
fp.write(cert)
|
2020-05-16 16:58:53 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def _get_handler_cmd(self, mimetype):
|
|
|
|
|
# Now look for a handler for this mimetype
|
|
|
|
|
# Consider exact matches before wildcard matches
|
|
|
|
|
exact_matches = []
|
|
|
|
|
wildcard_matches = []
|
|
|
|
|
for handled_mime, cmd_str in _MIME_HANDLERS.items():
|
|
|
|
|
if "*" in handled_mime:
|
|
|
|
|
wildcard_matches.append((handled_mime, cmd_str))
|
|
|
|
|
else:
|
|
|
|
|
exact_matches.append((handled_mime, cmd_str))
|
|
|
|
|
for handled_mime, cmd_str in exact_matches + wildcard_matches:
|
|
|
|
|
if fnmatch.fnmatch(mimetype, handled_mime):
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
# Use "xdg-open" as a last resort.
|
2022-02-20 22:22:25 +00:00
|
|
|
|
if _HAS_XDGOPEN:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
cmd_str = "xdg-open %s"
|
2022-02-20 22:22:25 +00:00
|
|
|
|
else:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
cmd_str = "echo \"Can’t find how to open \"%s"
|
2022-02-20 22:22:25 +00:00
|
|
|
|
print("Please install xdg-open (usually from xdg-util package)")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self._debug("Using handler: %s" % cmd_str)
|
|
|
|
|
return cmd_str
|
|
|
|
|
|
2022-01-17 22:37:12 +00:00
|
|
|
|
#TODO: remove format_geminiitem
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def _format_geminiitem(self, index, gi, url=False):
|
2022-11-01 16:04:30 +00:00
|
|
|
|
if not gi:
|
|
|
|
|
line = "[%s] - No valid URL"%index
|
|
|
|
|
else:
|
|
|
|
|
protocol = "" if gi.scheme == "gemini" else " %s" % gi.scheme
|
|
|
|
|
line = "[%d%s] %s" % (index, protocol, gi.name or gi.url)
|
|
|
|
|
if gi.name and url:
|
|
|
|
|
line += " (%s)" % gi.url
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return line
|
|
|
|
|
|
2022-11-20 11:19:03 +00:00
|
|
|
|
@needs_gi
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def _show_lookup(self, offset=0, end=None, url=False):
|
2022-10-07 12:45:44 +00:00
|
|
|
|
for n, gi in enumerate(self.gi.get_links()[offset:end]):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
print(self._format_geminiitem(n+offset+1, gi, url))
|
|
|
|
|
|
|
|
|
|
def _update_history(self, gi):
|
2022-01-31 14:32:34 +00:00
|
|
|
|
# We never update while in sync_only
|
|
|
|
|
if self.sync_only:
|
|
|
|
|
return
|
2022-01-30 18:11:23 +00:00
|
|
|
|
# We don’t add lists to history
|
2022-02-12 23:37:31 +00:00
|
|
|
|
#if not gi or os.path.join(_DATA_DIR,"lists") in gi.url:
|
|
|
|
|
# return
|
2022-01-26 18:35:05 +00:00
|
|
|
|
histlist = self.get_list("history")
|
|
|
|
|
links = self.list_get_links("history")
|
|
|
|
|
# avoid duplicate
|
|
|
|
|
length = len(links)
|
|
|
|
|
if length > self.options["history_size"]:
|
|
|
|
|
length = self.options["history_size"]
|
|
|
|
|
if length > 0 and links[self.hist_index] == gi:
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return
|
2022-01-26 18:35:05 +00:00
|
|
|
|
self.list_add_top("history",limit=self.options["history_size"],truncate_lines=self.hist_index)
|
|
|
|
|
self.hist_index = 0
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def _log_visit(self, gi, address, size):
|
|
|
|
|
if not address:
|
|
|
|
|
return
|
|
|
|
|
self.log["requests"] += 1
|
|
|
|
|
self.log["bytes_recvd"] += size
|
|
|
|
|
self.visited_hosts.add(address)
|
|
|
|
|
if address[0] == socket.AF_INET:
|
|
|
|
|
self.log["ipv4_requests"] += 1
|
|
|
|
|
self.log["ipv4_bytes_recvd"] += size
|
|
|
|
|
elif address[0] == socket.AF_INET6:
|
|
|
|
|
self.log["ipv6_requests"] += 1
|
|
|
|
|
self.log["ipv6_bytes_recvd"] += size
|
|
|
|
|
|
|
|
|
|
def _debug(self, debug_text):
|
|
|
|
|
if not self.options["debug"]:
|
|
|
|
|
return
|
|
|
|
|
debug_text = "\x1b[0;32m[DEBUG] " + debug_text + "\x1b[0m"
|
|
|
|
|
print(debug_text)
|
|
|
|
|
|
2020-05-10 14:09:54 +00:00
|
|
|
|
def _load_client_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Interactively load a TLS client certificate from the filesystem in PEM
|
|
|
|
|
format.
|
|
|
|
|
"""
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Loading client certificate file, in PEM format (blank line to cancel)")
|
|
|
|
|
certfile = input("Certfile path: ").strip()
|
|
|
|
|
if not certfile:
|
|
|
|
|
print("Aborting.")
|
|
|
|
|
return
|
2020-08-30 14:50:52 +00:00
|
|
|
|
certfile = os.path.expanduser(certfile)
|
|
|
|
|
if not os.path.isfile(certfile):
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Certificate file {} does not exist.".format(certfile))
|
|
|
|
|
return
|
|
|
|
|
print("Loading private key file, in PEM format (blank line to cancel)")
|
|
|
|
|
keyfile = input("Keyfile path: ").strip()
|
|
|
|
|
if not keyfile:
|
|
|
|
|
print("Aborting.")
|
|
|
|
|
return
|
2020-08-30 14:50:52 +00:00
|
|
|
|
keyfile = os.path.expanduser(keyfile)
|
|
|
|
|
if not os.path.isfile(keyfile):
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Private key file {} does not exist.".format(keyfile))
|
|
|
|
|
return
|
|
|
|
|
self._activate_client_cert(certfile, keyfile)
|
|
|
|
|
|
2020-05-11 20:22:24 +00:00
|
|
|
|
def _generate_transient_cert_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Use `openssl` command to generate a new transient client certificate
|
|
|
|
|
with 24 hours of validity.
|
|
|
|
|
"""
|
2022-01-18 17:04:29 +00:00
|
|
|
|
certdir = os.path.join(_CONFIG_DIR, "transient_certs")
|
2020-05-11 20:22:24 +00:00
|
|
|
|
name = str(uuid.uuid4())
|
2020-05-17 10:18:09 +00:00
|
|
|
|
self._generate_client_cert(certdir, name, transient=True)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
self.active_is_transient = True
|
|
|
|
|
self.transient_certs_created.append(name)
|
|
|
|
|
|
|
|
|
|
def _generate_persistent_client_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Interactively use `openssl` command to generate a new persistent client
|
|
|
|
|
certificate with one year of validity.
|
|
|
|
|
"""
|
2022-01-18 17:04:29 +00:00
|
|
|
|
certdir = os.path.join(_CONFIG_DIR, "client_certs")
|
2020-05-10 15:25:03 +00:00
|
|
|
|
print("What do you want to name this new certificate?")
|
2020-08-14 20:29:21 +00:00
|
|
|
|
print("Answering `mycert` will create `{0}/mycert.crt` and `{0}/mycert.key`".format(certdir))
|
|
|
|
|
name = input("> ")
|
2020-05-10 15:25:03 +00:00
|
|
|
|
if not name.strip():
|
|
|
|
|
print("Aborting.")
|
|
|
|
|
return
|
2020-05-27 06:57:44 +00:00
|
|
|
|
self._generate_client_cert(certdir, name)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
|
2020-05-17 10:18:09 +00:00
|
|
|
|
def _generate_client_cert(self, certdir, basename, transient=False):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Use `openssl` binary to generate a client certificate (which may be
|
|
|
|
|
transient or persistent) and save the certificate and private key to the
|
|
|
|
|
specified directory with the specified basename.
|
|
|
|
|
"""
|
2020-05-10 15:25:03 +00:00
|
|
|
|
if not os.path.exists(certdir):
|
|
|
|
|
os.makedirs(certdir)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
certfile = os.path.join(certdir, basename+".crt")
|
|
|
|
|
keyfile = os.path.join(certdir, basename+".key")
|
2020-05-17 10:18:09 +00:00
|
|
|
|
cmd = "openssl req -x509 -newkey rsa:2048 -days {} -nodes -keyout {} -out {}".format(1 if transient else 365, keyfile, certfile)
|
|
|
|
|
if transient:
|
2020-05-31 08:58:45 +00:00
|
|
|
|
cmd += " -subj '/CN={}'".format(basename)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
os.system(cmd)
|
2020-05-10 15:25:03 +00:00
|
|
|
|
self._activate_client_cert(certfile, keyfile)
|
|
|
|
|
|
2020-05-11 21:27:48 +00:00
|
|
|
|
def _choose_client_cert(self):
|
2020-06-07 18:42:19 +00:00
|
|
|
|
"""
|
|
|
|
|
Interactively select a previously generated client certificate and
|
|
|
|
|
activate it.
|
|
|
|
|
"""
|
2022-01-18 17:04:29 +00:00
|
|
|
|
certdir = os.path.join(_CONFIG_DIR, "client_certs")
|
2020-05-11 21:27:48 +00:00
|
|
|
|
certs = glob.glob(os.path.join(certdir, "*.crt"))
|
2020-08-14 20:29:21 +00:00
|
|
|
|
if len(certs) == 0:
|
|
|
|
|
print("There are no previously generated certificates.")
|
|
|
|
|
return
|
2020-05-11 21:27:48 +00:00
|
|
|
|
certdir = {}
|
|
|
|
|
for n, cert in enumerate(certs):
|
|
|
|
|
certdir[str(n+1)] = (cert, os.path.splitext(cert)[0] + ".key")
|
|
|
|
|
print("{}. {}".format(n+1, os.path.splitext(os.path.basename(cert))[0]))
|
|
|
|
|
choice = input("> ").strip()
|
|
|
|
|
if choice in certdir:
|
|
|
|
|
certfile, keyfile = certdir[choice]
|
|
|
|
|
self._activate_client_cert(certfile, keyfile)
|
|
|
|
|
else:
|
|
|
|
|
print("What?")
|
|
|
|
|
|
2020-05-10 12:17:35 +00:00
|
|
|
|
def _activate_client_cert(self, certfile, keyfile):
|
|
|
|
|
self.client_certs["active"] = (certfile, keyfile)
|
|
|
|
|
self.active_cert_domains = []
|
2021-02-03 07:06:27 +00:00
|
|
|
|
self.prompt = self.cert_prompt + "+" + os.path.basename(certfile).replace('.crt','') + "> " + "\x1b[0m"
|
2020-05-10 12:17:35 +00:00
|
|
|
|
self._debug("Using ID {} / {}.".format(*self.client_certs["active"]))
|
|
|
|
|
|
2020-05-10 11:48:25 +00:00
|
|
|
|
def _deactivate_client_cert(self):
|
2020-05-11 20:22:24 +00:00
|
|
|
|
if self.active_is_transient:
|
|
|
|
|
for filename in self.client_certs["active"]:
|
|
|
|
|
os.remove(filename)
|
|
|
|
|
for domain in self.active_cert_domains:
|
|
|
|
|
self.client_certs.pop(domain)
|
2020-05-10 11:48:25 +00:00
|
|
|
|
self.client_certs["active"] = None
|
|
|
|
|
self.active_cert_domains = []
|
|
|
|
|
self.prompt = self.no_cert_prompt
|
2020-05-11 20:22:24 +00:00
|
|
|
|
self.active_is_transient = False
|
2020-05-10 11:48:25 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Cmd implementation follows
|
|
|
|
|
|
|
|
|
|
def default(self, line):
|
|
|
|
|
if line.strip() == "EOF":
|
|
|
|
|
return self.onecmd("quit")
|
|
|
|
|
elif line.strip() == "..":
|
|
|
|
|
return self.do_up()
|
|
|
|
|
elif line.startswith("/"):
|
2022-01-24 16:04:03 +00:00
|
|
|
|
return self.do_find(line[1:])
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Expand abbreviated commands
|
|
|
|
|
first_word = line.split()[0].strip()
|
|
|
|
|
if first_word in _ABBREVS:
|
|
|
|
|
full_cmd = _ABBREVS[first_word]
|
|
|
|
|
expanded = line.replace(first_word, full_cmd, 1)
|
|
|
|
|
return self.onecmd(expanded)
|
2022-01-18 13:42:09 +00:00
|
|
|
|
# Try to access it like an URL
|
|
|
|
|
if looks_like_url(line):
|
|
|
|
|
return self.do_go(line)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Try to parse numerical index for lookup table
|
|
|
|
|
try:
|
|
|
|
|
n = int(line.strip())
|
|
|
|
|
except ValueError:
|
|
|
|
|
print("What?")
|
|
|
|
|
return
|
|
|
|
|
try:
|
2022-10-07 12:45:44 +00:00
|
|
|
|
gi = self.gi.get_link(n)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
except IndexError:
|
|
|
|
|
print ("Index too high!")
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
self.index_index = n
|
|
|
|
|
self._go_to_gi(gi)
|
|
|
|
|
|
|
|
|
|
### Settings
|
2022-07-21 15:02:36 +00:00
|
|
|
|
def do_redirect(self,line):
|
|
|
|
|
"""Display and manage the list of redirected URLs. This features is mostly useful to use privacy-friendly frontends for popular websites."""
|
|
|
|
|
if len(line.split()) == 1:
|
|
|
|
|
if line in self.redirects:
|
|
|
|
|
print("%s is redirected to %s" %(line,self.redirects[line]))
|
|
|
|
|
else:
|
|
|
|
|
print("Please add a destination to redirect %s" %line)
|
|
|
|
|
elif len(line.split()) >= 2:
|
|
|
|
|
orig, dest = line.split(" ",1)
|
|
|
|
|
if dest.lower() == "none":
|
|
|
|
|
if orig in self.redirects:
|
|
|
|
|
self.redirects.pop(orig)
|
|
|
|
|
print("Redirection for %s has been removed"%orig)
|
|
|
|
|
else:
|
|
|
|
|
print("%s was not redirected. Nothing has changed."%orig)
|
|
|
|
|
elif dest.lower() == "block":
|
|
|
|
|
self.redirects[orig] = "blocked"
|
|
|
|
|
print("%s will now be blocked"%orig)
|
|
|
|
|
else:
|
|
|
|
|
self.redirects[orig] = dest
|
|
|
|
|
print("%s will now be redirected to %s" %(orig,dest))
|
|
|
|
|
else:
|
|
|
|
|
toprint="Current redirections:\n"
|
|
|
|
|
toprint+="--------------------\n"
|
|
|
|
|
for r in self.redirects:
|
|
|
|
|
toprint += ("%s\t->\t%s\n" %(r,self.redirects[r]))
|
|
|
|
|
toprint +="\nTo add new, use \"redirect origine.com destination.org\""
|
|
|
|
|
toprint +="\nTo remove a redirect, use \"redirect origine.com NONE\""
|
|
|
|
|
toprint +="\nTo completely block a website, use \"redirect origine.com BLOCK\""
|
|
|
|
|
print(toprint)
|
2022-11-30 19:57:23 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def do_set(self, line):
|
|
|
|
|
"""View or set various options."""
|
|
|
|
|
if not line.strip():
|
|
|
|
|
# Show all current settings
|
|
|
|
|
for option in sorted(self.options.keys()):
|
|
|
|
|
print("%s %s" % (option, self.options[option]))
|
2022-03-30 13:46:25 +00:00
|
|
|
|
elif len(line.split()) == 1 :
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# Show current value of one specific setting
|
2019-06-22 12:58:21 +00:00
|
|
|
|
option = line.strip()
|
2022-07-21 15:02:36 +00:00
|
|
|
|
if option in self.options:
|
2019-06-22 12:58:21 +00:00
|
|
|
|
print("%s %s" % (option, self.options[option]))
|
|
|
|
|
else:
|
|
|
|
|
print("Unrecognised option %s" % option)
|
|
|
|
|
else:
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# Set value of one specific setting
|
2019-06-22 12:58:21 +00:00
|
|
|
|
option, value = line.split(" ", 1)
|
|
|
|
|
if option not in self.options:
|
|
|
|
|
print("Unrecognised option %s" % option)
|
|
|
|
|
return
|
2019-08-13 10:04:07 +00:00
|
|
|
|
# Validate / convert values
|
2020-05-19 21:14:09 +00:00
|
|
|
|
elif option == "tls_mode":
|
|
|
|
|
if value.lower() not in ("ca", "tofu"):
|
|
|
|
|
print("TLS mode must be `ca` or `tofu`!")
|
|
|
|
|
return
|
2022-11-16 13:44:47 +00:00
|
|
|
|
elif option == "accept_bad_ssl_certificates":
|
2022-11-17 09:30:12 +00:00
|
|
|
|
if not _DO_HTTP :
|
|
|
|
|
print("accepting bad certificates only makes sense with HTTP requests")
|
|
|
|
|
print("You need to install python3-request and other dependancies")
|
|
|
|
|
print("Type \"version\" for more information")
|
|
|
|
|
return
|
|
|
|
|
elif value.lower() == "false":
|
2022-11-16 13:44:47 +00:00
|
|
|
|
print("Only high security certificates are now accepted")
|
|
|
|
|
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:@SECLEVEL=2'
|
|
|
|
|
elif value.lower() == "true":
|
|
|
|
|
print("Low security SSL certificates are now accepted")
|
|
|
|
|
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:@SECLEVEL=1'
|
|
|
|
|
else:
|
|
|
|
|
print("accept_bad_ssl_certificates should be True or False")
|
|
|
|
|
return
|
2022-02-12 12:12:37 +00:00
|
|
|
|
elif option == "width":
|
|
|
|
|
if value.isnumeric():
|
|
|
|
|
value = int(value)
|
|
|
|
|
print("changing width to ",value)
|
|
|
|
|
global TERM_WIDTH
|
|
|
|
|
TERM_WIDTH = value
|
|
|
|
|
else:
|
|
|
|
|
print("%s is not a valid width (integer required)"%value)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif value.isnumeric():
|
|
|
|
|
value = int(value)
|
|
|
|
|
elif value.lower() == "false":
|
|
|
|
|
value = False
|
|
|
|
|
elif value.lower() == "true":
|
|
|
|
|
value = True
|
|
|
|
|
else:
|
|
|
|
|
try:
|
|
|
|
|
value = float(value)
|
|
|
|
|
except ValueError:
|
|
|
|
|
pass
|
|
|
|
|
self.options[option] = value
|
|
|
|
|
|
2020-05-10 10:35:46 +00:00
|
|
|
|
def do_cert(self, line):
|
2020-05-23 11:35:13 +00:00
|
|
|
|
"""Manage client certificates"""
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Managing client certificates")
|
2020-05-10 10:59:26 +00:00
|
|
|
|
if self.client_certs["active"]:
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Active certificate: {}".format(self.client_certs["active"][0]))
|
|
|
|
|
print("1. Deactivate client certificate.")
|
2020-05-11 21:27:48 +00:00
|
|
|
|
print("2. Generate new certificate.")
|
|
|
|
|
print("3. Load previously generated certificate.")
|
2020-05-23 11:35:13 +00:00
|
|
|
|
print("4. Load externally created client certificate from file.")
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Enter blank line to exit certificate manager.")
|
|
|
|
|
choice = input("> ").strip()
|
|
|
|
|
if choice == "1":
|
2020-05-10 10:35:46 +00:00
|
|
|
|
print("Deactivating client certificate.")
|
2020-05-10 11:48:25 +00:00
|
|
|
|
self._deactivate_client_cert()
|
2020-05-10 14:09:54 +00:00
|
|
|
|
elif choice == "2":
|
2020-05-11 21:27:48 +00:00
|
|
|
|
self._generate_persistent_client_cert()
|
2020-05-10 14:09:54 +00:00
|
|
|
|
elif choice == "3":
|
2020-05-11 21:27:48 +00:00
|
|
|
|
self._choose_client_cert()
|
|
|
|
|
elif choice == "4":
|
|
|
|
|
self._load_client_cert()
|
2020-05-10 10:35:46 +00:00
|
|
|
|
else:
|
2020-05-10 14:09:54 +00:00
|
|
|
|
print("Aborting.")
|
2020-05-10 10:35:46 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def do_handler(self, line):
|
|
|
|
|
"""View or set handler commands for different MIME types."""
|
|
|
|
|
if not line.strip():
|
|
|
|
|
# Show all current handlers
|
|
|
|
|
for mime in sorted(_MIME_HANDLERS.keys()):
|
|
|
|
|
print("%s %s" % (mime, _MIME_HANDLERS[mime]))
|
|
|
|
|
elif len(line.split()) == 1:
|
|
|
|
|
mime = line.strip()
|
|
|
|
|
if mime in _MIME_HANDLERS:
|
|
|
|
|
print("%s %s" % (mime, _MIME_HANDLERS[mime]))
|
|
|
|
|
else:
|
|
|
|
|
print("No handler set for MIME type %s" % mime)
|
|
|
|
|
else:
|
|
|
|
|
mime, handler = line.split(" ", 1)
|
|
|
|
|
_MIME_HANDLERS[mime] = handler
|
|
|
|
|
if "%s" not in handler:
|
|
|
|
|
print("Are you sure you don't want to pass the filename to the handler?")
|
|
|
|
|
|
2020-05-27 13:16:22 +00:00
|
|
|
|
def do_abbrevs(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Print all Offpunk command abbreviations."""
|
2020-06-13 10:39:18 +00:00
|
|
|
|
header = "Command Abbreviations:"
|
|
|
|
|
self.stdout.write("\n{}\n".format(str(header)))
|
2020-05-27 13:16:22 +00:00
|
|
|
|
if self.ruler:
|
|
|
|
|
self.stdout.write("{}\n".format(str(self.ruler * len(header))))
|
|
|
|
|
for k, v in _ABBREVS.items():
|
|
|
|
|
self.stdout.write("{:<7} {}\n".format(k, v))
|
2020-06-13 10:39:18 +00:00
|
|
|
|
self.stdout.write("\n")
|
2020-05-27 13:16:22 +00:00
|
|
|
|
|
2021-12-06 15:43:16 +00:00
|
|
|
|
def do_offline(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Use Offpunk offline by only accessing cached content"""
|
2021-12-06 15:43:16 +00:00
|
|
|
|
if self.offline_only:
|
2021-12-16 09:43:25 +00:00
|
|
|
|
print("Offline and undisturbed.")
|
2021-12-06 15:43:16 +00:00
|
|
|
|
else:
|
|
|
|
|
self.offline_only = True
|
2021-12-10 14:24:26 +00:00
|
|
|
|
self.prompt = self.offline_prompt
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Offpunk is now offline and will only access cached content")
|
2021-12-16 09:43:25 +00:00
|
|
|
|
|
|
|
|
|
def do_online(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Use Offpunk online with a direct connection"""
|
2021-12-16 09:43:25 +00:00
|
|
|
|
if self.offline_only:
|
|
|
|
|
self.offline_only = False
|
|
|
|
|
self.prompt = self.no_cert_prompt
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Offpunk is online and will access the network")
|
2021-12-16 09:43:25 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Already online. Try offline.")
|
2021-12-06 15:43:16 +00:00
|
|
|
|
|
2022-03-15 22:43:21 +00:00
|
|
|
|
def do_copy(self, arg):
|
2022-01-05 20:12:59 +00:00
|
|
|
|
"""Copy the content of the last visited page as gemtext in the clipboard.
|
|
|
|
|
Use with "url" as argument to only copy the adress.
|
2022-03-14 21:36:34 +00:00
|
|
|
|
Use with "raw" to copy ANSI content as seen in your terminal (not gemtext).
|
|
|
|
|
Use with "cache" to copy the path of the cached content."""
|
2022-01-05 20:12:59 +00:00
|
|
|
|
if self.gi:
|
2022-02-14 22:02:53 +00:00
|
|
|
|
if _HAS_XSEL:
|
2022-03-15 22:43:21 +00:00
|
|
|
|
args = arg.split()
|
2022-01-05 20:12:59 +00:00
|
|
|
|
if args and args[0] == "url":
|
2022-03-15 22:43:21 +00:00
|
|
|
|
if len(args) > 1 and args[1].isdecimal():
|
|
|
|
|
gi = self.index[int(args[1])-1]
|
|
|
|
|
url = gi.url
|
|
|
|
|
else:
|
|
|
|
|
url = self.gi.url
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run("xsel -b -i", input=url, direct_output=True)
|
2022-01-05 20:12:59 +00:00
|
|
|
|
elif args and args[0] == "raw":
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run("xsel -b -i", input=open(self.gi.get_temp_filename(), "rb"), direct_output=True)
|
2022-03-14 21:36:34 +00:00
|
|
|
|
elif args and args[0] == "cache":
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run("xsel -b -i", input=self.gi.get_cache_path(), direct_output=True)
|
2022-01-05 20:12:59 +00:00
|
|
|
|
else:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run("xsel -b -i", input=open(self.gi.get_body(as_file=True), "rb"), direct_output=True)
|
2022-01-05 20:12:59 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Please install xsel to use copy")
|
|
|
|
|
else:
|
|
|
|
|
print("No content to copy, visit a page first")
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
### Stuff for getting around
|
|
|
|
|
def do_go(self, line):
|
|
|
|
|
"""Go to a gemini URL or marked item."""
|
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line:
|
2021-12-22 12:20:10 +00:00
|
|
|
|
if shutil.which('xsel'):
|
|
|
|
|
clipboards = []
|
|
|
|
|
urls = []
|
2022-03-15 23:41:56 +00:00
|
|
|
|
for selec in ["-p","-s","-b"]:
|
|
|
|
|
try:
|
2022-03-30 17:23:44 +00:00
|
|
|
|
clipboards.append(run("xsel "+selec))
|
2022-03-15 23:41:56 +00:00
|
|
|
|
except Exception as err:
|
|
|
|
|
#print("Skippink clipboard %s because %s"%(selec,err))
|
|
|
|
|
pass
|
2021-12-22 12:20:10 +00:00
|
|
|
|
for u in clipboards:
|
2022-03-15 23:41:56 +00:00
|
|
|
|
if "://" in u and looks_like_url(u) and u not in urls :
|
2021-12-22 12:20:10 +00:00
|
|
|
|
urls.append(u)
|
|
|
|
|
if len(urls) > 1:
|
2022-10-07 12:45:44 +00:00
|
|
|
|
stri = "URLs in your clipboard\n"
|
|
|
|
|
counter = 0
|
2021-12-22 12:20:10 +00:00
|
|
|
|
for u in urls:
|
2022-10-07 12:45:44 +00:00
|
|
|
|
counter += 1
|
|
|
|
|
stri += "[%s] %s\n"%(counter,u)
|
|
|
|
|
stri += "Where do you want to go today ?> "
|
|
|
|
|
ans = input(stri)
|
|
|
|
|
if ans.isdigit() and 0 < int(ans) <= len(urls):
|
|
|
|
|
self.do_go(urls[int(ans)-1])
|
2021-12-22 12:20:10 +00:00
|
|
|
|
elif len(urls) == 1:
|
|
|
|
|
self.do_go(urls[0])
|
|
|
|
|
else:
|
2022-03-15 23:41:56 +00:00
|
|
|
|
print("Go where? (hint: simply copy an URL in your clipboard)")
|
2021-12-22 12:20:10 +00:00
|
|
|
|
else:
|
|
|
|
|
print("Go where? (hint: install xsel to go to copied URLs)")
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# First, check for possible marks
|
|
|
|
|
elif line in self.marks:
|
|
|
|
|
gi = self.marks[line]
|
|
|
|
|
self._go_to_gi(gi)
|
|
|
|
|
# or a local file
|
|
|
|
|
elif os.path.exists(os.path.expanduser(line)):
|
2021-11-18 11:02:00 +00:00
|
|
|
|
self._go_to_gi(GeminiItem(line))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# If this isn't a mark, treat it as a URL
|
2022-01-11 13:04:20 +00:00
|
|
|
|
elif looks_like_url(line):
|
2019-08-13 16:56:15 +00:00
|
|
|
|
self._go_to_gi(GeminiItem(line))
|
2022-03-22 22:17:47 +00:00
|
|
|
|
else:
|
|
|
|
|
print("%s is not a valid URL to go"%line)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_reload(self, *args):
|
|
|
|
|
"""Reload the current URL."""
|
2021-12-14 15:33:17 +00:00
|
|
|
|
if self.offline_only:
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.get_list("to_fetch")
|
2022-02-15 13:56:35 +00:00
|
|
|
|
r = self.list_add_line("to_fetch",gi=self.gi,verbose=False)
|
|
|
|
|
if r:
|
|
|
|
|
print("%s marked for syncing" %self.gi.url)
|
|
|
|
|
else:
|
|
|
|
|
print("%s already marked for syncing" %self.gi.url)
|
2021-12-14 15:33:17 +00:00
|
|
|
|
else:
|
|
|
|
|
self._go_to_gi(self.gi, check_cache=False)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_up(self, *args):
|
2022-03-08 21:21:44 +00:00
|
|
|
|
"""Go up one directory in the path.
|
|
|
|
|
Take an integer as argument to go up multiple times."""
|
|
|
|
|
level = 1
|
|
|
|
|
if args[0].isnumeric():
|
|
|
|
|
level = int(args[0])
|
|
|
|
|
elif args[0] != "":
|
|
|
|
|
print("Up only take integer as arguments")
|
|
|
|
|
self._go_to_gi(self.gi.up(level=level))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def do_back(self, *args):
|
|
|
|
|
"""Go back to the previous gemini item."""
|
2022-01-26 18:35:05 +00:00
|
|
|
|
histfile = self.get_list("history")
|
|
|
|
|
links = self.list_get_links("history")
|
|
|
|
|
if self.hist_index >= len(links) -1:
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return
|
2022-01-26 18:35:05 +00:00
|
|
|
|
self.hist_index += 1
|
|
|
|
|
gi = links[self.hist_index]
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self._go_to_gi(gi, update_hist=False)
|
|
|
|
|
|
|
|
|
|
def do_forward(self, *args):
|
|
|
|
|
"""Go forward to the next gemini item."""
|
2022-01-26 18:35:05 +00:00
|
|
|
|
histfile = self.get_list("history")
|
|
|
|
|
links = self.list_get_links("history")
|
|
|
|
|
if self.hist_index <= 0:
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return
|
2022-01-26 18:35:05 +00:00
|
|
|
|
self.hist_index -= 1
|
|
|
|
|
gi = links[self.hist_index]
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self._go_to_gi(gi, update_hist=False)
|
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_root(self, *args):
|
|
|
|
|
"""Go to root selector of the server hosting current item."""
|
2019-08-13 17:39:55 +00:00
|
|
|
|
self._go_to_gi(self.gi.root())
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def do_tour(self, line):
|
|
|
|
|
"""Add index items as waypoints on a tour, which is basically a FIFO
|
|
|
|
|
queue of gemini items.
|
|
|
|
|
|
2022-04-06 10:14:57 +00:00
|
|
|
|
`tour` or `t` alone brings you to the next item in your tour.
|
2019-06-22 12:58:21 +00:00
|
|
|
|
Items can be added with `tour 1 2 3 4` or ranges like `tour 1-4`.
|
|
|
|
|
All items in current menu can be added with `tour *`.
|
2022-09-25 21:33:36 +00:00
|
|
|
|
All items in $LIST can be added with `tour $LIST`.
|
2022-01-03 15:40:52 +00:00
|
|
|
|
Current item can be added back to the end of the tour with `tour .`.
|
2019-06-22 12:58:21 +00:00
|
|
|
|
Current tour can be listed with `tour ls` and scrubbed with `tour clear`."""
|
2022-01-23 22:14:06 +00:00
|
|
|
|
# Creating the tour list if needed
|
2022-01-23 16:09:05 +00:00
|
|
|
|
self.get_list("tour")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line:
|
|
|
|
|
# Fly to next waypoint on tour
|
2022-01-23 22:14:06 +00:00
|
|
|
|
if len(self.list_get_links("tour")) < 1:
|
2019-06-22 12:58:21 +00:00
|
|
|
|
print("End of tour.")
|
|
|
|
|
else:
|
2022-02-15 20:57:14 +00:00
|
|
|
|
url = self.list_go_to_line("1","tour")
|
|
|
|
|
if url:
|
|
|
|
|
self.list_rm_url(url,"tour")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif line == "ls":
|
2022-01-23 16:09:05 +00:00
|
|
|
|
self.list_show("tour")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif line == "clear":
|
2022-01-23 22:14:06 +00:00
|
|
|
|
for l in self.list_get_links("tour"):
|
2022-03-23 13:28:19 +00:00
|
|
|
|
self.list_rm_url(l.url_mode(),"tour")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif line == "*":
|
2022-10-07 12:45:44 +00:00
|
|
|
|
for l in self.gi.get_links():
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_add_line("tour",gi=l,verbose=False)
|
2022-01-03 15:40:52 +00:00
|
|
|
|
elif line == ".":
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_add_line("tour",verbose=False)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif looks_like_url(line):
|
2022-01-23 16:09:05 +00:00
|
|
|
|
self.list_add_line("tour",gi=GeminiItem(line))
|
2022-09-25 21:33:36 +00:00
|
|
|
|
elif line in self.list_lists():
|
|
|
|
|
list_path = self.list_path(line)
|
|
|
|
|
if not list_path:
|
|
|
|
|
print("List %s does not exist. Cannot add it to tour"%(list))
|
|
|
|
|
else:
|
|
|
|
|
gi = GeminiItem("list:///%s"%line)
|
|
|
|
|
display = not self.sync_only
|
|
|
|
|
if gi:
|
|
|
|
|
for l in gi.get_links():
|
|
|
|
|
self.list_add_line("tour",gi=l,verbose=False)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
for index in line.split():
|
|
|
|
|
try:
|
|
|
|
|
pair = index.split('-')
|
|
|
|
|
if len(pair) == 1:
|
|
|
|
|
# Just a single index
|
|
|
|
|
n = int(index)
|
2022-10-07 12:45:44 +00:00
|
|
|
|
gi = self.gi.get_link(n)
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_add_line("tour",gi=gi,verbose=False)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif len(pair) == 2:
|
|
|
|
|
# Two endpoints for a range of indices
|
2021-08-25 06:01:16 +00:00
|
|
|
|
if int(pair[0]) < int(pair[1]):
|
|
|
|
|
for n in range(int(pair[0]), int(pair[1]) + 1):
|
2022-10-07 12:45:44 +00:00
|
|
|
|
gi = self.gi.get_link(n)
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_add_line("tour",gi=gi,verbose=False)
|
2021-08-25 06:01:16 +00:00
|
|
|
|
else:
|
|
|
|
|
for n in range(int(pair[0]), int(pair[1]) - 1, -1):
|
2022-10-07 12:45:44 +00:00
|
|
|
|
gi = self.gi.get_link(n)
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_add_line("tour",gi=gi,verbose=False)
|
2021-08-25 06:01:16 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
# Syntax error
|
|
|
|
|
print("Invalid use of range syntax %s, skipping" % index)
|
|
|
|
|
except ValueError:
|
|
|
|
|
print("Non-numeric index %s, skipping." % index)
|
|
|
|
|
except IndexError:
|
|
|
|
|
print("Invalid index %d, skipping." % n)
|
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_mark(self, line):
|
|
|
|
|
"""Mark the current item with a single letter. This letter can then
|
|
|
|
|
be passed to the 'go' command to return to the current item later.
|
2022-03-08 21:21:44 +00:00
|
|
|
|
Think of it like marks in vi: 'mark a'='ma' and 'go a'=''a'.
|
|
|
|
|
Marks are temporary until shutdown (not saved to disk)."""
|
2019-06-22 12:58:21 +00:00
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line:
|
|
|
|
|
for mark, gi in self.marks.items():
|
2019-08-13 16:56:15 +00:00
|
|
|
|
print("[%s] %s (%s)" % (mark, gi.name, gi.url))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif line.isalpha() and len(line) == 1:
|
|
|
|
|
self.marks[line] = self.gi
|
|
|
|
|
else:
|
|
|
|
|
print("Invalid mark, must be one letter")
|
2022-02-18 14:52:27 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_info(self,line):
|
|
|
|
|
"""Display information about current page."""
|
2022-03-22 22:17:47 +00:00
|
|
|
|
out = self.gi.get_page_title() + "\n\n"
|
2022-02-18 14:52:27 +00:00
|
|
|
|
out += "URL : " + self.gi.url + "\n"
|
2022-02-19 21:09:57 +00:00
|
|
|
|
out += "Path : " + self.gi.path + "\n"
|
2022-02-18 14:52:27 +00:00
|
|
|
|
out += "Mime : " + self.gi.get_mime() + "\n"
|
|
|
|
|
out += "Cache : " + self.gi.get_cache_path() + "\n"
|
2022-03-15 23:13:45 +00:00
|
|
|
|
tmp = self.gi.get_temp_filename()
|
|
|
|
|
if tmp != self.gi.get_cache_path():
|
|
|
|
|
out += "Tempfile : " + self.gi.get_temp_filename() + "\n"
|
2022-02-18 14:52:27 +00:00
|
|
|
|
if self.gi.renderer :
|
|
|
|
|
rend = str(self.gi.renderer.__class__)
|
|
|
|
|
rend = rend.lstrip("<class '__main__.").rstrip("'>")
|
|
|
|
|
else:
|
2022-03-06 15:02:30 +00:00
|
|
|
|
rend = "None"
|
2022-02-19 20:16:47 +00:00
|
|
|
|
out += "Renderer : " + rend + "\n\n"
|
|
|
|
|
lists = []
|
|
|
|
|
for l in self.list_lists():
|
|
|
|
|
if self.list_has_url(self.gi.url,l):
|
|
|
|
|
lists.append(l)
|
|
|
|
|
if len(lists) > 0:
|
|
|
|
|
out += "Page appeard in following lists :\n"
|
|
|
|
|
for l in lists:
|
|
|
|
|
if not self.list_is_system(l):
|
|
|
|
|
status = "normal list"
|
|
|
|
|
if self.list_is_subscribed(l):
|
|
|
|
|
status = "subscription"
|
|
|
|
|
elif self.list_is_frozen(l):
|
|
|
|
|
status = "frozen list"
|
2022-03-19 19:58:18 +00:00
|
|
|
|
out += " • %s\t(%s)\n" %(l,status)
|
2022-02-19 20:16:47 +00:00
|
|
|
|
for l in lists:
|
|
|
|
|
if self.list_is_system(l):
|
2022-03-19 19:58:18 +00:00
|
|
|
|
out += " • %s\n" %l
|
2022-02-19 20:16:47 +00:00
|
|
|
|
else:
|
|
|
|
|
out += "Page is not save in any list"
|
2022-02-18 14:52:27 +00:00
|
|
|
|
print(out)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2020-05-10 12:34:48 +00:00
|
|
|
|
def do_version(self, line):
|
2022-02-14 22:02:53 +00:00
|
|
|
|
"""Display version and system information."""
|
|
|
|
|
def has(value):
|
|
|
|
|
if value:
|
|
|
|
|
return "\t\x1b[1;32mInstalled\x1b[0m\n"
|
|
|
|
|
else:
|
|
|
|
|
return "\t\x1b[1;31mNot Installed\x1b[0m\n"
|
|
|
|
|
output = "Offpunk " + _VERSION + "\n"
|
|
|
|
|
output += "===========\n"
|
2022-03-29 11:28:20 +00:00
|
|
|
|
output += "Highly recommended:\n"
|
2022-02-14 22:02:53 +00:00
|
|
|
|
output += " - python-cryptography : " + has(_HAS_CRYPTOGRAPHY)
|
2022-03-29 11:28:20 +00:00
|
|
|
|
output += " - xdg-open : " + has(_HAS_XDGOPEN)
|
|
|
|
|
output += "\nWeb browsing:\n"
|
2022-02-14 22:02:53 +00:00
|
|
|
|
output += " - python-requests : " + has(_DO_HTTP)
|
|
|
|
|
output += " - python-feedparser : " + has(_DO_FEED)
|
|
|
|
|
output += " - python-bs4 : " + has(_HAS_SOUP)
|
|
|
|
|
output += " - python-readability : " + has(_HAS_READABILITY)
|
2022-10-08 21:20:30 +00:00
|
|
|
|
output += " - timg 1.3.2+ : " + has(_NEW_TIMG)
|
2022-03-13 15:01:12 +00:00
|
|
|
|
if _NEW_CHAFA:
|
|
|
|
|
output += " - chafa 1.10+ : " + has(_HAS_CHAFA)
|
|
|
|
|
else:
|
|
|
|
|
output += " - chafa : " + has(_HAS_CHAFA)
|
|
|
|
|
output += " - python-pil : " + has(_HAS_PIL)
|
2022-03-29 11:28:20 +00:00
|
|
|
|
output += "\nNice to have:\n"
|
|
|
|
|
output += " - python-setproctitle : " + has(_HAS_SETPROCTITLE)
|
|
|
|
|
output += " - xsel : " + has(_HAS_XSEL)
|
2022-02-14 22:02:53 +00:00
|
|
|
|
|
|
|
|
|
output += "\nFeatures :\n"
|
2022-03-22 08:07:37 +00:00
|
|
|
|
if _NEW_CHAFA:
|
|
|
|
|
output += " - Render images (chafa or timg) : " + has(_RENDER_IMAGE)
|
|
|
|
|
else:
|
|
|
|
|
output += " - Render images (python-pil, chafa or timg) : " + has(_RENDER_IMAGE)
|
2022-03-21 21:43:33 +00:00
|
|
|
|
output += " - Render HTML (bs4, readability) : " + has(_DO_HTML)
|
|
|
|
|
output += " - Render Atom/RSS feeds (feedparser) : " + has(_DO_FEED)
|
|
|
|
|
output += " - Connect to http/https (requests) : " + has(_DO_HTTP)
|
|
|
|
|
output += " - copy to/from clipboard (xsel) : " + has(_HAS_XSEL)
|
|
|
|
|
output += " - restore last position (less 572+) : " + has(_LESS_RESTORE_POSITION)
|
2022-02-14 22:02:53 +00:00
|
|
|
|
output += "\n"
|
|
|
|
|
output += "Config directory : " + _CONFIG_DIR + "\n"
|
|
|
|
|
output += "User Data directory : " + _DATA_DIR + "\n"
|
2022-02-20 22:22:25 +00:00
|
|
|
|
output += "Cache directoy : " + _CACHE_PATH
|
2022-02-14 22:02:53 +00:00
|
|
|
|
|
|
|
|
|
print(output)
|
2020-05-10 12:34:48 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
### Stuff that modifies the lookup table
|
|
|
|
|
def do_ls(self, line):
|
|
|
|
|
"""List contents of current index.
|
|
|
|
|
Use 'ls -l' to see URLs."""
|
|
|
|
|
self._show_lookup(url = "-l" in line)
|
|
|
|
|
self.page_index = 0
|
|
|
|
|
|
2022-11-14 20:33:40 +00:00
|
|
|
|
def do_search(self,line):
|
|
|
|
|
"""Search on Gemini using the engine configured (by default kennedy.gemi.dev)
|
|
|
|
|
You can configure it using "set search URL".
|
|
|
|
|
URL should contains one "%s" that will be replaced by the search term."""
|
|
|
|
|
search = urllib.parse.quote(line)
|
|
|
|
|
url = self.options["search"]%search
|
|
|
|
|
gi = GeminiItem(url)
|
2022-11-15 16:01:13 +00:00
|
|
|
|
self._go_to_gi(gi)
|
2022-11-14 20:33:40 +00:00
|
|
|
|
|
|
|
|
|
def do_wikipedia(self,line):
|
|
|
|
|
"""Search on wikipedia using the configured Gemini interface.
|
|
|
|
|
The first word should be the two letters code for the language.
|
|
|
|
|
Exemple : "wikipedia en Gemini protocol"
|
|
|
|
|
But you can also use abbreviations to go faster:
|
|
|
|
|
"wen Gemini protocol". (your abbreviation might be missing, report the bug)
|
|
|
|
|
The interface used can be modified with the command:
|
|
|
|
|
"set wikipedia URL" where URL should contains two "%s", the first
|
|
|
|
|
one used for the language, the second for the search string."""
|
|
|
|
|
words = line.split(" ",maxsplit=1)
|
|
|
|
|
if len(words[0]) == 2:
|
|
|
|
|
lang = words[0]
|
|
|
|
|
search = urllib.parse.quote(words[1])
|
|
|
|
|
else:
|
|
|
|
|
lang = "en"
|
|
|
|
|
search = urllib.parse.quote(line)
|
|
|
|
|
url = self.options["wikipedia"]%(lang,search)
|
|
|
|
|
gi = GeminiItem(url)
|
|
|
|
|
self._go_to_gi(gi)
|
|
|
|
|
|
2020-05-15 11:38:51 +00:00
|
|
|
|
def do_gus(self, line):
|
2021-04-27 07:55:12 +00:00
|
|
|
|
"""Submit a search query to the geminispace.info search engine."""
|
|
|
|
|
gus = GeminiItem("gemini://geminispace.info/search")
|
2020-05-15 11:38:51 +00:00
|
|
|
|
self._go_to_gi(gus.query(line))
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def do_history(self, *args):
|
|
|
|
|
"""Display history."""
|
2022-01-26 18:35:05 +00:00
|
|
|
|
self.list_show("history")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2022-03-25 12:37:23 +00:00
|
|
|
|
@needs_gi
|
2022-01-24 16:04:03 +00:00
|
|
|
|
def do_find(self, searchterm):
|
2022-03-25 12:37:23 +00:00
|
|
|
|
"""Find in current page by displaying only relevant lines (grep)."""
|
|
|
|
|
self.gi.display(grep=searchterm)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
def emptyline(self):
|
|
|
|
|
"""Page through index ten lines at a time."""
|
|
|
|
|
i = self.page_index
|
2022-10-07 12:45:44 +00:00
|
|
|
|
if not self.gi or i > len(self.gi.get_links()):
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return
|
|
|
|
|
self._show_lookup(offset=i, end=i+10)
|
|
|
|
|
self.page_index += 10
|
|
|
|
|
|
|
|
|
|
### Stuff that does something to most recently viewed item
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_cat(self, *args):
|
|
|
|
|
"""Run most recently visited item through "cat" command."""
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run("cat", input=open(self.gi.get_temp_filename(), "rb"), direct_output=True)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
2022-03-07 22:16:17 +00:00
|
|
|
|
def do_view(self, *args):
|
2022-02-17 16:00:24 +00:00
|
|
|
|
"""Run most recently visited item through "less" command, restoring \
|
|
|
|
|
previous position.
|
2022-03-23 15:19:23 +00:00
|
|
|
|
Use "view normal" to see the default article view on html page.
|
2022-03-07 22:16:17 +00:00
|
|
|
|
Use "view full" to see a complete html page instead of the article view.
|
|
|
|
|
Use "view feed" to see the the linked feed of the page (in any).
|
|
|
|
|
Use "view feeds" to see available feeds on this page.
|
|
|
|
|
(full, feed, feeds have no effect on non-html content)."""
|
|
|
|
|
if self.gi and args and args[0] != "":
|
2022-03-19 16:44:28 +00:00
|
|
|
|
if args[0] in ["full","debug"]:
|
|
|
|
|
self._go_to_gi(self.gi,mode=args[0])
|
2022-03-23 15:19:23 +00:00
|
|
|
|
elif args[0] in ["normal","readable"]:
|
|
|
|
|
self._go_to_gi(self.gi,mode="readable")
|
2022-03-07 22:16:17 +00:00
|
|
|
|
elif args[0] == "feed":
|
|
|
|
|
subs = self.gi.get_subscribe_links()
|
|
|
|
|
if len(subs) > 1:
|
|
|
|
|
self.do_go(subs[1][0])
|
2022-03-13 15:01:12 +00:00
|
|
|
|
elif "rss" in subs[0][1] or "atom" in subs[0][1]:
|
|
|
|
|
print("%s is already a feed" %self.gi.url)
|
2022-03-07 22:16:17 +00:00
|
|
|
|
else:
|
|
|
|
|
print("No other feed found on %s"%self.gi.url)
|
|
|
|
|
elif args[0] == "feeds":
|
|
|
|
|
subs = self.gi.get_subscribe_links()
|
2022-10-07 12:45:44 +00:00
|
|
|
|
stri = "Available views :\n"
|
2022-03-07 22:16:17 +00:00
|
|
|
|
counter = 0
|
|
|
|
|
for s in subs:
|
|
|
|
|
counter += 1
|
|
|
|
|
stri += "[%s] %s [%s]\n"%(counter,s[0],s[1])
|
|
|
|
|
stri += "Which view do you want to see ? >"
|
|
|
|
|
ans = input(stri)
|
|
|
|
|
if ans.isdigit() and 0 < int(ans) <= len(subs):
|
|
|
|
|
self.do_go(subs[int(ans)-1][0])
|
|
|
|
|
else:
|
2022-03-23 15:19:23 +00:00
|
|
|
|
print("Valid argument for view are : normal, full, feed, feeds")
|
2022-01-25 10:49:31 +00:00
|
|
|
|
else:
|
2022-03-23 15:19:23 +00:00
|
|
|
|
self._go_to_gi(self.gi)
|
|
|
|
|
|
2022-02-12 13:24:43 +00:00
|
|
|
|
@needs_gi
|
|
|
|
|
def do_open(self, *args):
|
|
|
|
|
"""Open current item with the configured handler or xdg-open.
|
2022-04-29 08:41:08 +00:00
|
|
|
|
Uses "open url" to open current URL in a browser.
|
|
|
|
|
see "handler" command to set your handler."""
|
|
|
|
|
if args[0] == "url":
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run("xdg-open %s", parameter=self.gi.url, direct_output=True)
|
2022-04-29 08:41:08 +00:00
|
|
|
|
else:
|
|
|
|
|
cmd_str = self._get_handler_cmd(self.gi.get_mime())
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run(cmd_str, parameter=self.gi.get_body(as_file=True), direct_output=True)
|
2022-02-12 13:24:43 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
@needs_gi
|
|
|
|
|
def do_shell(self, line):
|
2022-03-08 21:21:44 +00:00
|
|
|
|
"""'cat' most recently visited item through a shell pipeline.
|
|
|
|
|
'!' is an useful shortcut."""
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
run(line, input=open(self.gi.get_temp_filename(), "rb"), direct_output=True)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_save(self, line):
|
|
|
|
|
"""Save an item to the filesystem.
|
|
|
|
|
'save n filename' saves menu item n to the specified filename.
|
|
|
|
|
'save filename' saves the last viewed item to the specified filename.
|
|
|
|
|
'save n' saves menu item n to an automagic filename."""
|
|
|
|
|
args = line.strip().split()
|
|
|
|
|
|
|
|
|
|
# First things first, figure out what our arguments are
|
|
|
|
|
if len(args) == 0:
|
|
|
|
|
# No arguments given at all
|
|
|
|
|
# Save current item, if there is one, to a file whose name is
|
|
|
|
|
# inferred from the gemini path
|
2022-01-08 20:32:25 +00:00
|
|
|
|
if not self.gi.is_cache_valid():
|
2022-01-01 21:05:02 +00:00
|
|
|
|
print("You cannot save if not cached!")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
return
|
|
|
|
|
else:
|
|
|
|
|
index = None
|
|
|
|
|
filename = None
|
|
|
|
|
elif len(args) == 1:
|
|
|
|
|
# One argument given
|
|
|
|
|
# If it's numeric, treat it as an index, and infer the filename
|
|
|
|
|
try:
|
|
|
|
|
index = int(args[0])
|
|
|
|
|
filename = None
|
|
|
|
|
# If it's not numeric, treat it as a filename and
|
|
|
|
|
# save the current item
|
|
|
|
|
except ValueError:
|
|
|
|
|
index = None
|
|
|
|
|
filename = os.path.expanduser(args[0])
|
|
|
|
|
elif len(args) == 2:
|
|
|
|
|
# Two arguments given
|
|
|
|
|
# Treat first as an index and second as filename
|
|
|
|
|
index, filename = args
|
|
|
|
|
try:
|
|
|
|
|
index = int(index)
|
|
|
|
|
except ValueError:
|
|
|
|
|
print("First argument is not a valid item index!")
|
|
|
|
|
return
|
|
|
|
|
filename = os.path.expanduser(filename)
|
|
|
|
|
else:
|
|
|
|
|
print("You must provide an index, a filename, or both.")
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
# Next, fetch the item to save, if it's not the current one.
|
|
|
|
|
if index:
|
|
|
|
|
last_gi = self.gi
|
|
|
|
|
try:
|
2022-10-07 12:45:44 +00:00
|
|
|
|
gi = self.gi.get_link(index)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
self._go_to_gi(gi, update_hist = False, handle = False)
|
|
|
|
|
except IndexError:
|
|
|
|
|
print ("Index too high!")
|
|
|
|
|
self.gi = last_gi
|
|
|
|
|
return
|
|
|
|
|
else:
|
|
|
|
|
gi = self.gi
|
|
|
|
|
|
|
|
|
|
# Derive filename from current GI's path, if one hasn't been set
|
|
|
|
|
if not filename:
|
2022-01-08 20:46:57 +00:00
|
|
|
|
filename = gi.get_filename()
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Check for filename collisions and actually do the save if safe
|
|
|
|
|
if os.path.exists(filename):
|
|
|
|
|
print("File %s already exists!" % filename)
|
|
|
|
|
else:
|
|
|
|
|
# Don't use _get_active_tmpfile() here, because we want to save the
|
2021-12-30 15:03:08 +00:00
|
|
|
|
# "source code" of menus, not the rendered view - this way Offpunk
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# can navigate to it later.
|
2022-03-19 11:17:02 +00:00
|
|
|
|
path = gi.get_body(as_file=True)
|
|
|
|
|
if os.path.isdir(path):
|
|
|
|
|
print("Can’t save %s because it’s a folder, not a file"%path)
|
|
|
|
|
else:
|
|
|
|
|
print("Saved to %s" % filename)
|
|
|
|
|
shutil.copyfile(path, filename)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
# Restore gi if necessary
|
|
|
|
|
if index != None:
|
|
|
|
|
self._go_to_gi(last_gi, handle=False)
|
|
|
|
|
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_url(self, *args):
|
|
|
|
|
"""Print URL of most recently visited item."""
|
2019-08-13 16:56:15 +00:00
|
|
|
|
print(self.gi.url)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
### Bookmarking stuff
|
|
|
|
|
@needs_gi
|
|
|
|
|
def do_add(self, line):
|
2022-01-21 16:42:14 +00:00
|
|
|
|
"""Add the current URL to the list specied as argument.
|
|
|
|
|
If no argument given, URL is added to Bookmarks."""
|
|
|
|
|
args = line.split()
|
|
|
|
|
if len(args) < 1 :
|
2022-03-31 09:00:55 +00:00
|
|
|
|
list = "bookmarks"
|
|
|
|
|
if not self.list_path(list):
|
|
|
|
|
self.list_create(list)
|
|
|
|
|
self.list_add_line(list)
|
2022-01-21 16:42:14 +00:00
|
|
|
|
else:
|
|
|
|
|
self.list_add_line(args[0])
|
2022-01-23 16:09:05 +00:00
|
|
|
|
|
|
|
|
|
# Get the list file name, creating or migrating it if needed.
|
|
|
|
|
# Migrate bookmarks/tour/to_fetch from XDG_CONFIG to XDG_DATA
|
|
|
|
|
# We migrate only if the file exists in XDG_CONFIG and not XDG_DATA
|
|
|
|
|
def get_list(self,list):
|
|
|
|
|
list_path = self.list_path(list)
|
|
|
|
|
if not list_path:
|
|
|
|
|
old_file_gmi = os.path.join(_CONFIG_DIR,list + ".gmi")
|
|
|
|
|
old_file_nogmi = os.path.join(_CONFIG_DIR,list)
|
|
|
|
|
target = os.path.join(_DATA_DIR,"lists")
|
|
|
|
|
if os.path.exists(old_file_gmi):
|
|
|
|
|
shutil.move(old_file_gmi,target)
|
|
|
|
|
elif os.path.exists(old_file_nogmi):
|
|
|
|
|
targetgmi = os.path.join(target,list+".gmi")
|
|
|
|
|
shutil.move(old_file_nogmi,targetgmi)
|
|
|
|
|
else:
|
2022-01-24 14:33:33 +00:00
|
|
|
|
if list == "subscribed":
|
2022-02-15 13:56:35 +00:00
|
|
|
|
title = "Subscriptions #subscribed (new links in those pages will be added to tour)"
|
2022-01-24 14:33:33 +00:00
|
|
|
|
elif list == "to_fetch":
|
|
|
|
|
title = "Links requested and to be fetched during the next --sync"
|
|
|
|
|
else:
|
|
|
|
|
title = None
|
2022-11-05 22:14:41 +00:00
|
|
|
|
self.list_create(list, title=title,quite=True)
|
2022-01-23 16:09:05 +00:00
|
|
|
|
list_path = self.list_path(list)
|
|
|
|
|
return list_path
|
2022-01-23 22:14:06 +00:00
|
|
|
|
|
|
|
|
|
def do_subscribe(self,line):
|
|
|
|
|
"""Subscribe to current page by saving it in the "subscribed" list.
|
|
|
|
|
If a new link is found in the page during a --sync, the new link is automatically
|
|
|
|
|
fetched and added to your next tour.
|
|
|
|
|
To unsubscribe, remove the page from the "subscribed" list."""
|
2022-02-18 13:11:09 +00:00
|
|
|
|
subs = self.gi.get_subscribe_links()
|
|
|
|
|
if len(subs) > 1:
|
2022-02-27 21:20:42 +00:00
|
|
|
|
stri = "Multiple feeds have been found :\n"
|
2022-03-13 15:01:12 +00:00
|
|
|
|
elif "rss" in subs[0][1] or "atom" in subs[0][1] :
|
|
|
|
|
stri = "This page is already a feed:\n"
|
2022-02-27 21:20:42 +00:00
|
|
|
|
else:
|
|
|
|
|
stri = "No feed detected. You can still watch the page :\n"
|
|
|
|
|
counter = 0
|
|
|
|
|
for l in subs:
|
2022-03-09 10:34:41 +00:00
|
|
|
|
link = l[0]
|
2022-02-27 21:32:25 +00:00
|
|
|
|
already = []
|
|
|
|
|
for li in self.list_lists():
|
|
|
|
|
if self.list_is_subscribed(li):
|
|
|
|
|
if self.list_has_url(link,li):
|
|
|
|
|
already.append(li)
|
2022-02-27 21:20:42 +00:00
|
|
|
|
stri += "[%s] %s [%s]\n"%(counter+1,link,l[1])
|
2022-02-27 21:32:25 +00:00
|
|
|
|
if len(already) > 0:
|
|
|
|
|
stri += "\t -> (already subscribed through lists %s)\n"%(str(already))
|
2022-02-27 21:20:42 +00:00
|
|
|
|
counter += 1
|
|
|
|
|
stri += "\n"
|
2022-07-18 12:53:49 +00:00
|
|
|
|
stri += "Which feed do you want to subscribe ? > "
|
2022-02-27 21:20:42 +00:00
|
|
|
|
ans = input(stri)
|
|
|
|
|
if ans.isdigit() and 0 < int(ans) <= len(subs):
|
|
|
|
|
sublink,mime,title = subs[int(ans)-1]
|
2022-01-24 14:33:33 +00:00
|
|
|
|
else:
|
2022-02-27 21:20:42 +00:00
|
|
|
|
sublink,title = None,None
|
2022-02-18 13:11:09 +00:00
|
|
|
|
if sublink:
|
2022-02-23 11:06:53 +00:00
|
|
|
|
sublink = self.gi.absolutise_url(sublink)
|
2022-02-18 13:11:09 +00:00
|
|
|
|
gi = GeminiItem(sublink,name=title)
|
|
|
|
|
list_path = self.get_list("subscribed")
|
|
|
|
|
added = self.list_add_line("subscribed",gi=gi,verbose=False)
|
|
|
|
|
if added :
|
|
|
|
|
print("Subscribed to %s" %sublink)
|
|
|
|
|
else:
|
|
|
|
|
print("You are already subscribed to %s"%sublink)
|
2022-02-27 21:20:42 +00:00
|
|
|
|
else:
|
|
|
|
|
print("No subscription registered")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2020-03-24 19:41:37 +00:00
|
|
|
|
def do_bookmarks(self, line):
|
|
|
|
|
"""Show or access the bookmarks menu.
|
|
|
|
|
'bookmarks' shows all bookmarks.
|
|
|
|
|
'bookmarks n' navigates immediately to item n in the bookmark menu.
|
2020-05-10 20:51:33 +00:00
|
|
|
|
Bookmarks are stored using the 'add' command."""
|
2022-01-23 16:09:05 +00:00
|
|
|
|
list_path = self.get_list("bookmarks")
|
2020-03-24 19:41:37 +00:00
|
|
|
|
args = line.strip()
|
|
|
|
|
if len(args.split()) > 1 or (args and not args.isnumeric()):
|
|
|
|
|
print("bookmarks command takes a single integer argument!")
|
2022-01-23 12:52:34 +00:00
|
|
|
|
elif args:
|
|
|
|
|
self.list_go_to_line(args,"bookmarks")
|
|
|
|
|
else:
|
|
|
|
|
self.list_show("bookmarks")
|
2022-01-17 13:32:46 +00:00
|
|
|
|
|
2022-01-26 17:52:38 +00:00
|
|
|
|
def do_archive(self,args):
|
|
|
|
|
"""Archive current page by removing it from every list and adding it to
|
|
|
|
|
archives, which is a special historical list limited in size. It is similar to `move archives`."""
|
|
|
|
|
for li in self.list_lists():
|
|
|
|
|
if li not in ["archives", "history"]:
|
2022-03-23 13:28:19 +00:00
|
|
|
|
deleted = self.list_rm_url(self.gi.url_mode(),li)
|
2022-01-26 17:52:38 +00:00
|
|
|
|
if deleted:
|
|
|
|
|
print("Removed from %s"%li)
|
|
|
|
|
self.list_add_top("archives",limit=self.options["archives_size"])
|
2022-03-22 22:17:47 +00:00
|
|
|
|
print("Archiving: %s"%self.gi.get_page_title())
|
2022-02-04 23:26:51 +00:00
|
|
|
|
print("\x1b[2;34mCurrent maximum size of archives : %s\x1b[0m" %self.options["archives_size"])
|
2022-01-26 17:52:38 +00:00
|
|
|
|
|
2022-01-23 22:14:06 +00:00
|
|
|
|
def list_add_line(self,list,gi=None,verbose=True):
|
2022-01-22 14:08:06 +00:00
|
|
|
|
list_path = self.list_path(list)
|
2022-11-05 22:14:41 +00:00
|
|
|
|
if not list_path and self.list_is_system(list):
|
|
|
|
|
self.list_create(list,quite=True)
|
2022-11-09 22:30:54 +00:00
|
|
|
|
list_path = self.list_path(list)
|
|
|
|
|
if not list_path:
|
2022-01-17 13:32:46 +00:00
|
|
|
|
print("List %s does not exist. Create it with ""list create %s"""%(list,list))
|
2022-01-22 14:08:06 +00:00
|
|
|
|
return False
|
2022-01-17 13:32:46 +00:00
|
|
|
|
else:
|
2022-01-23 16:09:05 +00:00
|
|
|
|
if not gi:
|
|
|
|
|
gi = self.gi
|
2022-01-22 14:08:06 +00:00
|
|
|
|
# first we check if url already exists in the file
|
|
|
|
|
with open(list_path,"r") as l_file:
|
|
|
|
|
lines = l_file.readlines()
|
|
|
|
|
l_file.close()
|
|
|
|
|
for l in lines:
|
|
|
|
|
sp = l.split()
|
2022-03-23 13:28:19 +00:00
|
|
|
|
if gi.url_mode() in sp:
|
2022-01-23 22:14:06 +00:00
|
|
|
|
if verbose:
|
|
|
|
|
print("%s already in %s."%(gi.url,list))
|
2022-01-22 14:08:06 +00:00
|
|
|
|
return False
|
2022-01-21 16:42:14 +00:00
|
|
|
|
with open(list_path,"a") as l_file:
|
2022-01-23 16:09:05 +00:00
|
|
|
|
l_file.write(gi.to_map_line())
|
2022-01-17 13:32:46 +00:00
|
|
|
|
l_file.close()
|
2022-01-23 22:14:06 +00:00
|
|
|
|
if verbose:
|
|
|
|
|
print("%s added to %s" %(gi.url,list))
|
2022-01-22 14:08:06 +00:00
|
|
|
|
return True
|
2022-01-26 17:52:38 +00:00
|
|
|
|
|
2022-01-26 18:35:05 +00:00
|
|
|
|
def list_add_top(self,list,limit=0,truncate_lines=0):
|
2022-01-28 12:25:55 +00:00
|
|
|
|
if not self.gi:
|
|
|
|
|
return
|
2022-01-30 18:11:23 +00:00
|
|
|
|
stri = self.gi.to_map_line().strip("\n")
|
2022-01-26 17:52:38 +00:00
|
|
|
|
if list == "archives":
|
|
|
|
|
stri += ", archived on "
|
|
|
|
|
elif list == "history":
|
|
|
|
|
stri += ", visited on "
|
|
|
|
|
else:
|
|
|
|
|
stri += ", added to %s on "%list
|
|
|
|
|
stri += time.ctime() + "\n"
|
|
|
|
|
list_path = self.get_list(list)
|
|
|
|
|
with open(list_path,"r") as l_file:
|
|
|
|
|
lines = l_file.readlines()
|
|
|
|
|
l_file.close()
|
|
|
|
|
with open(list_path,"w") as l_file:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
l_file.write("#%s\n"%list)
|
2022-01-26 17:52:38 +00:00
|
|
|
|
l_file.write(stri)
|
|
|
|
|
counter = 0
|
2022-03-22 22:17:47 +00:00
|
|
|
|
# Truncating is useful in case we open a new branch
|
|
|
|
|
# after a few back in history
|
2022-01-26 18:35:05 +00:00
|
|
|
|
to_truncate = truncate_lines
|
2022-01-26 17:52:38 +00:00
|
|
|
|
for l in lines:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
if not l.startswith("#"):
|
|
|
|
|
if to_truncate > 0:
|
|
|
|
|
to_truncate -= 1
|
|
|
|
|
elif limit == 0 or counter < limit:
|
|
|
|
|
l_file.write(l)
|
|
|
|
|
counter += 1
|
2022-01-26 17:52:38 +00:00
|
|
|
|
l_file.close()
|
|
|
|
|
|
2022-01-17 13:32:46 +00:00
|
|
|
|
|
2022-01-22 14:08:06 +00:00
|
|
|
|
# remove an url from a list.
|
|
|
|
|
# return True if the URL was removed
|
|
|
|
|
# return False if the URL was not found
|
|
|
|
|
def list_rm_url(self,url,list):
|
2022-02-19 20:16:47 +00:00
|
|
|
|
return self.list_has_url(url,list,deletion=True)
|
|
|
|
|
|
|
|
|
|
# deletion and has_url are so similar, I made them the same method
|
|
|
|
|
def list_has_url(self,url,list,deletion=False):
|
2022-01-22 14:08:06 +00:00
|
|
|
|
list_path = self.list_path(list)
|
|
|
|
|
if list_path:
|
|
|
|
|
to_return = False
|
|
|
|
|
with open(list_path,"r") as lf:
|
|
|
|
|
lines = lf.readlines()
|
|
|
|
|
lf.close()
|
2022-02-19 20:16:47 +00:00
|
|
|
|
to_write = []
|
2022-04-10 21:45:34 +00:00
|
|
|
|
# let’s remove the mode
|
|
|
|
|
url = url.split("##offpunk_mode=")[0]
|
2022-02-19 20:16:47 +00:00
|
|
|
|
for l in lines:
|
|
|
|
|
# we separate components of the line
|
|
|
|
|
# to ensure we identify a complete URL, not a part of it
|
|
|
|
|
splitted = l.split()
|
2022-04-12 13:03:26 +00:00
|
|
|
|
if url not in splitted and len(splitted) > 1:
|
2022-04-10 21:45:34 +00:00
|
|
|
|
current = splitted[1].split("##offpunk_mode=")[0]
|
2022-02-19 20:16:47 +00:00
|
|
|
|
#sometimes, we must remove the ending "/"
|
2022-04-10 21:45:34 +00:00
|
|
|
|
if url == current:
|
|
|
|
|
to_return = True
|
|
|
|
|
elif url.endswith("/") and url[:-1] == current:
|
2022-01-22 14:08:06 +00:00
|
|
|
|
to_return = True
|
2022-02-19 20:16:47 +00:00
|
|
|
|
else:
|
|
|
|
|
to_write.append(l)
|
|
|
|
|
else:
|
|
|
|
|
to_return = True
|
|
|
|
|
if deletion :
|
|
|
|
|
with open(list_path,"w") as lf:
|
|
|
|
|
for l in to_write:
|
|
|
|
|
lf.write(l)
|
|
|
|
|
lf.close()
|
2022-01-22 14:08:06 +00:00
|
|
|
|
return to_return
|
|
|
|
|
else:
|
|
|
|
|
return False
|
2022-01-17 13:32:46 +00:00
|
|
|
|
|
2022-01-23 22:14:06 +00:00
|
|
|
|
def list_get_links(self,list):
|
2022-01-23 16:09:05 +00:00
|
|
|
|
list_path = self.list_path(list)
|
|
|
|
|
if list_path:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
gi = GeminiItem("list:///%s"%list)
|
2022-01-23 22:14:06 +00:00
|
|
|
|
return gi.get_links()
|
|
|
|
|
else:
|
|
|
|
|
return []
|
2022-01-23 16:09:05 +00:00
|
|
|
|
|
2022-01-18 21:19:43 +00:00
|
|
|
|
def list_go_to_line(self,line,list):
|
2022-01-22 14:08:06 +00:00
|
|
|
|
list_path = self.list_path(list)
|
|
|
|
|
if not list_path:
|
2022-01-18 21:19:43 +00:00
|
|
|
|
print("List %s does not exist. Create it with ""list create %s"""%(list,list))
|
|
|
|
|
elif not line.isnumeric():
|
|
|
|
|
print("go_to_line requires a number as parameter")
|
|
|
|
|
else:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
gi = GeminiItem("list:///%s"%list)
|
2022-01-23 12:52:34 +00:00
|
|
|
|
gi = gi.get_link(int(line))
|
2022-03-07 14:36:16 +00:00
|
|
|
|
display = not self.sync_only
|
|
|
|
|
if gi:
|
|
|
|
|
self._go_to_gi(gi,handle=display)
|
2022-03-23 13:28:19 +00:00
|
|
|
|
return gi.url_mode()
|
2022-01-18 21:19:43 +00:00
|
|
|
|
|
2022-01-17 13:32:46 +00:00
|
|
|
|
def list_show(self,list):
|
2022-01-22 14:08:06 +00:00
|
|
|
|
list_path = self.list_path(list)
|
|
|
|
|
if not list_path:
|
2022-01-17 13:32:46 +00:00
|
|
|
|
print("List %s does not exist. Create it with ""list create %s"""%(list,list))
|
|
|
|
|
else:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
gi = GeminiItem("list:///%s"%list)
|
2022-01-18 21:19:43 +00:00
|
|
|
|
display = not self.sync_only
|
2022-01-18 17:04:29 +00:00
|
|
|
|
self._go_to_gi(gi,handle=display)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2022-01-22 14:08:06 +00:00
|
|
|
|
#return the path of the list file if list exists.
|
|
|
|
|
#return None if the list doesn’t exist.
|
|
|
|
|
def list_path(self,list):
|
2022-01-18 17:04:29 +00:00
|
|
|
|
listdir = os.path.join(_DATA_DIR,"lists")
|
2022-01-17 13:32:46 +00:00
|
|
|
|
list_path = os.path.join(listdir, "%s.gmi"%list)
|
2022-01-22 14:08:06 +00:00
|
|
|
|
if os.path.exists(list_path):
|
|
|
|
|
return list_path
|
|
|
|
|
else:
|
|
|
|
|
return None
|
|
|
|
|
|
2022-11-05 22:14:41 +00:00
|
|
|
|
def list_create(self,list,title=None,quite=False):
|
2022-01-22 14:08:06 +00:00
|
|
|
|
list_path = self.list_path(list)
|
2022-02-15 15:50:33 +00:00
|
|
|
|
if list in ["create","edit","delete","help"]:
|
2022-01-22 14:08:06 +00:00
|
|
|
|
print("%s is not allowed as a name for a list"%list)
|
|
|
|
|
elif not list_path:
|
|
|
|
|
listdir = os.path.join(_DATA_DIR,"lists")
|
|
|
|
|
os.makedirs(listdir,exist_ok=True)
|
|
|
|
|
list_path = os.path.join(listdir, "%s.gmi"%list)
|
2022-01-21 16:42:14 +00:00
|
|
|
|
with open(list_path,"a") as lfile:
|
2022-01-17 13:32:46 +00:00
|
|
|
|
if title:
|
2022-01-21 16:42:14 +00:00
|
|
|
|
lfile.write("# %s\n"%title)
|
2022-01-17 13:32:46 +00:00
|
|
|
|
else:
|
2022-01-21 16:42:14 +00:00
|
|
|
|
lfile.write("# %s\n"%list)
|
2022-01-17 13:32:46 +00:00
|
|
|
|
lfile.close()
|
2022-11-05 22:14:41 +00:00
|
|
|
|
if not quite:
|
|
|
|
|
print("list created. Display with `list %s`"%list)
|
2022-01-21 16:42:14 +00:00
|
|
|
|
else:
|
|
|
|
|
print("list %s already exists" %list)
|
2022-01-22 14:08:06 +00:00
|
|
|
|
|
|
|
|
|
def do_move(self,arg):
|
|
|
|
|
"""move LIST will add the current page to the list LIST.
|
|
|
|
|
With a major twist: current page will be removed from all other lists.
|
|
|
|
|
If current page was not in a list, this command is similar to `add LIST`."""
|
|
|
|
|
if not arg:
|
|
|
|
|
print("LIST argument is required as the target for your move")
|
2022-01-26 17:52:38 +00:00
|
|
|
|
elif arg[0] == "archives":
|
|
|
|
|
self.do_archive()
|
2022-01-22 14:08:06 +00:00
|
|
|
|
else:
|
|
|
|
|
args = arg.split()
|
|
|
|
|
list_path = self.list_path(args[0])
|
|
|
|
|
if not list_path:
|
|
|
|
|
print("%s is not a list, aborting the move" %args[0])
|
|
|
|
|
else:
|
2022-01-26 17:52:38 +00:00
|
|
|
|
lists = self.list_lists()
|
2022-01-22 14:08:06 +00:00
|
|
|
|
for l in lists:
|
2022-01-26 17:52:38 +00:00
|
|
|
|
if l != args[0] and l not in ["archives", "history"]:
|
2022-03-23 13:28:19 +00:00
|
|
|
|
isremoved = self.list_rm_url(self.gi.url_mode(),l)
|
2022-01-22 14:08:06 +00:00
|
|
|
|
if isremoved:
|
2022-01-26 17:52:38 +00:00
|
|
|
|
print("Removed from %s"%l)
|
2022-01-22 14:08:06 +00:00
|
|
|
|
self.list_add_line(args[0])
|
2022-01-23 22:14:06 +00:00
|
|
|
|
|
|
|
|
|
def list_lists(self):
|
|
|
|
|
listdir = os.path.join(_DATA_DIR,"lists")
|
|
|
|
|
to_return = []
|
|
|
|
|
if os.path.exists(listdir):
|
|
|
|
|
lists = os.listdir(listdir)
|
|
|
|
|
if len(lists) > 0:
|
|
|
|
|
for l in lists:
|
|
|
|
|
#removing the .gmi at the end of the name
|
|
|
|
|
to_return.append(l[:-4])
|
|
|
|
|
return to_return
|
|
|
|
|
|
2022-02-15 13:56:35 +00:00
|
|
|
|
def list_has_status(self,list,status):
|
|
|
|
|
path = self.list_path(list)
|
|
|
|
|
toreturn = False
|
|
|
|
|
if path:
|
|
|
|
|
with open(path) as f:
|
|
|
|
|
line = f.readline().strip()
|
|
|
|
|
f.close()
|
|
|
|
|
if line.startswith("#") and status in line:
|
|
|
|
|
toreturn = True
|
|
|
|
|
return toreturn
|
|
|
|
|
|
|
|
|
|
def list_is_subscribed(self,list):
|
2022-02-15 14:05:08 +00:00
|
|
|
|
return self.list_has_status(list,"#subscribed")
|
2022-02-15 13:56:35 +00:00
|
|
|
|
def list_is_frozen(self,list):
|
2022-02-15 14:05:08 +00:00
|
|
|
|
return self.list_has_status(list,"#frozen")
|
2022-02-15 13:56:35 +00:00
|
|
|
|
def list_is_system(self,list):
|
|
|
|
|
return list in ["history","to_fetch","archives","tour"]
|
|
|
|
|
|
|
|
|
|
# This modify the status of a list to one of :
|
|
|
|
|
# normal, frozen, subscribed
|
|
|
|
|
# action is either #frozen, #subscribed or None
|
|
|
|
|
def list_modify(self,list,action=None):
|
|
|
|
|
path = self.list_path(list)
|
|
|
|
|
with open(path) as f:
|
|
|
|
|
lines = f.readlines()
|
|
|
|
|
f.close()
|
|
|
|
|
if lines[0].strip().startswith("#"):
|
|
|
|
|
first_line = lines.pop(0).strip("\n")
|
|
|
|
|
else:
|
|
|
|
|
first_line = "# %s "%list
|
|
|
|
|
first_line = first_line.replace("#subscribed","").replace("#frozen","")
|
|
|
|
|
if action:
|
|
|
|
|
first_line += " " + action
|
|
|
|
|
print("List %s has been marked as %s"%(list,action))
|
|
|
|
|
else:
|
|
|
|
|
print("List %s is now a normal list" %list)
|
|
|
|
|
first_line += "\n"
|
|
|
|
|
lines.insert(0,first_line)
|
|
|
|
|
with open(path,"w") as f:
|
|
|
|
|
for line in lines:
|
|
|
|
|
f.write(line)
|
|
|
|
|
f.close()
|
2022-01-21 16:42:14 +00:00
|
|
|
|
def do_list(self,arg):
|
|
|
|
|
"""Manage list of bookmarked pages.
|
2022-01-24 11:24:19 +00:00
|
|
|
|
- list : display available lists
|
|
|
|
|
- list $LIST : display pages in $LIST
|
|
|
|
|
- list create $NEWLIST : create a new list
|
|
|
|
|
- list edit $LIST : edit the list
|
2022-02-15 13:56:35 +00:00
|
|
|
|
- list subscribe $LIST : during sync, add new links found in listed pages to tour
|
2022-02-15 14:18:07 +00:00
|
|
|
|
- list freeze $LIST : don’t update pages in list during sync if a cache already exists
|
2022-02-15 13:56:35 +00:00
|
|
|
|
- list normal $LIST : update pages in list during sync but don’t add anything to tour
|
2022-01-24 14:09:05 +00:00
|
|
|
|
- list delete $LIST : delete a list permanently (a confirmation is required)
|
2022-02-15 15:50:33 +00:00
|
|
|
|
- list help : print this help
|
2022-01-24 11:24:19 +00:00
|
|
|
|
See also :
|
2022-02-14 22:02:53 +00:00
|
|
|
|
- add $LIST (to add current page to $LIST or, by default, to bookmarks)
|
2022-01-30 18:23:50 +00:00
|
|
|
|
- move $LIST (to add current page to list while removing from all others)
|
2022-03-08 21:21:44 +00:00
|
|
|
|
- archive (to remove current page from all lists while adding to archives)
|
|
|
|
|
Note: There’s no "delete" on purpose. The use of "archive" is recommended."""
|
2022-01-21 16:42:14 +00:00
|
|
|
|
listdir = os.path.join(_DATA_DIR,"lists")
|
|
|
|
|
os.makedirs(listdir,exist_ok=True)
|
|
|
|
|
if not arg:
|
2022-01-23 22:14:06 +00:00
|
|
|
|
lists = self.list_lists()
|
2022-01-21 16:42:14 +00:00
|
|
|
|
if len(lists) > 0:
|
2022-03-22 22:17:47 +00:00
|
|
|
|
lgi = GeminiItem("list:///")
|
2022-02-12 23:37:31 +00:00
|
|
|
|
self._go_to_gi(lgi)
|
2022-01-21 16:42:14 +00:00
|
|
|
|
else:
|
|
|
|
|
print("No lists yet. Use `list create`")
|
|
|
|
|
else:
|
|
|
|
|
args = arg.split()
|
2022-01-22 14:08:06 +00:00
|
|
|
|
if args[0] == "create":
|
2022-01-21 16:42:14 +00:00
|
|
|
|
if len(args) > 2:
|
|
|
|
|
name = " ".join(args[2:])
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_create(args[1].lower(),title=name)
|
2022-01-22 14:08:06 +00:00
|
|
|
|
elif len(args) == 2:
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_create(args[1].lower())
|
2022-01-21 16:42:14 +00:00
|
|
|
|
else:
|
2022-01-22 14:08:06 +00:00
|
|
|
|
print("A name is required to create a new list. Use `list create NAME`")
|
2022-01-24 11:24:19 +00:00
|
|
|
|
elif args[0] == "edit":
|
2022-03-29 11:08:35 +00:00
|
|
|
|
editor = None
|
|
|
|
|
if "editor" in self.options and self.options["editor"]:
|
|
|
|
|
editor = self.options["editor"]
|
|
|
|
|
elif os.environ.get("VISUAL"):
|
|
|
|
|
editor = os.environ.get("VISUAL")
|
|
|
|
|
elif os.environ.get("EDITOR"):
|
|
|
|
|
editor = os.environ.get("EDITOR")
|
|
|
|
|
if editor:
|
|
|
|
|
if len(args) > 1 and args[1] in self.list_lists():
|
2022-01-24 11:24:19 +00:00
|
|
|
|
path = os.path.join(listdir,args[1]+".gmi")
|
2022-03-29 11:08:35 +00:00
|
|
|
|
try:
|
Avoid passing improperly escaped paths to shell
This should fix https://notabug.org/ploum/offpunk/issues/9 . This
involves a few closely-related changes to subprogram execution:
- If a path, url or file contents were being passed using `cat` or
`echo`, the code was changed to pass the file/string on stdin. This
also makes several pipelines into single programs and should allow for
the removal of `shell=True` in the future.
- For `file`, `xdg-open` and `less`, which either can't accept their
input on stdin or otherwise use the path, the paths are now being
escaped with `shlex.quote()`.
- Finally, the environment variable $LESSHISTFILE is now being set in
python code, where escaping is not necessary.
Notably, the argument to `grep` in `less_cmd()` is not quoted in this
commit, since I was unsure of how it was meant to be used. If the
argument is not already quoted, this should probably be passed through
`shlex.quote()`.
This does not do the following, which may be desired:
- This does not disable `shell=True` anywhere, since `subprocess.run()`
requires the command to be already split into a list of strings. I
think this would just require a `shlex.split()` in `run()` when this
is disabled, but it may require more thought.
- Some of the invoked programs (with the notable exception of `echo` and
`xdg-open`) support the use of "--" to prevent any following arguments
from being treated as program flags if they start with "-". I don't
believe there are any paths that start with "-", but it may make sense
to include this where possible.
I have briefly tested this commit, but it touches quite a few code
paths, so there might be bugs that I missed.
2022-11-30 22:14:03 +00:00
|
|
|
|
# Note that we intentionally don't quote the editor.
|
|
|
|
|
# In the unlikely case `editor` includes a percent
|
|
|
|
|
# sign, we also escape it for the %-formatting.
|
|
|
|
|
cmd = editor.replace("%", "%%") + " %s"
|
|
|
|
|
run(cmd, parameter=path, direct_output=True)
|
2022-03-29 11:08:35 +00:00
|
|
|
|
except Exception as err:
|
|
|
|
|
print(err)
|
|
|
|
|
print("Please set a valid editor with \"set editor\"")
|
2022-01-24 11:24:19 +00:00
|
|
|
|
else:
|
|
|
|
|
print("A valid list name is required to edit a list")
|
|
|
|
|
else:
|
2022-03-29 11:08:35 +00:00
|
|
|
|
print("No valid editor has been found.")
|
|
|
|
|
print("You can use the following command to set your favourite editor:")
|
|
|
|
|
print("set editor EDITOR")
|
|
|
|
|
print("or use the $VISUAL or $EDITOR environment variables.")
|
2022-01-24 14:09:05 +00:00
|
|
|
|
elif args[0] == "delete":
|
|
|
|
|
if len(args) > 1:
|
2022-02-15 13:56:35 +00:00
|
|
|
|
if self.list_is_system(args[1]):
|
2022-01-24 14:33:33 +00:00
|
|
|
|
print("%s is a system list which cannot be deleted"%args[1])
|
2022-01-24 14:09:05 +00:00
|
|
|
|
elif args[1] in self.list_lists():
|
|
|
|
|
size = len(self.list_get_links(args[1]))
|
|
|
|
|
stri = "Are you sure you want to delete %s ?\n"%args[1]
|
|
|
|
|
confirm = "YES"
|
|
|
|
|
if size > 0:
|
|
|
|
|
stri += "! %s items in the list will be lost !\n"%size
|
|
|
|
|
confirm = "YES DELETE %s" %size
|
|
|
|
|
else :
|
|
|
|
|
stri += "The list is empty, it should be safe to delete it.\n"
|
|
|
|
|
stri += "Type \"%s\" (in capital, without quotes) to confirm :"%confirm
|
|
|
|
|
answer = input(stri)
|
|
|
|
|
if answer == confirm:
|
|
|
|
|
path = os.path.join(listdir,args[1]+".gmi")
|
|
|
|
|
os.remove(path)
|
|
|
|
|
print("* * * %s has been deleted" %args[1])
|
|
|
|
|
else:
|
|
|
|
|
print("A valid list name is required to be deleted")
|
|
|
|
|
else:
|
|
|
|
|
print("A valid list name is required to be deleted")
|
2022-02-15 13:56:35 +00:00
|
|
|
|
elif args[0] in ["subscribe","freeze","normal"]:
|
|
|
|
|
if len(args) > 1:
|
|
|
|
|
if self.list_is_system(args[1]):
|
|
|
|
|
print("You cannot modify %s which is a system list"%args[1])
|
|
|
|
|
elif args[1] in self.list_lists():
|
|
|
|
|
if args[0] == "subscribe":
|
|
|
|
|
action = "#subscribed"
|
|
|
|
|
elif args[0] == "freeze":
|
|
|
|
|
action = "#frozen"
|
|
|
|
|
else:
|
|
|
|
|
action = None
|
|
|
|
|
self.list_modify(args[1],action=action)
|
|
|
|
|
else:
|
|
|
|
|
print("A valid list name is required after %s" %args[0])
|
2022-02-15 15:50:33 +00:00
|
|
|
|
elif args[0] == "help":
|
|
|
|
|
self.onecmd("help list")
|
2022-01-23 12:52:34 +00:00
|
|
|
|
elif len(args) == 1:
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_show(args[0].lower())
|
2022-01-23 12:52:34 +00:00
|
|
|
|
else:
|
2022-01-23 22:14:06 +00:00
|
|
|
|
self.list_go_to_line(args[1],args[0].lower())
|
2022-01-21 16:42:14 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
def do_help(self, arg):
|
|
|
|
|
"""ALARM! Recursion detected! ALARM! Prepare to eject!"""
|
|
|
|
|
if arg == "!":
|
|
|
|
|
print("! is an alias for 'shell'")
|
|
|
|
|
elif arg == "?":
|
|
|
|
|
print("? is an alias for 'help'")
|
2022-01-05 20:12:59 +00:00
|
|
|
|
elif arg in _ABBREVS:
|
|
|
|
|
full_cmd = _ABBREVS[arg]
|
2022-03-07 22:16:17 +00:00
|
|
|
|
print("%s is an alias for '%s'" %(arg,full_cmd))
|
2022-01-05 20:12:59 +00:00
|
|
|
|
print("See the list of aliases with 'abbrevs'")
|
|
|
|
|
print("'help %s':"%full_cmd)
|
|
|
|
|
cmd.Cmd.do_help(self, full_cmd)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
cmd.Cmd.do_help(self, arg)
|
|
|
|
|
|
|
|
|
|
### Flight recorder
|
|
|
|
|
def do_blackbox(self, *args):
|
|
|
|
|
"""Display contents of flight recorder, showing statistics for the
|
|
|
|
|
current gemini browsing session."""
|
|
|
|
|
lines = []
|
|
|
|
|
# Compute flight time
|
|
|
|
|
now = time.time()
|
|
|
|
|
delta = now - self.log["start_time"]
|
2020-06-14 10:28:34 +00:00
|
|
|
|
hours, remainder = divmod(delta, 3600)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
minutes, seconds = divmod(remainder, 60)
|
|
|
|
|
# Count hosts
|
|
|
|
|
ipv4_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET])
|
|
|
|
|
ipv6_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET6])
|
|
|
|
|
# Assemble lines
|
2020-05-10 16:47:07 +00:00
|
|
|
|
lines.append(("Patrol duration", "%02d:%02d:%02d" % (hours, minutes, seconds)))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
lines.append(("Requests sent:", self.log["requests"]))
|
|
|
|
|
lines.append((" IPv4 requests:", self.log["ipv4_requests"]))
|
|
|
|
|
lines.append((" IPv6 requests:", self.log["ipv6_requests"]))
|
|
|
|
|
lines.append(("Bytes received:", self.log["bytes_recvd"]))
|
|
|
|
|
lines.append((" IPv4 bytes:", self.log["ipv4_bytes_recvd"]))
|
|
|
|
|
lines.append((" IPv6 bytes:", self.log["ipv6_bytes_recvd"]))
|
|
|
|
|
lines.append(("Unique hosts visited:", len(self.visited_hosts)))
|
|
|
|
|
lines.append((" IPv4 hosts:", ipv4_hosts))
|
|
|
|
|
lines.append((" IPv6 hosts:", ipv6_hosts))
|
|
|
|
|
lines.append(("DNS failures:", self.log["dns_failures"]))
|
|
|
|
|
lines.append(("Timeouts:", self.log["timeouts"]))
|
|
|
|
|
lines.append(("Refused connections:", self.log["refused_connections"]))
|
|
|
|
|
lines.append(("Reset connections:", self.log["reset_connections"]))
|
2020-09-01 19:14:17 +00:00
|
|
|
|
lines.append(("Cache hits:", self.log["cache_hits"]))
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Print
|
|
|
|
|
for key, value in lines:
|
|
|
|
|
print(key.ljust(24) + str(value).rjust(8))
|
|
|
|
|
|
2022-02-18 21:55:32 +00:00
|
|
|
|
|
|
|
|
|
def do_sync(self, line):
|
|
|
|
|
"""Synchronize all bookmarks lists.
|
|
|
|
|
- New elements in pages in subscribed lists will be added to tour
|
|
|
|
|
- Elements in list to_fetch will be retrieved and added to tour
|
|
|
|
|
- Normal lists will be synchronized and updated
|
|
|
|
|
- Frozen lists will be fetched only if not present.
|
|
|
|
|
|
|
|
|
|
Argument : duration of cache validity (in seconds)."""
|
|
|
|
|
if self.offline_only:
|
|
|
|
|
print("Sync can only be achieved online. Change status with `online`.")
|
|
|
|
|
return
|
|
|
|
|
args = line.split()
|
|
|
|
|
if len(args) > 0:
|
|
|
|
|
if not args[0].isdigit():
|
|
|
|
|
print("sync argument should be the cache validity expressed in seconds")
|
|
|
|
|
return
|
|
|
|
|
else:
|
|
|
|
|
validity = int(args[0])
|
|
|
|
|
else:
|
|
|
|
|
validity = 0
|
|
|
|
|
self.call_sync(refresh_time=validity)
|
|
|
|
|
|
|
|
|
|
def call_sync(self,refresh_time=0,depth=1):
|
|
|
|
|
# fetch_gitem is the core of the sync algorithm.
|
|
|
|
|
# It takes as input :
|
|
|
|
|
# - a GeminiItem to be fetched
|
|
|
|
|
# - depth : the degree of recursion to build the cache (0 means no recursion)
|
|
|
|
|
# - validity : the age, in seconds, existing caches need to have before
|
|
|
|
|
# being refreshed (0 = never refreshed if it already exists)
|
|
|
|
|
# - savetotour : if True, newly cached items are added to tour
|
|
|
|
|
def add_to_tour(gitem):
|
2022-11-03 11:59:15 +00:00
|
|
|
|
if gitem and gitem.is_cache_valid():
|
2022-03-19 14:58:28 +00:00
|
|
|
|
toprint = " -> adding to tour: %s" %gitem.url
|
|
|
|
|
width = term_width() - 1
|
|
|
|
|
toprint = toprint[:width]
|
|
|
|
|
toprint += " "*(width-len(toprint))
|
|
|
|
|
print(toprint)
|
2022-02-18 21:55:32 +00:00
|
|
|
|
self.list_add_line("tour",gi=gitem,verbose=False)
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
return False
|
|
|
|
|
def fetch_gitem(gitem,depth=0,validity=0,savetotour=False,count=[0,0],strin=""):
|
|
|
|
|
#savetotour = True will save to tour newly cached content
|
|
|
|
|
# else, do not save to tour
|
|
|
|
|
#regardless of valitidy
|
2022-10-07 14:07:03 +00:00
|
|
|
|
if not gitem: return
|
2022-02-18 21:55:32 +00:00
|
|
|
|
if not gitem.is_cache_valid(validity=validity):
|
|
|
|
|
if strin != "":
|
|
|
|
|
endline = '\r'
|
|
|
|
|
else:
|
|
|
|
|
endline = None
|
|
|
|
|
#Did we already had a cache (even an old one) ?
|
|
|
|
|
isnew = not gitem.is_cache_valid()
|
2022-03-19 14:58:28 +00:00
|
|
|
|
toprint = "%s [%s/%s] Fetch "%(strin,count[0],count[1]) + gitem.url
|
|
|
|
|
width = term_width() - 1
|
|
|
|
|
toprint = toprint[:width]
|
|
|
|
|
toprint += " "*(width-len(toprint))
|
|
|
|
|
print(toprint,end=endline)
|
2022-03-03 15:16:32 +00:00
|
|
|
|
#If not saving to tour, then we should limit download size
|
|
|
|
|
limit = not savetotour
|
|
|
|
|
self._go_to_gi(gitem,update_hist=False,limit_size=limit)
|
2022-02-18 21:55:32 +00:00
|
|
|
|
if savetotour and isnew and gitem.is_cache_valid():
|
|
|
|
|
#we add to the next tour only if we managed to cache
|
|
|
|
|
#the ressource
|
|
|
|
|
add_to_tour(gitem)
|
|
|
|
|
#Now, recursive call, even if we didn’t refresh the cache
|
2022-10-20 20:53:33 +00:00
|
|
|
|
# This recursive call is impacting performances a lot but is needed
|
|
|
|
|
# For the case when you add a address to a list to read later
|
|
|
|
|
# You then expect the links to be loaded during next refresh, even
|
|
|
|
|
# if the link itself is fresh enough
|
|
|
|
|
# see fetch_list()
|
2022-02-18 21:55:32 +00:00
|
|
|
|
if depth > 0:
|
2022-03-03 15:16:32 +00:00
|
|
|
|
#we should only savetotour at the first level of recursion
|
|
|
|
|
# The code for this was removed so, currently, we savetotour
|
|
|
|
|
# at every level of recursion.
|
2022-10-20 20:53:33 +00:00
|
|
|
|
links = gitem.get_links(mode="links_only")
|
2022-02-18 21:55:32 +00:00
|
|
|
|
subcount = [0,len(links)]
|
|
|
|
|
d = depth - 1
|
|
|
|
|
for k in links:
|
|
|
|
|
#recursive call (validity is always 0 in recursion)
|
|
|
|
|
substri = strin + " -->"
|
|
|
|
|
subcount[0] += 1
|
|
|
|
|
fetch_gitem(k,depth=d,validity=0,savetotour=savetotour,\
|
|
|
|
|
count=subcount,strin=substri)
|
|
|
|
|
def fetch_list(list,validity=0,depth=1,tourandremove=False,tourchildren=False):
|
|
|
|
|
links = self.list_get_links(list)
|
|
|
|
|
end = len(links)
|
|
|
|
|
counter = 0
|
|
|
|
|
print(" * * * %s to fetch in %s * * *" %(end,list))
|
|
|
|
|
for l in links:
|
|
|
|
|
counter += 1
|
2022-10-20 20:53:33 +00:00
|
|
|
|
# If cache for a link is newer than the list
|
2022-02-18 21:55:32 +00:00
|
|
|
|
fetch_gitem(l,depth=depth,validity=validity,savetotour=tourchildren,count=[counter,end])
|
|
|
|
|
if tourandremove:
|
|
|
|
|
if add_to_tour(l):
|
2022-03-23 13:28:19 +00:00
|
|
|
|
self.list_rm_url(l.url_mode(),list)
|
2022-02-18 21:55:32 +00:00
|
|
|
|
|
|
|
|
|
self.sync_only = True
|
|
|
|
|
lists = self.list_lists()
|
|
|
|
|
# We will fetch all the lists except "archives" and "history"
|
|
|
|
|
# We keep tour for the last round
|
|
|
|
|
subscriptions = []
|
|
|
|
|
normal_lists = []
|
|
|
|
|
fridge = []
|
|
|
|
|
for l in lists:
|
|
|
|
|
if not self.list_is_system(l):
|
|
|
|
|
if self.list_is_frozen(l):
|
|
|
|
|
fridge.append(l)
|
|
|
|
|
elif self.list_is_subscribed(l):
|
|
|
|
|
subscriptions.append(l)
|
|
|
|
|
else:
|
|
|
|
|
normal_lists.append(l)
|
|
|
|
|
# We start with the "subscribed" as we need to find new items
|
2022-03-04 11:38:41 +00:00
|
|
|
|
starttime = int(time.time())
|
2022-02-18 21:55:32 +00:00
|
|
|
|
for l in subscriptions:
|
|
|
|
|
fetch_list(l,validity=refresh_time,depth=depth,tourchildren=True)
|
|
|
|
|
#Then the fetch list (item are removed from the list after fetch)
|
2022-03-04 11:38:41 +00:00
|
|
|
|
# We fetch regarless of the refresh_time
|
2022-02-18 21:55:32 +00:00
|
|
|
|
if "to_fetch" in lists:
|
2022-03-04 11:38:41 +00:00
|
|
|
|
nowtime = int(time.time())
|
|
|
|
|
short_valid = nowtime - starttime
|
|
|
|
|
fetch_list("to_fetch",validity=short_valid,depth=depth,tourandremove=True)
|
2022-02-18 21:55:32 +00:00
|
|
|
|
#then we fetch all the rest (including bookmarks and tour)
|
|
|
|
|
for l in normal_lists:
|
|
|
|
|
fetch_list(l,validity=refresh_time,depth=depth)
|
|
|
|
|
for l in fridge:
|
|
|
|
|
fetch_list(l,validity=0,depth=depth)
|
|
|
|
|
#tour should be the last one as item my be added to it by others
|
|
|
|
|
fetch_list("tour",validity=refresh_time,depth=depth)
|
|
|
|
|
print("End of sync")
|
|
|
|
|
self.sync_only = False
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
### The end!
|
|
|
|
|
def do_quit(self, *args):
|
2021-12-30 15:03:08 +00:00
|
|
|
|
"""Exit Offpunk."""
|
2022-02-17 16:47:16 +00:00
|
|
|
|
def unlink(filename):
|
|
|
|
|
if filename and os.path.exists(filename):
|
|
|
|
|
os.unlink(filename)
|
2020-05-16 16:58:53 +00:00
|
|
|
|
# Close TOFU DB
|
|
|
|
|
self.db_conn.commit()
|
|
|
|
|
self.db_conn.close()
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Clean up after ourself
|
2020-08-30 18:21:15 +00:00
|
|
|
|
|
2020-05-11 20:22:24 +00:00
|
|
|
|
for cert in self.transient_certs_created:
|
|
|
|
|
for ext in (".crt", ".key"):
|
2022-01-18 17:04:29 +00:00
|
|
|
|
certfile = os.path.join(_CONFIG_DIR, "transient_certs", cert+ext)
|
2020-05-11 20:22:24 +00:00
|
|
|
|
if os.path.exists(certfile):
|
|
|
|
|
os.remove(certfile)
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("You can close your screen!")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
sys.exit()
|
|
|
|
|
|
|
|
|
|
do_exit = do_quit
|
|
|
|
|
|
2022-02-18 21:55:32 +00:00
|
|
|
|
|
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Main function
|
|
|
|
|
def main():
|
|
|
|
|
|
|
|
|
|
# Parse args
|
|
|
|
|
parser = argparse.ArgumentParser(description='A command line gemini client.')
|
|
|
|
|
parser.add_argument('--bookmarks', action='store_true',
|
|
|
|
|
help='start with your list of bookmarks')
|
2020-03-23 02:12:00 +00:00
|
|
|
|
parser.add_argument('--tls-cert', metavar='FILE', help='TLS client certificate file')
|
|
|
|
|
parser.add_argument('--tls-key', metavar='FILE', help='TLS client certificate private key file')
|
2021-12-14 13:06:07 +00:00
|
|
|
|
parser.add_argument('--sync', action='store_true',
|
|
|
|
|
help='run non-interactively to build cache by exploring bookmarks')
|
2022-01-22 14:33:39 +00:00
|
|
|
|
parser.add_argument('--assume-yes', action='store_true',
|
2022-11-30 20:02:17 +00:00
|
|
|
|
help='assume-yes when asked questions about certificates/redirections during sync (lower security)')
|
2022-02-04 14:55:22 +00:00
|
|
|
|
parser.add_argument('--disable-http',action='store_true',
|
|
|
|
|
help='do not try to get http(s) links (but already cached will be displayed)')
|
2022-01-19 14:21:28 +00:00
|
|
|
|
parser.add_argument('--fetch-later', action='store_true',
|
|
|
|
|
help='run non-interactively with an URL as argument to fetch it later')
|
2022-02-04 00:14:22 +00:00
|
|
|
|
parser.add_argument('--depth',
|
|
|
|
|
help='depth of the cache to build. Default is 1. More is crazy. Use at your own risks!')
|
2021-12-16 14:58:05 +00:00
|
|
|
|
parser.add_argument('--cache-validity',
|
|
|
|
|
help='duration for which a cache is valid before sync (seconds)')
|
2020-05-10 12:34:48 +00:00
|
|
|
|
parser.add_argument('--version', action='store_true',
|
|
|
|
|
help='display version information and quit')
|
2022-07-03 13:38:29 +00:00
|
|
|
|
parser.add_argument('--features', action='store_true',
|
|
|
|
|
help='display available features and dependancies then quit')
|
2019-06-22 12:58:21 +00:00
|
|
|
|
parser.add_argument('url', metavar='URL', nargs='*',
|
|
|
|
|
help='start with this URL')
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
2020-05-10 12:34:48 +00:00
|
|
|
|
# Handle --version
|
|
|
|
|
if args.version:
|
2021-12-30 15:03:08 +00:00
|
|
|
|
print("Offpunk " + _VERSION)
|
2020-05-10 12:34:48 +00:00
|
|
|
|
sys.exit()
|
2022-07-03 13:38:29 +00:00
|
|
|
|
elif args.features:
|
|
|
|
|
GeminiClient.do_version(None,None)
|
|
|
|
|
sys.exit()
|
2022-07-03 13:31:33 +00:00
|
|
|
|
else:
|
|
|
|
|
for f in [_CONFIG_DIR, _CACHE_PATH, _DATA_DIR]:
|
|
|
|
|
if not os.path.exists(f):
|
|
|
|
|
print("Creating config directory {}".format(f))
|
|
|
|
|
os.makedirs(f)
|
2020-05-10 12:34:48 +00:00
|
|
|
|
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Instantiate client
|
2022-11-30 19:57:23 +00:00
|
|
|
|
gc = GeminiClient(synconly=args.sync)
|
2022-02-18 21:55:32 +00:00
|
|
|
|
torun_queue = []
|
|
|
|
|
|
2022-10-06 16:22:34 +00:00
|
|
|
|
# Interactive if offpunk started normally
|
|
|
|
|
# False if started with --sync
|
|
|
|
|
# Queue is a list of command (potentially empty)
|
|
|
|
|
def read_config(queue,interactive=True):
|
|
|
|
|
rcfile = os.path.join(_CONFIG_DIR, "offpunkrc")
|
|
|
|
|
if os.path.exists(rcfile):
|
|
|
|
|
print("Using config %s" % rcfile)
|
|
|
|
|
with open(rcfile, "r") as fp:
|
|
|
|
|
for line in fp:
|
|
|
|
|
line = line.strip()
|
|
|
|
|
if ((args.bookmarks or args.url) and
|
|
|
|
|
any((line.startswith(x) for x in ("go", "g", "tour", "t")))
|
|
|
|
|
):
|
|
|
|
|
if args.bookmarks:
|
|
|
|
|
print("Skipping rc command \"%s\" due to --bookmarks option." % line)
|
|
|
|
|
else:
|
|
|
|
|
print("Skipping rc command \"%s\" due to provided URLs." % line)
|
|
|
|
|
continue
|
|
|
|
|
# We always consider redirect
|
|
|
|
|
# for the rest, we need to be interactive
|
|
|
|
|
if line.startswith("redirect") or interactive:
|
|
|
|
|
queue.append(line)
|
|
|
|
|
return queue
|
2019-06-22 12:58:21 +00:00
|
|
|
|
# Act on args
|
2020-03-23 02:12:00 +00:00
|
|
|
|
if args.tls_cert:
|
|
|
|
|
# If tls_key is None, python will attempt to load the key from tls_cert.
|
2020-05-27 07:00:42 +00:00
|
|
|
|
gc._activate_client_cert(args.tls_cert, args.tls_key)
|
2019-06-22 12:58:21 +00:00
|
|
|
|
if args.bookmarks:
|
2022-02-18 21:55:32 +00:00
|
|
|
|
torun_queue.append("bookmarks")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
elif args.url:
|
|
|
|
|
if len(args.url) == 1:
|
2022-02-18 21:55:32 +00:00
|
|
|
|
torun_queue.append("go %s" % args.url[0])
|
2019-06-22 12:58:21 +00:00
|
|
|
|
else:
|
|
|
|
|
for url in args.url:
|
2022-02-18 21:55:32 +00:00
|
|
|
|
torun_queue.append("tour %s" % url)
|
|
|
|
|
torun_queue.append("tour")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
2022-02-04 14:55:22 +00:00
|
|
|
|
if args.disable_http:
|
2022-02-12 13:41:05 +00:00
|
|
|
|
gc.support_http = False
|
2022-02-04 14:55:22 +00:00
|
|
|
|
|
2022-02-18 16:13:20 +00:00
|
|
|
|
# Endless interpret loop (except while --sync or --fetch-later)
|
2022-01-19 14:21:28 +00:00
|
|
|
|
if args.fetch_later:
|
|
|
|
|
if args.url:
|
|
|
|
|
gc.sync_only = True
|
|
|
|
|
for u in args.url:
|
2022-02-14 10:28:12 +00:00
|
|
|
|
gi = GeminiItem(u)
|
|
|
|
|
if gi and gi.is_cache_valid():
|
|
|
|
|
gc.list_add_line("tour",gi)
|
|
|
|
|
else:
|
|
|
|
|
gc.list_add_line("to_fetch",gi)
|
2022-01-19 14:21:28 +00:00
|
|
|
|
else:
|
2022-01-23 16:09:05 +00:00
|
|
|
|
print("--fetch-later requires an URL (or a list of URLS) as argument")
|
2022-01-19 14:21:28 +00:00
|
|
|
|
elif args.sync:
|
2022-01-22 14:33:39 +00:00
|
|
|
|
if args.assume_yes:
|
2022-01-23 09:44:19 +00:00
|
|
|
|
gc.automatic_choice = "y"
|
2022-11-30 20:02:17 +00:00
|
|
|
|
gc.onecmd("set accept_bad_ssl_certificates True")
|
2021-12-16 14:58:05 +00:00
|
|
|
|
if args.cache_validity:
|
|
|
|
|
refresh_time = int(args.cache_validity)
|
|
|
|
|
else:
|
2022-01-09 20:21:09 +00:00
|
|
|
|
# if no refresh time, a default of 0 is used (which means "infinite")
|
2021-12-18 09:16:19 +00:00
|
|
|
|
refresh_time = 0
|
2022-02-04 00:14:22 +00:00
|
|
|
|
if args.depth:
|
|
|
|
|
depth = int(args.depth)
|
|
|
|
|
else:
|
|
|
|
|
depth = 1
|
2022-10-06 16:22:34 +00:00
|
|
|
|
read_config(torun_queue, interactive=False)
|
|
|
|
|
for line in torun_queue:
|
|
|
|
|
gc.onecmd(line)
|
2022-02-18 21:55:32 +00:00
|
|
|
|
gc.call_sync(refresh_time=refresh_time,depth=depth)
|
2021-12-09 16:23:50 +00:00
|
|
|
|
gc.onecmd("blackbox")
|
2021-12-09 14:12:32 +00:00
|
|
|
|
else:
|
2022-02-18 21:55:32 +00:00
|
|
|
|
# We are in the normal mode. First process config file
|
2022-10-06 16:22:34 +00:00
|
|
|
|
torun_queue = read_config(torun_queue,interactive=True)
|
2022-02-18 21:55:32 +00:00
|
|
|
|
print("Welcome to Offpunk!")
|
|
|
|
|
print("Type `help` to get the list of available command.")
|
|
|
|
|
for line in torun_queue:
|
|
|
|
|
gc.onecmd(line)
|
2022-02-23 09:43:46 +00:00
|
|
|
|
|
|
|
|
|
while True:
|
|
|
|
|
try:
|
|
|
|
|
gc.cmdloop()
|
|
|
|
|
except KeyboardInterrupt:
|
2022-02-25 13:36:46 +00:00
|
|
|
|
print("")
|
2019-06-22 12:58:21 +00:00
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
main()
|