add support for xclip and solving a crash found by Klaus Alexander Seistrup

This commit add support for Wayland clipboard through the wl-clipboard
package.

Wl-clipboard is thus a new suggested dependency.

All the clipboard code has been refactored to support both xsel and
wl-clipboard at the same time.

CHANGELOG

@@ -1,5 +1,15 @@
 # Offpunk History
 
+## 2.3 - Unreleased
+- Wayland clipboard support through wl-clipboard (new suggested dependency)
+- Xclip clipborad support (in case xsel is missing)
+- offpunk/netcache: fix IPv6 as an URL (bug #40)
+- ansicat: display empty files (instead of opening them with xdg-open)
+- fix escape sequence warning in python 3.12 (by Étienne Mollier) (Debian #1064209)
+- ansicat : fix crash when feedparser is crashing on bad RSS
+- netcache: fix spartan protocol error
+- opnk: fix a crash when caching returns None
+- ansicat: remove the treshold argument when launching chafa (strange artifacts with new version)
 
 ## 2.2 - February 13th 2023
 - cache folder is now configurable through $OFFPUNK_CACHE_PATH environment variable (by prx)

README.md

@@ -91,7 +91,8 @@ Older dependencies which are only needed on older systems (where chafa < 1.10)
 * [Python-pil](http://python-pillow.github.io/) is required to only display the first frame of animated gif with chafa if chafa version is lower than 1.10. Might be deprecated in the future.
 
 Nice to have (packagers should may make those optional):
-* [Xsel](http://www.vergenet.net/~conrad/software/xsel/) allows to `go` to the URL copied in the clipboard without having to paste it (both X and traditional clipboards are supported). Also needed to use the `copy` command. (apt-get install xsel)
+* [Xsel](http://www.vergenet.net/~conrad/software/xsel/) allows to `go` to the URL copied in the clipboard without having to paste it (both X and traditional clipboards are supported). Also needed to use the `copy` command. (apt-get install xsel). Xclip can be used too.
+* [Wl-clipboard](https://github.com/bugaevc/wl-clipboard) allows the same feature than xsel but under Wayland 
 * [Python-setproctitle](https://github.com/dvarrazzo/py-setproctitle) will change the process name from "python" to "offpunk". Useful to kill it without killing every python service.
 
 ## Features

ansicat.py

@@ -141,9 +141,9 @@ def terminal_image(img_file):
     # it is also centered
     cmds = []
     if _NEW_CHAFA:
-        cmds.append("chafa -C on -d 0 --bg white -t 1 -w 1")
+        cmds.append("chafa -C on -d 0 --bg white -w 1")
     elif _HAS_CHAFA:
-        cmds.append("chafa -d 0 --bg white -t 1 -w 1")
+        cmds.append("chafa -d 0 --bg white -w 1")
     if _NEW_TIMG:
         cmds.append("timg --loops=1 -C")
     image_success = False
@@ -727,6 +727,13 @@ class GemtextRenderer(AbstractRenderer):
         links += hidden_links
         return r.get_final(), links
 
+class EmptyRenderer(GemtextRenderer):
+    def get_mime(self):
+        return "text/empty"
+    def prepare(self,body,mode=None):
+        text= "(empty file)"
+        return [[text, "GemtextRenderer"]]
+
 class GopherRenderer(AbstractRenderer):
     def get_mime(self):
         return "text/gopher"
@@ -870,10 +877,15 @@ class FeedRenderer(GemtextRenderer):
         return "application/rss+xml"
     def is_valid(self):
         if _DO_FEED:
-            parsed = feedparser.parse(self.body)
+            try:
+                parsed = feedparser.parse(self.body)
+            except:
+                parsed = False
         else:
             return False
-        if parsed.bozo:
+        if not parsed:
+            return False
+        elif parsed.bozo:
             return False
         else:
             #If no content, then fallback to HTML
@@ -1312,11 +1324,16 @@ _FORMAT_RENDERERS = {
     "text/gopher": GopherRenderer,
     "image/*": ImageRenderer,
     "application/javascript": HtmlRenderer,
+    "application/json": HtmlRenderer,
+    "text/empty": EmptyRenderer,
 }
 def get_mime(path,url=None):
     #Beware, this one is really a shaddy ad-hoc function
     if not path:
         return None
+    #If the file is empty, simply returns it
+    elif os.path.exists(path) and os.stat(path).st_size == 0:
+        return "text/empty"
     elif url and url.startswith("gopher://"):
         #special case for gopher
         #code copy/pasted from netcache

netcache.py

@@ -656,6 +656,9 @@ def _fetch_gemini(url,timeout=DEFAULT_TIMEOUT,interactive=True,accept_bad_ssl_ce
     # Send request and wrap response in a file descriptor
     url = urllib.parse.urlparse(url)
     new_netloc = host
+    #Handle IPV6 hostname
+    if ":" in new_netloc:
+        new_netloc = "[" + new_netloc + "]"
     if port != standard_ports["gemini"]:
         new_netloc += ":" + str(port)
     url = urllib.parse.urlunparse(url._replace(netloc=new_netloc))
@@ -799,8 +802,10 @@ def fetch(url,offline=False,download_image_first=True,images_mode="readable",val
                 path=_fetch_finger(url,**kwargs)
             elif scheme == "gemini":
                 path,newurl=_fetch_gemini(url,**kwargs)
+            elif scheme == "spartan":
+                path,newurl=_fetch_spartan(url,**kwargs)
             else:
-                print("scheme %s not implemented yet")
+                print("scheme %s not implemented yet"%scheme)
         except UserAbortException:
             return None, newurl
         except Exception as err:

offpunk.py

@@ -35,7 +35,50 @@ try:
     _HAS_SETPROCTITLE = True
 except ModuleNotFoundError:
     _HAS_SETPROCTITLE = False
-_HAS_XSEL = shutil.which('xsel')
+
+#This method copy a string to the system clipboard
+def clipboard_copy(to_copy):
+    copied = False
+    if shutil.which('xsel'):
+        run("xsel -b -i", input=to_copy, direct_output=True)
+        copied = True
+    if shutil.which('xclip'):
+        run("xclip -selection clipboard", input=to_copy, direct_output=True)
+        copied = True
+    if shutil.which('wl-copy'):
+        run("wl-copy", input=to_copy, direct_output=True)
+        copied = True
+    if not copied:
+        print("Install xsel/xclip (X11) or wl-clipboard (Wayland) to use copy")
+#This method returns an array with all the values in all system clipboards
+def clipboard_paste():
+    #We use a set to avoid duplicates
+    clipboards = set()
+    cmds = set()
+    pasted = False
+    if shutil.which('xsel'):
+        pasted = True
+        for selec in ["-p","-s","-b"]:
+            cmds.add("xsel "+selec)
+    if shutil.which('xclip'):
+        pasted = True
+        for selec in ["clipboard","primary","secondary"]:
+            cmds.add("xsel "+selec)
+    if shutil.which('wl-paste'):
+        pasted = True
+        for selec in ["", "-p"]:
+            cmds.add("wl-paste "+selec)
+    for cmd in cmds:
+        try:
+            clipboards.add(run(cmd))
+        except Exception as err:
+            #print("Skippink clipboard %s because %s"%(selec,err))
+            pass
+    if not pasted: 
+        print("Install xsel/xclip (X11) or wl-clipboard (Wayland) to get URLs from your clipboard")
+    return list(clipboards)
+
+
 ## }}} end of imports
 
 # Command abbreviations
@@ -564,37 +607,31 @@ Use with "title" to copy the title of the page.
 Use with "link" to copy a link in the gemtext format to that page with the title.
 """
         if self.current_url:
-            if _HAS_XSEL:
+            args = arg.split()
-                args = arg.split()
+            if args and args[0] == "url":
-                if args and args[0] == "url":
+                if len(args) > 1 and args[1].isdecimal():
-                    if len(args) > 1 and args[1].isdecimal():
+                    url = self.get_renderer().get_link(int(args[1])-1)
-                        url = self.get_renderer().get_link(int(args[1])-1)
-                    else:
-                        url,mode = unmode_url(self.current_url)
-                    print(url)
-                    run("xsel -b -i", input=url, direct_output=True)
-                elif args and args[0] == "raw":
-                    tmp = self.opencache.get_temp_filename(self.current_url)
-                    if tmp:
-                        run("xsel -b -i", input=open(tmp, "rb"),\
-                                                direct_output=True)
-                elif args and args[0] == "cache":
-                    run("xsel -b -i", input=netcache.get_cache_path(self.current_url),\
-                                                                    direct_output=True)
-                elif args and args[0] == "title":
-                    title = self.get_renderer().get_page_title()
-                    run("xsel -b -i",input=title, direct_output=True)
-                    print(title)
-                elif args and args[0] == "link":
-                    link = "=> %s %s"%(unmode_url(self.current_url)[0],\
-                                        self.get_renderer().get_page_title())
-                    print(link)
-                    run("xsel -b -i", input=link,direct_output=True)
                 else:
-                    run("xsel -b -i", input=open(netcache.get_cache_path(self.current_url), "rb"),\
+                    url,mode = unmode_url(self.current_url)
-                                                direct_output=True)
+                print(url)
+                clipboard_copy(url)
+            elif args and args[0] == "raw":
+                tmp = self.opencache.get_temp_filename(self.current_url)
+                if tmp:
+                    clipboard_copy(open(tmp,"rb"))
+            elif args and args[0] == "cache":
+                clipboard_copy(netcache.get_cache_path(self.current_url))
+            elif args and args[0] == "title":
+                title = self.get_renderer().get_page_title()
+                clipboard_copy(title)
+                print(title)
+            elif args and args[0] == "link":
+                link = "=> %s %s"%(unmode_url(self.current_url)[0],\
+                                    self.get_renderer().get_page_title())
+                print(link)
+                clipboard_copy(link)
             else:
-                print("Please install xsel to use copy")
+                clipboard_copy(open(netcache.get_cache_path(self.current_url),"rb"))
         else:
             print("No content to copy, visit a page first")
 
@@ -603,34 +640,25 @@ Use with "link" to copy a link in the gemtext format to that page with the title
         """Go to a gemini URL or marked item."""
         line = line.strip()
         if not line:
-            if shutil.which('xsel'):
+            clipboards = clipboard_paste()
-                clipboards = []
+            urls = []
-                urls = []
+            for u in clipboards:
-                for selec in ["-p","-s","-b"]:
+                if "://" in u and looks_like_url(u) and u not in urls :
-                    try:
+                    urls.append(u)
-                        clipboards.append(run("xsel "+selec))
+            if len(urls) > 1:
-                    except Exception as err:
+                stri = "URLs in your clipboard\n"
-                        #print("Skippink clipboard %s because %s"%(selec,err))
+                counter = 0
-                        pass
+                for u in urls:
-                for u in clipboards:
+                    counter += 1
-                    if "://" in u and looks_like_url(u) and u not in urls :
+                    stri += "[%s] %s\n"%(counter,u)
-                        urls.append(u)
+                stri += "Where do you want to go today ?> "
-                if len(urls) > 1:
+                ans = input(stri)
-                    stri = "URLs in your clipboard\n"
+                if ans.isdigit() and 0 < int(ans) <= len(urls):
-                    counter = 0
+                    self.do_go(urls[int(ans)-1])
-                    for u in urls:
+            elif len(urls) == 1:
-                        counter += 1
+                self.do_go(urls[0])
-                        stri += "[%s] %s\n"%(counter,u)
-                    stri += "Where do you want to go today ?> "
-                    ans = input(stri)
-                    if ans.isdigit() and 0 < int(ans) <= len(urls):
-                        self.do_go(urls[int(ans)-1])
-                elif len(urls) == 1:
-                    self.do_go(urls[0])
-                else:
-                    print("Go where? (hint: simply copy an URL in your clipboard)")
             else:
-                print("Go where? (hint: install xsel to go to copied URLs)")
+                print("Go where? (hint: simply copy an URL in your clipboard)")
 
         # First, check for possible marks
         elif line in self.marks:
@@ -862,8 +890,10 @@ Marks are temporary until shutdown (not saved to disk)."""
             output += " - chafa               : " + has(ansicat._HAS_CHAFA)
             output += " - python-pil          : " + has(ansicat._HAS_PIL)
         output += "\nNice to have:\n"
-        output += " - python-setproctitle : " + has(_HAS_SETPROCTITLE)
+        output += " - python-setproctitle       : " + has(_HAS_SETPROCTITLE)
-        output += " - xsel                : " + has(_HAS_XSEL)
+        clip_support = shutil.which("xsel") or shutil.which("xclip")
+        output += " - X11 clipboard (xsel or xclip)   : " + has(clip_support)
+        output += " - Wayland clipboard (wl-clipboard): " + has(shutil.which("wl-copy"))
 
         output += "\nFeatures :\n"
         if ansicat._NEW_CHAFA:
@@ -874,7 +904,6 @@ Marks are temporary until shutdown (not saved to disk)."""
         output += " - Render Atom/RSS feeds (feedparser)         : " + has(ansicat._DO_FEED)
         output += " - Connect to http/https (requests)           : " + has(netcache._DO_HTTP)
         output += " - Detect text encoding (python-chardet)      : " + has(netcache._HAS_CHARDET)
-        output += " - copy to/from clipboard (xsel)              : " + has(_HAS_XSEL)
         output += " - restore last position (less 572+)          : " + has(opnk._LESS_RESTORE_POSITION)
         output += "\n"
         output += "Config directory    : " +  xdg("config") + "\n"

offutils.py

@@ -99,7 +99,7 @@ def fix_ipv6_url(url):
         netloc, rest = schemaless.split("/",1)
         if netloc.count(":") > 2 and "[" not in netloc and "]" not in netloc:
             schemaless = "[" + netloc + "]" + "/" + rest
-    elif schemaless.count(":") > 2:
+    elif schemaless.count(":") > 2 and "[" not in schemaless and "]" not in schemaless:
         schemaless = "[" + schemaless + "]/"
     if schema:
         return schema + "://" + schemaless
@@ -121,7 +121,16 @@ def looks_like_url(word):
         if mailto:
             return "@" in word
         elif not local:
-            return start and ("." in word or "localhost" in word)
+            if start:
+                #IPv4
+                if "." in word or "localhost" in word:
+                    return True
+                #IPv6
+                elif "[" in word and ":" in word and "]" in word:
+                    return True
+                else: return False
+            else:   return False
+            return start and ("." in word or "localhost" in word or ":" in word)
         else:
             return "/" in word
     except ValueError:

opnk.py

@@ -49,7 +49,7 @@ else:
 # are there on purpose (surch in asciiart)
 #--incsearch : incremental search starting rev581
 def less_cmd(file, histfile=None,cat=False,grep=None):
-    less_prompt = "page %%d/%%D- lines %%lb/%%L - %%Pb\%%"
+    less_prompt = "page %%d/%%D- lines %%lb/%%L - %%Pb\\%%"
     if less_version >= 581:
         less_base = "less --incsearch --save-marks -~ -XRfWiS -P \"%s\""%less_prompt
     elif less_version >= 572:
@@ -159,7 +159,10 @@ class opencache():
                 if inpath in self.renderer_time.keys():
                     last_downloaded = netcache.cache_last_modified(inpath)
                     last_cached = self.renderer_time[inpath]
-                    usecache = last_cached > last_downloaded
+                    if last_cached and last_downloaded:
+                        usecache = last_cached > last_downloaded
+                    else:
+                        usecache = False
                 else:
                     usecache = False
             if not usecache:

ubuntu_dependencies.txt

@@ -1 +1 @@
-sudo apt install less file xdg-utils xsel chafa timg python3-cryptography python3-requests python3-feedparser python3-bs4 python3-readability python3-pil python3-setproctitle python3-chardet file
+sudo apt install less file xdg-utils xsel chafa timg python3-cryptography python3-requests python3-feedparser python3-bs4 python3-readability python3-pil python3-setproctitle python3-chardet file python3-lxml-html-clean wl-clipboard