sw-convert and sw-fetch tools

README.gmi

@@ -16,7 +16,7 @@ sliderule is the toolkit for working with the small web in Go.
                             =====+=====
 ```
 
-Think of it as a net/http for small web protocols. You still have to write your server, but you can focus on the logic you want to implement knowing the protocol is already dealt with. It's been said of gemini that you can write your server in a day. Now you can write it in well under an hour.
+You still have to write your server, but you can focus on the logic you want to implement knowing the protocol is already dealt with. It's been said of gemini that you can write your server in a day. Now you can write it in well under an hour.
 
 ## The slide rule
 
@@ -81,6 +81,15 @@ The sub-packages include:
 * tlsauth contains middlewares and bool functions for authenticating against TLS client certificates
 * ...with more to come
 
+## Tools
+
+The tools directory contains main packages that implement useful cli tools for working with the small web. You can install them all at once with:
+```
+go install tildegit.org/tjp/sliderule/tools/...
+```
+
+They all have useful output when given the -h/--help flag.
+
 ## Get it
 
 ### Using sliderule in your project

README.md

@@ -17,7 +17,7 @@ sliderule is the toolkit for working with the small web in Go.
                             =====+=====
 ```
 
-Think of it as a net/http for small web protocols. You still have to write your server, but you can focus on the logic you want to implement knowing the protocol is already dealt with. It's been said of gemini that you can write your server in a day. Now you can write it in well under an hour.
+You still have to write your server, but you can focus on the logic you want to implement knowing the protocol is already dealt with. It's been said of gemini that you can write your server in a day. Now you can write it in well under an hour.
 
 ## The slide rule
 
@@ -85,6 +85,16 @@ The sub-packages include:
 * tlsauth contains middlewares and bool functions for authenticating against TLS client certificates
 * ...with more to come
 
+## Tools
+
+The tools directory contains main packages that implement useful cli tools for working with the small web. You can install them all at once with:
+
+```
+go install tildegit.org/tjp/sliderule/tools/...
+```
+
+They all have useful output when given the -h/--help flag.
+
 ## Get it
 
 ### Using sliderule in your project

tools/sw-convert/main.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	htemplate "html/template"
+	"net/url"
+	"os"
+	"text/template"
+
+	"tildegit.org/tjp/sliderule/gemini/gemtext"
+	"tildegit.org/tjp/sliderule/gemini/gemtext/atomconv"
+	gem_htmlconv "tildegit.org/tjp/sliderule/gemini/gemtext/htmlconv"
+	gem_mdconv "tildegit.org/tjp/sliderule/gemini/gemtext/mdconv"
+	goph_htmlconv "tildegit.org/tjp/sliderule/gopher/gophermap/htmlconv"
+	"tildegit.org/tjp/sliderule/gopher/gophermap"
+	goph_mdconv "tildegit.org/tjp/sliderule/gopher/gophermap/mdconv"
+)
+
+const usage = `Conversions for small web formats.
+
+Usage:
+  sw-convert (-h | --help)
+  sw-convert --list-formats
+  sw-convert (-i INPUT | --input INPUT) (-o OUTPUT | --output OUTPUT) [-t PATH | --template PATH]
+
+Options:
+  -h --help           Show this screen.
+  --list-formats      List supported input and output formats.
+  -i --input INPUT    Format with which to parse standard input.
+  -o --output OUTPUT  Which format to write to standard output.
+  -t --template PATH  Path to a template file. May be specified more than once.
+  -l --location URL   URL the source came from. Required only for gemtext to atom conversion.
+
+Templates:
+  Template files provided by the -t/--template option should be in the golang template formats.
+  When converting to markdown they will be parsed by text/template (https://pkg.go.dev/text/template),
+  and for html conversions they will be parsed by html/template (https://pkg.go.dev/html/template).
+
+  The template names available for override depend on the type of the source document.
+
+  Gemtext:
+    "header" is rendered at the beginning of the document and is passed the full Document.
+    "footer" is rendered at the end of the document and is passed the full Document.
+    "textline" is rendered once for each plain text line, and is passed the Line.
+    "linkline" is rendered for each link line and is passed the Line.
+    "preformattedtextlines" is rendered for each block of pre-formatted text, and is passed a slice of Lines.
+    "heading1line", "heading2line", and "heading3line" are rendered for heading lines and are passed the Line.
+    "listitemlines" is rendered for any contiguous group of list item lines, and is passed a slice of the Lines.
+    "quoteline" is rendered for each block-quote line and is passed the Line.
+
+    The default gemtext->html templates define an HTML5 document with all HTML nodes given a class of "gemtext".
+
+    Document:
+      The header and footer templates are given the full document, which can be iterated over to produce all the lines.
+    Line:
+      Line-specific or line-group-specific templates are passed Line objects. These all have Type, Raw, and String methods, and some have type-specific additional methods.
+        - link lines also have URL() and Label() methods.
+        - heading, list item, and quote lines have a Body() method which omit the leading prefixes.
+
+  Gophermap:
+    "header" is rendered at the beginning of the document and is passed the full Document.
+    "footer" is rendered at the end of the document and is passed the full Document.
+    "message" is rendered for any contiguous group of info-message lines, and is passed a string of the newline-separated lines.
+    "image" is rendered for any gif file, bitmap file, png file, or image line types, and is passed the Map Item.
+    "link" is rendered for all other line types and is passed the Map Item.
+
+    The default gophermap->html templates define an HTML5 document with all HTML nodes given a class of "gophermap".
+
+    Document:
+      The full gophermap document object is a list of map items, additionally with a String() method which serializes the full document back into the gophermap format.
+    Map Item:
+      An individual gophermap item has attributes Type, Display, Selector, Hostname, and Port, and can re-serialize itself into a gophermap CRLF-terminated line with the String() method.
+`
+
+const formats = `Inputs:
+  gemtext
+  gophermap
+
+Outputs:
+  markdown
+  html
+  atom (with gemtext input only)
+`
+
+func main() {
+	conf := configure()
+
+	tover, hover, err := overrides(conf)
+	if err != nil {
+		fail("template loading failed")
+	}
+
+	switch conf.input + "-" + conf.output {
+	case "gemtext-markdown":
+		doc, err := gemtext.Parse(os.Stdin)
+		if err != nil {
+			fail("gemtext reading failed")
+		}
+		if err := gem_mdconv.Convert(os.Stdout, doc, tover); err != nil {
+			fail("markdown writing failed")
+		}
+	case "gemtext-html":
+		doc, err := gemtext.Parse(os.Stdin)
+		if err != nil {
+			fail("gemtext reading failed")
+		}
+		if err := gem_htmlconv.Convert(os.Stdout, doc, hover); err != nil {
+			fail("html writing failed")
+		}
+	case "gemtext-atom":
+		u, err := url.Parse(conf.location)
+		if conf.location == "" || err != nil {
+			fail("-l|--location must be provided and valid for gemtext to atom conversion")
+		}
+		doc, err := gemtext.Parse(os.Stdin)
+		if err != nil {
+			fail("gemtext reading failed")
+		}
+		if err := atomconv.Convert(os.Stdout, doc, u); err != nil {
+			fail("atom writing failed")
+		}
+	case "gophermap-markdown":
+		doc, err := gophermap.Parse(os.Stdin)
+		if err != nil {
+			fail("gophermap reading failed")
+		}
+		if err := goph_mdconv.Convert(os.Stdout, doc, tover); err != nil {
+			fail("markdown wriiting failed")
+		}
+	case "gophermap-html":
+		doc, err := gophermap.Parse(os.Stdin)
+		if err != nil {
+			fail("gophermap reading failed")
+		}
+		if err := goph_htmlconv.Convert(os.Stdout, doc, hover); err != nil {
+			fail("html writing failed")
+		}
+	default:
+		os.Stderr.WriteString("unsupported input/output combination\n")
+		fail(formats)
+	}
+}
+
+type config struct {
+	input    string
+	output   string
+	location string
+	template []string
+}
+
+func configure() config {
+	conf := config{}
+
+	for i := 1; i < len(os.Args); i += 1 {
+		switch os.Args[i] {
+		case "-h", "--help":
+			os.Stdout.WriteString(usage)
+			os.Exit(0)
+		case "--list-formats":
+			os.Stdout.WriteString(formats)
+			os.Exit(0)
+		case "-i", "--input":
+			if i == len(os.Args) {
+				fail(usage)
+			}
+			i += 1
+			conf.input = os.Args[i]
+		case "-o", "--output":
+			if i == len(os.Args) {
+				fail(usage)
+			}
+			i += 1
+			conf.output = os.Args[i]
+		case "-l", "--location":
+			if i == len(os.Args) {
+				fail(usage)
+			}
+			i += 1
+			conf.location = os.Args[i]
+		case "-t", "--template":
+			if i == len(os.Args) {
+				fail(usage)
+			}
+			i += 1
+			conf.template = append(conf.template, os.Args[i])
+		}
+	}
+
+	if conf.input == "" || conf.output == "" {
+		fail("both -i|--input and -o|--output are required\n")
+	}
+
+	return conf
+}
+
+func overrides(conf config) (*template.Template, *htemplate.Template, error) {
+	if len(conf.template) == 0 {
+		return nil, nil, nil
+	}
+
+	switch conf.output {
+	case "markdown":
+		tmpl, err := template.New("mdconv").ParseFiles(conf.template...)
+		if err != nil {
+			return nil, nil, err
+		}
+		return tmpl, nil, nil
+
+	case "html":
+		tmpl, err := htemplate.New("htmlconv").ParseFiles(conf.template...)
+		if err != nil {
+			return nil, nil, err
+		}
+		return nil, tmpl, err
+	}
+
+	return nil, nil, nil
+}
+
+func fail(msg string) {
+	os.Stderr.WriteString(msg)
+	os.Exit(1)
+}

tools/sw-fetch/main.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+
+	"tildegit.org/tjp/sliderule"
+	"tildegit.org/tjp/sliderule/gemini"
+)
+
+const usage = `Resource fetcher for the small web.
+
+Usage:
+  sw-fetch (-h | --help)
+  sw-fetch [-v | --verbose] [-o PATH | --output PATH] [-k | --keyfile PATH] [ -c | --certfile PATH ] [ -s | --skip-verify ] URL
+
+Options:
+  -h --help           Show this screen.
+  -v --verbose        Display more diagnostic information on standard error.
+  -o --output PATH    Send the fetched resource to PATH instead of standard out.
+  -k --keyfile PATH   Path to the TLS key file to use.
+  -c --certfile PATH  Path to the TLS certificate file to use.
+  -s --skip-verify    Don't verify server TLS certificates.
+`
+
+func main() {
+	conf := configure()
+	cl := sliderule.NewClient(conf.clientTLS)
+
+	response, err := cl.Fetch(conf.url.String())
+	if err != nil {
+		fail(err.Error() + "\n")
+	}
+	defer func() {
+		_ = response.Close()
+		_ = conf.output.Close()
+	}()
+
+	_, _ = io.Copy(conf.output, response.Body)
+}
+
+type config struct {
+	verbose   bool
+	output    io.WriteCloser
+	url       *url.URL
+	clientTLS *tls.Config
+}
+
+func configure() config {
+	if len(os.Args) == 1 {
+		fail(usage)
+	}
+
+	conf := config{output: os.Stdout}
+	key := ""
+	cert := ""
+	verify := true
+
+	for i := 1; i <= len(os.Args)-1; i += 1 {
+		switch os.Args[i] {
+		case "-h", "--help":
+			os.Stdout.WriteString(usage)
+			os.Exit(0)
+		case "-v", "--verbose":
+			conf.verbose = true
+		case "-o", "--output":
+			if i+1 == len(os.Args)-1 {
+				fail(usage)
+			}
+
+			out := os.Args[i+1]
+			if out != "-" {
+				output, err := os.OpenFile(out, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+				if err != nil {
+					fmt.Println(err.Error())
+					failf("'%s' is not a valid path\n", out)
+				}
+				conf.output = output
+			}
+
+			i += 1
+		case "-k", "--keyfile":
+			if i+1 == len(os.Args)-1 {
+				fail(usage)
+			}
+
+			i += 1
+			key = os.Args[i]
+		case "-c", "--certfile":
+			if i+1 == len(os.Args)-1 {
+				fail(usage)
+			}
+
+			i += 1
+			cert = os.Args[i]
+		case "-s", "--skip-verify":
+			verify = false
+		}
+	}
+
+	conf.clientTLS = &tls.Config{}
+	if key != "" || cert != "" {
+		if key == "" || cert == "" {
+			fail("-k|--keyfile and -c|--certfile must both be present, or neither\n")
+		}
+		tlsConf, err := gemini.FileTLS(cert, key)
+		if err != nil {
+			failf("failed to load TLS key pair")
+		}
+		conf.clientTLS = tlsConf
+	}
+	conf.clientTLS.InsecureSkipVerify = !verify
+
+	u, err := url.Parse(os.Args[len(os.Args)-1])
+	if err != nil || u.Scheme == "" {
+		fail(usage)
+	}
+	conf.url = u
+
+	return conf
+}
+
+func fail(msg string) {
+	os.Stderr.WriteString(msg)
+	os.Exit(1)
+}
+
+func failf(msg string, args ...any) {
+	fmt.Fprintf(os.Stderr, msg, args...)
+	os.Exit(1)
+}