sitegenerator/main.go

// A website generator tool
package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
	"tildegit.org/nihilazo/go-gemtext"
	"github.com/gorilla/feeds"
	t "text/template"
)

// TODO replace these hardcoded paths with config
var root string = "/home/nico/website/wiki"
var geminiPrefix string = "gemini://breadpunk.club/~bagel"
var htmlPrefix string = "https://itwont.work"
var geminiOutputDir string = filepath.Join(root, "public_gemini")
var templateDir string = filepath.Join(root, "templates")
var htmlOutputDir string = filepath.Join(root, "public_html")
var inputDir string = filepath.Join(root, "content")
var tagFile string = filepath.Join(root, "tags.json")
var feedFile string = filepath.Join(root, "feeds.json")

var templates *t.Template = t.Must(t.ParseGlob(templateDir + "/*"))

type filePathInfo struct {
	Path string
	New bool
}

type pageInfo struct {
	Path  string
	Title string
}

type templateTag struct {
	Name  string
	Pages []pageInfo
} // Only used for tagpages

const (
	GEMINI = iota
	HTML
)

// RemoveIndex is a reslicing function.
func RemoveIndex(s []pageInfo, index int) []pageInfo {
	if index == len(s) - 1 {
		return s[:index]
	} else {
	return append(s[:index], s[index+1:]...)
	}
}
// RemoveIndexFeed is a reslicing function.
func RemoveIndexFeed(s []*feeds.Item, index int) []*feeds.Item {
	if index > len(s) {
		return s[:len(s)-1]
	} else {
	return append(s[:index], s[index+1:]...)
	}
}

// publishedAfter returns true if a was updated after b.
func publishedAfter(a, b *feeds.Item) bool {
	return b.Updated.Before(a.Updated)
}

var tagData map[string][]pageInfo = make(map[string][]pageInfo)

var feed *feeds.Feed = &feeds.Feed{
	Title: "lipu pi jan Niko",
	Author: &feeds.Author{Name: "Nico", Email: "nico@itwont.work"},
	}

var files []filePathInfo // Files to process (those that are new or have been edited)

func copy(src, dst string) (int64, error) {
	sourceFileStat, err := os.Stat(src)
	if err != nil {
		return 0, err
	}

	if !sourceFileStat.Mode().IsRegular() {
		return 0, fmt.Errorf("%s is not a regular file", src)
	}

	source, err := os.Open(src)
	if err != nil {
		return 0, err
	}
	defer source.Close()

	destination, err := os.Create(dst)
	if err != nil {
		return 0, err
	}
	defer destination.Close()
	nBytes, err := io.Copy(destination, source)
	return nBytes, err
}

// isTagged returns true if the page is a tagged gemtext page, false otherwise
func isTagged(page *gemtext.GemtextPage) bool {
	p := *page
	if len(p) > 2 {
		if p[0].Type == gemtext.HEADING && p[1].Type == gemtext.TEXT {
			if strings.HasPrefix(p[1].Text, "Tags:") {
				return true
			}
		}
	}
	return false
}

// addPrefix adds the prefix "pref" to a path if it is absolute
// this is to allow protocol-independent absolute links.
func addPrefix(pref string, link string) string {
	if filepath.IsAbs(link) {
		return pref + link
	} else {
		return link
	}
}

// processLink takes a LINK GemtextObject and returns it processed for either gemini or HTML output
func processLink(o gemtext.GemtextObject, mode int) gemtext.GemtextObject {
	if mode == GEMINI {
		o.Path = addPrefix(geminiPrefix, o.Path)
	} else if mode == HTML {
		o.Path = addPrefix(htmlPrefix, strings.Replace(o.Path, ".gmi", ".html", -1))
	}
	return o
}

func processLinkString(s string, mode int) (string, error) {
	o, err := filepath.Rel(inputDir, s)
	if err != nil {
		return "", err
	}
	if mode == GEMINI {
		o = geminiPrefix + "/" + o
	} else if mode == HTML {
		o = htmlPrefix + "/" + strings.Replace(o, ".gmi", ".html", -1)
	}
	return o, nil
}

// processPage finds a page, gets the tags, and writes it out to HTML and gemini output directories.
// TODO BUG: Fix removing old feed entries for the same item (make it not t)
func processPage(f filePathInfo) error {

	// Delete any existing tag references to this file.
	if filepath.Ext(f.Path) == ".gmi" {
		for i, d := range tagData { // For each tag in tagdata
			if len(d) != 0 {
				for j, _ := range d { // for each file in the tag
					if d[j].Path == f.Path {
						tagData[i] = RemoveIndex(d, j)
					}
				}
			}
		}
		l := &feeds.Link{Href: f.Path}
		for i, d := range feed.Items {
			if *l == *d.Link {
				fmt.Println("got dupe!")
				feed.Items = RemoveIndexFeed(feed.Items,i)
			}
		}
		// Open the file and parse the gemtext
		data, err := ioutil.ReadFile(f.Path)
		if err != nil {
			return err
		}
		parse, err := gemtext.ParsePage(string(data))
		if err != nil {
			return err
		}
		var info pageInfo
		if isTagged(&parse) {
			title := parse[0].Text
			info = pageInfo{Title: title, Path: f.Path}
			tags := strings.Split(parse[1].Text[5:], ",") // Remove the Tags: prefix and split
			for _, tag := range tags {
				tagData[tag] = append(tagData[tag], info) // For each tag, append the data to it
			}
			s, err := os.Stat(f.Path)
			if err != nil {
				return err
			}
			feed.Add(&feeds.Item{
				Title: info.Title,
				Link: &feeds.Link{Href: f.Path},
				Updated: s.ModTime(),
				Description: "New/Updated Page: " + info.Title,
			})
		}
		var gemlinks gemtext.GemtextPage  // Page with prefixed links for gemini
		var htmllinks gemtext.GemtextPage // Page with prefixed links for html
		for _, line := range parse {
			if line.Type == gemtext.LINK {
				gemlinks = append(gemlinks, processLink(line, GEMINI))
				htmllinks = append(htmllinks, processLink(line, HTML))
			} else {
				gemlinks = append(gemlinks, line)
				htmllinks = append(htmllinks, line)
			}
		}
		geminiRender, err := gemtext.RenderGemtext(&gemlinks)
		if err != nil {
			return err
		}
		geminiRender = strings.Trim(geminiRender,"\n")
		rel, err := filepath.Rel(inputDir, f.Path)
		if err != nil {
			return err
		}
		output, err := os.Create(filepath.Join(geminiOutputDir, rel))
		defer output.Close()
		err = templates.ExecuteTemplate(output, "page.gmi", geminiRender)
		if err != nil {
			return err
		}
		htmlRender, err := gemtext.RenderHTML(&htmllinks)
		if err != nil {
			return err
		}
		output, err = os.Create(strings.Replace(filepath.Join(htmlOutputDir, rel), ".gmi", ".html", -1))
		if err != nil {
			return err
		}
		err = templates.ExecuteTemplate(output, "page.html", htmlRender)
		if err != nil {
			return err
		}
	} else { // Non-gemtext files
		rel, err := filepath.Rel(inputDir, f.Path)
		if err != nil {
			return err
		}
		copy(f.Path, filepath.Join(htmlOutputDir, rel))
		copy(f.Path, filepath.Join(geminiOutputDir, rel))
	}

	return nil
}

// walkInputDir is called when walking the input directory, finds what files are new or have been edited, adds them to files
func walkInputDir(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if !info.IsDir() {
		rel, err := filepath.Rel(inputDir, path)
		if err != nil {
			return err
		}
		outPath := filepath.Join(geminiOutputDir, rel)
		outFileInfo, err := os.Stat(outPath)
		if err != nil {
			if os.IsNotExist(err) {
				files = append(files, filePathInfo{Path: path, New: true})
			} else {
				return err
			}
		} else {
			if info.ModTime().After(outFileInfo.ModTime()) {
				files = append(files, filePathInfo{Path: path})
			}
		}

	}
	return nil
}

// writeTagPage writes a tag page.
func writeTagPage(tag string, pages *[]pageInfo, basePath string, linkType int) error {
	var outPath string
	var tmpl string
	if linkType == HTML {
		outPath = filepath.Join(basePath, tag+".html")
		tmpl = "tag.html"
	} else {
		outPath = filepath.Join(basePath, tag+".gmi")
		tmpl = "tag.gmi"
	}
	f, err := os.Create(filepath.Join(outPath))
	defer f.Close()
	if err != nil {
		return err
	}
	np := make([]pageInfo, len(*pages))
	for i, p := range *pages {
		np[i].Path, err = processLinkString(p.Path, linkType)
		np[i].Title = p.Title
		if err != nil {
			return err
		}
	}
	err = templates.ExecuteTemplate(f, tmpl, templateTag{Name: tag, Pages: np})
	if err != nil {
		return err
	}
	f.Sync()
	return nil
}

func main() {
	loadingJson := true
	if _, err := os.Stat(tagFile); err != nil {
		if os.IsNotExist(err) {
			loadingJson = false
			fmt.Println("Not loading tags file, it doesn't exist yet")
		} else {
			panic(err)
		}
	}
	tagFileData, err := ioutil.ReadFile(tagFile)
	if loadingJson {
		err = json.Unmarshal([]byte(tagFileData), &tagData)
		if err != nil {
			panic(err)
		}
	}
	loadingJson = true
	if _, err := os.Stat(feedFile); err != nil {
		if os.IsNotExist(err) {
			loadingJson = false
			fmt.Println("Not loading feed file, it doesn't exist yet")
		} else {
			panic(err)
		}
	}
	feedFileData, err := ioutil.ReadFile(feedFile)
	if loadingJson {
		err = json.Unmarshal([]byte(feedFileData), feed)
		if err != nil {
			panic(err)
		}
	}
	feed.Updated = time.Now()
	err = filepath.Walk(inputDir, walkInputDir) // walks the tree, creates the files slice and updates tagData
	if err != nil {
		panic(err)
	}
	for _, file := range files {
		err := processPage(file) // Process all the updated files.
		if err != nil {
			panic(err)
		}
	}
	if err := os.Mkdir(filepath.Join(htmlOutputDir, "_tags"), 0664); err != nil {
		if !os.IsExist(err) {
			panic(err)
		}
	}
	if err := os.Mkdir(filepath.Join(geminiOutputDir, "_tags"), 0664); err != nil {
		if !os.IsExist(err) {
			panic(err)
		}
	}
	f, err := os.Create(filepath.Join(geminiOutputDir, "_tags/_tags.gmi"))
	defer f.Close()
	if err != nil {
		panic(err)
	}
	err = templates.ExecuteTemplate(f, "_tags.gmi", tagData)
	if err != nil {
		panic(err)
	}
	f.Sync()

	// Write out HTML _tags page
	hf, err := os.Create(filepath.Join(htmlOutputDir, "_tags.html"))
	defer hf.Close()
	if err != nil {
		panic(err)
	}
	err = templates.ExecuteTemplate(hf, "_tags.html", tagData)
	if err != nil {
		panic(err)
	}
	hf.Sync()
	// Write tag pages
	for tag, pages := range tagData {
		err = writeTagPage(tag, &pages, filepath.Join(htmlOutputDir, "/_tags/"), HTML)
		if err != nil {
			panic(err)
		}
		err = writeTagPage(tag, &pages, filepath.Join(geminiOutputDir, "/_tags/"), GEMINI)
		if err != nil {
			panic(err)
		}
	}

	// Write Feeds
	gemFeed := &feeds.Feed{
		Title: feed.Title,
		Author: feed.Author,
		Link: &feeds.Link{Href: geminiPrefix},
		}
	HTMLFeed := &feeds.Feed{
		Title: feed.Title,
		Author: feed.Author,
		Link: &feeds.Link{Href: htmlPrefix},
		}
	for _, i := range feed.Items {
		gi := *i // gemini item
		hi := *i // HTML item
		hl, err :=  processLinkString(i.Link.Href,HTML)
		if err != nil {
			panic(err)
		}
		gl, err := processLinkString(i.Link.Href, GEMINI)
		if err != nil {
			panic(err)
		}
		hi.Link = &feeds.Link{Href: hl}
		gi.Link = &feeds.Link{Href: gl}
		gemFeed.Add(&gi)
		HTMLFeed.Add(&hi)
	}
	gemFeed.Sort(publishedAfter)
	HTMLFeed.Sort(publishedAfter)
	fe, err := gemFeed.ToAtom()
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile(filepath.Join(geminiOutputDir,"atom.xml"), []byte(fe), 0644)
	if err != nil {
		panic(err)
	}
	fe, err = HTMLFeed.ToAtom()
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile(filepath.Join(htmlOutputDir,"atom.xml"), []byte(fe), 0644)
	if err != nil {
		panic(err)
	}
	// Write recently edited pages
	gp, err := os.Create(filepath.Join(geminiOutputDir, "edits.gmi"))
	defer gp.Close()
	if err != nil {
		panic(err)
	}
	err = templates.ExecuteTemplate(gp, "edits.gmi", gemFeed)
	if err != nil {
		panic(err)
	}
	gp, err = os.Create(filepath.Join(htmlOutputDir, "edits.html"))
	if err != nil {
		panic(err)
	}
	err = templates.ExecuteTemplate(gp, "edits.html", HTMLFeed)
	if err != nil {
		panic(err)
	}
	j, err := json.Marshal(tagData)
	err = ioutil.WriteFile(tagFile, j, 0644)
	if err != nil {
		panic(err)
    }
	j, err = json.Marshal(feed)
	err = ioutil.WriteFile(feedFile, j, 0644)
	if err != nil {
		panic(err)
	}

}