// A website generator tool package main import ( "encoding/json" "fmt" "io" "io/ioutil" "os" "path/filepath" "strings" "time" "tildegit.org/nihilazo/go-gemtext" "github.com/gorilla/feeds" t "text/template" ) // TODO replace these hardcoded paths with config var root string = "/home/nico/website/wiki" var geminiPrefix string = "gemini://breadpunk.club/~bagel" var htmlPrefix string = "https://itwont.work" var geminiOutputDir string = filepath.Join(root, "public_gemini") var templateDir string = filepath.Join(root, "templates") var htmlOutputDir string = filepath.Join(root, "public_html") var inputDir string = filepath.Join(root, "content") var tagFile string = filepath.Join(root, "tags.json") var feedFile string = filepath.Join(root, "feeds.json") var templates *t.Template = t.Must(t.ParseGlob(templateDir + "/*")) type filePathInfo struct { Path string New bool } type pageInfo struct { Path string Title string } type templateTag struct { Name string Pages []pageInfo } // Only used for tagpages const ( GEMINI = iota HTML ) // RemoveIndex is a reslicing function. func RemoveIndex(s []pageInfo, index int) []pageInfo { if index == len(s) - 1 { return s[:index] } else { return append(s[:index], s[index+1:]...) } } // RemoveIndexFeed is a reslicing function. func RemoveIndexFeed(s []*feeds.Item, index int) []*feeds.Item { if index > len(s) { return s[:len(s)-1] } else { return append(s[:index], s[index+1:]...) } } // publishedAfter returns true if a was updated after b. func publishedAfter(a, b *feeds.Item) bool { return b.Updated.Before(a.Updated) } var tagData map[string][]pageInfo = make(map[string][]pageInfo) var feed *feeds.Feed = &feeds.Feed{ Title: "lipu pi jan Niko", Author: &feeds.Author{Name: "Nico", Email: "nico@itwont.work"}, } var files []filePathInfo // Files to process (those that are new or have been edited) func copy(src, dst string) (int64, error) { sourceFileStat, err := os.Stat(src) if err != nil { return 0, err } if !sourceFileStat.Mode().IsRegular() { return 0, fmt.Errorf("%s is not a regular file", src) } source, err := os.Open(src) if err != nil { return 0, err } defer source.Close() destination, err := os.Create(dst) if err != nil { return 0, err } defer destination.Close() nBytes, err := io.Copy(destination, source) return nBytes, err } // isTagged returns true if the page is a tagged gemtext page, false otherwise func isTagged(page *gemtext.GemtextPage) bool { p := *page if len(p) > 2 { if p[0].Type == gemtext.HEADING && p[1].Type == gemtext.TEXT { if strings.HasPrefix(p[1].Text, "Tags:") { return true } } } return false } // addPrefix adds the prefix "pref" to a path if it is absolute // this is to allow protocol-independent absolute links. func addPrefix(pref string, link string) string { if filepath.IsAbs(link) { return pref + link } else { return link } } // processLink takes a LINK GemtextObject and returns it processed for either gemini or HTML output func processLink(o gemtext.GemtextObject, mode int) gemtext.GemtextObject { if mode == GEMINI { o.Path = addPrefix(geminiPrefix, o.Path) } else if mode == HTML { o.Path = addPrefix(htmlPrefix, strings.Replace(o.Path, ".gmi", ".html", -1)) } return o } func processLinkString(s string, mode int) (string, error) { o, err := filepath.Rel(inputDir, s) if err != nil { return "", err } if mode == GEMINI { o = geminiPrefix + "/" + o } else if mode == HTML { o = htmlPrefix + "/" + strings.Replace(o, ".gmi", ".html", -1) } return o, nil } // processPage finds a page, gets the tags, and writes it out to HTML and gemini output directories. // TODO BUG: Fix removing old feed entries for the same item (make it not t) func processPage(f filePathInfo) error { // Delete any existing tag references to this file. if filepath.Ext(f.Path) == ".gmi" { for i, d := range tagData { // For each tag in tagdata if len(d) != 0 { for j, _ := range d { // for each file in the tag if d[j].Path == f.Path { tagData[i] = RemoveIndex(d, j) } } } } l := &feeds.Link{Href: f.Path} for i, d := range feed.Items { if *l == *d.Link { fmt.Println("got dupe!") feed.Items = RemoveIndexFeed(feed.Items,i) } } // Open the file and parse the gemtext data, err := ioutil.ReadFile(f.Path) if err != nil { return err } parse, err := gemtext.ParsePage(string(data)) if err != nil { return err } var info pageInfo if isTagged(&parse) { title := parse[0].Text info = pageInfo{Title: title, Path: f.Path} tags := strings.Split(parse[1].Text[5:], ",") // Remove the Tags: prefix and split for _, tag := range tags { tagData[tag] = append(tagData[tag], info) // For each tag, append the data to it } s, err := os.Stat(f.Path) if err != nil { return err } feed.Add(&feeds.Item{ Title: info.Title, Link: &feeds.Link{Href: f.Path}, Updated: s.ModTime(), Description: "New/Updated Page: " + info.Title, }) } var gemlinks gemtext.GemtextPage // Page with prefixed links for gemini var htmllinks gemtext.GemtextPage // Page with prefixed links for html for _, line := range parse { if line.Type == gemtext.LINK { gemlinks = append(gemlinks, processLink(line, GEMINI)) htmllinks = append(htmllinks, processLink(line, HTML)) } else { gemlinks = append(gemlinks, line) htmllinks = append(htmllinks, line) } } geminiRender, err := gemtext.RenderGemtext(&gemlinks) if err != nil { return err } geminiRender = strings.Trim(geminiRender,"\n") rel, err := filepath.Rel(inputDir, f.Path) if err != nil { return err } output, err := os.Create(filepath.Join(geminiOutputDir, rel)) defer output.Close() err = templates.ExecuteTemplate(output, "page.gmi", geminiRender) if err != nil { return err } htmlRender, err := gemtext.RenderHTML(&htmllinks) if err != nil { return err } output, err = os.Create(strings.Replace(filepath.Join(htmlOutputDir, rel), ".gmi", ".html", -1)) if err != nil { return err } err = templates.ExecuteTemplate(output, "page.html", htmlRender) if err != nil { return err } } else { // Non-gemtext files rel, err := filepath.Rel(inputDir, f.Path) if err != nil { return err } copy(f.Path, filepath.Join(htmlOutputDir, rel)) copy(f.Path, filepath.Join(geminiOutputDir, rel)) } return nil } // walkInputDir is called when walking the input directory, finds what files are new or have been edited, adds them to files func walkInputDir(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { rel, err := filepath.Rel(inputDir, path) if err != nil { return err } outPath := filepath.Join(geminiOutputDir, rel) outFileInfo, err := os.Stat(outPath) if err != nil { if os.IsNotExist(err) { files = append(files, filePathInfo{Path: path, New: true}) } else { return err } } else { if info.ModTime().After(outFileInfo.ModTime()) { files = append(files, filePathInfo{Path: path}) } } } return nil } // writeTagPage writes a tag page. func writeTagPage(tag string, pages *[]pageInfo, basePath string, linkType int) error { var outPath string var tmpl string if linkType == HTML { outPath = filepath.Join(basePath, tag+".html") tmpl = "tag.html" } else { outPath = filepath.Join(basePath, tag+".gmi") tmpl = "tag.gmi" } f, err := os.Create(filepath.Join(outPath)) defer f.Close() if err != nil { return err } np := make([]pageInfo, len(*pages)) for i, p := range *pages { np[i].Path, err = processLinkString(p.Path, linkType) np[i].Title = p.Title if err != nil { return err } } err = templates.ExecuteTemplate(f, tmpl, templateTag{Name: tag, Pages: np}) if err != nil { return err } f.Sync() return nil } func main() { loadingJson := true if _, err := os.Stat(tagFile); err != nil { if os.IsNotExist(err) { loadingJson = false fmt.Println("Not loading tags file, it doesn't exist yet") } else { panic(err) } } tagFileData, err := ioutil.ReadFile(tagFile) if loadingJson { err = json.Unmarshal([]byte(tagFileData), &tagData) if err != nil { panic(err) } } loadingJson = true if _, err := os.Stat(feedFile); err != nil { if os.IsNotExist(err) { loadingJson = false fmt.Println("Not loading feed file, it doesn't exist yet") } else { panic(err) } } feedFileData, err := ioutil.ReadFile(feedFile) if loadingJson { err = json.Unmarshal([]byte(feedFileData), feed) if err != nil { panic(err) } } feed.Updated = time.Now() err = filepath.Walk(inputDir, walkInputDir) // walks the tree, creates the files slice and updates tagData if err != nil { panic(err) } for _, file := range files { err := processPage(file) // Process all the updated files. if err != nil { panic(err) } } if err := os.Mkdir(filepath.Join(htmlOutputDir, "_tags"), 0664); err != nil { if !os.IsExist(err) { panic(err) } } if err := os.Mkdir(filepath.Join(geminiOutputDir, "_tags"), 0664); err != nil { if !os.IsExist(err) { panic(err) } } f, err := os.Create(filepath.Join(geminiOutputDir, "_tags/_tags.gmi")) defer f.Close() if err != nil { panic(err) } err = templates.ExecuteTemplate(f, "_tags.gmi", tagData) if err != nil { panic(err) } f.Sync() // Write out HTML _tags page hf, err := os.Create(filepath.Join(htmlOutputDir, "_tags.html")) defer hf.Close() if err != nil { panic(err) } err = templates.ExecuteTemplate(hf, "_tags.html", tagData) if err != nil { panic(err) } hf.Sync() // Write tag pages for tag, pages := range tagData { err = writeTagPage(tag, &pages, filepath.Join(htmlOutputDir, "/_tags/"), HTML) if err != nil { panic(err) } err = writeTagPage(tag, &pages, filepath.Join(geminiOutputDir, "/_tags/"), GEMINI) if err != nil { panic(err) } } // Write Feeds gemFeed := &feeds.Feed{ Title: feed.Title, Author: feed.Author, Link: &feeds.Link{Href: geminiPrefix}, } HTMLFeed := &feeds.Feed{ Title: feed.Title, Author: feed.Author, Link: &feeds.Link{Href: htmlPrefix}, } for _, i := range feed.Items { gi := *i // gemini item hi := *i // HTML item hl, err := processLinkString(i.Link.Href,HTML) if err != nil { panic(err) } gl, err := processLinkString(i.Link.Href, GEMINI) if err != nil { panic(err) } hi.Link = &feeds.Link{Href: hl} gi.Link = &feeds.Link{Href: gl} gemFeed.Add(&gi) HTMLFeed.Add(&hi) } gemFeed.Sort(publishedAfter) HTMLFeed.Sort(publishedAfter) fe, err := gemFeed.ToAtom() if err != nil { panic(err) } err = ioutil.WriteFile(filepath.Join(geminiOutputDir,"atom.xml"), []byte(fe), 0644) if err != nil { panic(err) } fe, err = HTMLFeed.ToAtom() if err != nil { panic(err) } err = ioutil.WriteFile(filepath.Join(htmlOutputDir,"atom.xml"), []byte(fe), 0644) if err != nil { panic(err) } // Write recently edited pages gp, err := os.Create(filepath.Join(geminiOutputDir, "edits.gmi")) defer gp.Close() if err != nil { panic(err) } err = templates.ExecuteTemplate(gp, "edits.gmi", gemFeed) if err != nil { panic(err) } gp, err = os.Create(filepath.Join(htmlOutputDir, "edits.html")) if err != nil { panic(err) } err = templates.ExecuteTemplate(gp, "edits.html", HTMLFeed) if err != nil { panic(err) } j, err := json.Marshal(tagData) err = ioutil.WriteFile(tagFile, j, 0644) if err != nil { panic(err) } j, err = json.Marshal(feed) err = ioutil.WriteFile(feedFile, j, 0644) if err != nil { panic(err) } }