bombadillo/http/lynx_mode.go

92 lines
1.9 KiB
Go

package http
import (
"fmt"
"io/ioutil"
"net/http"
"os/exec"
"strings"
)
type page struct {
Content string
Links []string
}
func Visit(url string, width int) (page, error) {
if width > 80 {
width = 80
}
w := fmt.Sprintf("-width=%d", width)
c, err := exec.Command("lynx", "-dump", w, url).Output()
if err != nil {
return page{}, err
}
return parseLinks(string(c)), nil
}
// Returns false on err or non-text type
// Else returns true
func IsTextFile(url string) bool {
c, err := exec.Command("lynx", "-dump", "-head", url).Output()
if err != nil {
return false
}
content := string(c)
content = strings.ToLower(content)
headers := strings.Split(content, "\n")
for _, header := range headers {
if strings.Contains(header, "content-type:") && strings.Contains(header, "text") {
return true
} else if strings.Contains(header, "content-type:") {
return false
}
}
// If we made it here, there is no content-type header.
// So in the event of the unknown, lets render to the
// screen. This will allow redirects to get rendered
// as well.
return true
}
func parseLinks(c string) page {
var out page
contentUntil := strings.LastIndex(c, "References")
if contentUntil >= 1 {
out.Content = c[:contentUntil]
} else {
out.Content = c
out.Links = make([]string, 0)
return out
}
links := c[contentUntil+11:]
links = strings.TrimSpace(links)
linkSlice := strings.Split(links, "\n")
out.Links = make([]string, 0, len(linkSlice))
for _, link := range linkSlice {
ls := strings.SplitN(link, ".", 2)
if len(ls) < 2 {
continue
}
out.Links = append(out.Links, strings.TrimSpace(ls[1]))
}
return out
}
func Fetch(url string) ([]byte, error) {
resp, err := http.Get(url)
if err != nil {
return []byte{}, err
}
defer resp.Body.Close()
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return []byte{}, err
}
return bodyBytes, nil
}