tweaks, thumbnails, more spaghetti for world domination

This commit is contained in:
Jan Delta 2022-08-13 21:29:18 +09:00
parent e06a727d8b
commit 7b4244674b
13 changed files with 292 additions and 65 deletions

3
.gitignore vendored
View File

@ -1,2 +1,3 @@
amlform.sqlite
config.toml
config.toml
media

View File

@ -10,16 +10,26 @@ probably also questionable security-wise but shhhh
## Running it
(if for some reason you feel like running this) its made it go so you'll need the golang, other than that you just need to configure the configuration config
```toml
DiscordWebhook=""
NotifTopics=[""]
HostEmail=""
BaseURL="https://example.com"
ThumbnailDir=""
[Submissions]
DiscordWebhook=""
NotifTopics=[""]
[Audit]
DiscordWebhook=""
NotifTopics=[""]
```
- baseURL is what you'd expect and is required
- BaseURL is what you'd expect and is required
- HostEmail is added to the `From` header of the automated requests made to ntfy and to sites submitted for the thumbnail fetcher (see [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/From) for why)
- DiscordWebhook is a webhook that new submissions and other action notifications are sent to
- NotifTopics are [ntfy](https://ntfy.sh) topics for sending push notifications about new submissions to any ntfy compatible destination
you don't need both NotifTopics and DiscordWebhook, but you do need at least one of them configured, otherwise you'll never be able to get the URLs to actually approve any submissions
- ThumbnailDir if set creates a directory that it'll put thumbnails into
- (Submissions and Audit)
- DiscordWebhook is a webhook that new submissions and other action notifications are sent to
- NotifTopics are [ntfy](https://ntfy.sh) topics for sending push notifications about new submissions to any ntfy compatible destination
- Submissions sends notifications for new submissions
- Audit sends notifications about actions done (approvals, unapprovals, deletions)
for submissions you don't need both NotifTopics and DiscordWebhook, but you do need at least one of them configured, otherwise you'll never be able to get the URLs to actually approve any submissions
for audit setting any notification reporter is entirely optional (you don't need it, its just there if you want to keep tabs on what gets approved)
then you'll just need to build the thing from the standalone folder, and run it with the address you want it to listen to (eg `:7070` or `localhost:7070`)

View File

@ -8,16 +8,33 @@ import (
"log"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"codeberg.org/eviedelta/dwhook"
)
var picrewRegex = regexp.MustCompile(`^https?\:\/\/picrew\.me\/image_maker\/(\d+)$`)
var thumbExtractor = regexp.MustCompile(`\<meta[^\>]*property=\"(?:og\:)?image\" content=\"(https?\:\/\/\w+\.\w+\/[^>]+.png)\"\>`)
var thumbExtractor = regexp.MustCompile(`\<meta[^\>]*property=\"(?:og\:)?image\" content=\"(https?\:\/\/\w+\.\w+\/[^>]+\.(?:png|jpeg|jpg))\"\>`)
var nameExtractor = regexp.MustCompile(`\<meta[^\>]*property=\"(?:og\:)?title\" content=\"([^">]+)\"\>`)
func getThumbURL(url string) (thumb string, name string) {
if EnableThumbs {
mapMu.Lock()
defer mapMu.Unlock()
if !thumbRatelimit[url].IsZero() && time.Now().Sub(thumbRatelimit[url]) < time.Hour*24 {
id, err := GetMakerID(url)
if err != nil {
return
}
thumb = Conf.BaseURL + "/media/" + strconv.Itoa(id) + ".jpeg"
name = nameCache[url]
return
}
thumbRatelimit[url] = time.Now()
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Println(err)
@ -39,9 +56,16 @@ func getThumbURL(url string) (thumb string, name string) {
if thumbExtractor.Match(dat) {
thumb = thumbExtractor.FindStringSubmatch(string(dat))[1]
if EnableThumbs && (strings.HasSuffix(thumb, ".png") || strings.HasSuffix(thumb, ".jpeg") || strings.HasSuffix(thumb, ".jpg")) {
go addThumbnail(url, thumb)
}
}
if nameExtractor.Match(dat) {
name = nameExtractor.FindStringSubmatch(string(dat))[1]
nameCache[url] = name
}
if EnableThumbs {
thumbRatelimit[url] = time.Now()
}
return
}
@ -57,6 +81,7 @@ func newNotification(s Submission, where string) {
title := s.FmtName()
thumb, name := getThumbURL(s.URI)
// fmt.Println(thumb, name)
if Hook != nil {
err := func() error {
@ -106,7 +131,7 @@ func newNotification(s Submission, where string) {
}
}
for _, x := range Conf.NotifTopics {
for _, x := range Conf.Submissions.NotifTopics {
err := func() error {
req, err := http.NewRequest("POST", "https://ntfy.sh/"+x,
strings.NewReader(title+"\nBy: "+ifdef(s.Submitter, "anon")+"\nName: "+name+"\nTags:\n"+s.Tags))
@ -168,7 +193,7 @@ func logAction(s Submission, what uint, remote string) {
Value: "> " + s.Tags,
}}
dat, err := Hook.SendWait(dwhook.Message{
dat, err := AuditHook.Send(dwhook.Message{
Embeds: []dwhook.Embed{{
Title: message + ": " + s.ID.String(),
URL: Conf.BaseURL + "/pending?id=" + s.ID.String(),
@ -183,12 +208,7 @@ func logAction(s Submission, what uint, remote string) {
return err
}
msg, err := dwhook.UnmarshalResponse(dat, nil)
if err != nil {
return err
}
return AddSubmissionWebhook(s.ID, msg.ID)
return nil
}()
if err != nil {
log.Println(err)
@ -196,33 +216,33 @@ func logAction(s Submission, what uint, remote string) {
}
// i have decided my phone buzzing every time somebody does something would be too annoying
/*
for _, x := range Conf.NotifTopics {
err := func() error {
req, err := http.NewRequest("POST", "https://ntfy.sh/"+x,
strings.NewReader(message+" "+s.ID.String()+"\nBy: "+who+"\nMaker: "+title+"\nTags: "+s.Tags))
if err != nil {
return err
}
req.Header.Set("X-Title", "ACD "+message)
req.Header.Set("X-Priority", "min")
req.Header.Set("X-Tags", "inbox_tray")
req.Header.Set("X-Click", Conf.BaseURL+"/pending?id="+s.ID.String())
req.Header.Set("user-agent", "ACD Submissions")
req.Header.Set("from", Conf.HostEmail)
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
return nil
}()
// edit: i have still deemed it would but too annoying, i have just decided to put it in its own config
for _, x := range Conf.Audit.NotifTopics {
err := func() error {
req, err := http.NewRequest("POST", "https://ntfy.sh/"+x,
strings.NewReader(message+" "+s.ID.String()+"\nBy: "+who+"\nMaker: "+title+"\nTags: "+s.Tags))
if err != nil {
log.Println(err)
return err
}
}*/
req.Header.Set("X-Title", "ACD "+message)
req.Header.Set("X-Priority", "min")
req.Header.Set("X-Tags", "inbox_tray")
req.Header.Set("X-Click", Conf.BaseURL+"/pending?id="+s.ID.String())
req.Header.Set("user-agent", "ACD Submissions")
req.Header.Set("from", Conf.HostEmail)
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
return nil
}()
if err != nil {
log.Println(err)
}
}
}
func SplitAndDeduplicate(s string) []string {

View File

@ -3,12 +3,21 @@ package acdform
import "codeberg.org/eviedelta/dwhook"
var Hook *dwhook.Webhook
var AuditHook *dwhook.Webhook
var Conf Config
type Config struct {
NotifTopics []string
DiscordWebhook string
HostEmail string // used for the thumbnail fetcher
BaseURL string
Submissions struct {
NotifTopics []string
DiscordWebhook string
}
Audit struct {
NotifTopics []string
DiscordWebhook string
}
HostEmail string // used for the thumbnail fetcher
BaseURL string
ThumbnailDir string
}

View File

@ -79,6 +79,11 @@ func AddSubmission(s Submission) (id ID, err error) {
id = GenID()
s.ID = id
err = AddMaker(s.URI)
if err != nil {
return 0, err
}
_, err = DB.Exec(`INSERT INTO submissions (ID, URI, Tags, Submitter, ApprovalKey, Webhook, Approved) VALUES ($1, $2, $3, $4, $5, '', false)`,
s.ID, s.URI, s.Tags, s.Submitter, s.ApprovalKey)
return id, err
@ -94,6 +99,7 @@ type List struct {
URI string
Count int
Tags string
ID int
}
func (l List) Split() []string {
@ -116,7 +122,7 @@ func (s List) Fragment() string {
func ListSubmissions() ([]List, error) {
var list = []List{}
rows, err := DB.Queryx(`SELECT URI, count(ID) as count, group_concat(tags) FROM submissions WHERE approved=true GROUP BY URI ORDER BY count DESC`)
rows, err := DB.Queryx(`SELECT makers.URI, count(submissions.ID) as count, group_concat(tags), makers.ID FROM submissions LEFT JOIN makers ON submissions.URI=makers.URI WHERE approved=true GROUP BY makers.URI ORDER BY count DESC`)
if err != nil {
return nil, err
}
@ -195,6 +201,26 @@ func AddDiscovery(user, where string) error {
return err
}
type Maker struct {
URI string
ID int
}
func AddMaker(url string) error {
_, err := DB.Exec(`INSERT OR IGNORE INTO makers (URI) VALUES (?)`, url)
return err
}
func GetMaker(url string) (m Maker, err error) {
err = DB.Select(&m, `SELECT * FROM makers WHERE URI = ?`, url)
return
}
func GetMakerID(url string) (id int, err error) {
err = DB.Get(&id, `SELECT ID FROM makers WHERE URI = ?`, url)
return
}
// i just realised i didn't need any of this and could just do it in post :derp:
/*type Creator struct {
URI string

13
form.go
View File

@ -35,6 +35,17 @@ func Handle(w http.ResponseWriter, r *http.Request) {
}
}
func HandleNoCookie(w http.ResponseWriter, r *http.Request) {
http.SetCookie(w, &http.Cookie{
Name: "acd-identity",
Value: "",
SameSite: http.SameSiteStrictMode,
MaxAge: -100,
})
w.Header().Set("location", "/")
w.WriteHeader(301)
}
func HandleSubmit(w http.ResponseWriter, r *http.Request) {
if r.ContentLength > 4000 {
w.WriteHeader(http.StatusBadRequest)
@ -91,7 +102,7 @@ func HandleSubmit(w http.ResponseWriter, r *http.Request) {
Tags: sub.SplitTags(),
}
log.Println(r.FormValue("robot?"))
// log.Println(r.FormValue("robot?"))
// i assume robots are gonna click the field called "robot?"
// cause its usually "i am not a robot" so, i've done it backwards

2
go.mod
View File

@ -6,7 +6,9 @@ require github.com/pelletier/go-toml/v2 v2.0.2
require (
codeberg.org/eviedelta/dwhook v0.0.0-20210103160441-0562c0cbc295 // indirect
github.com/disintegration/imaging v1.6.2 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/mattn/go-sqlite3 v1.14.14 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect
)

5
go.sum
View File

@ -1,6 +1,8 @@
codeberg.org/eviedelta/dwhook v0.0.0-20210103160441-0562c0cbc295 h1:Y1hZlecHBEcQBMSEcuEqdgOS0FeJCNKU6UlrzJoFShA=
codeberg.org/eviedelta/dwhook v0.0.0-20210103160441-0562c0cbc295/go.mod h1:Fnh32YdiYc52GYb+yLu2fbtRQ94Tghu3oM1QVVBoQkA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
@ -15,5 +17,8 @@ github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuw
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

118
media.go Normal file
View File

@ -0,0 +1,118 @@
package acdform
import (
"bytes"
"log"
"net/http"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"image/jpeg"
_ "image/png"
"github.com/disintegration/imaging"
)
var EnableThumbs bool
var Mediadirecty string
func InitMedia() error {
mediadir := Conf.ThumbnailDir
if mediadir == "" {
return nil
}
mediadir, err := filepath.Abs(mediadir)
if err != nil {
return err
}
er2 := os.MkdirAll(mediadir, 0775)
if err != nil {
return er2 // :D silly code solely for alignment
}
EnableThumbs = true
Mediadirecty = mediadir
return nil
}
func init() {
// vacuum
go func() {
for {
time.Sleep(time.Hour)
mapMu.Lock()
for k, v := range thumbRatelimit {
if time.Now().Sub(v) > time.Hour*24 {
delete(thumbRatelimit, k)
}
}
mapMu.Unlock()
}
}()
}
var thumbRatelimit = make(map[string]time.Time, 100)
var nameCache = make(map[string]string, 100)
var mapMu sync.Mutex
func addThumbnail(maker, url string) {
if !EnableThumbs {
return
}
mapMu.Lock()
if !thumbRatelimit[url].IsZero() && time.Now().Sub(thumbRatelimit[url]) < time.Hour*24 {
mapMu.Unlock()
return
}
thumbRatelimit[url] = time.Now()
mapMu.Unlock()
id, err := GetMakerID(maker)
if err != nil {
log.Println(err)
return
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Println(err)
return
}
req.Header.Set("user-agent", "Thumbnail Fetcher for ACD Submissions")
req.Header.Set("from", Conf.HostEmail)
bod, err := http.DefaultClient.Do(req)
if err != nil {
log.Println(err)
return
}
defer bod.Body.Close()
img, err := imaging.Decode(bod.Body)
if err != nil {
log.Println(err)
return
}
img = imaging.Resize(img, 0, 128, imaging.Linear)
b := new(bytes.Buffer)
err = jpeg.Encode(b, img, &jpeg.Options{
Quality: 50, // not pretty but tiny
})
if err != nil {
log.Println(err)
return
}
err = os.WriteFile(filepath.Join(Mediadirecty, strconv.Itoa(id))+".jpeg", b.Bytes(), 0664)
if err != nil {
log.Println(err)
return
}
}

View File

@ -22,6 +22,12 @@ CREATE TABLE submissions (
-- TimesSubmitted int
--);
CREATE TABLE "makers" (
"URI" TEXT NOT NULL UNIQUE,
"ID" INTEGER,
PRIMARY KEY("ID" AUTOINCREMENT)
);
CREATE TABLE discoveries (
User text,
Source text

View File

@ -42,18 +42,34 @@ func main() {
log.Fatalln(err)
}
if acdform.Conf.DiscordWebhook != "" {
acdform.Hook, err = dwhook.New(acdform.Conf.DiscordWebhook)
if acdform.Conf.Submissions.DiscordWebhook != "" {
acdform.Hook, err = dwhook.New(acdform.Conf.Submissions.DiscordWebhook)
if err != nil {
log.Fatalln(err)
}
}
if acdform.Conf.Audit.DiscordWebhook != "" {
acdform.AuditHook, err = dwhook.New(acdform.Conf.Audit.DiscordWebhook)
if err != nil {
log.Fatalln(err)
}
}
if err := acdform.InitMedia(); err != nil {
log.Fatalln(err)
}
http.HandleFunc("/", acdform.HandleHome)
http.HandleFunc("/submit", acdform.HandleSubmit)
http.HandleFunc("/list", acdform.HandleList)
http.HandleFunc("/list.json", acdform.HandleListJSON)
http.HandleFunc("/pending", acdform.HandlePending)
http.HandleFunc("/nocookie", acdform.HandleNoCookie)
if acdform.EnableThumbs {
http.Handle("/media/", http.StripPrefix("/media/", http.FileServer(http.Dir(acdform.Mediadirecty))))
} else {
http.Handle("/media/", http.NotFoundHandler())
}
var conn net.Listener
if strings.HasSuffix(listen, ".sock") {

View File

@ -1,12 +1,11 @@
{{define "navigation"}}
<nav class="navbar" role="navigation">
<div class="navbar-start">
<a class="navbar-item" href="/">Submit</a>
<a class="navbar-item" href="/list">Approved</a>
<a class="navbar-item" href="/list.json">JSON</a>
<a class="navbar-item button is-rounded is-danger" href="/">Submit</a>
<a class="navbar-item button is-rounded is-info" href="/list">View Data</a>
<a class="navbar-item button is-rounded has-background-link-light" href="/list.json">JSON</a>
</div>
</nav>
<hr/>
{{end}}
{{define "home"}}
@ -22,23 +21,24 @@
{{template "navigation"}}
<form action="/submit" method="post">
<label for="furl"><b>Picrew URL</b><br/><sup><i>(picrews only currently, other character makers will be considered when the actual site is live)</i></sup></label><br/>
<input id="furl" name="furl" type="url" placeholder="https://picrew.me/image_maker/*****" class="input is-link" required><br/>
<label for="ftags"><b>What tags would you give to this picrew?</b> (like what features does it have, etc)<br/>
<input id="furl" name="furl" type="url" placeholder="https://picrew.me/image_maker/*****" class="input is-link" required maxlength="128"><br/>
<label for="ftags"><b>What tags would you give to this picrew?</b> (like what features does it have, etc, especially features that are less common)<br/>
<sup><i>(separate individual tags with commas, tags with a space <q>curly hair</q> keep the space as is <code>curly hair</code>)</i></sup></label><br/>
<textarea id="ftags" name="ftags" class="input is-info" required placeholder="curly hair, dark skin, animal ears"></textarea><br>
<textarea id="ftags" name="ftags" class="input is-info" required placeholder="curly hair, dark skin, animal ears" rows="4" maxlength="2000"></textarea><br>
<hr/>
{{if .NoCookie}}
<label><i><b>Who are you?</b> (this is optional and mostly just for curiosity, and you only have to fill it the first time)</i><br/>
<sup><i>(we'll set a cookie to not show this to you in the future, if you don't give us a name the cookie will not be able to identify you)</i></sup></label><br/>
<label for="fuser"><b>Name</b> (can be your username, or anything really)</label><br/>
<input id="fuser" name="fuser" type="text" class="input" placeholder="@yourname"><br/>
<input id="fuser" name="fuser" type="text" class="input" placeholder="@yourname" maxlength="100"><br/>
<label for="fwhere"><b>How did you find this form?</b></label><br/>
<textarea id="fwhere" name="fwhere" type="text" class="input" placeholder="(you can be as specific or nonspecific as you want)"></textarea><br/>
<hr/>
<textarea id="fwhere" name="fwhere" type="text" class="input" placeholder="(you can be as specific or nonspecific as you want)" rows="2" maxlength="1000"></textarea><br/>
{{else}}
<p>Name cached as {{if eq .Whomst "anon"}}Anonymous{{else}}<code>{{.Whomst}}</code>{{end}} <a href="/nocookie">click here to clear</a></p>
<input hidden="true" type="text" name="fname" value="{{.Whomst}}">
{{end}}
<label>are you an evil robot that desires to stuff our mailbox with junk? if so click the box below, otherwise leave it unchecked</label><br/>
<hr/>
<label>are you an evil robot that desires to stuff our mailbox with junk? if so click the box below, <i><b class="has-text-warning-dark has-background-danger-light">otherwise leave it unchecked</b></i> (your submittion will be ignored if you check it)</label><br/>
<label for="frobot"><b>I am a robot</b></label>
<input type="checkbox" id="frobot" name="robot?" class="checkbox is-small"><label></label><br/><hr/>
<input type="submit" value="Submit" class="button is-dark"><br/>

View File

@ -9,19 +9,22 @@
</head>
<body>
{{template "navigation"}}
<p class="tag is-light">{{len .List}} Entries</p>
{{range .List}}
<div class="box" id="{{.Fragment}}">
<a href="/list#{{.Fragment}}">§</a>
<a class="button is-link" href="{{.URI}}"><p>{{.FmtName}}</p></a><hr/>
<div class="box media" id="{{.ID}}">
<div class="media-content">
<a href="/list#{{.ID}}">§</a>
<a class="tag is-link" href="{{.URI}}"><p>{{.FmtName}}</p></a>
<p class="tag is-light">Submitted {{.Count}} Times</p>
<div class="box">
{{range .Split}}
<p class="tag is-info">{{.}}</p>
{{end}}
</div>
<p class="tag is-light">Submitted {{.Count}} Times</p>
</div>
<img class="media-right image is-128x128" src="/media/{{.ID}}.jpeg" alt="thumbnail" loading="lazy"/>
</div>
{{end}}
<p class="tag is-light">{{len .List}} Entries</p>
</body>
</html>
{{end}}