Merge branch 'getwtxt:master' into master

This commit is contained in:
~creme 2021-10-22 08:57:11 +02:00 committed by GitHub
commit f29bf2702e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 210 additions and 53 deletions

1
.gitignore vendored
View File

@ -4,3 +4,4 @@ logs/
local/
*.db/
*.db
/.idea

View File

@ -16,12 +16,6 @@ clean:
go clean
@printf "\n%s\n" "...Done!"
.PHONY: update
update:
@printf "\n%s\n\n" "Updating from upstream repository..."
git pull --rebase origin master
@printf "\n%s\n" "...Done!"
.PHONY: install
install:
@printf "\n%s\n" "Installing getwtxt..."
@ -34,7 +28,8 @@ install:
@printf "\n%s\n" "Copying files..."
install -m755 getwtxt $(BINDIR)
@if [ -f "$(BINDIR)/getwtxt.yml" ]; then printf "%s\n" "getwtxt.yml exists. Skipping ..."; else printf "%s\n" "getwtxt.yml ..." && install -m644 getwtxt.yml "$(BINDIR)"; fi
@if [ -f "$(BINDIR)/getwtxt.yml" ]; then printf "%s\n" "getwtxt.yml exists. Skipping ..."; else printf "%s\n" "getwtxt.yml ..." && install -m600 getwtxt.yml "$(BINDIR)"; fi
chmod 600 $(BINDIR)/getwtxt.yml
@if [ -f "$(BINDIR)/assets/style.css" ]; then printf "%s\n" "style.css exists. Skipping ..."; else printf "%s\n" "style.css ..." && install -m644 assets/style.css "$(BINDIR)/assets/style.css"; fi
@if [ -f "$(BINDIR)/assets/tmpl/index.html" ]; then printf "%s\n" "tmpl/index.html exists. Skipping ..."; else printf "%s\n" "tmpl/index.html ..." && install -m644 assets/tmpl/index.html "$(BINDIR)/assets/tmpl/index.html"; fi
install -m644 static/kognise.water.css.dark.min.css $(BINDIR)/static

View File

@ -1,10 +1,15 @@
# getwtxt  [![builds.sr.ht status](https://builds.sr.ht/~gbmor/getwtxt.svg)](https://builds.sr.ht/~gbmor/getwtxt?) [![Build Status](https://travis-ci.com/getwtxt/getwtxt.svg?branch=master)](https://travis-ci.com/getwtxt/getwtxt) [![Go Report Card](https://goreportcard.com/badge/github.com/getwtxt/getwtxt)](https://goreportcard.com/report/github.com/getwtxt/getwtxt) [![Code Climate Maintainability](https://api.codeclimate.com/v1/badges/0e48bd9002de0f84b24e/maintainability)](https://codeclimate.com/github/getwtxt/getwtxt/maintainability)
# getwtxt
[![builds.sr.ht status](https://builds.sr.ht/~gbmor/getwtxt.svg)](https://builds.sr.ht/~gbmor/getwtxt?)
[![Build Status](https://travis-ci.com/getwtxt/getwtxt.svg?branch=master)](https://travis-ci.com/getwtxt/getwtxt)
[![Go Report Card](https://goreportcard.com/badge/github.com/getwtxt/getwtxt)](https://goreportcard.com/report/github.com/getwtxt/getwtxt)
[![Code Climate Maintainability](https://api.codeclimate.com/v1/badges/0e48bd9002de0f84b24e/maintainability)](https://codeclimate.com/github/getwtxt/getwtxt/maintainability)
twtxt registry written in Go!
[twtxt](https://github.com/buckket/twtxt) is a decentralized microblogging platform
for hackers based on text files. The user is "followed" and "mentioned" by referencing
the URL to their `twtxt.txt` file and a nickname.
Registries are designed to aggregate several users' statuses into a single location,
facilitating the discovery of new users to follow and allowing the search of statuses
for tags and key words.
@ -72,29 +77,16 @@ $ sudo make install
## Upgrading
Upgrading is a fairly simple process. First, we need to commit your local changes
to the configuration file.
Upgrading is nearly a identical process. Pull the changes, check out the
latest tag, and rebuild.
```
$ cp /usr/local/getwtxt/getwtxt.yml .
$ git add getwtxt.yml
$ git commit -m 'my local config'
```
systemd might yell at you about running `systemctl daemon-reload` when you
go to restart getwtxt.
Now, we need to either run `make update` or `git pull --rebase origin master`
```
$ make update
...
```
Afterwards, follow the normal instructions for building and installing.
If no configuration changes have been made since your last upgrade,
you will not need to commit them again. While `getwtxt` is pre-`1.0`, any
patch-level updates (`v0.4.x`) will not change configuration values.
Of course, you can also just back up your configuration file, then copy it
back into `/usr/local/getwtxt/` after installing the new version.
While getwtxt is pre-`1.0`, any patch-level updates (`v0.4.x`) will not
change configuration values. If a minor version increase has happened, for
example `v0.4.x -> v0.5.x`, then check if you need to update the config
file before restarting getwtxt.
## Configuration
@ -113,20 +105,24 @@ the template.
### Proxying
Though getwtxt will run perfectly fine facing the internet directly, it does not
understand virtual hosts, nor does it use TLS. You'll probably want to proxy it behind
understand virtual hosts, nor does it use TLS. You'll probably want to proxy it
behind
`Caddy` or `nginx` for this reason.
`Caddy` is ludicrously easy to set up, and automatically handles `TLS` certificates. Here's the config:
`Caddy` is ludicrously easy to set up, and automatically handles `TLS`
certificates. Here's the config:
```caddyfile
twtxt.example.com
proxy / example.com:9001
```
If you're using `nginx`, here's a skeleton config to get you started. Don't forget to change
the 5 instances of `twtxt.example.com` to the (sub)domain you'll be using to access the registry,
generate SSL/TLS certificates using LetsEncrypt, and change the port in `proxy_pass` to whichever
port you specified when modifying the configuration file. Currently, it's set to the default port `9001`
If you're using `nginx`, here's a skeleton config to get you started. Don't
forget to change the 5 instances of `twtxt.example.com` to the (sub)domain
you'll be using to access the registry, generate SSL/TLS certificates using
LetsEncrypt, and change the port in `proxy_pass` to whichever port you
specified when modifying the configuration file. Currently, it's set to the
default port `9001`
```nginx
server {
@ -166,13 +162,14 @@ $ sudo systemctl start getwtxt
## Using the Registry
The following examples will all apply to using `curl` from a `Linux`, `BSD`, or `macOS` terminal.
All timestamps are in `RFC3339` format, per the twtxt registry specification. Additionally, all
queries support the `?page=N` parameter, where `N` is a positive integer, that will retrieve page
`N` of results in groups of twenty.
The following examples will all apply to using `curl` from a `Linux`, `BSD`, or
`macOS` terminal. All timestamps are in `RFC3339` format, per the twtxt registry
specification. Additionally, all queries support the `?page=N` parameter, where
`N` is a positive integer, that will retrieve page `N` of results in groups of
twenty.
The example API calls can also be found on the landing page of any getwtxt instance, assuming
the admin has not customized the landing page.
The example API calls can also be found on the landing page of any getwtxt
instance, assuming the admin has not customized the landing page.
### Adding a User
Both nickname and URL are required
@ -266,6 +263,14 @@ $ curl 'https://twtxt.example.com/api/plain/tags/programming'
foo https://example.com/twtxt.txt 2019-03-01T09:31:02.000Z I love #programming!
```
### Delete a User
```
$ curl -X DELETE -H 'X-Auth: password_in_getwtxt.yml' 'https://twtxt.example.com/api/admin/users?url=https://example.com/twtxt.txt'
200 OK
```
## Benchmarks
* [bombardier](https://github.com/codesenberg/bombardier)
@ -289,9 +294,10 @@ Statistics Avg Stdev Max
## Other Documentation
In addition to what is provided here, additional information, particularly regarding the configuration
file, may be found by running getwtxt with the `-m` or `--manual` flags. You will likely want to pipe the output
to `less` as it is quite long.
In addition to what is provided here, additional information, particularly
regarding the configuration file, may be found by running getwtxt with the `-m`
or `--manual` flags. You will likely want to pipe the output to `less` as it is
quite long.
```
$ ./getwtxt -m | less
@ -319,7 +325,7 @@ Registry Specification: [`twtxt.readthedocs.io/en/latest/user/registry.html`](ht
Special thanks to [`github.com/kognise/water.css`](https://github.com/kognise/water.css) for open-sourcing a pleasant, easy-to-use, importable stylesheet
### Contributing
## Contributing
All contributions are greatly appreciated!

View File

@ -43,6 +43,10 @@
<pre><code>$ curl '{{.URL}}/api/plain/version'
getwtxt {{.Vers}}
</code></pre>
<p>Delete a user by issuing a <code>DELETE</code> request to the <code>/api/admin/users</code> endpoint. This
must include the <code>X-Auth</code> header with the password specified during configuration.</p>
<pre><code>$ curl -X DELETE -H 'X-Auth: mypassword' '{{.URL}}/api/admin/users?url=https://foo.ext/twtxt.txt'
200 OK</code></pre>
<p>Add new user by submitting a <code>POST</code> request to the <code>/api/plain/users</code> endpoint.
If both <code>?url=X</code> and <code>?nickname=X</code> are not passed, or the user already exists in
this registry, you will receive <code>400 Bad Request</code> as a response. If you are unsure what went

View File

@ -40,6 +40,9 @@ DatabasePath: "getwtxt.db"
## changes are detected. ##
#############################################################
# Administrator password for certain destructive actions
AdminPassword: "please_change_me"
# The path to the assets directory, which contains:
# style.css
# tmpl/index.html

3
go.mod
View File

@ -10,5 +10,6 @@ require (
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.0
github.com/syndtr/goleveldb v1.0.0
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1
)

14
go.sum
View File

@ -203,6 +203,8 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -235,8 +237,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -260,12 +263,15 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

16
svc/common.go Normal file
View File

@ -0,0 +1,16 @@
package svc
import "golang.org/x/crypto/bcrypt"
// HashPass returns the bcrypt hash of the provided string.
// If an empty string is provided, return an empty string.
func HashPass(s string) (string, error) {
if s == "" {
return "", nil
}
h, err := bcrypt.GenerateFromPassword([]byte(s), 14)
if err != nil {
return "", err
}
return string(h), nil
}

34
svc/common_test.go Normal file
View File

@ -0,0 +1,34 @@
package svc
import (
"testing"
)
func TestHashPass(t *testing.T) {
cases := []struct {
in, name string
shouldFail bool
}{
{
in: "foo",
name: "non-empty password",
shouldFail: false,
},
{
in: "",
name: "empty password",
shouldFail: true,
},
}
for _, v := range cases {
t.Run(v.name, func(t *testing.T) {
out, err := HashPass(v.in)
if err != nil && !v.shouldFail {
t.Errorf("Shouldn't have failed: Case %s, Error: %s", v.name, err)
}
if out == "" && v.in != "" {
t.Errorf("Got empty out for case %s input %s", v.name, v.in)
}
})
}
}

View File

@ -20,6 +20,7 @@ along with Getwtxt. If not, see <https://www.gnu.org/licenses/>.
package svc // import "git.sr.ht/~gbmor/getwtxt/svc"
import (
"fmt"
"log"
"os"
"path/filepath"
@ -43,6 +44,7 @@ type Configuration struct {
DBPath string `yaml:"DatabasePath"`
AssetsDir string `yaml:"AssetsDirectory"`
StaticDir string `yaml:"StaticFilesDirectory"`
AdminPassHash string `yaml:"-"`
StdoutLogging bool `yaml:"StdoutLogging"`
CacheInterval time.Duration `yaml:"StatusFetchInterval"`
DBInterval time.Duration `yaml:"DatabasePushInterval"`
@ -126,6 +128,7 @@ func setConfigDefaults() {
viper.SetDefault("StdoutLogging", false)
viper.SetDefault("ReCacheInterval", "1h")
viper.SetDefault("DatabasePushInterval", "5m")
viper.SetDefault("AdminPassword", "please_change_me")
viper.SetDefault("Instance.SiteName", "getwtxt")
viper.SetDefault("Instance.OwnerName", "Anonymous Microblogger")
@ -173,6 +176,16 @@ func bindConfig() {
confObj.StdoutLogging = viper.GetBool("StdoutLogging")
confObj.CacheInterval = viper.GetDuration("StatusFetchInterval")
confObj.DBInterval = viper.GetDuration("DatabasePushInterval")
txtPass := viper.GetString("AdminPassword")
if txtPass == "please_change_me" || strings.TrimSpace(txtPass) == "" {
fmt.Println("Please set AdminPassword in getwtxt.yml")
os.Exit(1)
}
passHash, err := HashPass(txtPass)
if err != nil {
errFatal("Failed to hash administrator password: ", err)
}
confObj.AdminPassHash = passHash
confObj.Instance.Vers = Vers
confObj.Instance.Name = viper.GetString("Instance.SiteName")

View File

@ -39,6 +39,7 @@ import (
type dbase interface {
push() error
pull()
delUser(string) error
}
// Opens a new connection to the specified
@ -96,3 +97,13 @@ func pullDB() {
dbChan <- db
log.Printf("Database pull took: %v\n", time.Since(start))
}
func delUser(userURL string) error {
db := <-dbChan
err := db.delUser(userURL)
dbChan <- db
if err != nil {
return err
}
return twtxtCache.DelUser(userURL)
}

View File

@ -20,15 +20,18 @@ along with Getwtxt. If not, see <https://www.gnu.org/licenses/>.
package svc // import "git.sr.ht/~gbmor/getwtxt/svc"
import (
"errors"
"fmt"
"hash/fnv"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"git.sr.ht/~gbmor/getwtxt/registry"
"github.com/gorilla/mux"
"golang.org/x/crypto/bcrypt"
)
// Takes the modtime of one of the static files, derives
@ -242,3 +245,38 @@ func apiTagsHandler(w http.ResponseWriter, r *http.Request) {
}
log200(r)
}
func handleUserDelete(w http.ResponseWriter, r *http.Request) {
pass := r.Header.Get("X-Auth")
if pass == "" {
errHTTP(w, r, errors.New("unauthorized"), http.StatusUnauthorized)
return
}
confObj.Mu.RLock()
adminHash := []byte(confObj.AdminPassHash)
confObj.Mu.RUnlock()
if err := bcrypt.CompareHashAndPassword(adminHash, []byte(pass)); err != nil {
errHTTP(w, r, errors.New("unauthorized"), http.StatusUnauthorized)
return
}
r.ParseForm()
userURL := strings.TrimSpace(r.Form.Get("url"))
if userURL == "" {
errHTTP(w, r, errors.New("bad request"), http.StatusBadRequest)
return
}
if _, err := url.Parse(userURL); err != nil {
errHTTP(w, r, errors.New("bad request"), http.StatusBadRequest)
return
}
if err := delUser(userURL); err != nil {
return
}
w.WriteHeader(200)
w.Write([]byte("200 OK\n"))
log200(r)
}

View File

@ -33,6 +33,27 @@ type dbLevel struct {
db *leveldb.DB
}
func (lvl *dbLevel) delUser(userURL string) error {
twtxtCache.Mu.RLock()
defer twtxtCache.Mu.RUnlock()
userStatuses := twtxtCache.Users[userURL].Status
var dbBasket = &leveldb.Batch{}
dbBasket.Delete([]byte(userURL + "*Nick"))
dbBasket.Delete([]byte(userURL + "*URL"))
dbBasket.Delete([]byte(userURL + "*IP"))
dbBasket.Delete([]byte(userURL + "*Date"))
dbBasket.Delete([]byte(userURL + "*LastModified"))
for i := range userStatuses {
rfc := i.Format(time.RFC3339)
dbBasket.Delete([]byte(userURL + "*Status*" + rfc))
}
return lvl.db.Write(dbBasket, nil)
}
// Called intermittently to commit registry data to
// a LevelDB database.
func (lvl *dbLevel) push() error {
@ -53,9 +74,9 @@ func (lvl *dbLevel) push() error {
}
}
for k, v := range remoteRegistries.List {
dbBasket.Put([]byte("remote*"+string(k)), []byte(v))
}
//for k, v := range remoteRegistries.List {
//dbBasket.Put([]byte("remote*"+string(rune(k))), []byte(v))
//}
return lvl.db.Write(dbBasket, nil)
}

View File

@ -64,6 +64,10 @@ func initSqlite() *dbSqlite {
}
}
func (lite *dbSqlite) delUser(userURL string) error {
return nil
}
// Commits data from memory to a SQLite database intermittently.
func (lite *dbSqlite) push() error {
if err := lite.db.Ping(); err != nil {

View File

@ -91,6 +91,10 @@ func setIndexRouting(index *mux.Router) {
}
func setEndpointRouting(api *mux.Router) {
api.Path("/admin/users").
Methods("DELETE").
HandlerFunc(handleUserDelete)
// May add support for other formats later.
// Making this future-proof.
api.Path("/{format:(?:plain)}").