From 538e305925b9b04102ef0a4fb7cca19a6c116142 Mon Sep 17 00:00:00 2001 From: Ben Morrison Date: Sat, 20 Jun 2020 02:27:31 -0400 Subject: [PATCH] updating module to live at sourcehut Also moving the 'registry' library into this repo, rather than maintaining them separately. It will still be decoupled, just live in this repository. --- .build.yml | 10 + Makefile | 2 +- assets/tmpl/index.html | 2 +- getwtxt.go | 2 +- go.mod | 7 +- go.sum | 185 +++++++++++++++-- registry/README.md | 48 +++++ registry/fetch.go | 277 +++++++++++++++++++++++++ registry/fetch_test.go | 286 +++++++++++++++++++++++++ registry/init_test.go | 93 +++++++++ registry/integ_test.go | 98 +++++++++ registry/query.go | 196 ++++++++++++++++++ registry/query_test.go | 459 +++++++++++++++++++++++++++++++++++++++++ registry/revive.toml | 30 +++ registry/types.go | 148 +++++++++++++ registry/user.go | 270 ++++++++++++++++++++++++ registry/user_test.go | 349 +++++++++++++++++++++++++++++++ svc/README.md | 8 - svc/cache.go | 2 +- svc/cache_test.go | 6 +- svc/conf.go | 2 +- svc/db.go | 2 +- svc/db_test.go | 22 +- svc/handlers.go | 4 +- svc/handlers_test.go | 2 +- svc/help.go | 6 +- svc/http.go | 2 +- svc/init.go | 4 +- svc/init_test.go | 16 +- svc/leveldb.go | 4 +- svc/periodic.go | 2 +- svc/post.go | 4 +- svc/post_test.go | 8 +- svc/query.go | 4 +- svc/query_test.go | 14 +- svc/sqlite.go | 4 +- svc/svc.go | 2 +- testdata/twtxt.txt | 2 +- 38 files changed, 2493 insertions(+), 89 deletions(-) create mode 100644 .build.yml create mode 100644 registry/README.md create mode 100644 registry/fetch.go create mode 100644 registry/fetch_test.go create mode 100644 registry/init_test.go create mode 100644 registry/integ_test.go create mode 100644 registry/query.go create mode 100644 registry/query_test.go create mode 100644 registry/revive.toml create mode 100644 registry/types.go create mode 100644 registry/user.go create mode 100644 registry/user_test.go delete mode 100644 svc/README.md diff --git a/.build.yml b/.build.yml new file mode 100644 index 0000000..d4677a8 --- /dev/null +++ b/.build.yml @@ -0,0 +1,10 @@ +image: alpine/edge +packages: + - go +sources: + - https://git.sr.ht/~gbmor/getwtxt +tasks: + - build: | + cd getwtxt + go test -v + go test -v --bench . --benchmem diff --git a/Makefile b/Makefile index 9d83b15..319357b 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PREFIX?=/usr/local _INSTDIR=$(PREFIX) BINDIR?=$(_INSTDIR)/getwtxt VERSION?=$(shell git tag | grep ^v | sort -V | tail -n 1) -GOFLAGS?=-ldflags '-X github.com/getwtxt/getwtxt/svc.Vers=${VERSION}' +GOFLAGS?=-ldflags '-X git.sr.ht/~gbmor/getwtxt/svc.Vers=${VERSION}' getwtxt: getwtxt.go go.mod go.sum @echo diff --git a/assets/tmpl/index.html b/assets/tmpl/index.html index 9006bec..0318810 100644 --- a/assets/tmpl/index.html +++ b/assets/tmpl/index.html @@ -90,7 +90,7 @@ foo_barrington https://example3.com/twtxt.txt 2019-02-26T11:06:44.000Z foo https://example.com/twtxt.txt 2019-02-26T11:06:44.000Z @<foo_barrington https://example3.com/twtxt.txt> Hey!! Are you still working on that project? diff --git a/getwtxt.go b/getwtxt.go index 45d04c3..5e8e52c 100644 --- a/getwtxt.go +++ b/getwtxt.go @@ -19,7 +19,7 @@ along with Getwtxt. If not, see . package main -import "github.com/getwtxt/getwtxt/svc" +import "git.sr.ht/~gbmor/getwtxt/svc" func main() { svc.Start() diff --git a/go.mod b/go.mod index 1d60931..2013e93 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,14 @@ -module github.com/getwtxt/getwtxt +module git.sr.ht/~gbmor/getwtxt go 1.11 require ( github.com/fsnotify/fsnotify v1.4.9 - github.com/getwtxt/registry v0.4.2 github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.4 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.6.2 + github.com/spf13/viper v1.7.0 github.com/syndtr/goleveldb v1.0.0 - golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 + golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 ) diff --git a/go.sum b/go.sum index 647704a..fdd4551 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,47 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/getwtxt/registry v0.4.2 h1:6qp7KkBi60CZaJpFC21ez29QwiIcCnzazAG3w9OdXaU= -github.com/getwtxt/registry v0.4.2/go.mod h1:BGSIALOFqIRj+ACLB8etWGUOgFAKN8oFDpCsw6YOdYQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -34,28 +51,60 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -70,11 +119,22 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -82,11 +142,14 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -97,6 +160,9 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -113,62 +179,139 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= @@ -181,3 +324,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/registry/README.md b/registry/README.md new file mode 100644 index 0000000..34cd37e --- /dev/null +++ b/registry/README.md @@ -0,0 +1,48 @@ +# `getwtxt/registry` + +### twtxt Registry Library for Go + +`getwtxt/registry` helps you implement twtxt registries in Go. +It uses no third-party dependencies whatsoever, only the standard library, +and has no global state. +Specifying your own `http.Client` for requests is encouraged, with a sensible +default available by passing `nil` to the constructor. + +## Using the Library + +Just add it to your imports list in the file(s) where it's needed. + +```go +import ( + "git.sr.ht/~gbmor/getwtxt/registry" +) +``` + +## Documentation + +The code is commented, so feel free to browse the files themselves. +Alternatively, the generated documentation can be found at: + +[pkg.go.dev/git.sr.ht/~gbmor/getwtxt/registry](https://pkg.go.dev/git.sr.ht/~gbmor/getwtxt/registry) + +## Contributions + +All contributions are very welcome! Please specify that you are referring to `getwtxt/registry` +when using the following: + +* Mailing list (patches, discussion) + * [https://lists.sr.ht/~gbmor/getwtxt](https://lists.sr.ht/~gbmor/getwtxt) +* Ticket tracker + * [https://todo.sr.ht/~gbmor/getwtxt](https://todo.sr.ht/~gbmor/getwtxt) + +## Notes + +* getwtxt - parent project: + * [sr.ht/~gbmor/getwtxt](https://sr.ht/~gbmor/getwtxt) + +* twtxt repository: + * [github.com/buckket/twtxt](https://github.com/buckket/twtxt) +* twtxt documentation: + * [twtxt.readthedocs.io/en/latest/](https://twtxt.readthedocs.io/en/latest/) +* twtxt registry documentation: + * [twtxt.readthedocs.io/en/latest/user/registry.html](https://twtxt.readthedocs.io/en/latest/user/registry.html) diff --git a/registry/fetch.go b/registry/fetch.go new file mode 100644 index 0000000..9adf4ec --- /dev/null +++ b/registry/fetch.go @@ -0,0 +1,277 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "net/http" + "regexp" + "strings" + "sync" + "time" +) + +// GetTwtxt fetches the raw twtxt file data from the user's +// provided URL, after validating the URL. If the returned +// boolean value is false, the fetched URL is a single user's +// twtxt file. If true, the fetched URL is the output of +// another registry's /api/plain/tweets. The output of +// GetTwtxt should be passed to either ParseUserTwtxt or +// ParseRegistryTwtxt, respectively. +// Generally, the *http.Client inside a given Registry instance should +// be passed to GetTwtxt. If the *http.Client passed is nil, +// Registry will use a preconstructed client with a +// timeout of 10s and all other values set to default. +func GetTwtxt(urlKey string, client *http.Client) ([]byte, bool, error) { + if !strings.HasPrefix(urlKey, "http://") && !strings.HasPrefix(urlKey, "https://") { + return nil, false, fmt.Errorf("invalid URL: %v", urlKey) + } + + res, err := doReq(urlKey, "GET", "", client) + if err != nil { + return nil, false, err + } + defer res.Body.Close() + + var textPlain bool + for _, v := range res.Header["Content-Type"] { + if strings.Contains(v, "text/plain") { + textPlain = true + break + } + } + if !textPlain { + return nil, false, fmt.Errorf("received non-text/plain response body from %v", urlKey) + } + + if res.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("didn't get 200 from remote server, received %v: %v", res.StatusCode, urlKey) + } + + twtxt, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, false, fmt.Errorf("error reading response body from %v: %v", urlKey, err) + } + + // Signal that we're adding another twtxt registry as a "user" + if strings.HasSuffix(urlKey, "/api/plain/tweets") || strings.HasSuffix(urlKey, "/api/plain/tweets/all") { + return twtxt, true, nil + } + + return twtxt, false, nil +} + +// DiffTwtxt issues a HEAD request on the user's +// remote twtxt data. It then checks the Content-Length +// header. If it's different from the stored result of +// the previous Content-Length header, update the stored +// value for a given user and return true. +// Otherwise, return false. In some error conditions, +// such as the user not being in the registry, it returns true. +// In other error conditions considered "unrecoverable," +// such as the supplied URL being invalid, it returns false. +func (registry *Registry) DiffTwtxt(urlKey string) (bool, error) { + if !strings.HasPrefix(urlKey, "http://") && !strings.HasPrefix(urlKey, "https://") { + return false, fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.Lock() + user, ok := registry.Users[urlKey] + if !ok { + return true, fmt.Errorf("user not in registry") + } + + user.Mu.Lock() + + defer func() { + registry.Users[urlKey] = user + user.Mu.Unlock() + registry.Mu.Unlock() + }() + + res, err := doReq(urlKey, "HEAD", user.LastModified, registry.HTTPClient) + if err != nil { + return false, err + } + + switch res.StatusCode { + case http.StatusOK: + for _, e := range res.Header["Last-Modified"] { + if e != "" { + user.LastModified = e + break + } + } + return true, nil + + case http.StatusNotModified: + return false, nil + } + + return false, nil +} + +// internal function. boilerplate for http requests. +func doReq(urlKey, method, modTime string, client *http.Client) (*http.Response, error) { + if client == nil { + client = &http.Client{ + Transport: nil, + CheckRedirect: nil, + Jar: nil, + Timeout: 10 * time.Second, + } + } + + var b []byte + buf := bytes.NewBuffer(b) + req, err := http.NewRequest(method, urlKey, buf) + if err != nil { + return nil, err + } + + if modTime != "" { + req.Header.Set("If-Modified-Since", modTime) + } + + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("couldn't %v %v: %v", method, urlKey, err) + } + + return res, nil +} + +// ParseUserTwtxt takes a fetched twtxt file in the form of +// a slice of bytes, parses it, and returns it as a +// TimeMap. The output may then be passed to Index.AddUser() +func ParseUserTwtxt(twtxt []byte, nickname, urlKey string) (TimeMap, error) { + var erz []byte + if len(twtxt) == 0 { + return nil, fmt.Errorf("no data to parse in twtxt file") + } + + reader := bytes.NewReader(twtxt) + scanner := bufio.NewScanner(reader) + timemap := NewTimeMap() + + for scanner.Scan() { + nopadding := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(nopadding, "#") || nopadding == "" { + continue + } + + columns := strings.Split(nopadding, "\t") + if len(columns) != 2 { + return nil, fmt.Errorf("improperly formatted data in twtxt file") + } + + normalizedDatestamp := fixTimestamp(columns[0]) + thetime, err := time.Parse(time.RFC3339, normalizedDatestamp) + if err != nil { + erz = append(erz, []byte(fmt.Sprintf("unable to retrieve date: %v\n", err))...) + } + + timemap[thetime] = nickname + "\t" + urlKey + "\t" + nopadding + } + + if len(erz) == 0 { + return timemap, nil + } + return timemap, fmt.Errorf("%v", string(erz)) +} + +func fixTimestamp(ts string) string { + normalizeTimestamp := regexp.MustCompile(`[\+][\d][\d][:][\d][\d]`) + return strings.TrimSpace(normalizeTimestamp.ReplaceAllString(ts, "Z")) +} + +// ParseRegistryTwtxt takes output from a remote registry and outputs +// the accessible user data via a slice of Users. +func ParseRegistryTwtxt(twtxt []byte) ([]*User, error) { + var erz []byte + if len(twtxt) == 0 { + return nil, fmt.Errorf("received no data") + } + + reader := bytes.NewReader(twtxt) + scanner := bufio.NewScanner(reader) + userdata := []*User{} + + for scanner.Scan() { + + nopadding := strings.TrimSpace(scanner.Text()) + + if strings.HasPrefix(nopadding, "#") || nopadding == "" { + continue + } + + columns := strings.Split(nopadding, "\t") + if len(columns) != 4 { + return nil, fmt.Errorf("improperly formatted data") + } + + thetime, err := time.Parse(time.RFC3339, columns[2]) + if err != nil { + erz = append(erz, []byte(fmt.Sprintf("%v\n", err))...) + continue + } + + parsednickname := columns[0] + dataIndex := 0 + parsedurl := columns[1] + inIndex := false + + for i, e := range userdata { + if e.Nick == parsednickname || e.URL == parsedurl { + dataIndex = i + inIndex = true + break + } + } + + if inIndex { + tmp := userdata[dataIndex] + tmp.Status[thetime] = nopadding + userdata[dataIndex] = tmp + } else { + timeNowRFC := time.Now().Format(time.RFC3339) + if err != nil { + erz = append(erz, []byte(fmt.Sprintf("%v\n", err))...) + } + + tmp := &User{ + Mu: sync.RWMutex{}, + Nick: parsednickname, + URL: parsedurl, + Date: timeNowRFC, + Status: TimeMap{ + thetime: nopadding, + }, + } + + userdata = append(userdata, tmp) + } + } + + return userdata, fmt.Errorf("%v", erz) +} diff --git a/registry/fetch_test.go b/registry/fetch_test.go new file mode 100644 index 0000000..4eab2a4 --- /dev/null +++ b/registry/fetch_test.go @@ -0,0 +1,286 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry + +import ( + "bufio" + "fmt" + "net/http" + "os" + "strings" + "testing" + "time" +) + +func constructTwtxt() []byte { + registry := initTestEnv() + var resp []byte + // iterates through each mock user's mock statuses + for _, v := range registry.Users { + for _, e := range v.Status { + split := strings.Split(e, "\t") + status := []byte(split[2] + "\t" + split[3] + "\n") + resp = append(resp, status...) + } + } + return resp +} + +// this is just dumping all the mock statuses. +// it'll be served under fake paths as +// "remote" twtxt.txt files +func twtxtHandler(w http.ResponseWriter, _ *http.Request) { + // prepare the response + resp := constructTwtxt() + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + n, err := w.Write(resp) + if err != nil || n == 0 { + fmt.Printf("Got error or wrote zero bytes: %v bytes, %v\n", n, err) + } +} + +var getTwtxtCases = []struct { + name string + url string + wantErr bool + localOnly bool +}{ + { + name: "Constructed Local twtxt.txt", + url: "http://localhost:8080/twtxt.txt", + wantErr: false, + localOnly: true, + }, + { + name: "Inaccessible Site With twtxt.txt", + url: "https://example33333333333.com/twtxt.txt", + wantErr: true, + localOnly: false, + }, + { + name: "Inaccessible Site Without twtxt.txt", + url: "https://example333333333333.com", + wantErr: true, + localOnly: false, + }, + { + name: "Local File Inclusion 1", + url: "file://init_test.go", + wantErr: true, + localOnly: false, + }, + { + name: "Local File Inclusion 2", + url: "/etc/passwd", + wantErr: true, + localOnly: false, + }, + { + name: "Remote File Inclusion", + url: "https://example.com/file.cgi", + wantErr: true, + localOnly: false, + }, + { + name: "Remote Registry", + url: "https://twtxt.tilde.institute/api/plain/tweets/", + wantErr: false, + localOnly: false, + }, + { + name: "Garbage Data", + url: "this will be replaced with garbage data", + wantErr: true, + localOnly: true, + }, +} + +// Test the function that yoinks the /twtxt.txt file +// for a given user. +func Test_GetTwtxt(t *testing.T) { + var buf = make([]byte, 256) + // read random data into case 4 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + getTwtxtCases[7].url = string(buf) + + if !getTwtxtCases[0].localOnly { + http.Handle("/twtxt.txt", http.HandlerFunc(twtxtHandler)) + go fmt.Println(http.ListenAndServe(":8080", nil)) + } + + for _, tt := range getTwtxtCases { + t.Run(tt.name, func(t *testing.T) { + if tt.localOnly { + t.Skipf("Local-only test. Skipping ... \n") + } + out, _, err := GetTwtxt(tt.url, nil) + if tt.wantErr && err == nil { + t.Errorf("Expected error: %v\n", tt.url) + } + if !tt.wantErr && err != nil { + t.Errorf("Unexpected error: %v %v\n", tt.url, err) + } + if !tt.wantErr && out == nil { + t.Errorf("Incorrect data received: %v\n", out) + } + }) + } + +} + +// running the benchmarks separately for each case +// as they have different properties (allocs, time) +func Benchmark_GetTwtxt(b *testing.B) { + + for i := 0; i < b.N; i++ { + _, _, err := GetTwtxt("https://gbmor.dev/twtxt.txt", nil) + if err != nil { + continue + } + } +} + +var parseTwtxtCases = []struct { + name string + data []byte + wantErr bool + localOnly bool +}{ + { + name: "Constructed twtxt file", + data: constructTwtxt(), + wantErr: false, + localOnly: false, + }, + { + name: "Incorrectly formatted date", + data: []byte("2019 April 23rd\tI love twtxt!!!11"), + wantErr: true, + localOnly: false, + }, + { + name: "No data", + data: []byte{}, + wantErr: true, + localOnly: false, + }, + { + name: "Variant rfc3339 datestamp", + data: []byte("2020-02-04T21:28:21.868659+00:00\tWill this work?"), + wantErr: false, + localOnly: false, + }, + { + name: "Random/garbage data", + wantErr: true, + localOnly: true, + }, +} + +// See if we can break ParseTwtxt or get it +// to throw an unexpected error +func Test_ParseUserTwtxt(t *testing.T) { + var buf = make([]byte, 256) + // read random data into case 4 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + parseTwtxtCases[4].data = buf + + for _, tt := range parseTwtxtCases { + if tt.localOnly { + t.Skipf("Local-only test: Skipping ... \n") + } + t.Run(tt.name, func(t *testing.T) { + timemap, errs := ParseUserTwtxt(tt.data, "testuser", "testurl") + if errs == nil && tt.wantErr { + t.Errorf("Expected error(s), received none.\n") + } + + if !tt.wantErr { + if errs != nil { + t.Errorf("Unexpected error: %v\n", errs) + } + + for k, v := range timemap { + if k == (time.Time{}) || v == "" { + t.Errorf("Empty status or empty timestamp: %v, %v\n", k, v) + } + } + } + }) + } +} + +func Benchmark_ParseUserTwtxt(b *testing.B) { + var buf = make([]byte, 256) + // read random data into case 4 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + b.Errorf("Couldn't set up benchmark: %v\n", err) + } + parseTwtxtCases[3].data = buf + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range parseTwtxtCases { + _, _ = ParseUserTwtxt(tt.data, "testuser", "testurl") + } + } +} + +var timestampCases = []struct { + name string + orig string + expected string +}{ + { + name: "Timezone appended", + orig: "2020-01-13T16:08:25.544735+00:00", + expected: "2020-01-13T16:08:25.544735Z", + }, + { + name: "It's fine already", + orig: "2020-01-14T00:19:45.092344Z", + expected: "2020-01-14T00:19:45.092344Z", + }, +} + +func Test_fixTimestamp(t *testing.T) { + for _, tt := range timestampCases { + t.Run(tt.name, func(t *testing.T) { + tsout := fixTimestamp(tt.orig) + if tsout != tt.expected { + t.Errorf("Failed :: %s :: got %s expected %s", tt.name, tsout, tt.expected) + } + }) + } +} diff --git a/registry/init_test.go b/registry/init_test.go new file mode 100644 index 0000000..ab9d494 --- /dev/null +++ b/registry/init_test.go @@ -0,0 +1,93 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry //import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "fmt" + "log" + "os" + "time" +) + +func quickErr(err error) { + if err != nil { + fmt.Printf("%v\n", err) + } +} + +// Sets up mock users and statuses +func initTestEnv() *Registry { + hush, err := os.Open("/dev/null") + quickErr(err) + log.SetOutput(hush) + + // this is a bit tedious, but set up fake dates + // for the mock users' join and status timestamps + timeMonthPrev := time.Now().AddDate(0, -1, 0) + timeMonthPrevRFC := timeMonthPrev.Format(time.RFC3339) + + timeTwoMonthsPrev := time.Now().AddDate(0, -2, 0) + timeTwoMonthsPrevRFC := timeTwoMonthsPrev.Format(time.RFC3339) + + timeThreeMonthsPrev := time.Now().AddDate(0, -3, 0) + timeThreeMonthsPrevRFC := timeThreeMonthsPrev.Format(time.RFC3339) + + timeFourMonthsPrev := time.Now().AddDate(0, -4, 0) + timeFourMonthsPrevRFC := timeFourMonthsPrev.Format(time.RFC3339) + + var mockusers = []struct { + url string + nick string + date string + apidate []byte + status TimeMap + }{ + { + url: "https://example3.com/twtxt.txt", + nick: "foo_barrington", + date: timeTwoMonthsPrevRFC, + status: TimeMap{ + timeTwoMonthsPrev: "foo_barrington\thttps://example3.com/twtxt.txt\t" + timeTwoMonthsPrevRFC + "\tJust got started with #twtxt!", + timeMonthPrev: "foo_barrington\thttps://example3.com/twtxt.txt\t" + timeMonthPrevRFC + "\tHey <@foo https://example.com/twtxt.txt>, I love programming. Just FYI.", + }, + }, + { + url: "https://example.com/twtxt.txt", + nick: "foo", + date: timeFourMonthsPrevRFC, + status: TimeMap{ + timeFourMonthsPrev: "foo\thttps://example.com/twtxt.txt\t" + timeFourMonthsPrevRFC + "\tThis is so much better than #twitter", + timeThreeMonthsPrev: "foo\thttps://example.com/twtxt.txt\t" + timeThreeMonthsPrevRFC + "\tI can't wait to start on my next programming #project with <@foo_barrington https://example3.com/twtxt.txt>", + }, + }, + } + registry := New(nil) + + // fill the test registry with the mock users + for _, e := range mockusers { + data := &User{} + data.Nick = e.nick + data.Date = e.date + data.Status = e.status + registry.Users[e.url] = data + } + + return registry +} diff --git a/registry/integ_test.go b/registry/integ_test.go new file mode 100644 index 0000000..2cfbb13 --- /dev/null +++ b/registry/integ_test.go @@ -0,0 +1,98 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry + +import ( + "strings" + "testing" +) + +// This tests all the operations on an registry. +func Test_Integration(t *testing.T) { + var integration = func(t *testing.T) { + t.Logf("Creating registry object ...\n") + registry := New(nil) + + t.Logf("Fetching remote twtxt file ...\n") + mainregistry, _, err := GetTwtxt("https://gbmor.dev/twtxt.txt", nil) + if err != nil { + t.Errorf("%v\n", err) + } + + t.Logf("Parsing remote twtxt file ...\n") + parsed, errz := ParseUserTwtxt(mainregistry, "gbmor", "https://gbmor.dev/twtxt.txt") + if errz != nil { + t.Errorf("%v\n", errz) + } + + t.Logf("Adding new user to registry ...\n") + err = registry.AddUser("TestRegistry", "https://gbmor.dev/twtxt.txt", nil, parsed) + if err != nil { + t.Errorf("%v\n", err) + } + + t.Logf("Querying user statuses ...\n") + queryuser, err := registry.QueryUser("TestRegistry") + if err != nil { + t.Errorf("%v\n", err) + } + for _, e := range queryuser { + if !strings.Contains(e, "TestRegistry") { + t.Errorf("QueryUser() returned incorrect data\n") + } + } + + t.Logf("Querying for keyword in statuses ...\n") + querystatus, err := registry.QueryInStatus("morning") + if err != nil { + t.Errorf("%v\n", err) + } + for _, e := range querystatus { + if !strings.Contains(e, "morning") { + t.Errorf("QueryInStatus() returned incorrect data\n") + } + } + + t.Logf("Querying for all statuses ...\n") + allstatus, err := registry.QueryAllStatuses() + if err != nil { + t.Errorf("%v\n", err) + } + if len(allstatus) == 0 || allstatus == nil { + t.Errorf("Got nil/zero from QueryAllStatuses") + } + + t.Logf("Querying for all users ...\n") + allusers, err := registry.QueryUser("") + if err != nil { + t.Errorf("%v\n", err) + } + if len(allusers) == 0 || allusers == nil { + t.Errorf("Got nil/zero users on empty QueryUser() query") + } + + t.Logf("Deleting user ...\n") + err = registry.DelUser("https://gbmor.dev/twtxt.txt") + if err != nil { + t.Errorf("%v\n", err) + } + } + t.Run("Integration Test", integration) +} diff --git a/registry/query.go b/registry/query.go new file mode 100644 index 0000000..604b974 --- /dev/null +++ b/registry/query.go @@ -0,0 +1,196 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// QueryUser checks the Registry for usernames +// or user URLs that contain the term provided as an argument. Entries +// are returned sorted by the date they were added to the Registry. If +// the argument provided is blank, return all users. +func (registry *Registry) QueryUser(term string) ([]string, error) { + if registry == nil { + return nil, fmt.Errorf("can't query empty registry for user") + } + + term = strings.ToLower(term) + timekey := NewTimeMap() + keys := make(TimeSlice, 0) + var users []string + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + for k, v := range registry.Users { + if registry.Users[k] == nil { + continue + } + v.Mu.RLock() + if strings.Contains(strings.ToLower(v.Nick), term) || strings.Contains(strings.ToLower(k), term) { + thetime, err := time.Parse(time.RFC3339, v.Date) + if err != nil { + v.Mu.RUnlock() + continue + } + timekey[thetime] = v.Nick + "\t" + k + "\t" + v.Date + "\n" + keys = append(keys, thetime) + } + v.Mu.RUnlock() + } + + sort.Sort(keys) + for _, e := range keys { + users = append(users, timekey[e]) + } + + return users, nil +} + +// QueryInStatus returns all statuses in the Registry +// that contain the provided substring (tag, mention URL, etc). +func (registry *Registry) QueryInStatus(substring string) ([]string, error) { + if substring == "" { + return nil, fmt.Errorf("cannot query for empty tag") + } else if registry == nil { + return nil, fmt.Errorf("can't query statuses of empty registry") + } + + statusmap := make([]TimeMap, 0) + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + for _, v := range registry.Users { + statusmap = append(statusmap, v.FindInStatus(substring)) + } + + sorted, err := SortByTime(statusmap...) + if err != nil { + return nil, err + } + + return sorted, nil +} + +// QueryAllStatuses returns all statuses in the Registry +// as a slice of strings sorted by timestamp. +func (registry *Registry) QueryAllStatuses() ([]string, error) { + if registry == nil { + return nil, fmt.Errorf("can't get latest statuses from empty registry") + } + + statusmap, err := registry.GetStatuses() + if err != nil { + return nil, err + } + + sorted, err := SortByTime(statusmap) + if err != nil { + return nil, err + } + + if sorted == nil { + sorted = make([]string, 1) + } + + return sorted, nil +} + +// ReduceToPage returns the passed 'page' worth of output. +// One page is twenty items. For example, if 2 is passed, +// it will return data[20:40]. According to the twtxt +// registry specification, queries should accept a "page" +// value. +func ReduceToPage(page int, data []string) []string { + end := 20 * page + if end > len(data) || end < 1 { + end = len(data) + } + + beg := end - 20 + if beg > len(data)-1 || beg < 0 { + beg = 0 + } + + return data[beg:end] +} + +// FindInStatus takes a user's statuses and looks for a given substring. +// Returns the statuses that include the substring as a TimeMap. +func (userdata *User) FindInStatus(substring string) TimeMap { + if userdata == nil { + return nil + } else if len(substring) > 140 { + return nil + } + + substring = strings.ToLower(substring) + statuses := NewTimeMap() + + userdata.Mu.RLock() + defer userdata.Mu.RUnlock() + + for k, e := range userdata.Status { + if _, ok := userdata.Status[k]; !ok { + continue + } + + parts := strings.Split(strings.ToLower(e), "\t") + if strings.Contains(parts[3], substring) { + statuses[k] = e + } + } + + return statuses +} + +// SortByTime returns a string slice of the query results, +// sorted by timestamp in descending order (newest first). +func SortByTime(tm ...TimeMap) ([]string, error) { + if tm == nil { + return nil, fmt.Errorf("can't sort nil TimeMaps") + } + + var times = make(TimeSlice, 0) + var data []string + + for _, e := range tm { + for k := range e { + times = append(times, k) + } + } + + sort.Sort(times) + + for k := range tm { + for _, e := range times { + if _, ok := tm[k][e]; ok { + data = append(data, tm[k][e]) + } + } + } + + return data, nil +} diff --git a/registry/query_test.go b/registry/query_test.go new file mode 100644 index 0000000..7eed2cd --- /dev/null +++ b/registry/query_test.go @@ -0,0 +1,459 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry + +import ( + "bufio" + "os" + "strings" + "testing" + "time" +) + +var queryUserCases = []struct { + name string + term string + wantErr bool +}{ + { + name: "Valid User", + term: "foo", + wantErr: false, + }, + { + name: "Empty Query", + term: "", + wantErr: false, + }, + { + name: "Nonexistent User", + term: "doesntexist", + wantErr: true, + }, + { + name: "Garbage Data", + term: "will be replaced with garbage data", + wantErr: true, + }, +} + +// Checks if Registry.QueryUser() returns users that +// match the provided substring. +func Test_Registry_QueryUser(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 8 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + queryUserCases[3].term = string(buf) + + for n, tt := range queryUserCases { + + t.Run(tt.name, func(t *testing.T) { + out, err := registry.QueryUser(tt.term) + + if out == nil && err != nil && !tt.wantErr { + t.Errorf("Received nil output or an error when unexpected. Case %v, %v, %v\n", n, tt.term, err) + } + + if out != nil && tt.wantErr { + t.Errorf("Received unexpected nil output when an error was expected. Case %v, %v\n", n, tt.term) + } + + for _, e := range out { + one := strings.Split(e, "\t") + + if !strings.Contains(one[0], tt.term) && !strings.Contains(one[1], tt.term) { + t.Errorf("Received incorrect output: %v != %v\n", tt.term, e) + } + } + }) + } +} +func Benchmark_Registry_QueryUser(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range queryUserCases { + _, err := registry.QueryUser(tt.term) + if err != nil { + b.Errorf("%v\n", err) + } + } + } +} + +var queryInStatusCases = []struct { + name string + substr string + wantNil bool + wantErr bool +}{ + { + name: "Tag in Status", + substr: "twtxt", + wantNil: false, + wantErr: false, + }, + { + name: "Valid URL", + substr: "https://example.com/twtxt.txt", + wantNil: false, + wantErr: false, + }, + { + name: "Multiple Words in Status", + substr: "next programming", + wantNil: false, + wantErr: false, + }, + { + name: "Multiple Words, Not in Status", + substr: "explosive bananas from antarctica", + wantNil: true, + wantErr: false, + }, + { + name: "Empty Query", + substr: "", + wantNil: true, + wantErr: true, + }, + { + name: "Nonsense", + substr: "ahfiurrenkhfkajdhfao", + wantNil: true, + wantErr: false, + }, + { + name: "Invalid URL", + substr: "https://doesnt.exist/twtxt.txt", + wantNil: true, + wantErr: false, + }, + { + name: "Garbage Data", + substr: "will be replaced with garbage data", + wantNil: true, + wantErr: false, + }, +} + +// This tests whether we can find a substring in all of +// the known status messages, disregarding the metadata +// stored with each status. +func Test_Registry_QueryInStatus(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 8 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + queryInStatusCases[7].substr = string(buf) + + for _, tt := range queryInStatusCases { + + t.Run(tt.name, func(t *testing.T) { + + out, err := registry.QueryInStatus(tt.substr) + if err != nil && !tt.wantErr { + t.Errorf("Caught unexpected error: %v\n", err) + } + + if !tt.wantErr && out == nil && !tt.wantNil { + t.Errorf("Got nil when expecting output\n") + } + + if err == nil && tt.wantErr { + t.Errorf("Expecting error, got nil.\n") + } + + for _, e := range out { + split := strings.Split(strings.ToLower(e), "\t") + + if e != "" { + if !strings.Contains(split[3], strings.ToLower(tt.substr)) { + t.Errorf("Status without substring returned\n") + } + } + } + }) + } + +} +func Benchmark_Registry_QueryInStatus(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range queryInStatusCases { + _, err := registry.QueryInStatus(tt.substr) + if err != nil { + continue + } + } + } +} + +// Tests whether we can retrieve the 20 most +// recent statuses in the registry +func Test_QueryAllStatuses(t *testing.T) { + registry := initTestEnv() + t.Run("Latest Statuses", func(t *testing.T) { + out, err := registry.QueryAllStatuses() + if out == nil || err != nil { + t.Errorf("Got no statuses, or more than 20: %v, %v\n", len(out), err) + } + }) +} +func Benchmark_QueryAllStatuses(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := registry.QueryAllStatuses() + if err != nil { + continue + } + } +} + +var get20cases = []struct { + name string + page int + wantErr bool +}{ + { + name: "First Page", + page: 1, + wantErr: false, + }, + { + name: "High Page Number", + page: 256, + wantErr: false, + }, + { + name: "Illegal Page Number", + page: -23, + wantErr: false, + }, +} + +func Test_ReduceToPage(t *testing.T) { + registry := initTestEnv() + for _, tt := range get20cases { + t.Run(tt.name, func(t *testing.T) { + out, err := registry.QueryAllStatuses() + if err != nil && !tt.wantErr { + t.Errorf("%v\n", err.Error()) + } + out = ReduceToPage(tt.page, out) + if len(out) > 20 || len(out) == 0 { + t.Errorf("Page-Reduce Malfunction: length of data %v\n", len(out)) + } + }) + } +} + +func Benchmark_ReduceToPage(b *testing.B) { + registry := initTestEnv() + out, _ := registry.QueryAllStatuses() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, tt := range get20cases { + ReduceToPage(tt.page, out) + } + } +} + +// This tests whether we can find a substring in the +// given user's status messages, disregarding the metadata +// stored with each status. +func Test_User_FindInStatus(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 8 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + queryInStatusCases[7].substr = string(buf) + + data := make([]*User, 0) + + for _, v := range registry.Users { + data = append(data, v) + } + + for _, tt := range queryInStatusCases { + t.Run(tt.name, func(t *testing.T) { + for _, e := range data { + + tag := e.FindInStatus(tt.substr) + if tag == nil && !tt.wantNil { + t.Errorf("Got nil tag\n") + } + } + }) + } + +} +func Benchmark_User_FindInStatus(b *testing.B) { + registry := initTestEnv() + data := make([]*User, 0) + + for _, v := range registry.Users { + data = append(data, v) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range data { + for _, v := range queryInStatusCases { + tt.FindInStatus(v.substr) + } + } + } +} + +func Test_SortByTime_Slice(t *testing.T) { + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + t.Errorf("Failed to finish test initialization: %v\n", err) + } + + t.Run("Sort By Time ([]TimeMap)", func(t *testing.T) { + sorted, err := SortByTime(statusmap) + if err != nil { + t.Errorf("%v\n", err) + } + split := strings.Split(sorted[0], "\t") + firsttime, _ := time.Parse("RFC3339", split[0]) + + for i := range sorted { + if i < len(sorted)-1 { + + nextsplit := strings.Split(sorted[i+1], "\t") + nexttime, _ := time.Parse("RFC3339", nextsplit[0]) + + if firsttime.Before(nexttime) { + t.Errorf("Timestamps out of order: %v\n", sorted) + } + + firsttime = nexttime + } + } + }) +} + +// Benchmarking a sort of 1000000 statuses by timestamp. +// Right now it's at roughly 2000ns per 2 statuses. +// Set sortMultiplier to be the number of desired +// statuses divided by four. +func Benchmark_SortByTime_Slice(b *testing.B) { + // I set this to 250,000,000 and it hard-locked + // my laptop. Oops. + sortMultiplier := 250 + b.Logf("Benchmarking SortByTime with a constructed slice of %v statuses ...\n", sortMultiplier*4) + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + b.Errorf("Failed to finish benchmark initialization: %v\n", err) + } + + // Constructed registry has four statuses. This + // makes a TimeMapSlice of 1000000 statuses. + statusmaps := make([]TimeMap, sortMultiplier*4) + for i := 0; i < sortMultiplier; i++ { + statusmaps = append(statusmaps, statusmap) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := SortByTime(statusmaps...) + if err != nil { + b.Errorf("%v\n", err) + } + } +} + +func Test_SortByTime_Single(t *testing.T) { + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + t.Errorf("Failed to finish test initialization: %v\n", err) + } + + t.Run("Sort By Time (TimeMap)", func(t *testing.T) { + sorted, err := SortByTime(statusmap) + if err != nil { + t.Errorf("%v\n", err) + } + split := strings.Split(sorted[0], "\t") + firsttime, _ := time.Parse("RFC3339", split[0]) + + for i := range sorted { + if i < len(sorted)-1 { + + nextsplit := strings.Split(sorted[i+1], "\t") + nexttime, _ := time.Parse("RFC3339", nextsplit[0]) + + if firsttime.Before(nexttime) { + t.Errorf("Timestamps out of order: %v\n", sorted) + } + + firsttime = nexttime + } + } + }) +} + +func Benchmark_SortByTime_Single(b *testing.B) { + registry := initTestEnv() + + statusmap, err := registry.GetStatuses() + if err != nil { + b.Errorf("Failed to finish benchmark initialization: %v\n", err) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := SortByTime(statusmap) + if err != nil { + b.Errorf("%v\n", err) + } + } +} diff --git a/registry/revive.toml b/registry/revive.toml new file mode 100644 index 0000000..f9e2405 --- /dev/null +++ b/registry/revive.toml @@ -0,0 +1,30 @@ +ignoreGeneratedHeader = false +severity = "warning" +confidence = 0.8 +errorCode = 0 +warningCode = 0 + +[rule.blank-imports] +[rule.context-as-argument] +[rule.context-keys-type] +[rule.dot-imports] +[rule.error-return] +[rule.error-strings] +[rule.error-naming] +[rule.exported] +[rule.if-return] +[rule.increment-decrement] +[rule.var-naming] +[rule.var-declaration] +[rule.package-comments] +[rule.range] +[rule.receiver-naming] +[rule.time-naming] +[rule.unexported-return] +[rule.indent-error-flow] +[rule.errorf] +[rule.empty-block] +[rule.superfluous-else] +[rule.unused-parameter] +[rule.unreachable-code] +[rule.redefines-builtin-id] diff --git a/registry/types.go b/registry/types.go new file mode 100644 index 0000000..eb8eee1 --- /dev/null +++ b/registry/types.go @@ -0,0 +1,148 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +// Package registry implements functions and types that assist +// in the creation and management of a twtxt registry. +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "net" + "net/http" + "sync" + "time" +) + +// Registrar implements the minimum amount of methods +// for a functioning Registry. +type Registrar interface { + Put(user *User) error + Get(urlKey string) (*User, error) + DelUser(urlKey string) error + UpdateUser(urlKey string) error + GetUserStatuses(urlKey string) (TimeMap, error) + GetStatuses() (TimeMap, error) +} + +// User holds a given user's information +// and statuses. +type User struct { + // Provided to aid in concurrency-safe + // reads and writes. In most cases, the + // mutex in the associated Index should be + // used instead. This mutex is provided + // should the library user need to access + // a User independently of an Index. + Mu sync.RWMutex + + // Nick is the user-specified nickname. + Nick string + + // The URL of the user's twtxt file + URL string + + // The reported last modification date + // of the user's twtxt.txt file. + LastModified string + + // The IP address of the user is optionally + // recorded when submitted via POST. + IP net.IP + + // The timestamp, in RFC3339 format, + // reflecting when the user was added. + Date string + + // A TimeMap of the user's statuses + // from their twtxt file. + Status TimeMap +} + +// Registry enables the bulk of a registry's +// user data storage and access. +type Registry struct { + // Provided to aid in concurrency-safe + // reads and writes to a given registry + // Users map. + Mu sync.RWMutex + + // The registry's user data is contained + // in this map. The functions within this + // library expect the key to be the URL of + // a given user's twtxt file. + Users map[string]*User + + // The client to use for HTTP requests. + // If nil is passed to NewIndex(), a + // client with a 10 second timeout + // and all other values as default is + // used. + HTTPClient *http.Client +} + +// TimeMap holds extracted and processed user data as a +// string. A time.Time value is used as the key. +type TimeMap map[time.Time]string + +// TimeSlice is a slice of time.Time used for sorting +// a TimeMap by timestamp. +type TimeSlice []time.Time + +// NewUser returns a pointer to an initialized User +func NewUser() *User { + return &User{ + Mu: sync.RWMutex{}, + Status: NewTimeMap(), + } +} + +// New returns an initialized Registry instance. +func New(client *http.Client) *Registry { + return &Registry{ + Mu: sync.RWMutex{}, + Users: make(map[string]*User), + HTTPClient: client, + } +} + +// NewTimeMap returns an initialized TimeMap. +func NewTimeMap() TimeMap { + return make(TimeMap) +} + +// Len returns the length of the TimeSlice to be sorted. +// This helps satisfy sort.Interface. +func (t TimeSlice) Len() int { + return len(t) +} + +// Less returns true if the timestamp at index i is after +// the timestamp at index j in a given TimeSlice. This results +// in a descending (reversed) sort order for timestamps rather +// than ascending. +// This helps satisfy sort.Interface. +func (t TimeSlice) Less(i, j int) bool { + return t[i].After(t[j]) +} + +// Swap transposes the timestamps at the two given indices +// for the TimeSlice receiver. +// This helps satisfy sort.Interface. +func (t TimeSlice) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} diff --git a/registry/user.go b/registry/user.go new file mode 100644 index 0000000..329b6e3 --- /dev/null +++ b/registry/user.go @@ -0,0 +1,270 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "fmt" + "net" + "strings" + "sync" + "time" +) + +// AddUser inserts a new user into the Registry. +func (registry *Registry) AddUser(nickname, urlKey string, ipAddress net.IP, statuses TimeMap) error { + + if registry == nil { + return fmt.Errorf("can't add user to uninitialized registry") + + } else if nickname == "" || urlKey == "" { + return fmt.Errorf("both URL and Nick must be specified") + + } else if !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.Lock() + defer registry.Mu.Unlock() + + if _, ok := registry.Users[urlKey]; ok { + return fmt.Errorf("user %v already exists", urlKey) + } + + registry.Users[urlKey] = &User{ + Mu: sync.RWMutex{}, + Nick: nickname, + URL: urlKey, + LastModified: "", + IP: ipAddress, + Date: time.Now().Format(time.RFC3339), + Status: statuses} + + return nil +} + +// Put inserts a given User into an Registry. The User +// being pushed need only have the URL field filled. +// All other fields may be empty. +// This can be destructive: an existing User in the +// Registry will be overwritten if its User.URL is the +// same as the User.URL being pushed. +func (registry *Registry) Put(user *User) error { + if user == nil { + return fmt.Errorf("can't push nil data to registry") + } + if registry == nil || registry.Users == nil { + return fmt.Errorf("can't push data to registry: registry uninitialized") + } + user.Mu.RLock() + if user.URL == "" { + user.Mu.RUnlock() + return fmt.Errorf("can't push data to registry: missing URL for key") + } + urlKey := user.URL + registry.Mu.Lock() + registry.Users[urlKey] = user + registry.Mu.Unlock() + user.Mu.RUnlock() + + return nil +} + +// Get returns the User associated with the +// provided URL key in the Registry. +func (registry *Registry) Get(urlKey string) (*User, error) { + if registry == nil { + return nil, fmt.Errorf("can't pop from nil registry") + } + if urlKey == "" { + return nil, fmt.Errorf("can't pop unless provided a key") + } + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + if _, ok := registry.Users[urlKey]; !ok { + return nil, fmt.Errorf("provided url key doesn't exist in registry") + } + + registry.Users[urlKey].Mu.RLock() + userGot := registry.Users[urlKey] + registry.Users[urlKey].Mu.RUnlock() + + return userGot, nil +} + +// DelUser removes a user and all associated data from +// the Registry. +func (registry *Registry) DelUser(urlKey string) error { + + if registry == nil { + return fmt.Errorf("can't delete user from empty registry") + + } else if urlKey == "" { + return fmt.Errorf("can't delete blank user") + + } else if !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.Lock() + defer registry.Mu.Unlock() + + if _, ok := registry.Users[urlKey]; !ok { + return fmt.Errorf("can't delete user %v, user doesn't exist", urlKey) + } + + delete(registry.Users, urlKey) + + return nil +} + +// UpdateUser scrapes an existing user's remote twtxt.txt +// file. Any new statuses are added to the user's entry +// in the Registry. If the remote twtxt data's reported +// Content-Length does not differ from what is stored, +// an error is returned. +func (registry *Registry) UpdateUser(urlKey string) error { + if urlKey == "" || !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + diff, err := registry.DiffTwtxt(urlKey) + if err != nil { + return err + } else if !diff { + return fmt.Errorf("no new statuses available for %v", urlKey) + } + + out, isRemoteRegistry, err := GetTwtxt(urlKey, registry.HTTPClient) + if err != nil { + return err + } + + if isRemoteRegistry { + return fmt.Errorf("attempting to update registry URL - users should be updated individually") + } + + registry.Mu.Lock() + defer registry.Mu.Unlock() + user := registry.Users[urlKey] + + user.Mu.Lock() + defer user.Mu.Unlock() + nick := user.Nick + + data, err := ParseUserTwtxt(out, nick, urlKey) + if err != nil { + return err + } + + for i, e := range data { + user.Status[i] = e + } + + registry.Users[urlKey] = user + + return nil +} + +// CrawlRemoteRegistry scrapes all nicknames and user URLs +// from a provided registry. The urlKey passed to this function +// must be in the form of https://registry.example.com/api/plain/users +func (registry *Registry) CrawlRemoteRegistry(urlKey string) error { + if urlKey == "" || !strings.HasPrefix(urlKey, "http") { + return fmt.Errorf("invalid URL: %v", urlKey) + } + + out, isRemoteRegistry, err := GetTwtxt(urlKey, registry.HTTPClient) + if err != nil { + return err + } + + if !isRemoteRegistry { + return fmt.Errorf("can't add single user via call to CrawlRemoteRegistry") + } + + users, err := ParseRegistryTwtxt(out) + if err != nil { + return err + } + + // only add new users so we don't overwrite data + // we already have (and lose statuses, etc) + registry.Mu.Lock() + defer registry.Mu.Unlock() + for _, e := range users { + if _, ok := registry.Users[e.URL]; !ok { + registry.Users[e.URL] = e + } + } + + return nil +} + +// GetUserStatuses returns a TimeMap containing single user's statuses +func (registry *Registry) GetUserStatuses(urlKey string) (TimeMap, error) { + if registry == nil { + return nil, fmt.Errorf("can't get statuses from an empty registry") + } else if urlKey == "" || !strings.HasPrefix(urlKey, "http") { + return nil, fmt.Errorf("invalid URL: %v", urlKey) + } + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + if _, ok := registry.Users[urlKey]; !ok { + return nil, fmt.Errorf("can't retrieve statuses of nonexistent user") + } + + registry.Users[urlKey].Mu.RLock() + status := registry.Users[urlKey].Status + registry.Users[urlKey].Mu.RUnlock() + + return status, nil +} + +// GetStatuses returns a TimeMap containing all statuses +// from all users in the Registry. +func (registry *Registry) GetStatuses() (TimeMap, error) { + if registry == nil { + return nil, fmt.Errorf("can't get statuses from an empty registry") + } + + statuses := NewTimeMap() + + registry.Mu.RLock() + defer registry.Mu.RUnlock() + + for _, v := range registry.Users { + v.Mu.RLock() + if v.Status == nil || len(v.Status) == 0 { + v.Mu.RUnlock() + continue + } + for a, b := range v.Status { + if _, ok := v.Status[a]; ok { + statuses[a] = b + } + } + v.Mu.RUnlock() + } + + return statuses, nil +} diff --git a/registry/user_test.go b/registry/user_test.go new file mode 100644 index 0000000..f0c9622 --- /dev/null +++ b/registry/user_test.go @@ -0,0 +1,349 @@ +/* +Copyright (c) 2019 Ben Morrison (gbmor) + +This file is part of Registry. + +Registry is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Registry is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Registry. If not, see . +*/ + +package registry // import "git.sr.ht/~gbmor/getwtxt/registry" + +import ( + "bufio" + "fmt" + "net/http" + "os" + "reflect" + "testing" +) + +var addUserCases = []struct { + name string + nick string + url string + wantErr bool + localOnly bool +}{ + { + name: "Legitimate User (Local Only)", + nick: "testuser1", + url: "http://localhost:8080/twtxt.txt", + wantErr: false, + localOnly: true, + }, + { + name: "Empty Query", + nick: "", + url: "", + wantErr: true, + localOnly: false, + }, + { + name: "Invalid URL", + nick: "foo", + url: "foobarringtons", + wantErr: true, + localOnly: false, + }, + { + name: "Garbage Data", + nick: "", + url: "", + wantErr: true, + localOnly: false, + }, +} + +// Tests if we can successfully add a user to the registry +func Test_Registry_AddUser(t *testing.T) { + registry := initTestEnv() + if !addUserCases[0].localOnly { + http.Handle("/twtxt.txt", http.HandlerFunc(twtxtHandler)) + go fmt.Println(http.ListenAndServe(":8080", nil)) + } + var buf = make([]byte, 256) + // read random data into case 5 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + addUserCases[3].nick = string(buf) + addUserCases[3].url = string(buf) + + statuses, err := registry.GetStatuses() + if err != nil { + t.Errorf("Error setting up test: %v\n", err) + } + + for n, tt := range addUserCases { + t.Run(tt.name, func(t *testing.T) { + if tt.localOnly { + t.Skipf("Local-only test. Skipping ... ") + } + + err := registry.AddUser(tt.nick, tt.url, nil, statuses) + + // only run some checks if we don't want an error + if !tt.wantErr { + if err != nil { + t.Errorf("Got error: %v\n", err) + } + + // make sure we have *something* in the registry + if reflect.ValueOf(registry.Users[tt.url]).IsNil() { + t.Errorf("Failed to add user %v registry.\n", tt.url) + } + + // see if the nick in the registry is the same + // as the test case. verifies the URL and the nick + // since the URL is used as the key + data := registry.Users[tt.url] + if data.Nick != tt.nick { + t.Errorf("Incorrect user data added to registry for user %v.\n", tt.url) + } + } + // check for the cases that should throw an error + if tt.wantErr && err == nil { + t.Errorf("Expected error for case %v, got nil\n", n) + } + }) + } +} +func Benchmark_Registry_AddUser(b *testing.B) { + registry := initTestEnv() + statuses, err := registry.GetStatuses() + if err != nil { + b.Errorf("Error setting up test: %v\n", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, tt := range addUserCases { + err := registry.AddUser(tt.nick, tt.url, nil, statuses) + if err != nil { + continue + } + registry.Users[tt.url] = &User{} + } + } +} + +var delUserCases = []struct { + name string + url string + wantErr bool +}{ + { + name: "Valid User", + url: "https://example.com/twtxt.txt", + wantErr: false, + }, + { + name: "Valid User", + url: "https://example3.com/twtxt.txt", + wantErr: false, + }, + { + name: "Already Deleted User", + url: "https://example3.com/twtxt.txt", + wantErr: true, + }, + { + name: "Empty Query", + url: "", + wantErr: true, + }, + { + name: "Garbage Data", + url: "", + wantErr: true, + }, +} + +// Tests if we can successfully delete a user from the registry +func Test_Registry_DelUser(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 5 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + delUserCases[4].url = string(buf) + + for n, tt := range delUserCases { + t.Run(tt.name, func(t *testing.T) { + + err := registry.DelUser(tt.url) + if !reflect.ValueOf(registry.Users[tt.url]).IsNil() { + t.Errorf("Failed to delete user %v from registry.\n", tt.url) + } + if tt.wantErr && err == nil { + t.Errorf("Expected error but did not receive. Case %v\n", n) + } + if !tt.wantErr && err != nil { + t.Errorf("Unexpected error for case %v: %v\n", n, err) + } + }) + } +} +func Benchmark_Registry_DelUser(b *testing.B) { + registry := initTestEnv() + + data1 := &User{ + Nick: registry.Users[delUserCases[0].url].Nick, + Date: registry.Users[delUserCases[0].url].Date, + Status: registry.Users[delUserCases[0].url].Status, + } + + data2 := &User{ + Nick: registry.Users[delUserCases[1].url].Nick, + Date: registry.Users[delUserCases[1].url].Date, + Status: registry.Users[delUserCases[1].url].Status, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range delUserCases { + err := registry.DelUser(tt.url) + if err != nil { + continue + } + } + + registry.Users[delUserCases[0].url] = data1 + registry.Users[delUserCases[1].url] = data2 + } +} + +var getUserStatusCases = []struct { + name string + url string + wantErr bool +}{ + { + name: "Valid User", + url: "https://example.com/twtxt.txt", + wantErr: false, + }, + { + name: "Valid User", + url: "https://example3.com/twtxt.txt", + wantErr: false, + }, + { + name: "Nonexistent User", + url: "https://doesn't.exist/twtxt.txt", + wantErr: true, + }, + { + name: "Empty Query", + url: "", + wantErr: true, + }, + { + name: "Garbage Data", + url: "", + wantErr: true, + }, +} + +// Checks if we can retrieve a single user's statuses +func Test_Registry_GetUserStatuses(t *testing.T) { + registry := initTestEnv() + var buf = make([]byte, 256) + // read random data into case 5 + rando, _ := os.Open("/dev/random") + reader := bufio.NewReader(rando) + n, err := reader.Read(buf) + if err != nil || n == 0 { + t.Errorf("Couldn't set up test: %v\n", err) + } + getUserStatusCases[4].url = string(buf) + + for n, tt := range getUserStatusCases { + t.Run(tt.name, func(t *testing.T) { + + statuses, err := registry.GetUserStatuses(tt.url) + + if !tt.wantErr { + if reflect.ValueOf(statuses).IsNil() { + t.Errorf("Failed to pull statuses for user %v\n", tt.url) + } + // see if the function returns the same data + // that we already have + data := registry.Users[tt.url] + if !reflect.DeepEqual(data.Status, statuses) { + t.Errorf("Incorrect data retrieved as statuses for user %v.\n", tt.url) + } + } + + if tt.wantErr && err == nil { + t.Errorf("Expected error, received nil for case %v: %v\n", n, tt.url) + } + }) + } +} +func Benchmark_Registry_GetUserStatuses(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, tt := range getUserStatusCases { + _, err := registry.GetUserStatuses(tt.url) + if err != nil { + continue + } + } + } +} + +// Tests if we can retrieve all user statuses at once +func Test_Registry_GetStatuses(t *testing.T) { + registry := initTestEnv() + t.Run("Registry.GetStatuses()", func(t *testing.T) { + + statuses, err := registry.GetStatuses() + if reflect.ValueOf(statuses).IsNil() || err != nil { + t.Errorf("Failed to pull all statuses. %v\n", err) + } + + // Now do the same query manually to see + // if we get the same result + unionmap := NewTimeMap() + for _, v := range registry.Users { + for i, e := range v.Status { + unionmap[i] = e + } + } + if !reflect.DeepEqual(statuses, unionmap) { + t.Errorf("Incorrect data retrieved as statuses.\n") + } + }) +} +func Benchmark_Registry_GetStatuses(b *testing.B) { + registry := initTestEnv() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := registry.GetStatuses() + if err != nil { + continue + } + } +} diff --git a/svc/README.md b/svc/README.md deleted file mode 100644 index 797b7ab..0000000 --- a/svc/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# `getwtxt|svc` - -This is an internal package created for code organization -purposes. - -The file `getwtxt.go` in the parent directory calls -`svc.go::Start()`, which starts the registry server. - diff --git a/svc/cache.go b/svc/cache.go index f887d05..16287bf 100644 --- a/svc/cache.go +++ b/svc/cache.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" diff --git a/svc/cache_test.go b/svc/cache_test.go index 96c2638..c37f099 100644 --- a/svc/cache_test.go +++ b/svc/cache_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" @@ -27,7 +27,7 @@ import ( "reflect" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) func Test_initTemplates(t *testing.T) { @@ -49,7 +49,7 @@ func Test_cacheUpdate(t *testing.T) { killStatuses() cacheUpdate() - urls := "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt" + urls := testTwtxtURL newStatus := twtxtCache.Users[urls].Status t.Run("Checking for any data", func(t *testing.T) { diff --git a/svc/conf.go b/svc/conf.go index 8968ed4..7365b2b 100644 --- a/svc/conf.go +++ b/svc/conf.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "log" diff --git a/svc/db.go b/svc/db.go index 34b7c34..8cd05d1 100644 --- a/svc/db.go +++ b/svc/db.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "log" diff --git a/svc/db_test.go b/svc/db_test.go index f54d610..21fc972 100644 --- a/svc/db_test.go +++ b/svc/db_test.go @@ -17,30 +17,30 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "net" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) func Test_pushpullDatabase(t *testing.T) { initTestConf() initTestDB() - out, _, err := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) + out, _, err := registry.GetTwtxt(testTwtxtURL, nil) if err != nil { t.Errorf("Couldn't set up test: %v\n", err) } - statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", testTwtxtURL) if err != nil { t.Errorf("Couldn't set up test: %v\n", err) } - twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), statusmap) + twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), statusmap) remoteRegistries.List = append(remoteRegistries.List, "https://twtxt.tilde.institute/api/plain/users") @@ -52,7 +52,7 @@ func Test_pushpullDatabase(t *testing.T) { }) t.Run("Clearing Registry", func(t *testing.T) { - err := twtxtCache.DelUser("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + err := twtxtCache.DelUser(testTwtxtURL) if err != nil { t.Errorf("%v", err) } @@ -62,7 +62,7 @@ func Test_pushpullDatabase(t *testing.T) { pullDB() twtxtCache.Mu.RLock() - if _, ok := twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"]; !ok { + if _, ok := twtxtCache.Users[testTwtxtURL]; !ok { t.Errorf("Missing user previously pushed to database\n") } twtxtCache.Mu.RUnlock() @@ -73,18 +73,18 @@ func Benchmark_pushDatabase(b *testing.B) { initTestConf() initTestDB() - if _, ok := twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"]; !ok { - out, _, err := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) + if _, ok := twtxtCache.Users[testTwtxtURL]; !ok { + out, _, err := registry.GetTwtxt(testTwtxtURL, nil) if err != nil { b.Errorf("Couldn't set up benchmark: %v\n", err) } - statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + statusmap, err := registry.ParseUserTwtxt(out, "getwtxttest", testTwtxtURL) if err != nil { b.Errorf("Couldn't set up benchmark: %v\n", err) } - twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), statusmap) + twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), statusmap) } b.ResetTimer() diff --git a/svc/handlers.go b/svc/handlers.go index 7ce4b77..cb07349 100644 --- a/svc/handlers.go +++ b/svc/handlers.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/gorilla/mux" ) diff --git a/svc/handlers_test.go b/svc/handlers_test.go index a2d77e2..46151d7 100644 --- a/svc/handlers_test.go +++ b/svc/handlers_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" diff --git a/svc/help.go b/svc/help.go index 3cc60ff..6eb5be7 100644 --- a/svc/help.go +++ b/svc/help.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import "fmt" @@ -31,7 +31,7 @@ func titleScreen() { \__, |\___|\__| \_/\_/ \__/_/\_\\__| |___/ version ` + Vers + ` - github.com/getwtxt/getwtxt + git.sr.ht/~gbmor/getwtxt GPL v3 `) @@ -284,7 +284,7 @@ func manualScreen() { Adding a user: curl -X POST 'http://localhost:9001/api/plain/users\ - ?url=https://gbmor.dev/twtxt.txt&nickname=gbmor' + ?url=https://example.org/twtxt.txt&nickname=somebody' Retrieve user list: curl 'http://localhost:9001/api/plain/users' diff --git a/svc/http.go b/svc/http.go index 6350e9e..c137d08 100644 --- a/svc/http.go +++ b/svc/http.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "context" diff --git a/svc/init.go b/svc/init.go index 00e107b..3f9d505 100644 --- a/svc/init.go +++ b/svc/init.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "html/template" @@ -26,7 +26,7 @@ import ( "os/signal" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/spf13/pflag" ) diff --git a/svc/init_test.go b/svc/init_test.go index 2fbfd5e..0c255c5 100644 --- a/svc/init_test.go +++ b/svc/init_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "bytes" @@ -29,10 +29,12 @@ import ( "sync" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/spf13/viper" ) +const testTwtxtURL = "https://git.sr.ht/~gbmor/getwtxt/blob/master/testdata/twtxt.txt" + var ( testport string initTestOnce sync.Once @@ -111,21 +113,21 @@ func testConfig() { // user and their statuses, for testing. func mockRegistry() { twtxtCache = registry.New(nil) - statuses, _, _ := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) - parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") - _ = twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), parsed) + statuses, _, _ := registry.GetTwtxt(testTwtxtURL, nil) + parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", testTwtxtURL) + _ = twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), parsed) } // Empties the mock registry's user of statuses // for functions that test status modifications func killStatuses() { twtxtCache.Mu.Lock() - user := twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"] + user := twtxtCache.Users[testTwtxtURL] user.Mu.Lock() user.Status = registry.NewTimeMap() user.LastModified = "0" - twtxtCache.Users["https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt"] = user + twtxtCache.Users[testTwtxtURL] = user user.Mu.Unlock() twtxtCache.Mu.Unlock() diff --git a/svc/leveldb.go b/svc/leveldb.go index 16751ce..5fb4a45 100644 --- a/svc/leveldb.go +++ b/svc/leveldb.go @@ -17,14 +17,14 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "net" "strings" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/svc/periodic.go b/svc/periodic.go index c8b918b..c6818a6 100644 --- a/svc/periodic.go +++ b/svc/periodic.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "log" diff --git a/svc/post.go b/svc/post.go index db4afd7..bba8136 100644 --- a/svc/post.go +++ b/svc/post.go @@ -17,14 +17,14 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" "net/http" "strings" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) // Requests to apiEndpointPOSTHandler are passed off to this diff --git a/svc/post_test.go b/svc/post_test.go index cd15565..6f68a66 100644 --- a/svc/post_test.go +++ b/svc/post_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" @@ -27,7 +27,7 @@ import ( "strings" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) var apiPostUserCases = []struct { @@ -39,7 +39,7 @@ var apiPostUserCases = []struct { { name: "Known Good User", nick: "getwtxttest", - uri: "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", + uri: testTwtxtURL, wantErr: false, }, { @@ -100,7 +100,7 @@ func Benchmark_apiPostUser(b *testing.B) { twtxtCache = registry.New(nil) params := url.Values{} - params.Set("url", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") + params.Set("url", testTwtxtURL) params.Set("nickname", "gbmor") req, _ := http.NewRequest("POST", "https://localhost"+portnum+"/api/plain/users", strings.NewReader(params.Encode())) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") diff --git a/svc/query.go b/svc/query.go index 8f22838..6da3601 100644 --- a/svc/query.go +++ b/svc/query.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "crypto/sha256" @@ -28,7 +28,7 @@ import ( "strings" "sync" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" "github.com/gorilla/mux" ) diff --git a/svc/query_test.go b/svc/query_test.go index d2d7304..939592e 100644 --- a/svc/query_test.go +++ b/svc/query_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "net" @@ -25,7 +25,7 @@ import ( "strings" "testing" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" ) func Test_dedupe(t *testing.T) { @@ -61,7 +61,7 @@ func Benchmark_dedupe(b *testing.B) { func Test_parseQueryOut(t *testing.T) { initTestConf() - urls := "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt" + urls := testTwtxtURL nick := "getwtxttest" out, _, err := registry.GetTwtxt(urls, nil) @@ -95,7 +95,7 @@ func Test_parseQueryOut(t *testing.T) { func Benchmark_parseQueryOut(b *testing.B) { initTestConf() - urls := "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt" + urls := testTwtxtURL nick := "getwtxttest" out, _, err := registry.GetTwtxt(urls, nil) @@ -203,9 +203,9 @@ func Test_compositeStatusQuery(t *testing.T) { func Benchmark_compositeStatusQuery(b *testing.B) { initTestConf() - statuses, _, _ := registry.GetTwtxt("https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", nil) - parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt") - _ = twtxtCache.AddUser("getwtxttest", "https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt", net.ParseIP("127.0.0.1"), parsed) + statuses, _, _ := registry.GetTwtxt(testTwtxtURL, nil) + parsed, _ := registry.ParseUserTwtxt(statuses, "getwtxttest", testTwtxtURL) + _ = twtxtCache.AddUser("getwtxttest", testTwtxtURL, net.ParseIP("127.0.0.1"), parsed) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/svc/sqlite.go b/svc/sqlite.go index 58a804d..128aed3 100644 --- a/svc/sqlite.go +++ b/svc/sqlite.go @@ -17,14 +17,14 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "database/sql" "net" "time" - "github.com/getwtxt/registry" + "git.sr.ht/~gbmor/getwtxt/registry" _ "github.com/mattn/go-sqlite3" // for the sqlite3 driver ) diff --git a/svc/svc.go b/svc/svc.go index 8c1c63a..72ccdc3 100644 --- a/svc/svc.go +++ b/svc/svc.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with Getwtxt. If not, see . */ -package svc // import "github.com/getwtxt/getwtxt/svc" +package svc // import "git.sr.ht/~gbmor/getwtxt/svc" import ( "fmt" diff --git a/testdata/twtxt.txt b/testdata/twtxt.txt index ab28c66..1365751 100644 --- a/testdata/twtxt.txt +++ b/testdata/twtxt.txt @@ -18,7 +18,7 @@ # == Metadata == # # nick = getwtxttest -# url = https://github.com/getwtxt/getwtxt/raw/master/testdata/twtxt.txt +# url = https://git.sr.ht/~gbmor/getwtxt/blob/master/testdata/twtxt.txt # # == Content == #