From 76e5d8e77c1636d2cadbb48cb6176d8231aa69bf Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Thu, 2 Dec 2021 19:12:45 +0100 Subject: [PATCH 01/41] Add TODOs --- certificates.go | 9 ++++++--- handler.go | 3 +++ haproxy-sni/haproxy.cfg | 1 + main.go | 4 +++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/certificates.go b/certificates.go index db51020..73b4793 100644 --- a/certificates.go +++ b/certificates.go @@ -130,6 +130,7 @@ var tlsConfig = &tls.Config{ }, } +// TODO: clean up & move to init var keyCache = mcache.New() var keyDatabase, keyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{ BackgroundSyncInterval: 30 * time.Second, @@ -218,6 +219,7 @@ func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) { panic(err) } + // TODO: document & put into own function if !bytes.Equal(sni, MainDomainSuffix) { tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0]) if err != nil { @@ -226,6 +228,7 @@ func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) { // renew certificates 7 days before they expire if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) { + // TODO: add ValidUntil to custom res struct if res.CSR != nil && len(res.CSR) > 0 { // CSR stores the time when the renewal shall be tried again nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64) @@ -315,9 +318,8 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re PogrebPut(keyDatabase, []byte(name), renew) return tlsCertificate, nil } - } else { - return mockCert(domains[0], err.Error()), err } + return mockCert(domains[0], err.Error()), err } log.Printf("Obtained certificate for %v", domains) @@ -531,9 +533,10 @@ func setupCertificates() { for { err := keyDatabase.Sync() if err != nil { - log.Printf("[ERROR] Syncinc key database failed: %s", err) + log.Printf("[ERROR] Syncing key database failed: %s", err) } time.Sleep(5 * time.Minute) + // TODO: graceful exit } })() go (func() { diff --git a/handler.go b/handler.go index 626385d..4e9efbf 100644 --- a/handler.go +++ b/handler.go @@ -301,6 +301,7 @@ func returnErrorPage(ctx *fasthttp.RequestCtx, code int) { if code == fasthttp.StatusFailedDependency { message += " - target repo/branch doesn't exist or is private" } + // TODO: use template engine? ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message))) } @@ -351,6 +352,7 @@ func getBranchTimestamp(owner, repo, branch string) *branchTimestamp { if branch == "" { // Get default branch var body = make([]byte, 0) + // TODO: use header for API key? status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+GiteaApiToken, 5*time.Second) if err != nil || status != 200 { _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout) @@ -509,6 +511,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t if res.Header.ContentLength() > FileCacheSizeLimit { err = res.BodyWriteTo(ctx.Response.BodyWriter()) } else { + // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) } } else { diff --git a/haproxy-sni/haproxy.cfg b/haproxy-sni/haproxy.cfg index 869bae3..c8f3610 100644 --- a/haproxy-sni/haproxy.cfg +++ b/haproxy-sni/haproxy.cfg @@ -51,6 +51,7 @@ frontend https_sni_frontend ################################################### acl use_http_backend req.ssl_sni -i "codeberg.org" acl use_http_backend req.ssl_sni -i "join.codeberg.org" + # TODO: use this if no SNI exists use_backend https_termination_backend if use_http_backend ############################ diff --git a/main.go b/main.go index 44cec0f..1a4cb65 100644 --- a/main.go +++ b/main.go @@ -71,6 +71,7 @@ var IndexPages = []string{ // main sets up and starts the web server. func main() { + // TODO: CLI Library if len(os.Args) > 1 && os.Args[1] == "--remove-certificate" { if len(os.Args) < 2 { println("--remove-certificate requires at least one domain as an argument") @@ -105,7 +106,7 @@ func main() { server := &fasthttp.Server{ Handler: compressedHandler, - DisablePreParseMultipartForm: false, + DisablePreParseMultipartForm: true, MaxRequestBodySize: 0, NoDefaultServerHeader: true, NoDefaultDate: true, @@ -151,6 +152,7 @@ func main() { } // envOr reads an environment variable and returns a default value if it's empty. +// TODO: to helpers.go or use CLI framework func envOr(env string, or string) string { if v := os.Getenv(env); v != "" { return v From fb5726bd202e40a3de9e113d290e8ffb7100396a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 14:43:51 +0100 Subject: [PATCH 02/41] use zerolog instead of own logger --- debug-stepper/stepper.go | 68 ---------------------------------------- go.mod | 1 + go.sum | 10 ++++++ handler.go | 67 +++++++++++++++++++-------------------- 4 files changed, 44 insertions(+), 102 deletions(-) delete mode 100644 debug-stepper/stepper.go diff --git a/debug-stepper/stepper.go b/debug-stepper/stepper.go deleted file mode 100644 index 05506b6..0000000 --- a/debug-stepper/stepper.go +++ /dev/null @@ -1,68 +0,0 @@ -package debug_stepper - -import ( - "fmt" - "os" - "strings" - "time" -) - -var Enabled = strings.HasSuffix(os.Args[0], ".test") || os.Getenv("DEBUG") == "1" - -var Logger = func(s string, i ...interface{}) { - fmt.Printf(s, i...) -} - -type Stepper struct { - Name string - Start time.Time - LastStep time.Time - Completion time.Time -} - -func Start(name string) *Stepper { - if !Enabled { - return nil - } - t := time.Now() - Logger("%s: started at %s\n", name, t.Format(time.RFC3339)) - return &Stepper{ - Name: name, - Start: t, - LastStep: t, - } -} - -func (s *Stepper) Debug(text string) { - if !Enabled { - return - } - t := time.Now() - Logger("%s: %s (at %s, %s since last step, %s since start)\n", s.Name, text, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String()) -} - -func (s *Stepper) Step(description string) { - if !Enabled { - return - } - if s.Completion != (time.Time{}) { - Logger("%s: already completed all tasks.\n") - return - } - t := time.Now() - Logger("%s: completed %s at %s (%s)\n", s.Name, description, t.Format(time.RFC3339), t.Sub(s.LastStep).String()) - s.LastStep = t -} - -func (s *Stepper) Complete() { - if !Enabled { - return - } - if s.Completion != (time.Time{}) { - Logger("%s: already completed all tasks.\n") - return - } - t := time.Now() - Logger("%s: completed all tasks at %s (%s since last step; total time: %s)\n", s.Name, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String()) - s.Completion = t -} diff --git a/go.mod b/go.mod index fd52ef1..7a64921 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad + github.com/rs/zerolog v1.26.0 // indirect github.com/valyala/fasthttp v1.31.0 github.com/valyala/fastjson v1.6.3 ) diff --git a/go.sum b/go.sum index c93f985..8561c5c 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4= github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ= @@ -148,6 +149,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b h1:/vQ+oYKu+JoyaMPDsv5FzwuL2wwWBgBbtj/YLCi4LuA= github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -422,6 +424,9 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad h1:WtSUHi5zthjudjI github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe2Y+6iHTjIq/9HzUq7NII/Nffp0HkFrsAKq4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE= +github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI= @@ -499,6 +504,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -557,6 +563,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -591,6 +598,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -655,6 +663,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -708,6 +717,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/handler.go b/handler.go index 4e9efbf..55ae36d 100644 --- a/handler.go +++ b/handler.go @@ -2,23 +2,23 @@ package main import ( "bytes" - debug_stepper "codeberg.org/codeberg/pages/debug-stepper" "fmt" - "github.com/OrlovEvgeny/go-mcache" - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" "io" "mime" "path" "strconv" "strings" "time" + + "github.com/OrlovEvgeny/go-mcache" + "github.com/rs/zerolog/log" + "github.com/valyala/fasthttp" + "github.com/valyala/fastjson" ) // handler handles a single HTTP request to the web server. func handler(ctx *fasthttp.RequestCtx) { - s := debug_stepper.Start("handler") - defer s.Complete() + log := log.With().Str("handler", string(ctx.Request.Header.RequestURI())).Logger() ctx.Response.Header.Set("Server", "Codeberg Pages") @@ -129,11 +129,11 @@ func handler(ctx *fasthttp.RequestCtx) { } } - s.Step("preparations") + log.Debug().Msg("preparations") if RawDomain != nil && bytes.Equal(trimmedHost, RawDomain) { // Serve raw content from RawDomain - s.Debug("raw domain") + log.Debug().Msg("raw domain") targetOptions.TryIndexPages = false targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{} @@ -150,30 +150,30 @@ func handler(ctx *fasthttp.RequestCtx) { // raw.codeberg.org/example/myrepo/@main/index.html if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { - s.Step("raw domain preparations, now trying with specified branch") + log.Debug().Msg("raw domain preparations, now trying with specified branch") if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:], string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) { - s.Step("tryBranch, now trying upstream") + log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() return } - s.Debug("missing branch") + log.Debug().Msg("missing branch") returnErrorPage(ctx, fasthttp.StatusFailedDependency) return } else { - s.Step("raw domain preparations, now trying with default branch") + log.Debug().Msg("raw domain preparations, now trying with default branch") tryBranch(targetRepo, "", pathElements[2:], string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) - s.Step("tryBranch, now trying upstream") + log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() return } } else if bytes.HasSuffix(trimmedHost, MainDomainSuffix) { // Serve pages from subdomains of MainDomainSuffix - s.Debug("main domain suffix") + log.Debug().Msg("main domain suffix") pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") targetOwner = string(bytes.TrimSuffix(trimmedHost, MainDomainSuffix)) @@ -182,7 +182,7 @@ func handler(ctx *fasthttp.RequestCtx) { if targetOwner == "www" { // www.codeberg.page redirects to codeberg.page - ctx.Redirect("https://" + string(MainDomainSuffix[1:]) + string(ctx.Path()), fasthttp.StatusPermanentRedirect) + ctx.Redirect("https://"+string(MainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect) return } @@ -195,11 +195,11 @@ func handler(ctx *fasthttp.RequestCtx) { return } - s.Step("main domain preparations, now trying with specified repo & branch") + log.Debug().Msg("main domain preparations, now trying with specified repo & branch") if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:], "/"+pathElements[0]+"/%p", ) { - s.Step("tryBranch, now trying upstream") + log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() } else { returnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -210,9 +210,9 @@ func handler(ctx *fasthttp.RequestCtx) { // Check if the first directory is a branch for the "pages" repo // example.codeberg.page/@main/index.html if strings.HasPrefix(pathElements[0], "@") { - s.Step("main domain preparations, now trying with specified branch") + log.Debug().Msg("main domain preparations, now trying with specified branch") if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") { - s.Step("tryBranch, now trying upstream") + log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() } else { returnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -223,18 +223,18 @@ func handler(ctx *fasthttp.RequestCtx) { // Check if the first directory is a repo with a "pages" branch // example.codeberg.page/myrepo/index.html // example.codeberg.page/pages/... is not allowed here. - s.Step("main domain preparations, now trying with specified repo") + log.Debug().Msg("main domain preparations, now trying with specified repo") if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") { - s.Step("tryBranch, now trying upstream") + log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() return } // Try to use the "pages" repo on its default branch // example.codeberg.page/index.html - s.Step("main domain preparations, now trying with default repo/branch") + log.Debug().Msg("main domain preparations, now trying with default repo/branch") if tryBranch("pages", "", pathElements, "") { - s.Step("tryBranch, now trying upstream") + log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() return } @@ -261,7 +261,7 @@ func handler(ctx *fasthttp.RequestCtx) { } // Try to use the given repo on the given branch or the default branch - s.Step("custom domain preparations, now trying with details from DNS") + log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr) if !valid { @@ -279,7 +279,7 @@ func handler(ctx *fasthttp.RequestCtx) { } } - s.Step("tryBranch, now trying upstream") + log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() return } else { @@ -380,9 +380,8 @@ var upstreamClient = fasthttp.Client{ } // upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, targetBranch string, targetPath string, options *upstreamOptions) (final bool) { - s := debug_stepper.Start("upstream") - defer s.Complete() +func upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath string, options *upstreamOptions) (final bool) { + log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() if options.ForbiddenMimeTypes == nil { options.ForbiddenMimeTypes = map[string]struct{}{} @@ -412,7 +411,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t return true } } - s.Step("preparations") + log.Debug().Msg("preparations") // Make a GET request to the upstream URL uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath @@ -429,7 +428,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t res.SetBodyStream(&strings.Reader{}, -1) err = upstreamClient.Do(req, res) } - s.Step("acquisition") + log.Debug().Msg("acquisition") // Handle errors if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { @@ -449,7 +448,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.AppendTrailingSlash = false optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" - if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath + ".html", &optionsForIndexPages) { + if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", &optionsForIndexPages) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, FileCacheTimeout) @@ -485,7 +484,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect) return true } - s.Step("error handling") + log.Debug().Msg("error handling") // Set the MIME type mimeType := mime.TypeByExtension(path.Ext(targetPath)) @@ -503,7 +502,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t ctx.Response.SetStatusCode(fasthttp.StatusOK) ctx.Response.Header.SetLastModified(options.BranchTimestamp) - s.Step("response preparations") + log.Debug().Msg("response preparations") // Write the response body to the original request var cacheBodyWriter bytes.Buffer @@ -522,7 +521,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t returnErrorPage(ctx, fasthttp.StatusInternalServerError) return true } - s.Step("response") + log.Debug().Msg("response") if res != nil && ctx.Err() == nil { cachedResponse.exists = true From bdc2d0c2592c8c81358eef562a9eb1266f9aaf56 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 14:45:17 +0100 Subject: [PATCH 03/41] dont access global vars inject them --- go.mod | 2 +- go.sum | 4 +- handler.go | 544 --------------------- 404.html => html/404.html | 0 html/html.go | 6 + main.go | 52 +- certificates.go => server/certificates.go | 224 ++++----- domains.go => server/domains.go | 16 +- server/handler.go | 553 ++++++++++++++++++++++ handler_test.go => server/handler_test.go | 18 +- helpers.go => server/helpers.go | 17 +- 11 files changed, 730 insertions(+), 706 deletions(-) delete mode 100644 handler.go rename 404.html => html/404.html (100%) create mode 100644 html/html.go rename certificates.go => server/certificates.go (74%) rename domains.go => server/domains.go (83%) create mode 100644 server/handler.go rename handler_test.go => server/handler_test.go (80%) rename helpers.go => server/helpers.go (71%) diff --git a/go.mod b/go.mod index 7a64921..55e4675 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad - github.com/rs/zerolog v1.26.0 // indirect + github.com/rs/zerolog v1.26.0 github.com/valyala/fasthttp v1.31.0 github.com/valyala/fastjson v1.6.3 ) diff --git a/go.sum b/go.sum index 8561c5c..65da291 100644 --- a/go.sum +++ b/go.sum @@ -596,8 +596,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -661,8 +661,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/handler.go b/handler.go deleted file mode 100644 index 55ae36d..0000000 --- a/handler.go +++ /dev/null @@ -1,544 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io" - "mime" - "path" - "strconv" - "strings" - "time" - - "github.com/OrlovEvgeny/go-mcache" - "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" -) - -// handler handles a single HTTP request to the web server. -func handler(ctx *fasthttp.RequestCtx) { - log := log.With().Str("handler", string(ctx.Request.Header.RequestURI())).Logger() - - ctx.Response.Header.Set("Server", "Codeberg Pages") - - // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin - ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin") - - // Enable browser caching for up to 10 minutes - ctx.Response.Header.Set("Cache-Control", "public, max-age=600") - - trimmedHost := TrimHostPort(ctx.Request.Host()) - - // Add HSTS for RawDomain and MainDomainSuffix - if hsts := GetHSTSHeader(trimmedHost); hsts != "" { - ctx.Response.Header.Set("Strict-Transport-Security", hsts) - } - - // Block all methods not required for static pages - if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() { - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed) - return - } - - // Block blacklisted paths (like ACME challenges) - for _, blacklistedPath := range BlacklistedPaths { - if bytes.HasPrefix(ctx.Path(), blacklistedPath) { - returnErrorPage(ctx, fasthttp.StatusForbidden) - return - } - } - - // Allow CORS for specified domains - if ctx.IsOptions() { - allowCors := false - for _, allowedCorsDomain := range AllowedCorsDomains { - if bytes.Equal(trimmedHost, allowedCorsDomain) { - allowCors = true - break - } - } - if allowCors { - ctx.Response.Header.Set("Access-Control-Allow-Origin", "*") - ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD") - } - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent) - return - } - - // Prepare request information to Gitea - var targetOwner, targetRepo, targetBranch, targetPath string - var targetOptions = &upstreamOptions{ - ForbiddenMimeTypes: map[string]struct{}{}, - TryIndexPages: true, - } - - // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will - // also disallow search indexing and add a Link header to the canonical URL. - var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool { - if repo == "" { - return false - } - - // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch) - if branchTimestampResult == nil { - // branch doesn't exist - return false - } - - // Branch exists, use it - targetRepo = repo - targetPath = strings.Trim(strings.Join(path, "/"), "/") - targetBranch = branchTimestampResult.branch - - targetOptions.BranchTimestamp = branchTimestampResult.timestamp - - if canonicalLink != "" { - // Hide from search machines & add canonical link - ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex") - ctx.Response.Header.Set("Link", - strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+ - "; rel=\"canonical\"", - ) - } - - return true - } - - // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. - var tryUpstream = func() { - // check if a canonical domain exists on a request on MainDomain - if bytes.HasSuffix(trimmedHost, MainDomainSuffix) { - canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "") - if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(MainDomainSuffix)) { - canonicalPath := string(ctx.RequestURI()) - if targetRepo != "pages" { - canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2] - } - ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) - return - } - } - - // Try to request the file from the Gitea API - if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, targetOptions) { - returnErrorPage(ctx, ctx.Response.StatusCode()) - } - } - - log.Debug().Msg("preparations") - - if RawDomain != nil && bytes.Equal(trimmedHost, RawDomain) { - // Serve raw content from RawDomain - log.Debug().Msg("raw domain") - - targetOptions.TryIndexPages = false - targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{} - targetOptions.DefaultMimeType = "text/plain; charset=utf-8" - - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") - if len(pathElements) < 2 { - // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required - ctx.Redirect(RawInfoPage, fasthttp.StatusTemporaryRedirect) - return - } - targetOwner = pathElements[0] - targetRepo = pathElements[1] - - // raw.codeberg.org/example/myrepo/@main/index.html - if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { - log.Debug().Msg("raw domain preparations, now trying with specified branch") - if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:], - string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", - ) { - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() - return - } - log.Debug().Msg("missing branch") - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } else { - log.Debug().Msg("raw domain preparations, now trying with default branch") - tryBranch(targetRepo, "", pathElements[2:], - string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", - ) - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() - return - } - - } else if bytes.HasSuffix(trimmedHost, MainDomainSuffix) { - // Serve pages from subdomains of MainDomainSuffix - log.Debug().Msg("main domain suffix") - - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") - targetOwner = string(bytes.TrimSuffix(trimmedHost, MainDomainSuffix)) - targetRepo = pathElements[0] - targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/") - - if targetOwner == "www" { - // www.codeberg.page redirects to codeberg.page - ctx.Redirect("https://"+string(MainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect) - return - } - - // Check if the first directory is a repo with the second directory as a branch - // example.codeberg.page/myrepo/@main/index.html - if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { - if targetRepo == "pages" { - // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... - ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect) - return - } - - log.Debug().Msg("main domain preparations, now trying with specified repo & branch") - if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:], - "/"+pathElements[0]+"/%p", - ) { - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() - } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - } - return - } - - // Check if the first directory is a branch for the "pages" repo - // example.codeberg.page/@main/index.html - if strings.HasPrefix(pathElements[0], "@") { - log.Debug().Msg("main domain preparations, now trying with specified branch") - if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") { - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() - } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - } - return - } - - // Check if the first directory is a repo with a "pages" branch - // example.codeberg.page/myrepo/index.html - // example.codeberg.page/pages/... is not allowed here. - log.Debug().Msg("main domain preparations, now trying with specified repo") - if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") { - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() - return - } - - // Try to use the "pages" repo on its default branch - // example.codeberg.page/index.html - log.Debug().Msg("main domain preparations, now trying with default repo/branch") - if tryBranch("pages", "", pathElements, "") { - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() - return - } - - // Couldn't find a valid repo/branch - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } else { - trimmedHostStr := string(trimmedHost) - - // Serve pages from external domains - targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr) - if targetOwner == "" { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } - - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") - canonicalLink := "" - if strings.HasPrefix(pathElements[0], "@") { - targetBranch = pathElements[0][1:] - pathElements = pathElements[1:] - canonicalLink = "/%p" - } - - // Try to use the given repo on the given branch or the default branch - log.Debug().Msg("custom domain preparations, now trying with details from DNS") - if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr) - if !valid { - returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) - return - } else if canonicalDomain != trimmedHostStr { - // only redirect if the target is also a codeberg page! - targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0]) - if targetOwner != "" { - ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) - return - } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } - } - - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() - return - } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } - } -} - -// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced -// with the provided status code. -func returnErrorPage(ctx *fasthttp.RequestCtx, code int) { - ctx.Response.SetStatusCode(code) - ctx.Response.Header.SetContentType("text/html; charset=utf-8") - message := fasthttp.StatusMessage(code) - if code == fasthttp.StatusMisdirectedRequest { - message += " - domain not specified in .domains file" - } - if code == fasthttp.StatusFailedDependency { - message += " - target repo/branch doesn't exist or is private" - } - // TODO: use template engine? - ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message))) -} - -// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. -var DefaultBranchCacheTimeout = 15 * time.Minute - -// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter -// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be -// picked up faster, while still allowing the content to be cached longer if nothing changes. -var BranchExistanceCacheTimeout = 5 * time.Minute - -// branchTimestampCache stores branch timestamps for faster cache checking -var branchTimestampCache = mcache.New() - -type branchTimestamp struct { - branch string - timestamp time.Time -} - -// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending -// on your available memory. -var FileCacheTimeout = 5 * time.Minute - -// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. -var FileCacheSizeLimit = 1024 * 1024 - -// fileResponseCache stores responses from the Gitea server -// TODO: make this an MRU cache with a size limit -var fileResponseCache = mcache.New() - -type fileResponse struct { - exists bool - mimeType string - body []byte -} - -// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch -// (or nil if the branch doesn't exist) -func getBranchTimestamp(owner, repo, branch string) *branchTimestamp { - if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { - if result == nil { - return nil - } - return result.(*branchTimestamp) - } - result := &branchTimestamp{} - result.branch = branch - if branch == "" { - // Get default branch - var body = make([]byte, 0) - // TODO: use header for API key? - status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+GiteaApiToken, 5*time.Second) - if err != nil || status != 200 { - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout) - return nil - } - result.branch = fastjson.GetString(body, "default_branch") - } - - var body = make([]byte, 0) - status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+GiteaApiToken, 5*time.Second) - if err != nil || status != 200 { - return nil - } - - result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp")) - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout) - return result -} - -var upstreamClient = fasthttp.Client{ - ReadTimeout: 10 * time.Second, - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! -} - -// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath string, options *upstreamOptions) (final bool) { - log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() - - if options.ForbiddenMimeTypes == nil { - options.ForbiddenMimeTypes = map[string]struct{}{} - } - - // Check if the branch exists and when it was modified - if options.BranchTimestamp == (time.Time{}) { - branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch) - - if branch == nil { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - return true - } - targetBranch = branch.branch - options.BranchTimestamp = branch.timestamp - } - - if targetOwner == "" || targetRepo == "" || targetBranch == "" { - returnErrorPage(ctx, fasthttp.StatusBadRequest) - return true - } - - // Check if the browser has a cached version - if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { - if !ifModifiedSince.Before(options.BranchTimestamp) { - ctx.Response.SetStatusCode(fasthttp.StatusNotModified) - return true - } - } - log.Debug().Msg("preparations") - - // Make a GET request to the upstream URL - uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath - var req *fasthttp.Request - var res *fasthttp.Response - var cachedResponse fileResponse - var err error - if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { - cachedResponse = cachedValue.(fileResponse) - } else { - req = fasthttp.AcquireRequest() - req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + uri + "?access_token=" + GiteaApiToken) - res = fasthttp.AcquireResponse() - res.SetBodyStream(&strings.Reader{}, -1) - err = upstreamClient.Do(req, res) - } - log.Debug().Msg("acquisition") - - // Handle errors - if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { - if options.TryIndexPages { - // copy the options struct & try if an index page exists - optionsForIndexPages := *options - optionsForIndexPages.TryIndexPages = false - optionsForIndexPages.AppendTrailingSlash = true - for _, indexPage := range IndexPages { - if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, &optionsForIndexPages) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, - }, FileCacheTimeout) - return true - } - } - // compatibility fix for GitHub Pages (/example → /example.html) - optionsForIndexPages.AppendTrailingSlash = false - optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" - if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", &optionsForIndexPages) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, - }, FileCacheTimeout) - return true - } - } - ctx.Response.SetStatusCode(fasthttp.StatusNotFound) - if res != nil { - // Update cache if the request is fresh - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, - }, FileCacheTimeout) - } - return false - } - if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { - fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode()) - returnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true - } - - // Append trailing slash if missing (for index files), and redirect to fix filenames in general - // options.AppendTrailingSlash is only true when looking for index pages - if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { - ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) - return true - } - if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) { - ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) - return true - } - if options.RedirectIfExists != "" { - ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect) - return true - } - log.Debug().Msg("error handling") - - // Set the MIME type - mimeType := mime.TypeByExtension(path.Ext(targetPath)) - mimeTypeSplit := strings.SplitN(mimeType, ";", 2) - if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { - if options.DefaultMimeType != "" { - mimeType = options.DefaultMimeType - } else { - mimeType = "application/octet-stream" - } - } - ctx.Response.Header.SetContentType(mimeType) - - // Everything's okay so far - ctx.Response.SetStatusCode(fasthttp.StatusOK) - ctx.Response.Header.SetLastModified(options.BranchTimestamp) - - log.Debug().Msg("response preparations") - - // Write the response body to the original request - var cacheBodyWriter bytes.Buffer - if res != nil { - if res.Header.ContentLength() > FileCacheSizeLimit { - err = res.BodyWriteTo(ctx.Response.BodyWriter()) - } else { - // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? - err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) - } - } else { - _, err = ctx.Write(cachedResponse.body) - } - if err != nil { - fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err) - returnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true - } - log.Debug().Msg("response") - - if res != nil && ctx.Err() == nil { - cachedResponse.exists = true - cachedResponse.mimeType = mimeType - cachedResponse.body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout) - } - - return true -} - -// upstreamOptions provides various options for the upstream request. -type upstreamOptions struct { - DefaultMimeType string - ForbiddenMimeTypes map[string]struct{} - TryIndexPages bool - AppendTrailingSlash bool - RedirectIfExists string - BranchTimestamp time.Time -} diff --git a/404.html b/html/404.html similarity index 100% rename from 404.html rename to html/404.html diff --git a/html/html.go b/html/html.go new file mode 100644 index 0000000..d223e15 --- /dev/null +++ b/html/html.go @@ -0,0 +1,6 @@ +package html + +import _ "embed" + +//go:embed 404.html +var NotFoundPage []byte diff --git a/main.go b/main.go index 1a4cb65..501e0ab 100644 --- a/main.go +++ b/main.go @@ -26,31 +26,28 @@ import ( "os" "time" - _ "embed" - "github.com/valyala/fasthttp" + + pages_server "codeberg.org/codeberg/pages/server" ) // MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static // pages, or used for comparison in CNAME lookups. Static pages can be accessed through // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". -var MainDomainSuffix = []byte("." + envOr("PAGES_DOMAIN", "codeberg.page")) +var MainDomainSuffix = []byte("." + pages_server.EnvOr("PAGES_DOMAIN", "codeberg.page")) // GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash. -var GiteaRoot = []byte(envOr("GITEA_ROOT", "https://codeberg.org")) +var GiteaRoot = []byte(pages_server.EnvOr("GITEA_ROOT", "https://codeberg.org")) -var GiteaApiToken = envOr("GITEA_API_TOKEN", "") - -//go:embed 404.html -var NotFoundPage []byte +var GiteaApiToken = pages_server.EnvOr("GITEA_API_TOKEN", "") // RawDomain specifies the domain from which raw repository content shall be served in the following format: // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} // (set to []byte(nil) to disable raw content hosting) -var RawDomain = []byte(envOr("RAW_DOMAIN", "raw.codeberg.org")) +var RawDomain = []byte(pages_server.EnvOr("RAW_DOMAIN", "raw.codeberg.org")) // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). -var RawInfoPage = envOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/") +var RawInfoPage = pages_server.EnvOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/") // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. var AllowedCorsDomains = [][]byte{ @@ -64,11 +61,6 @@ var BlacklistedPaths = [][]byte{ []byte("/.well-known/acme-challenge/"), } -// IndexPages lists pages that may be considered as index pages for directories. -var IndexPages = []string{ - "index.html", -} - // main sets up and starts the web server. func main() { // TODO: CLI Library @@ -77,15 +69,15 @@ func main() { println("--remove-certificate requires at least one domain as an argument") os.Exit(1) } - if keyDatabaseErr != nil { - panic(keyDatabaseErr) + if pages_server.KeyDatabaseErr != nil { + panic(pages_server.KeyDatabaseErr) } for _, domain := range os.Args[2:] { - if err := keyDatabase.Delete([]byte(domain)); err != nil { + if err := pages_server.KeyDatabase.Delete([]byte(domain)); err != nil { panic(err) } } - if err := keyDatabase.Sync(); err != nil { + if err := pages_server.KeyDatabase.Sync(); err != nil { panic(err) } os.Exit(0) @@ -98,10 +90,13 @@ func main() { GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'}) // Use HOST and PORT environment variables to determine listening address - address := fmt.Sprintf("%s:%s", envOr("HOST", "[::]"), envOr("PORT", "443")) + address := fmt.Sprintf("%s:%s", pages_server.EnvOr("HOST", "[::]"), pages_server.EnvOr("PORT", "443")) log.Printf("Listening on https://%s", address) - // Enable compression by wrapping the handler() method with the compression function provided by FastHTTP + // Create handler based on settings + handler := pages_server.Handler(MainDomainSuffix, RawDomain, GiteaRoot, RawInfoPage, GiteaApiToken, BlacklistedPaths, AllowedCorsDomains) + + // Enable compression by wrapping the handler with the compression function provided by FastHTTP compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) server := &fasthttp.Server{ @@ -120,15 +115,15 @@ func main() { if err != nil { log.Fatalf("Couldn't create listener: %s", err) } - listener = tls.NewListener(listener, tlsConfig) + listener = tls.NewListener(listener, pages_server.TlsConfig(MainDomainSuffix, string(GiteaRoot), GiteaApiToken)) - setupCertificates() + pages_server.SetupCertificates(MainDomainSuffix) if os.Getenv("ENABLE_HTTP_SERVER") == "true" { go (func() { challengePath := []byte("/.well-known/acme-challenge/") err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) { if bytes.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := challengeCache.Get(string(TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) + challenge, ok := pages_server.ChallengeCache.Get(string(pages_server.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) if !ok || challenge == nil { ctx.SetStatusCode(http.StatusNotFound) ctx.SetBodyString("no challenge for this token") @@ -150,12 +145,3 @@ func main() { log.Fatalf("Couldn't start server: %s", err) } } - -// envOr reads an environment variable and returns a default value if it's empty. -// TODO: to helpers.go or use CLI framework -func envOr(env string, or string) string { - if v := os.Getenv(env); v != "" { - return v - } - return or -} diff --git a/certificates.go b/server/certificates.go similarity index 74% rename from certificates.go rename to server/certificates.go index 73b4793..c330439 100644 --- a/certificates.go +++ b/server/certificates.go @@ -1,4 +1,4 @@ -package main +package server import ( "bytes" @@ -37,102 +37,104 @@ import ( "github.com/go-acme/lego/v4/registration" ) -// tlsConfig contains the configuration for generating, serving and cleaning up Let's Encrypt certificates. -var tlsConfig = &tls.Config{ - // check DNS name & get certificate from Let's Encrypt - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - sni := strings.ToLower(strings.TrimSpace(info.ServerName)) - sniBytes := []byte(sni) - if len(sni) < 1 { - return nil, errors.New("missing sni") - } +// TlsConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. +func TlsConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken string) *tls.Config { + return &tls.Config{ + // check DNS name & get certificate from Let's Encrypt + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + sni := strings.ToLower(strings.TrimSpace(info.ServerName)) + sniBytes := []byte(sni) + if len(sni) < 1 { + return nil, errors.New("missing sni") + } - if info.SupportedProtos != nil { - for _, proto := range info.SupportedProtos { - if proto == tlsalpn01.ACMETLS1Protocol { - challenge, ok := challengeCache.Get(sni) - if !ok { - return nil, errors.New("no challenge for this domain") + if info.SupportedProtos != nil { + for _, proto := range info.SupportedProtos { + if proto == tlsalpn01.ACMETLS1Protocol { + challenge, ok := ChallengeCache.Get(sni) + if !ok { + return nil, errors.New("no challenge for this domain") + } + cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string)) + if err != nil { + return nil, err + } + return cert, nil } - cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string)) - if err != nil { - return nil, err - } - return cert, nil } } - } - targetOwner := "" - if bytes.HasSuffix(sniBytes, MainDomainSuffix) || bytes.Equal(sniBytes, MainDomainSuffix[1:]) { - // deliver default certificate for the main domain (*.codeberg.page) - sniBytes = MainDomainSuffix - sni = string(sniBytes) - } else { - var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni) - if targetOwner == "" { - // DNS not set up, return main certificate to redirect to the docs - sniBytes = MainDomainSuffix + targetOwner := "" + if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) { + // deliver default certificate for the main domain (*.codeberg.page) + sniBytes = mainDomainSuffix sni = string(sniBytes) } else { - _, _ = targetRepo, targetBranch - _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni) - if !valid { - sniBytes = MainDomainSuffix + var targetRepo, targetBranch string + targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni, string(mainDomainSuffix)) + if targetOwner == "" { + // DNS not set up, return main certificate to redirect to the docs + sniBytes = mainDomainSuffix sni = string(sniBytes) + } else { + _, _ = targetRepo, targetBranch + _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken) + if !valid { + sniBytes = mainDomainSuffix + sni = string(sniBytes) + } } } - } - if tlsCertificate, ok := keyCache.Get(sni); ok { - // we can use an existing certificate object - return tlsCertificate.(*tls.Certificate), nil - } - - var tlsCertificate tls.Certificate - var err error - var ok bool - if tlsCertificate, ok = retrieveCertFromDB(sniBytes); !ok { - // request a new certificate - if bytes.Equal(sniBytes, MainDomainSuffix) { - return nil, errors.New("won't request certificate for main domain, something really bad has happened") + if tlsCertificate, ok := keyCache.Get(sni); ok { + // we can use an existing certificate object + return tlsCertificate.(*tls.Certificate), nil } - tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner) + var tlsCertificate tls.Certificate + var err error + var ok bool + if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix); !ok { + // request a new certificate + if bytes.Equal(sniBytes, mainDomainSuffix) { + return nil, errors.New("won't request certificate for main domain, something really bad has happened") + } + + tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, mainDomainSuffix) + if err != nil { + return nil, err + } + } + + err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute) if err != nil { - return nil, err + panic(err) } - } + return &tlsCertificate, nil + }, + PreferServerCipherSuites: true, + NextProtos: []string{ + "http/1.1", + tlsalpn01.ACMETLS1Protocol, + }, - err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute) - if err != nil { - panic(err) - } - return &tlsCertificate, nil - }, - PreferServerCipherSuites: true, - NextProtos: []string{ - "http/1.1", - tlsalpn01.ACMETLS1Protocol, - }, - - // generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration - // https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6 - MinVersion: tls.VersionTLS12, - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - }, + // generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration + // https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6 + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + }, + } } // TODO: clean up & move to init var keyCache = mcache.New() -var keyDatabase, keyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{ +var KeyDatabase, KeyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{ BackgroundSyncInterval: 30 * time.Second, BackgroundCompactionInterval: 6 * time.Hour, FileSystem: fs.OSMMap, @@ -181,17 +183,17 @@ var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute) // rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests) var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second) -var challengeCache = mcache.New() +var ChallengeCache = mcache.New() type AcmeTLSChallengeProvider struct{} var _ challenge.Provider = AcmeTLSChallengeProvider{} func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { - return challengeCache.Set(domain, keyAuth, 1*time.Hour) + return ChallengeCache.Set(domain, keyAuth, 1*time.Hour) } func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { - challengeCache.Remove(domain) + ChallengeCache.Remove(domain) return nil } @@ -200,17 +202,17 @@ type AcmeHTTPChallengeProvider struct{} var _ challenge.Provider = AcmeHTTPChallengeProvider{} func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { - return challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) + return ChallengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) } func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { - challengeCache.Remove(domain + "/" + token) + ChallengeCache.Remove(domain + "/" + token) return nil } -func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix []byte) (tls.Certificate, bool) { // parse certificate from database res := &certificate.Resource{} - if !PogrebGet(keyDatabase, sni, res) { + if !PogrebGet(KeyDatabase, sni, res) { return tls.Certificate{}, false } @@ -220,7 +222,7 @@ func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) { } // TODO: document & put into own function - if !bytes.Equal(sni, MainDomainSuffix) { + if !bytes.Equal(sni, mainDomainSuffix) { tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0]) if err != nil { panic(err) @@ -238,7 +240,7 @@ func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) { } go (func() { res.CSR = nil // acme client doesn't like CSR to be set - tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "") + tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", mainDomainSuffix) if err != nil { log.Printf("Couldn't renew certificate for %s: %s", sni, err) } @@ -251,7 +253,7 @@ func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) { var obtainLocks = sync.Map{} -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string) (tls.Certificate, error) { +func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, mainDomainSuffix []byte) (tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") if os.Getenv("DNS_PROVIDER") == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] @@ -264,7 +266,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re time.Sleep(100 * time.Millisecond) _, working = obtainLocks.Load(name) } - cert, ok := retrieveCertFromDB([]byte(name)) + cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix) if !ok { return tls.Certificate{}, errors.New("certificate failed in synchronous request") } @@ -273,7 +275,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re defer obtainLocks.Delete(name) if acmeClient == nil { - return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!"), nil + return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix)), nil } // request actual cert @@ -315,15 +317,15 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) { // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) - PogrebPut(keyDatabase, []byte(name), renew) + PogrebPut(KeyDatabase, []byte(name), renew) return tlsCertificate, nil } } - return mockCert(domains[0], err.Error()), err + return mockCert(domains[0], err.Error(), string(mainDomainSuffix)), err } log.Printf("Obtained certificate for %v", domains) - PogrebPut(keyDatabase, []byte(name), res) + PogrebPut(KeyDatabase, []byte(name), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { return tls.Certificate{}, err @@ -331,7 +333,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re return tlsCertificate, nil } -func mockCert(domain string, msg string) tls.Certificate { +func mockCert(domain, msg, mainDomainSuffix string) tls.Certificate { key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048) if err != nil { panic(err) @@ -385,10 +387,10 @@ func mockCert(domain string, msg string) tls.Certificate { Domain: domain, } databaseName := domain - if domain == "*"+string(MainDomainSuffix) || domain == string(MainDomainSuffix[1:]) { - databaseName = string(MainDomainSuffix) + if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] { + databaseName = mainDomainSuffix } - PogrebPut(keyDatabase, []byte(databaseName), res) + PogrebPut(KeyDatabase, []byte(databaseName), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { @@ -397,9 +399,9 @@ func mockCert(domain string, msg string) tls.Certificate { return tlsCertificate } -func setupCertificates() { - if keyDatabaseErr != nil { - panic(keyDatabaseErr) +func SetupCertificates(mainDomainSuffix []byte) { + if KeyDatabaseErr != nil { + panic(KeyDatabaseErr) } if os.Getenv("ACME_ACCEPT_TERMS") != "true" || (os.Getenv("DNS_PROVIDER") == "" && os.Getenv("ACME_API") != "https://acme.mock.directory") { @@ -407,7 +409,7 @@ func setupCertificates() { } // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits - mainCertBytes, err := keyDatabase.Get(MainDomainSuffix) + mainCertBytes, err := KeyDatabase.Get(mainDomainSuffix) if err != nil { // key database is not working panic(err) @@ -423,7 +425,7 @@ func setupCertificates() { panic(err) } myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") + myAcmeConfig.CADirURL = EnvOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 _, err := lego.NewClient(myAcmeConfig) if err != nil { @@ -435,12 +437,12 @@ func setupCertificates() { panic(err) } myAcmeAccount = AcmeAccount{ - Email: envOr("ACME_EMAIL", "noreply@example.email"), + Email: EnvOr("ACME_EMAIL", "noreply@example.email"), Key: privateKey, KeyPEM: string(certcrypto.PEMEncode(privateKey)), } myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") + myAcmeConfig.CADirURL = EnvOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 tempClient, err := lego.NewClient(myAcmeConfig) if err != nil { @@ -523,7 +525,7 @@ func setupCertificates() { } if mainCertBytes == nil { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, nil, "") + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", mainDomainSuffix) if err != nil { log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err) } @@ -531,7 +533,7 @@ func setupCertificates() { go (func() { for { - err := keyDatabase.Sync() + err := KeyDatabase.Sync() if err != nil { log.Printf("[ERROR] Syncing key database failed: %s", err) } @@ -544,10 +546,10 @@ func setupCertificates() { // clean up expired certs now := time.Now() expiredCertCount := 0 - keyDatabaseIterator := keyDatabase.Items() + keyDatabaseIterator := KeyDatabase.Items() key, resBytes, err := keyDatabaseIterator.Next() for err == nil { - if !bytes.Equal(key, MainDomainSuffix) { + if !bytes.Equal(key, mainDomainSuffix) { resGob := bytes.NewBuffer(resBytes) resDec := gob.NewDecoder(resGob) res := &certificate.Resource{} @@ -558,7 +560,7 @@ func setupCertificates() { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) if err != nil || !tlsCertificates[0].NotAfter.After(now) { - err := keyDatabase.Delete(key) + err := KeyDatabase.Delete(key) if err != nil { log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) } else { @@ -571,7 +573,7 @@ func setupCertificates() { log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) // compact the database - result, err := keyDatabase.Compact() + result, err := KeyDatabase.Compact() if err != nil { log.Printf("[ERROR] Compacting key database failed: %s", err) } else { @@ -580,7 +582,7 @@ func setupCertificates() { // update main cert res := &certificate.Resource{} - if !PogrebGet(keyDatabase, MainDomainSuffix, res) { + if !PogrebGet(KeyDatabase, mainDomainSuffix, res) { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") } else { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) @@ -588,7 +590,7 @@ func setupCertificates() { // renew main certificate 30 days before it expires if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) { go (func() { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, res, "") + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", mainDomainSuffix) if err != nil { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err) } diff --git a/domains.go b/server/domains.go similarity index 83% rename from domains.go rename to server/domains.go index 0a5abc1..5673960 100644 --- a/domains.go +++ b/server/domains.go @@ -1,4 +1,4 @@ -package main +package server import ( "github.com/OrlovEvgeny/go-mcache" @@ -16,7 +16,7 @@ var dnsLookupCache = mcache.New() // getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. -func getTargetFromDNS(domain string) (targetOwner, targetRepo, targetBranch string) { +func getTargetFromDNS(domain, mainDomainSuffix string) (targetOwner, targetRepo, targetBranch string) { // Get CNAME or TXT var cname string var err error @@ -25,14 +25,14 @@ func getTargetFromDNS(domain string) (targetOwner, targetRepo, targetBranch stri } else { cname, err = net.LookupCNAME(domain) cname = strings.TrimSuffix(cname, ".") - if err != nil || !strings.HasSuffix(cname, string(MainDomainSuffix)) { + if err != nil || !strings.HasSuffix(cname, mainDomainSuffix) { cname = "" // TODO: check if the A record matches! names, err := net.LookupTXT(domain) if err == nil { for _, name := range names { name = strings.TrimSuffix(name, ".") - if strings.HasSuffix(name, string(MainDomainSuffix)) { + if strings.HasSuffix(name, mainDomainSuffix) { cname = name break } @@ -44,7 +44,7 @@ func getTargetFromDNS(domain string) (targetOwner, targetRepo, targetBranch stri if cname == "" { return } - cnameParts := strings.Split(strings.TrimSuffix(cname, string(MainDomainSuffix)), ".") + cnameParts := strings.Split(strings.TrimSuffix(cname, mainDomainSuffix), ".") targetOwner = cnameParts[len(cnameParts)-1] if len(cnameParts) > 1 { targetRepo = cnameParts[len(cnameParts)-2] @@ -69,7 +69,7 @@ var CanonicalDomainCacheTimeout = 15 * time.Minute var canonicalDomainCache = mcache.New() // checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`). -func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain string) (canonicalDomain string, valid bool) { +func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string) (canonicalDomain string, valid bool) { domains := []string{} if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { domains = cachedValue.([]string) @@ -81,7 +81,7 @@ func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain st } } else { req := fasthttp.AcquireRequest() - req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + GiteaApiToken) + req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaApiToken) res := fasthttp.AcquireResponse() err := upstreamClient.Do(req, res) @@ -99,7 +99,7 @@ func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain st } } } - domains = append(domains, targetOwner+string(MainDomainSuffix)) + domains = append(domains, targetOwner+mainDomainSuffix) if domains[len(domains)-1] == actualDomain { valid = true } diff --git a/server/handler.go b/server/handler.go new file mode 100644 index 0000000..f24227b --- /dev/null +++ b/server/handler.go @@ -0,0 +1,553 @@ +package server + +import ( + "bytes" + "fmt" + "io" + "mime" + "path" + "strconv" + "strings" + "time" + + "github.com/OrlovEvgeny/go-mcache" + "github.com/rs/zerolog/log" + "github.com/valyala/fasthttp" + "github.com/valyala/fastjson" + + "codeberg.org/codeberg/pages/html" +) + +// Handler handles a single HTTP request to the web server. +func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaApiToken string, blacklistedPaths, allowedCorsDomains [][]byte) func(ctx *fasthttp.RequestCtx) { + return func(ctx *fasthttp.RequestCtx) { + log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() + + ctx.Response.Header.Set("Server", "Codeberg Pages") + + // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin + ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin") + + // Enable browser caching for up to 10 minutes + ctx.Response.Header.Set("Cache-Control", "public, max-age=600") + + trimmedHost := TrimHostPort(ctx.Request.Host()) + + // Add HSTS for RawDomain and MainDomainSuffix + if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { + ctx.Response.Header.Set("Strict-Transport-Security", hsts) + } + + // Block all methods not required for static pages + if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() { + ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") + ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed) + return + } + + // Block blacklisted paths (like ACME challenges) + for _, blacklistedPath := range blacklistedPaths { + if bytes.HasPrefix(ctx.Path(), blacklistedPath) { + returnErrorPage(ctx, fasthttp.StatusForbidden) + return + } + } + + // Allow CORS for specified domains + if ctx.IsOptions() { + allowCors := false + for _, allowedCorsDomain := range allowedCorsDomains { + if bytes.Equal(trimmedHost, allowedCorsDomain) { + allowCors = true + break + } + } + if allowCors { + ctx.Response.Header.Set("Access-Control-Allow-Origin", "*") + ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD") + } + ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") + ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent) + return + } + + // Prepare request information to Gitea + var targetOwner, targetRepo, targetBranch, targetPath string + var targetOptions = &upstreamOptions{ + ForbiddenMimeTypes: map[string]struct{}{}, + TryIndexPages: true, + } + + // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will + // also disallow search indexing and add a Link header to the canonical URL. + var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool { + if repo == "" { + return false + } + + // Check if the branch exists, otherwise treat it as a file path + branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch, string(giteaRoot), giteaApiToken) + if branchTimestampResult == nil { + // branch doesn't exist + return false + } + + // Branch exists, use it + targetRepo = repo + targetPath = strings.Trim(strings.Join(path, "/"), "/") + targetBranch = branchTimestampResult.branch + + targetOptions.BranchTimestamp = branchTimestampResult.timestamp + + if canonicalLink != "" { + // Hide from search machines & add canonical link + ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex") + ctx.Response.Header.Set("Link", + strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+ + "; rel=\"canonical\"", + ) + } + + return true + } + + // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. + var tryUpstream = func() { + // check if a canonical domain exists on a request on MainDomain + if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { + canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), string(giteaRoot), giteaApiToken) + if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { + canonicalPath := string(ctx.RequestURI()) + if targetRepo != "pages" { + canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2] + } + ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) + return + } + } + + // Try to request the file from the Gitea API + if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, string(giteaRoot), giteaApiToken, targetOptions) { + returnErrorPage(ctx, ctx.Response.StatusCode()) + } + } + + log.Debug().Msg("preparations") + + if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) { + // Serve raw content from RawDomain + log.Debug().Msg("raw domain") + + targetOptions.TryIndexPages = false + targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{} + targetOptions.DefaultMimeType = "text/plain; charset=utf-8" + + pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") + if len(pathElements) < 2 { + // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required + ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect) + return + } + targetOwner = pathElements[0] + targetRepo = pathElements[1] + + // raw.codeberg.org/example/myrepo/@main/index.html + if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { + log.Debug().Msg("raw domain preparations, now trying with specified branch") + if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:], + string(giteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", + ) { + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream() + return + } + log.Debug().Msg("missing branch") + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + return + } else { + log.Debug().Msg("raw domain preparations, now trying with default branch") + tryBranch(targetRepo, "", pathElements[2:], + string(giteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", + ) + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream() + return + } + + } else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { + // Serve pages from subdomains of MainDomainSuffix + log.Debug().Msg("main domain suffix") + + pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") + targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix)) + targetRepo = pathElements[0] + targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/") + + if targetOwner == "www" { + // www.codeberg.page redirects to codeberg.page + ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect) + return + } + + // Check if the first directory is a repo with the second directory as a branch + // example.codeberg.page/myrepo/@main/index.html + if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { + if targetRepo == "pages" { + // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... + ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect) + return + } + + log.Debug().Msg("main domain preparations, now trying with specified repo & branch") + if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:], + "/"+pathElements[0]+"/%p", + ) { + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream() + } else { + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + } + return + } + + // Check if the first directory is a branch for the "pages" repo + // example.codeberg.page/@main/index.html + if strings.HasPrefix(pathElements[0], "@") { + log.Debug().Msg("main domain preparations, now trying with specified branch") + if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") { + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream() + } else { + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + } + return + } + + // Check if the first directory is a repo with a "pages" branch + // example.codeberg.page/myrepo/index.html + // example.codeberg.page/pages/... is not allowed here. + log.Debug().Msg("main domain preparations, now trying with specified repo") + if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") { + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream() + return + } + + // Try to use the "pages" repo on its default branch + // example.codeberg.page/index.html + log.Debug().Msg("main domain preparations, now trying with default repo/branch") + if tryBranch("pages", "", pathElements, "") { + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream() + return + } + + // Couldn't find a valid repo/branch + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + return + } else { + trimmedHostStr := string(trimmedHost) + + // Serve pages from external domains + targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix)) + if targetOwner == "" { + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + return + } + + pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") + canonicalLink := "" + if strings.HasPrefix(pathElements[0], "@") { + targetBranch = pathElements[0][1:] + pathElements = pathElements[1:] + canonicalLink = "/%p" + } + + // Try to use the given repo on the given branch or the default branch + log.Debug().Msg("custom domain preparations, now trying with details from DNS") + if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { + canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), string(giteaRoot), giteaApiToken) + if !valid { + returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) + return + } else if canonicalDomain != trimmedHostStr { + // only redirect if the target is also a codeberg page! + targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) + if targetOwner != "" { + ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) + return + } else { + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + return + } + } + + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream() + return + } else { + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + return + } + } + } +} + +// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced +// with the provided status code. +func returnErrorPage(ctx *fasthttp.RequestCtx, code int) { + ctx.Response.SetStatusCode(code) + ctx.Response.Header.SetContentType("text/html; charset=utf-8") + message := fasthttp.StatusMessage(code) + if code == fasthttp.StatusMisdirectedRequest { + message += " - domain not specified in .domains file" + } + if code == fasthttp.StatusFailedDependency { + message += " - target repo/branch doesn't exist or is private" + } + // TODO: use template engine? + ctx.Response.SetBody(bytes.ReplaceAll(html.NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message))) +} + +// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. +var DefaultBranchCacheTimeout = 15 * time.Minute + +// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter +// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be +// picked up faster, while still allowing the content to be cached longer if nothing changes. +var BranchExistanceCacheTimeout = 5 * time.Minute + +// branchTimestampCache stores branch timestamps for faster cache checking +var branchTimestampCache = mcache.New() + +type branchTimestamp struct { + branch string + timestamp time.Time +} + +// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending +// on your available memory. +var FileCacheTimeout = 5 * time.Minute + +// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. +var FileCacheSizeLimit = 1024 * 1024 + +// fileResponseCache stores responses from the Gitea server +// TODO: make this an MRU cache with a size limit +var fileResponseCache = mcache.New() + +type fileResponse struct { + exists bool + mimeType string + body []byte +} + +// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch +// (or nil if the branch doesn't exist) +func getBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string) *branchTimestamp { + if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { + if result == nil { + return nil + } + return result.(*branchTimestamp) + } + result := &branchTimestamp{} + result.branch = branch + if branch == "" { + // Get default branch + var body = make([]byte, 0) + // TODO: use header for API key? + status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second) + if err != nil || status != 200 { + _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout) + return nil + } + result.branch = fastjson.GetString(body, "default_branch") + } + + var body = make([]byte, 0) + status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second) + if err != nil || status != 200 { + return nil + } + + result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp")) + _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout) + return result +} + +var upstreamClient = fasthttp.Client{ + ReadTimeout: 10 * time.Second, + MaxConnDuration: 60 * time.Second, + MaxConnWaitTimeout: 1000 * time.Millisecond, + MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! +} + +// upstreamIndexPages lists pages that may be considered as index pages for directories. +var upstreamIndexPages = []string{ + "index.html", +} + +// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. +func upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *upstreamOptions) (final bool) { + log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() + + if options.ForbiddenMimeTypes == nil { + options.ForbiddenMimeTypes = map[string]struct{}{} + } + + // Check if the branch exists and when it was modified + if options.BranchTimestamp == (time.Time{}) { + branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken) + + if branch == nil { + returnErrorPage(ctx, fasthttp.StatusFailedDependency) + return true + } + targetBranch = branch.branch + options.BranchTimestamp = branch.timestamp + } + + if targetOwner == "" || targetRepo == "" || targetBranch == "" { + returnErrorPage(ctx, fasthttp.StatusBadRequest) + return true + } + + // Check if the browser has a cached version + if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { + if !ifModifiedSince.Before(options.BranchTimestamp) { + ctx.Response.SetStatusCode(fasthttp.StatusNotModified) + return true + } + } + log.Debug().Msg("preparations") + + // Make a GET request to the upstream URL + uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath + var req *fasthttp.Request + var res *fasthttp.Response + var cachedResponse fileResponse + var err error + if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { + cachedResponse = cachedValue.(fileResponse) + } else { + req = fasthttp.AcquireRequest() + req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaApiToken) + res = fasthttp.AcquireResponse() + res.SetBodyStream(&strings.Reader{}, -1) + err = upstreamClient.Do(req, res) + } + log.Debug().Msg("acquisition") + + // Handle errors + if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { + if options.TryIndexPages { + // copy the options struct & try if an index page exists + optionsForIndexPages := *options + optionsForIndexPages.TryIndexPages = false + optionsForIndexPages.AppendTrailingSlash = true + for _, indexPage := range upstreamIndexPages { + if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages) { + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + exists: false, + }, FileCacheTimeout) + return true + } + } + // compatibility fix for GitHub Pages (/example → /example.html) + optionsForIndexPages.AppendTrailingSlash = false + optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" + if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages) { + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + exists: false, + }, FileCacheTimeout) + return true + } + } + ctx.Response.SetStatusCode(fasthttp.StatusNotFound) + if res != nil { + // Update cache if the request is fresh + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + exists: false, + }, FileCacheTimeout) + } + return false + } + if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { + fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode()) + returnErrorPage(ctx, fasthttp.StatusInternalServerError) + return true + } + + // Append trailing slash if missing (for index files), and redirect to fix filenames in general + // options.AppendTrailingSlash is only true when looking for index pages + if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { + ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) + return true + } + if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) { + ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) + return true + } + if options.RedirectIfExists != "" { + ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect) + return true + } + log.Debug().Msg("error handling") + + // Set the MIME type + mimeType := mime.TypeByExtension(path.Ext(targetPath)) + mimeTypeSplit := strings.SplitN(mimeType, ";", 2) + if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { + if options.DefaultMimeType != "" { + mimeType = options.DefaultMimeType + } else { + mimeType = "application/octet-stream" + } + } + ctx.Response.Header.SetContentType(mimeType) + + // Everything's okay so far + ctx.Response.SetStatusCode(fasthttp.StatusOK) + ctx.Response.Header.SetLastModified(options.BranchTimestamp) + + log.Debug().Msg("response preparations") + + // Write the response body to the original request + var cacheBodyWriter bytes.Buffer + if res != nil { + if res.Header.ContentLength() > FileCacheSizeLimit { + err = res.BodyWriteTo(ctx.Response.BodyWriter()) + } else { + // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? + err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) + } + } else { + _, err = ctx.Write(cachedResponse.body) + } + if err != nil { + fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err) + returnErrorPage(ctx, fasthttp.StatusInternalServerError) + return true + } + log.Debug().Msg("response") + + if res != nil && ctx.Err() == nil { + cachedResponse.exists = true + cachedResponse.mimeType = mimeType + cachedResponse.body = cacheBodyWriter.Bytes() + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout) + } + + return true +} + +// upstreamOptions provides various options for the upstream request. +type upstreamOptions struct { + DefaultMimeType string + ForbiddenMimeTypes map[string]struct{} + TryIndexPages bool + AppendTrailingSlash bool + RedirectIfExists string + BranchTimestamp time.Time +} diff --git a/handler_test.go b/server/handler_test.go similarity index 80% rename from handler_test.go rename to server/handler_test.go index 70b655e..78357e7 100644 --- a/handler_test.go +++ b/server/handler_test.go @@ -1,4 +1,4 @@ -package main +package server import ( "fmt" @@ -8,6 +8,16 @@ import ( ) func TestHandlerPerformance(t *testing.T) { + testHandler := Handler( + []byte("codeberg.page"), + []byte("raw.codeberg.org"), + []byte("https://codeberg.org"), + "https://docs.codeberg.org/pages/raw-content/", + "", + [][]byte{[]byte("/.well-known/acme-challenge/")}, + [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, + ) + ctx := &fasthttp.RequestCtx{ Request: *fasthttp.AcquireRequest(), Response: *fasthttp.AcquireResponse(), @@ -15,7 +25,7 @@ func TestHandlerPerformance(t *testing.T) { ctx.Request.SetRequestURI("http://mondstern.codeberg.page/") fmt.Printf("Start: %v\n", time.Now()) start := time.Now() - handler(ctx) + testHandler(ctx) end := time.Now() fmt.Printf("Done: %v\n", time.Now()) if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 { @@ -28,7 +38,7 @@ func TestHandlerPerformance(t *testing.T) { ctx.Response.ResetBody() fmt.Printf("Start: %v\n", time.Now()) start = time.Now() - handler(ctx) + testHandler(ctx) end = time.Now() fmt.Printf("Done: %v\n", time.Now()) if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 { @@ -42,7 +52,7 @@ func TestHandlerPerformance(t *testing.T) { ctx.Request.SetRequestURI("http://example.momar.xyz/") fmt.Printf("Start: %v\n", time.Now()) start = time.Now() - handler(ctx) + testHandler(ctx) end = time.Now() fmt.Printf("Done: %v\n", time.Now()) if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 1 { diff --git a/helpers.go b/server/helpers.go similarity index 71% rename from helpers.go rename to server/helpers.go index 46a1492..354f15e 100644 --- a/helpers.go +++ b/server/helpers.go @@ -1,15 +1,17 @@ -package main +package server import ( "bytes" "encoding/gob" + "os" + "github.com/akrylysov/pogreb" ) // GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty // string for custom domains. -func GetHSTSHeader(host []byte) string { - if bytes.HasSuffix(host, MainDomainSuffix) || bytes.Equal(host, RawDomain) { +func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string { + if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) { return "max-age=63072000; includeSubdomains; preload" } else { return "" @@ -54,3 +56,12 @@ func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool { } return true } + +// EnvOr reads an environment variable and returns a default value if it's empty. +// TODO: to helpers.go or use CLI framework +func EnvOr(env string, or string) string { + if v := os.Getenv(env); v != "" { + return v + } + return or +} From ac93a5661c1fdc015c7bbabf3dd2c0c6789bb1f3 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 02:12:51 +0100 Subject: [PATCH 04/41] start using urfave/cli --- README.md | 18 +++++++ cmd/certs.go | 40 ++++++++++++++ cmd/flags.go | 37 +++++++++++++ cmd/main.go | 98 +++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 5 ++ main.go | 149 ++++++--------------------------------------------- 7 files changed, 216 insertions(+), 132 deletions(-) create mode 100644 cmd/certs.go create mode 100644 cmd/flags.go create mode 100644 cmd/main.go diff --git a/README.md b/README.md index 35c230e..044379e 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,21 @@ - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80. - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. See https://go-acme.github.io/lego/dns/ for available values & additional environment variables. + + +// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories. +// +// Mapping custom domains is not static anymore, but can be done with DNS: +// +// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The +// first line will be the canonical domain/URL; all other occurrences will be redirected to it. +// +// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to +// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else): +// www.example.org. IN CNAME main.pages.example.codeberg.page. +// +// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record +// for "example.org" (if your provider allows ALIAS or similar records): +// example.org IN ALIAS codeberg.page. +// +// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge. diff --git a/cmd/certs.go b/cmd/certs.go new file mode 100644 index 0000000..4676520 --- /dev/null +++ b/cmd/certs.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "os" + + "github.com/urfave/cli/v2" + + pages_server "codeberg.org/codeberg/pages/server" +) + +var Certs = &cli.Command{ + Name: "certs", + Usage: "manage certs manually", + Action: certs, +} + +func certs(ctx *cli.Context) error { + if ctx.Args().Len() >= 1 && ctx.Args().First() == "--remove-certificate" { + if ctx.Args().Len() == 1 { + println("--remove-certificate requires at least one domain as an argument") + os.Exit(1) + } + + domains := ctx.Args().Slice()[2:] + + if pages_server.KeyDatabaseErr != nil { + panic(pages_server.KeyDatabaseErr) + } + for _, domain := range domains { + if err := pages_server.KeyDatabase.Delete([]byte(domain)); err != nil { + panic(err) + } + } + if err := pages_server.KeyDatabase.Sync(); err != nil { + panic(err) + } + os.Exit(0) + } + return nil +} diff --git a/cmd/flags.go b/cmd/flags.go new file mode 100644 index 0000000..258307f --- /dev/null +++ b/cmd/flags.go @@ -0,0 +1,37 @@ +package cmd + +import ( + "codeberg.org/codeberg/pages/server" + "github.com/urfave/cli/v2" +) + +// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash. +var GiteaRoot = []byte(server.EnvOr("GITEA_ROOT", "https://codeberg.org")) + +var GiteaApiToken = server.EnvOr("GITEA_API_TOKEN", "") + +// RawDomain specifies the domain from which raw repository content shall be served in the following format: +// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} +// (set to []byte(nil) to disable raw content hosting) +var RawDomain = []byte(server.EnvOr("RAW_DOMAIN", "raw.codeberg.org")) + +// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). +var RawInfoPage = server.EnvOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/") + +var ServeFlags = []cli.Flag{ + // MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static + // pages, or used for comparison in CNAME lookups. Static pages can be accessed through + // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". + // var MainDomainSuffix = []byte("." + server.EnvOr("PAGES_DOMAIN", "codeberg.page")) + &cli.StringFlag{ + Name: "main-domain-suffix", + Aliases: nil, + Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", + EnvVars: []string{"PAGES_DOMAIN"}, + FilePath: "", + Required: false, + Hidden: false, + TakesFile: false, + Value: "codeberg.page", + }, +} diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..66f9e7b --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "bytes" + "crypto/tls" + "fmt" + "log" + "net" + "net/http" + "os" + "time" + + "github.com/urfave/cli/v2" + "github.com/valyala/fasthttp" + + "codeberg.org/codeberg/pages/server" +) + +// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. +var AllowedCorsDomains = [][]byte{ + RawDomain, + []byte("fonts.codeberg.org"), + []byte("design.codeberg.org"), +} + +// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. +var BlacklistedPaths = [][]byte{ + []byte("/.well-known/acme-challenge/"), +} + +// Serve sets up and starts the web server. +func Serve(ctx *cli.Context) error { + mainDomainSuffix := []byte(ctx.String("main-domain-suffix")) + // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash + if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) { + mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...) + } + + GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'}) + + // Use HOST and PORT environment variables to determine listening address + address := fmt.Sprintf("%s:%s", server.EnvOr("HOST", "[::]"), server.EnvOr("PORT", "443")) + log.Printf("Listening on https://%s", address) + + // Create handler based on settings + handler := server.Handler(mainDomainSuffix, RawDomain, GiteaRoot, RawInfoPage, GiteaApiToken, BlacklistedPaths, AllowedCorsDomains) + + // Enable compression by wrapping the handler with the compression function provided by FastHTTP + compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) + + fastServer := &fasthttp.Server{ + Handler: compressedHandler, + DisablePreParseMultipartForm: true, + MaxRequestBodySize: 0, + NoDefaultServerHeader: true, + NoDefaultDate: true, + ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge + Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea! + MaxConnsPerIP: 100, + } + + // Setup listener and TLS + listener, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("Couldn't create listener: %s", err) + } + listener = tls.NewListener(listener, server.TlsConfig(mainDomainSuffix, string(GiteaRoot), GiteaApiToken)) + + server.SetupCertificates(mainDomainSuffix) + if os.Getenv("ENABLE_HTTP_SERVER") == "true" { + go (func() { + challengePath := []byte("/.well-known/acme-challenge/") + err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) { + if bytes.HasPrefix(ctx.Path(), challengePath) { + challenge, ok := server.ChallengeCache.Get(string(server.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) + if !ok || challenge == nil { + ctx.SetStatusCode(http.StatusNotFound) + ctx.SetBodyString("no challenge for this token") + } + ctx.SetBodyString(challenge.(string)) + } else { + ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently) + } + }) + if err != nil { + log.Fatalf("Couldn't start HTTP fastServer: %s", err) + } + })() + } + + // Start the web fastServer + err = fastServer.Serve(listener) + if err != nil { + log.Fatalf("Couldn't start fastServer: %s", err) + } + + return nil +} diff --git a/go.mod b/go.mod index 55e4675..a2bd8ee 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-acme/lego/v4 v4.5.3 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.26.0 + github.com/urfave/cli/v2 v2.3.0 github.com/valyala/fasthttp v1.31.0 github.com/valyala/fastjson v1.6.3 ) diff --git a/go.sum b/go.sum index 65da291..d04f727 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4= github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -427,6 +428,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE= github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI= @@ -434,6 +436,7 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f h1:WSnaD0/cvbKJgSTYbjAPf4RJXVvNNDAwVm+W8wEmnGE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -483,7 +486,9 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= diff --git a/main.go b/main.go index 501e0ab..41aba22 100644 --- a/main.go +++ b/main.go @@ -1,147 +1,32 @@ -// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories. -// -// Mapping custom domains is not static anymore, but can be done with DNS: -// -// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The -// first line will be the canonical domain/URL; all other occurrences will be redirected to it. -// -// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to -// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else): -// www.example.org. IN CNAME main.pages.example.codeberg.page. -// -// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record -// for "example.org" (if your provider allows ALIAS or similar records): -// example.org IN ALIAS codeberg.page. -// -// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge. package main import ( - "bytes" - "crypto/tls" "fmt" - "log" - "net" - "net/http" "os" - "time" - "github.com/valyala/fasthttp" + "github.com/urfave/cli/v2" - pages_server "codeberg.org/codeberg/pages/server" + "codeberg.org/codeberg/pages/cmd" ) -// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static -// pages, or used for comparison in CNAME lookups. Static pages can be accessed through -// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". -var MainDomainSuffix = []byte("." + pages_server.EnvOr("PAGES_DOMAIN", "codeberg.page")) +var ( + // can be changed with -X on compile + version = "dev" +) -// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash. -var GiteaRoot = []byte(pages_server.EnvOr("GITEA_ROOT", "https://codeberg.org")) - -var GiteaApiToken = pages_server.EnvOr("GITEA_API_TOKEN", "") - -// RawDomain specifies the domain from which raw repository content shall be served in the following format: -// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} -// (set to []byte(nil) to disable raw content hosting) -var RawDomain = []byte(pages_server.EnvOr("RAW_DOMAIN", "raw.codeberg.org")) - -// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). -var RawInfoPage = pages_server.EnvOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/") - -// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. -var AllowedCorsDomains = [][]byte{ - RawDomain, - []byte("fonts.codeberg.org"), - []byte("design.codeberg.org"), -} - -// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. -var BlacklistedPaths = [][]byte{ - []byte("/.well-known/acme-challenge/"), -} - -// main sets up and starts the web server. func main() { - // TODO: CLI Library - if len(os.Args) > 1 && os.Args[1] == "--remove-certificate" { - if len(os.Args) < 2 { - println("--remove-certificate requires at least one domain as an argument") - os.Exit(1) - } - if pages_server.KeyDatabaseErr != nil { - panic(pages_server.KeyDatabaseErr) - } - for _, domain := range os.Args[2:] { - if err := pages_server.KeyDatabase.Delete([]byte(domain)); err != nil { - panic(err) - } - } - if err := pages_server.KeyDatabase.Sync(); err != nil { - panic(err) - } - os.Exit(0) + app := cli.NewApp() + app.Name = "pages-server" + app.Version = version + app.Usage = "pages server" + app.Action = cmd.Serve + app.Flags = cmd.ServeFlags + app.Commands = []*cli.Command{ + cmd.Certs, } - // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash - if !bytes.HasPrefix(MainDomainSuffix, []byte{'.'}) { - MainDomainSuffix = append([]byte{'.'}, MainDomainSuffix...) - } - GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'}) - - // Use HOST and PORT environment variables to determine listening address - address := fmt.Sprintf("%s:%s", pages_server.EnvOr("HOST", "[::]"), pages_server.EnvOr("PORT", "443")) - log.Printf("Listening on https://%s", address) - - // Create handler based on settings - handler := pages_server.Handler(MainDomainSuffix, RawDomain, GiteaRoot, RawInfoPage, GiteaApiToken, BlacklistedPaths, AllowedCorsDomains) - - // Enable compression by wrapping the handler with the compression function provided by FastHTTP - compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) - - server := &fasthttp.Server{ - Handler: compressedHandler, - DisablePreParseMultipartForm: true, - MaxRequestBodySize: 0, - NoDefaultServerHeader: true, - NoDefaultDate: true, - ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge - Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea! - MaxConnsPerIP: 100, - } - - // Setup listener and TLS - listener, err := net.Listen("tcp", address) - if err != nil { - log.Fatalf("Couldn't create listener: %s", err) - } - listener = tls.NewListener(listener, pages_server.TlsConfig(MainDomainSuffix, string(GiteaRoot), GiteaApiToken)) - - pages_server.SetupCertificates(MainDomainSuffix) - if os.Getenv("ENABLE_HTTP_SERVER") == "true" { - go (func() { - challengePath := []byte("/.well-known/acme-challenge/") - err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) { - if bytes.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := pages_server.ChallengeCache.Get(string(pages_server.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) - if !ok || challenge == nil { - ctx.SetStatusCode(http.StatusNotFound) - ctx.SetBodyString("no challenge for this token") - } - ctx.SetBodyString(challenge.(string)) - } else { - ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently) - } - }) - if err != nil { - log.Fatalf("Couldn't start HTTP server: %s", err) - } - })() - } - - // Start the web server - err = server.Serve(listener) - if err != nil { - log.Fatalf("Couldn't start server: %s", err) + if err := app.Run(os.Args); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) } } From 35e08d2252dee948c1a1d528e95a2085736a2210 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 03:05:38 +0100 Subject: [PATCH 05/41] remove EnvOr use flags --- cmd/flags.go | 83 ++++++++++++++++++++++++++++++------------ cmd/main.go | 38 +++++++++++-------- server/certificates.go | 8 ++-- server/handler.go | 14 +++---- server/helpers.go | 11 ------ 5 files changed, 93 insertions(+), 61 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 258307f..209228e 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -1,37 +1,72 @@ package cmd import ( - "codeberg.org/codeberg/pages/server" "github.com/urfave/cli/v2" ) -// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash. -var GiteaRoot = []byte(server.EnvOr("GITEA_ROOT", "https://codeberg.org")) - -var GiteaApiToken = server.EnvOr("GITEA_API_TOKEN", "") - -// RawDomain specifies the domain from which raw repository content shall be served in the following format: -// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} -// (set to []byte(nil) to disable raw content hosting) -var RawDomain = []byte(server.EnvOr("RAW_DOMAIN", "raw.codeberg.org")) - -// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). -var RawInfoPage = server.EnvOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/") - var ServeFlags = []cli.Flag{ // MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static // pages, or used for comparison in CNAME lookups. Static pages can be accessed through // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". - // var MainDomainSuffix = []byte("." + server.EnvOr("PAGES_DOMAIN", "codeberg.page")) &cli.StringFlag{ - Name: "main-domain-suffix", - Aliases: nil, - Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", - EnvVars: []string{"PAGES_DOMAIN"}, - FilePath: "", - Required: false, - Hidden: false, - TakesFile: false, - Value: "codeberg.page", + Name: "main-domain-suffix", + Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", + EnvVars: []string{"PAGES_DOMAIN"}, + Value: "codeberg.page", + }, + // GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash. + &cli.StringFlag{ + Name: "gitea-root", + Usage: "specifies the root URL of the Gitea instance, without a trailing slash.", + EnvVars: []string{"GITEA_ROOT"}, + Value: "https://codeberg.org", + }, + // GiteaApiToken specifies an api token for the Gitea instance + &cli.StringFlag{ + Name: "gitea-api-token", + Usage: "specifies an api token for the Gitea instance", + EnvVars: []string{"GITEA_API_TOKEN"}, + Value: "", + }, + // RawDomain specifies the domain from which raw repository content shall be served in the following format: + // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} + // (set to []byte(nil) to disable raw content hosting) + &cli.StringFlag{ + Name: "raw-domain", + Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", + EnvVars: []string{"RAW_DOMAIN"}, + Value: "raw.codeberg.org", + }, + // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). + &cli.StringFlag{ + Name: "raw-info-page", + Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)", + EnvVars: []string{"REDIRECT_RAW_INFO"}, + Value: "https://docs.codeberg.org/pages/raw-content/", + }, + + &cli.StringFlag{ + Name: "host", + Usage: "specifies host of listening address", + EnvVars: []string{"HOST"}, + Value: "[::]", + }, + &cli.StringFlag{ + Name: "port", + Usage: "specifies port of listening address", + EnvVars: []string{"PORT"}, + Value: "443", + }, + + // ACME_API + &cli.StringFlag{ + Name: "acme-api", + EnvVars: []string{"ACME_API"}, + Value: "https://acme-v02.api.letsencrypt.org/directory", + }, + &cli.StringFlag{ + Name: "acme-email", + EnvVars: []string{"ACME_EMAIL"}, + Value: "noreply@example.email", }, } diff --git a/cmd/main.go b/cmd/main.go index 66f9e7b..a46fdbc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -4,12 +4,13 @@ import ( "bytes" "crypto/tls" "fmt" - "log" "net" "net/http" "os" + "strings" "time" + "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "github.com/valyala/fasthttp" @@ -17,8 +18,8 @@ import ( ) // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. +// TODO: make it a flag var AllowedCorsDomains = [][]byte{ - RawDomain, []byte("fonts.codeberg.org"), []byte("design.codeberg.org"), } @@ -30,20 +31,26 @@ var BlacklistedPaths = [][]byte{ // Serve sets up and starts the web server. func Serve(ctx *cli.Context) error { + giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") + giteaAPIToken := ctx.String("gitea-api-token") + rawDomain := ctx.String("raw-domain") mainDomainSuffix := []byte(ctx.String("main-domain-suffix")) + rawInfoPage := ctx.String("raw-info-page") + listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) + acmeAPI := ctx.String("acme-api") + acmeMail := ctx.String("acme-email") + allowedCorsDomains := AllowedCorsDomains + if len(rawDomain) != 0 { + allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain)) + } + // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) { mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...) } - GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'}) - - // Use HOST and PORT environment variables to determine listening address - address := fmt.Sprintf("%s:%s", server.EnvOr("HOST", "[::]"), server.EnvOr("PORT", "443")) - log.Printf("Listening on https://%s", address) - // Create handler based on settings - handler := server.Handler(mainDomainSuffix, RawDomain, GiteaRoot, RawInfoPage, GiteaApiToken, BlacklistedPaths, AllowedCorsDomains) + handler := server.Handler(mainDomainSuffix, []byte(rawDomain), giteaRoot, rawInfoPage, giteaAPIToken, BlacklistedPaths, allowedCorsDomains) // Enable compression by wrapping the handler with the compression function provided by FastHTTP compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) @@ -60,13 +67,14 @@ func Serve(ctx *cli.Context) error { } // Setup listener and TLS - listener, err := net.Listen("tcp", address) + log.Info().Msgf("Listening on https://%s", listeningAddress) + listener, err := net.Listen("tcp", listeningAddress) if err != nil { - log.Fatalf("Couldn't create listener: %s", err) + return fmt.Errorf("couldn't create listener: %s", err) } - listener = tls.NewListener(listener, server.TlsConfig(mainDomainSuffix, string(GiteaRoot), GiteaApiToken)) + listener = tls.NewListener(listener, server.TlsConfig(mainDomainSuffix, giteaRoot, giteaAPIToken)) - server.SetupCertificates(mainDomainSuffix) + server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail) if os.Getenv("ENABLE_HTTP_SERVER") == "true" { go (func() { challengePath := []byte("/.well-known/acme-challenge/") @@ -83,7 +91,7 @@ func Serve(ctx *cli.Context) error { } }) if err != nil { - log.Fatalf("Couldn't start HTTP fastServer: %s", err) + log.Fatal().Err(err).Msg("Couldn't start HTTP fastServer") } })() } @@ -91,7 +99,7 @@ func Serve(ctx *cli.Context) error { // Start the web fastServer err = fastServer.Serve(listener) if err != nil { - log.Fatalf("Couldn't start fastServer: %s", err) + log.Fatal().Err(err).Msg("Couldn't start fastServer") } return nil diff --git a/server/certificates.go b/server/certificates.go index c330439..7b3e9d9 100644 --- a/server/certificates.go +++ b/server/certificates.go @@ -399,7 +399,7 @@ func mockCert(domain, msg, mainDomainSuffix string) tls.Certificate { return tlsCertificate } -func SetupCertificates(mainDomainSuffix []byte) { +func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { if KeyDatabaseErr != nil { panic(KeyDatabaseErr) } @@ -425,7 +425,7 @@ func SetupCertificates(mainDomainSuffix []byte) { panic(err) } myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = EnvOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") + myAcmeConfig.CADirURL = acmeAPI myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 _, err := lego.NewClient(myAcmeConfig) if err != nil { @@ -437,12 +437,12 @@ func SetupCertificates(mainDomainSuffix []byte) { panic(err) } myAcmeAccount = AcmeAccount{ - Email: EnvOr("ACME_EMAIL", "noreply@example.email"), + Email: acmeMail, Key: privateKey, KeyPEM: string(certcrypto.PEMEncode(privateKey)), } myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = EnvOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") + myAcmeConfig.CADirURL = acmeAPI myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 tempClient, err := lego.NewClient(myAcmeConfig) if err != nil { diff --git a/server/handler.go b/server/handler.go index f24227b..ca2872f 100644 --- a/server/handler.go +++ b/server/handler.go @@ -19,7 +19,7 @@ import ( ) // Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaApiToken string, blacklistedPaths, allowedCorsDomains [][]byte) func(ctx *fasthttp.RequestCtx) { +func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaApiToken string, blacklistedPaths, allowedCorsDomains [][]byte) func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) { log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() @@ -86,7 +86,7 @@ func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaAp } // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch, string(giteaRoot), giteaApiToken) + branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken) if branchTimestampResult == nil { // branch doesn't exist return false @@ -115,7 +115,7 @@ func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaAp var tryUpstream = func() { // check if a canonical domain exists on a request on MainDomain if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), string(giteaRoot), giteaApiToken) + canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { canonicalPath := string(ctx.RequestURI()) if targetRepo != "pages" { @@ -127,7 +127,7 @@ func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaAp } // Try to request the file from the Gitea API - if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, string(giteaRoot), giteaApiToken, targetOptions) { + if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions) { returnErrorPage(ctx, ctx.Response.StatusCode()) } } @@ -155,7 +155,7 @@ func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaAp if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { log.Debug().Msg("raw domain preparations, now trying with specified branch") if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:], - string(giteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", + giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) { log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() @@ -167,7 +167,7 @@ func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaAp } else { log.Debug().Msg("raw domain preparations, now trying with default branch") tryBranch(targetRepo, "", pathElements[2:], - string(giteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", + giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() @@ -266,7 +266,7 @@ func Handler(mainDomainSuffix, rawDomain, giteaRoot []byte, rawInfoPage, giteaAp // Try to use the given repo on the given branch or the default branch log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), string(giteaRoot), giteaApiToken) + canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken) if !valid { returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) return diff --git a/server/helpers.go b/server/helpers.go index 354f15e..ecb4bf8 100644 --- a/server/helpers.go +++ b/server/helpers.go @@ -3,8 +3,6 @@ package server import ( "bytes" "encoding/gob" - "os" - "github.com/akrylysov/pogreb" ) @@ -56,12 +54,3 @@ func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool { } return true } - -// EnvOr reads an environment variable and returns a default value if it's empty. -// TODO: to helpers.go or use CLI framework -func EnvOr(env string, or string) string { - if v := os.Getenv(env); v != "" { - return v - } - return or -} From 5b81a8b8bc1869b78563bca34c59e2b2cfd81407 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 03:34:50 +0100 Subject: [PATCH 06/41] remove os.Getenv() usage --- cmd/flags.go | 34 +++++++++++++++++++++++++++++- cmd/main.go | 19 +++++++++++++---- server/certificates.go | 48 +++++++++++++++++++----------------------- 3 files changed, 70 insertions(+), 31 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 209228e..a631492 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -45,6 +45,7 @@ var ServeFlags = []cli.Flag{ Value: "https://docs.codeberg.org/pages/raw-content/", }, + // Server &cli.StringFlag{ Name: "host", Usage: "specifies host of listening address", @@ -57,8 +58,13 @@ var ServeFlags = []cli.Flag{ EnvVars: []string{"PORT"}, Value: "443", }, + &cli.BoolFlag{ + Name: "enable-http-server", + // TODO: desc + EnvVars: []string{"ENABLE_HTTP_SERVER"}, + }, - // ACME_API + // ACME &cli.StringFlag{ Name: "acme-api", EnvVars: []string{"ACME_API"}, @@ -69,4 +75,30 @@ var ServeFlags = []cli.Flag{ EnvVars: []string{"ACME_EMAIL"}, Value: "noreply@example.email", }, + &cli.BoolFlag{ + Name: "acme-use-rate-limits", + // TODO: Usage + EnvVars: []string{"ACME_USE_RATE_LIMITS"}, + Value: true, + }, + &cli.BoolFlag{ + Name: "acme-accept-terms", + // TODO: Usage + EnvVars: []string{"ACME_ACCEPT_TERMS"}, + }, + &cli.StringFlag{ + Name: "acme-eab-kid", + // TODO: Usage + EnvVars: []string{"ACME_EAB_KID"}, + }, + &cli.StringFlag{ + Name: "acme-eab-hmac", + // TODO: Usage + EnvVars: []string{"ACME_EAB_HMAC"}, + }, + &cli.StringFlag{ + Name: "dns-provider", + // TODO: Usage + EnvVars: []string{"DNS_PROVIDER"}, + }, } diff --git a/cmd/main.go b/cmd/main.go index a46fdbc..be33940 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,10 +3,10 @@ package cmd import ( "bytes" "crypto/tls" + "errors" "fmt" "net" "net/http" - "os" "strings" "time" @@ -37,8 +37,19 @@ func Serve(ctx *cli.Context) error { mainDomainSuffix := []byte(ctx.String("main-domain-suffix")) rawInfoPage := ctx.String("raw-info-page") listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) + enableHTTPServer := ctx.Bool("enable-http-server") + acmeAPI := ctx.String("acme-api") acmeMail := ctx.String("acme-email") + acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") + acmeAcceptTerms := ctx.Bool("acme-accept-terms") + acmeEabKID := ctx.String("acme-eab-kid") + acmeEabHmac := ctx.String("acme-eab-hmac") + dnsProvider := ctx.String("dns-provider") + if acmeAcceptTerms || (dnsProvider == "" && acmeAPI != "https://acme.mock.directory") { + return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory") + } + allowedCorsDomains := AllowedCorsDomains if len(rawDomain) != 0 { allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain)) @@ -72,10 +83,10 @@ func Serve(ctx *cli.Context) error { if err != nil { return fmt.Errorf("couldn't create listener: %s", err) } - listener = tls.NewListener(listener, server.TlsConfig(mainDomainSuffix, giteaRoot, giteaAPIToken)) + listener = tls.NewListener(listener, server.TlsConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits)) - server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail) - if os.Getenv("ENABLE_HTTP_SERVER") == "true" { + server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer) + if enableHTTPServer { go (func() { challengePath := []byte("/.well-known/acme-challenge/") err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) { diff --git a/server/certificates.go b/server/certificates.go index 7b3e9d9..8ac158d 100644 --- a/server/certificates.go +++ b/server/certificates.go @@ -38,7 +38,7 @@ import ( ) // TlsConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. -func TlsConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken string) *tls.Config { +func TlsConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -94,13 +94,13 @@ func TlsConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken string) *tls.Co var tlsCertificate tls.Certificate var err error var ok bool - if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix); !ok { + if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits); !ok { // request a new certificate if bytes.Equal(sniBytes, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } - tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, mainDomainSuffix) + tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits) if err != nil { return nil, err } @@ -209,7 +209,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func retrieveCertFromDB(sni, mainDomainSuffix []byte) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool) (tls.Certificate, bool) { // parse certificate from database res := &certificate.Resource{} if !PogrebGet(KeyDatabase, sni, res) { @@ -240,7 +240,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte) (tls.Certificate, bool) { } go (func() { res.CSR = nil // acme client doesn't like CSR to be set - tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", mainDomainSuffix) + tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits) if err != nil { log.Printf("Couldn't renew certificate for %s: %s", sni, err) } @@ -253,9 +253,9 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte) (tls.Certificate, bool) { var obtainLocks = sync.Map{} -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, mainDomainSuffix []byte) (tls.Certificate, error) { +func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool) (tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") - if os.Getenv("DNS_PROVIDER") == "" && len(domains[0]) > 0 && domains[0][0] == '*' { + if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] } @@ -266,7 +266,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re time.Sleep(100 * time.Millisecond) _, working = obtainLocks.Load(name) } - cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix) + cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits) if !ok { return tls.Certificate{}, errors.New("certificate failed in synchronous request") } @@ -282,7 +282,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re var res *certificate.Resource var err error if renew != nil && renew.CertURL != "" { - if os.Getenv("ACME_USE_RATE_LIMITS") != "false" { + if acmeUseRateLimits { acmeClientRequestLimit.Take() } log.Printf("Renewing certificate for %v", domains) @@ -299,7 +299,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re } } - if os.Getenv("ACME_USE_RATE_LIMITS") != "false" { + if acmeUseRateLimits { acmeClientOrderLimit.Take() acmeClientRequestLimit.Take() } @@ -399,13 +399,9 @@ func mockCert(domain, msg, mainDomainSuffix string) tls.Certificate { return tlsCertificate } -func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { +func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer bool) { if KeyDatabaseErr != nil { - panic(KeyDatabaseErr) - } - - if os.Getenv("ACME_ACCEPT_TERMS") != "true" || (os.Getenv("DNS_PROVIDER") == "" && os.Getenv("ACME_API") != "https://acme.mock.directory") { - panic(errors.New("you must set ACME_ACCEPT_TERMS and DNS_PROVIDER, unless ACME_API is set to https://acme.mock.directory")) + panic(KeyDatabaseErr) // TODO: move it into own init and not panic on a unrelated topic!!!! } // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits @@ -449,8 +445,8 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { // accept terms & log in to EAB - if os.Getenv("ACME_EAB_KID") == "" || os.Getenv("ACME_EAB_HMAC") == "" { - reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true"}) + if acmeEabKID == "" || acmeEabHmac == "" { + reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) if err != nil { log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) } else { @@ -458,9 +454,9 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { } } else { reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ - TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true", - Kid: os.Getenv("ACME_EAB_KID"), - HmacEncoded: os.Getenv("ACME_EAB_HMAC"), + TermsOfServiceAgreed: acmeAcceptTerms, + Kid: acmeEabKID, + HmacEncoded: acmeEabHmac, }) if err != nil { log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) @@ -494,7 +490,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { if err != nil { log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err) } - if os.Getenv("ENABLE_HTTP_SERVER") == "true" { + if enableHTTPServer { err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{}) if err != nil { log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err) @@ -506,14 +502,14 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { - if os.Getenv("DNS_PROVIDER") == "" { + if dnsProvider == "" { // using mock server, don't use wildcard certs err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{}) if err != nil { log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err) } } else { - provider, err := dns.NewDNSChallengeProviderByName(os.Getenv("DNS_PROVIDER")) + provider, err := dns.NewDNSChallengeProviderByName(dnsProvider) if err != nil { log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err) } @@ -525,7 +521,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { } if mainCertBytes == nil { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", mainDomainSuffix) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits) if err != nil { log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err) } @@ -590,7 +586,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail string) { // renew main certificate 30 days before it expires if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) { go (func() { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", mainDomainSuffix) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits) if err != nil { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err) } From 690879440a05d7969c3d86eae4f26847c8766075 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 03:44:21 +0100 Subject: [PATCH 07/41] move helper func in related packages --- cmd/main.go | 3 ++- server/certificates.go | 24 ++++++++++++---------- server/database/helpers.go | 38 +++++++++++++++++++++++++++++++++++ server/handler.go | 3 ++- server/helpers.go | 41 -------------------------------------- server/utils/utils.go | 11 ++++++++++ 6 files changed, 66 insertions(+), 54 deletions(-) create mode 100644 server/database/helpers.go create mode 100644 server/utils/utils.go diff --git a/cmd/main.go b/cmd/main.go index be33940..a7d606d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,7 @@ import ( "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/server" + "codeberg.org/codeberg/pages/server/utils" ) // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. @@ -91,7 +92,7 @@ func Serve(ctx *cli.Context) error { challengePath := []byte("/.well-known/acme-challenge/") err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) { if bytes.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := server.ChallengeCache.Get(string(server.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) + challenge, ok := server.ChallengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) if !ok || challenge == nil { ctx.SetStatusCode(http.StatusNotFound) ctx.SetBodyString("no challenge for this token") diff --git a/server/certificates.go b/server/certificates.go index 8ac158d..5339375 100644 --- a/server/certificates.go +++ b/server/certificates.go @@ -14,12 +14,6 @@ import ( "encoding/json" "encoding/pem" "errors" - "github.com/OrlovEvgeny/go-mcache" - "github.com/akrylysov/pogreb/fs" - "github.com/go-acme/lego/v4/certificate" - "github.com/go-acme/lego/v4/challenge" - "github.com/go-acme/lego/v4/challenge/tlsalpn01" - "github.com/go-acme/lego/v4/providers/dns" "io/ioutil" "log" "math/big" @@ -29,12 +23,20 @@ import ( "sync" "time" + "github.com/OrlovEvgeny/go-mcache" "github.com/akrylysov/pogreb" + "github.com/akrylysov/pogreb/fs" "github.com/reugn/equalizer" "github.com/go-acme/lego/v4/certcrypto" + "github.com/go-acme/lego/v4/certificate" + "github.com/go-acme/lego/v4/challenge" + "github.com/go-acme/lego/v4/challenge/tlsalpn01" "github.com/go-acme/lego/v4/lego" + "github.com/go-acme/lego/v4/providers/dns" "github.com/go-acme/lego/v4/registration" + + "codeberg.org/codeberg/pages/server/database" ) // TlsConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. @@ -212,7 +214,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool) (tls.Certificate, bool) { // parse certificate from database res := &certificate.Resource{} - if !PogrebGet(KeyDatabase, sni, res) { + if !database.PogrebGet(KeyDatabase, sni, res) { return tls.Certificate{}, false } @@ -317,7 +319,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) { // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) - PogrebPut(KeyDatabase, []byte(name), renew) + database.PogrebPut(KeyDatabase, []byte(name), renew) return tlsCertificate, nil } } @@ -325,7 +327,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re } log.Printf("Obtained certificate for %v", domains) - PogrebPut(KeyDatabase, []byte(name), res) + database.PogrebPut(KeyDatabase, []byte(name), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { return tls.Certificate{}, err @@ -390,7 +392,7 @@ func mockCert(domain, msg, mainDomainSuffix string) tls.Certificate { if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] { databaseName = mainDomainSuffix } - PogrebPut(KeyDatabase, []byte(databaseName), res) + database.PogrebPut(KeyDatabase, []byte(databaseName), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { @@ -578,7 +580,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, // update main cert res := &certificate.Resource{} - if !PogrebGet(KeyDatabase, mainDomainSuffix, res) { + if !database.PogrebGet(KeyDatabase, mainDomainSuffix, res) { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") } else { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) diff --git a/server/database/helpers.go b/server/database/helpers.go new file mode 100644 index 0000000..b2eb017 --- /dev/null +++ b/server/database/helpers.go @@ -0,0 +1,38 @@ +package database + +import ( + "bytes" + "encoding/gob" + "github.com/akrylysov/pogreb" +) + +func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) { + var resGob bytes.Buffer + resEnc := gob.NewEncoder(&resGob) + err := resEnc.Encode(obj) + if err != nil { + panic(err) + } + err = db.Put(name, resGob.Bytes()) + if err != nil { + panic(err) + } +} + +func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool { + resBytes, err := db.Get(name) + if err != nil { + panic(err) + } + if resBytes == nil { + return false + } + + resGob := bytes.NewBuffer(resBytes) + resDec := gob.NewDecoder(resGob) + err = resDec.Decode(obj) + if err != nil { + panic(err) + } + return true +} diff --git a/server/handler.go b/server/handler.go index ca2872f..80c1c79 100644 --- a/server/handler.go +++ b/server/handler.go @@ -16,6 +16,7 @@ import ( "github.com/valyala/fastjson" "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/utils" ) // Handler handles a single HTTP request to the web server. @@ -31,7 +32,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp // Enable browser caching for up to 10 minutes ctx.Response.Header.Set("Cache-Control", "public, max-age=600") - trimmedHost := TrimHostPort(ctx.Request.Host()) + trimmedHost := utils.TrimHostPort(ctx.Request.Host()) // Add HSTS for RawDomain and MainDomainSuffix if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { diff --git a/server/helpers.go b/server/helpers.go index ecb4bf8..6d55ddf 100644 --- a/server/helpers.go +++ b/server/helpers.go @@ -2,8 +2,6 @@ package server import ( "bytes" - "encoding/gob" - "github.com/akrylysov/pogreb" ) // GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty @@ -15,42 +13,3 @@ func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string { return "" } } - -func TrimHostPort(host []byte) []byte { - i := bytes.IndexByte(host, ':') - if i >= 0 { - return host[:i] - } - return host -} - -func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) { - var resGob bytes.Buffer - resEnc := gob.NewEncoder(&resGob) - err := resEnc.Encode(obj) - if err != nil { - panic(err) - } - err = db.Put(name, resGob.Bytes()) - if err != nil { - panic(err) - } -} - -func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool { - resBytes, err := db.Get(name) - if err != nil { - panic(err) - } - if resBytes == nil { - return false - } - - resGob := bytes.NewBuffer(resBytes) - resDec := gob.NewDecoder(resGob) - err = resDec.Decode(obj) - if err != nil { - panic(err) - } - return true -} diff --git a/server/utils/utils.go b/server/utils/utils.go new file mode 100644 index 0000000..7be330f --- /dev/null +++ b/server/utils/utils.go @@ -0,0 +1,11 @@ +package utils + +import "bytes" + +func TrimHostPort(host []byte) []byte { + i := bytes.IndexByte(host, ':') + if i >= 0 { + return host[:i] + } + return host +} From 796f24262ecb4b7679b45e9c14ba0b30a59eb2a5 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 03:46:21 +0100 Subject: [PATCH 08/41] fix code format --- server/handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/handler_test.go b/server/handler_test.go index 78357e7..c772dfc 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -11,7 +11,7 @@ func TestHandlerPerformance(t *testing.T) { testHandler := Handler( []byte("codeberg.page"), []byte("raw.codeberg.org"), - []byte("https://codeberg.org"), + "https://codeberg.org", "https://docs.codeberg.org/pages/raw-content/", "", [][]byte{[]byte("/.well-known/acme-challenge/")}, From 5ca5020cfa78811c4784f405dd633791c46ac081 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 04:15:48 +0100 Subject: [PATCH 09/41] open key-database deterministic --- cmd/certs.go | 14 +++++--- cmd/main.go | 14 ++++++-- server/cache/interface.go | 8 +++++ server/cache/setup.go | 7 ++++ server/certificates.go | 63 ++++++++++++++---------------------- server/database/helpers.go | 5 ++- server/database/interface.go | 12 +++++++ server/database/setup.go | 19 +++++++++++ 8 files changed, 94 insertions(+), 48 deletions(-) create mode 100644 server/cache/interface.go create mode 100644 server/cache/setup.go create mode 100644 server/database/interface.go create mode 100644 server/database/setup.go diff --git a/cmd/certs.go b/cmd/certs.go index 4676520..89521da 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -1,11 +1,12 @@ package cmd import ( + "fmt" "os" "github.com/urfave/cli/v2" - pages_server "codeberg.org/codeberg/pages/server" + "codeberg.org/codeberg/pages/server/database" ) var Certs = &cli.Command{ @@ -23,15 +24,18 @@ func certs(ctx *cli.Context) error { domains := ctx.Args().Slice()[2:] - if pages_server.KeyDatabaseErr != nil { - panic(pages_server.KeyDatabaseErr) + // TODO: make "key-database.pogreb" set via flag + keyDatabase, err := database.New("key-database.pogreb") + if err != nil { + return fmt.Errorf("could not create database: %v", err) } + for _, domain := range domains { - if err := pages_server.KeyDatabase.Delete([]byte(domain)); err != nil { + if err := keyDatabase.Delete([]byte(domain)); err != nil { panic(err) } } - if err := pages_server.KeyDatabase.Sync(); err != nil { + if err := keyDatabase.Sync(); err != nil { panic(err) } os.Exit(0) diff --git a/cmd/main.go b/cmd/main.go index a7d606d..2f3d7ac 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,8 @@ import ( "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/server" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/database" "codeberg.org/codeberg/pages/server/utils" ) @@ -84,9 +86,17 @@ func Serve(ctx *cli.Context) error { if err != nil { return fmt.Errorf("couldn't create listener: %s", err) } - listener = tls.NewListener(listener, server.TlsConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits)) - server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer) + // TODO: make "key-database.pogreb" set via flag + keyDatabase, err := database.New("key-database.pogreb") + if err != nil { + return fmt.Errorf("could not create database: %v", err) + } + + keyCache := cache.NewKeyValueCache() + listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, keyDatabase)) + + server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, keyDatabase) if enableHTTPServer { go (func() { challengePath := []byte("/.well-known/acme-challenge/") diff --git a/server/cache/interface.go b/server/cache/interface.go new file mode 100644 index 0000000..37ae8f5 --- /dev/null +++ b/server/cache/interface.go @@ -0,0 +1,8 @@ +package cache + +import "time" + +type SetGetKey interface { + Set(key string, value interface{}, ttl time.Duration) error + Get(key string) (interface{}, bool) +} diff --git a/server/cache/setup.go b/server/cache/setup.go new file mode 100644 index 0000000..a5928b0 --- /dev/null +++ b/server/cache/setup.go @@ -0,0 +1,7 @@ +package cache + +import "github.com/OrlovEvgeny/go-mcache" + +func NewKeyValueCache() SetGetKey { + return mcache.New() +} diff --git a/server/certificates.go b/server/certificates.go index 5339375..d6b6b86 100644 --- a/server/certificates.go +++ b/server/certificates.go @@ -24,8 +24,6 @@ import ( "time" "github.com/OrlovEvgeny/go-mcache" - "github.com/akrylysov/pogreb" - "github.com/akrylysov/pogreb/fs" "github.com/reugn/equalizer" "github.com/go-acme/lego/v4/certcrypto" @@ -36,11 +34,12 @@ import ( "github.com/go-acme/lego/v4/providers/dns" "github.com/go-acme/lego/v4/registration" + "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/database" ) -// TlsConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. -func TlsConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool) *tls.Config { +// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. +func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool, keyCache cache.SetGetKey, keyDatabase database.KeyDB) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -96,13 +95,13 @@ func TlsConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider st var tlsCertificate tls.Certificate var err error var ok bool - if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits); !ok { + if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase); !ok { // request a new certificate if bytes.Equal(sniBytes, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } - tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits) + tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) if err != nil { return nil, err } @@ -134,14 +133,6 @@ func TlsConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider st } } -// TODO: clean up & move to init -var keyCache = mcache.New() -var KeyDatabase, KeyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{ - BackgroundSyncInterval: 30 * time.Second, - BackgroundCompactionInterval: 6 * time.Hour, - FileSystem: fs.OSMMap, -}) - func CheckUserLimit(user string) error { userLimit, ok := acmeClientCertificateLimitPerUser[user] if !ok { @@ -211,10 +202,10 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.KeyDB) (tls.Certificate, bool) { // parse certificate from database res := &certificate.Resource{} - if !database.PogrebGet(KeyDatabase, sni, res) { + if !database.PogrebGet(keyDatabase, sni, res) { return tls.Certificate{}, false } @@ -242,7 +233,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs } go (func() { res.CSR = nil // acme client doesn't like CSR to be set - tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits) + tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) if err != nil { log.Printf("Couldn't renew certificate for %s: %s", sni, err) } @@ -255,7 +246,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs var obtainLocks = sync.Map{} -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool) (tls.Certificate, error) { +func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.KeyDB) (tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] @@ -268,7 +259,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re time.Sleep(100 * time.Millisecond) _, working = obtainLocks.Load(name) } - cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits) + cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) if !ok { return tls.Certificate{}, errors.New("certificate failed in synchronous request") } @@ -277,7 +268,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re defer obtainLocks.Delete(name) if acmeClient == nil { - return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix)), nil + return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix), keyDatabase), nil } // request actual cert @@ -319,15 +310,15 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) { // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) - database.PogrebPut(KeyDatabase, []byte(name), renew) + database.PogrebPut(keyDatabase, []byte(name), renew) return tlsCertificate, nil } } - return mockCert(domains[0], err.Error(), string(mainDomainSuffix)), err + return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err } log.Printf("Obtained certificate for %v", domains) - database.PogrebPut(KeyDatabase, []byte(name), res) + database.PogrebPut(keyDatabase, []byte(name), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { return tls.Certificate{}, err @@ -335,7 +326,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re return tlsCertificate, nil } -func mockCert(domain, msg, mainDomainSuffix string) tls.Certificate { +func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.KeyDB) tls.Certificate { key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048) if err != nil { panic(err) @@ -392,7 +383,7 @@ func mockCert(domain, msg, mainDomainSuffix string) tls.Certificate { if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] { databaseName = mainDomainSuffix } - database.PogrebPut(KeyDatabase, []byte(databaseName), res) + database.PogrebPut(keyDatabase, []byte(databaseName), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { @@ -401,13 +392,9 @@ func mockCert(domain, msg, mainDomainSuffix string) tls.Certificate { return tlsCertificate } -func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer bool) { - if KeyDatabaseErr != nil { - panic(KeyDatabaseErr) // TODO: move it into own init and not panic on a unrelated topic!!!! - } - +func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer bool, keyDatabase database.KeyDB) { // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits - mainCertBytes, err := KeyDatabase.Get(mainDomainSuffix) + mainCertBytes, err := keyDatabase.Get(mainDomainSuffix) if err != nil { // key database is not working panic(err) @@ -523,7 +510,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, } if mainCertBytes == nil { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) if err != nil { log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err) } @@ -531,7 +518,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, go (func() { for { - err := KeyDatabase.Sync() + err := keyDatabase.Sync() if err != nil { log.Printf("[ERROR] Syncing key database failed: %s", err) } @@ -544,7 +531,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, // clean up expired certs now := time.Now() expiredCertCount := 0 - keyDatabaseIterator := KeyDatabase.Items() + keyDatabaseIterator := keyDatabase.Items() key, resBytes, err := keyDatabaseIterator.Next() for err == nil { if !bytes.Equal(key, mainDomainSuffix) { @@ -558,7 +545,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) if err != nil || !tlsCertificates[0].NotAfter.After(now) { - err := KeyDatabase.Delete(key) + err := keyDatabase.Delete(key) if err != nil { log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) } else { @@ -571,7 +558,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) // compact the database - result, err := KeyDatabase.Compact() + result, err := keyDatabase.Compact() if err != nil { log.Printf("[ERROR] Compacting key database failed: %s", err) } else { @@ -580,7 +567,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, // update main cert res := &certificate.Resource{} - if !database.PogrebGet(KeyDatabase, mainDomainSuffix, res) { + if !database.PogrebGet(keyDatabase, mainDomainSuffix, res) { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") } else { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) @@ -588,7 +575,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, // renew main certificate 30 days before it expires if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) { go (func() { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) if err != nil { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err) } diff --git a/server/database/helpers.go b/server/database/helpers.go index b2eb017..98ea3fa 100644 --- a/server/database/helpers.go +++ b/server/database/helpers.go @@ -3,10 +3,9 @@ package database import ( "bytes" "encoding/gob" - "github.com/akrylysov/pogreb" ) -func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) { +func PogrebPut(db KeyDB, name []byte, obj interface{}) { var resGob bytes.Buffer resEnc := gob.NewEncoder(&resGob) err := resEnc.Encode(obj) @@ -19,7 +18,7 @@ func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) { } } -func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool { +func PogrebGet(db KeyDB, name []byte, obj interface{}) bool { resBytes, err := db.Get(name) if err != nil { panic(err) diff --git a/server/database/interface.go b/server/database/interface.go new file mode 100644 index 0000000..2b582ae --- /dev/null +++ b/server/database/interface.go @@ -0,0 +1,12 @@ +package database + +import "github.com/akrylysov/pogreb" + +type KeyDB interface { + Sync() error + Put(key []byte, value []byte) error + Get(key []byte) ([]byte, error) + Delete(key []byte) error + Compact() (pogreb.CompactionResult, error) + Items() *pogreb.ItemIterator +} diff --git a/server/database/setup.go b/server/database/setup.go new file mode 100644 index 0000000..c16ff36 --- /dev/null +++ b/server/database/setup.go @@ -0,0 +1,19 @@ +package database + +import ( + "fmt" + "github.com/akrylysov/pogreb" + "github.com/akrylysov/pogreb/fs" + "time" +) + +func New(path string) (KeyDB, error) { + if path == "" { + return nil, fmt.Errorf("path not set") + } + return pogreb.Open(path, &pogreb.Options{ + BackgroundSyncInterval: 30 * time.Second, + BackgroundCompactionInterval: 6 * time.Hour, + FileSystem: fs.OSMMap, + }) +} From 0bc38b668f557e07834a283a3064cecceeaf8ff1 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 04:18:53 +0100 Subject: [PATCH 10/41] db Sync on exit --- cmd/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/main.go b/cmd/main.go index 2f3d7ac..ff778d1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -92,6 +92,7 @@ func Serve(ctx *cli.Context) error { if err != nil { return fmt.Errorf("could not create database: %v", err) } + defer keyDatabase.Sync() // database has no close ... sync behave like it keyCache := cache.NewKeyValueCache() listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, keyDatabase)) From f35c4d0f667cb2304f1dabf020c7e3f46664dae8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 3 Dec 2021 04:32:30 +0100 Subject: [PATCH 11/41] make mem cache an interface and inject --- cmd/main.go | 7 ++++--- server/cache/interface.go | 1 + server/certificates.go | 33 ++++++++++++++++++--------------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index ff778d1..98b65d2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -95,15 +95,16 @@ func Serve(ctx *cli.Context) error { defer keyDatabase.Sync() // database has no close ... sync behave like it keyCache := cache.NewKeyValueCache() - listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, keyDatabase)) + challengeCache := cache.NewKeyValueCache() + listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, challengeCache, keyDatabase)) - server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, keyDatabase) + server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase) if enableHTTPServer { go (func() { challengePath := []byte("/.well-known/acme-challenge/") err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) { if bytes.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := server.ChallengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) + challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) if !ok || challenge == nil { ctx.SetStatusCode(http.StatusNotFound) ctx.SetBodyString("no challenge for this token") diff --git a/server/cache/interface.go b/server/cache/interface.go index 37ae8f5..2952b29 100644 --- a/server/cache/interface.go +++ b/server/cache/interface.go @@ -5,4 +5,5 @@ import "time" type SetGetKey interface { Set(key string, value interface{}, ttl time.Duration) error Get(key string) (interface{}, bool) + Remove(key string) } diff --git a/server/certificates.go b/server/certificates.go index d6b6b86..12d42a1 100644 --- a/server/certificates.go +++ b/server/certificates.go @@ -23,7 +23,6 @@ import ( "sync" "time" - "github.com/OrlovEvgeny/go-mcache" "github.com/reugn/equalizer" "github.com/go-acme/lego/v4/certcrypto" @@ -39,7 +38,7 @@ import ( ) // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. -func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool, keyCache cache.SetGetKey, keyDatabase database.KeyDB) *tls.Config { +func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool, keyCache, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -52,7 +51,7 @@ func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider st if info.SupportedProtos != nil { for _, proto := range info.SupportedProtos { if proto == tlsalpn01.ACMETLS1Protocol { - challenge, ok := ChallengeCache.Get(sni) + challenge, ok := challengeCache.Get(sni) if !ok { return nil, errors.New("no challenge for this domain") } @@ -176,29 +175,33 @@ var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute) // rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests) var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second) -var ChallengeCache = mcache.New() - -type AcmeTLSChallengeProvider struct{} +type AcmeTLSChallengeProvider struct { + challengeCache cache.SetGetKey +} +// make sure AcmeTLSChallengeProvider match Provider interface var _ challenge.Provider = AcmeTLSChallengeProvider{} func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { - return ChallengeCache.Set(domain, keyAuth, 1*time.Hour) + return a.challengeCache.Set(domain, keyAuth, 1*time.Hour) } func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { - ChallengeCache.Remove(domain) + a.challengeCache.Remove(domain) return nil } -type AcmeHTTPChallengeProvider struct{} +type AcmeHTTPChallengeProvider struct { + challengeCache cache.SetGetKey +} +// make sure AcmeHTTPChallengeProvider match Provider interface var _ challenge.Provider = AcmeHTTPChallengeProvider{} func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { - return ChallengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) + return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) } func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { - ChallengeCache.Remove(domain + "/" + token) + a.challengeCache.Remove(domain + "/" + token) return nil } @@ -392,7 +395,7 @@ func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.KeyDB) return tlsCertificate } -func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer bool, keyDatabase database.KeyDB) { +func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer bool, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) { // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits mainCertBytes, err := keyDatabase.Get(mainDomainSuffix) if err != nil { @@ -475,12 +478,12 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { - err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{}) + err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) if err != nil { log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err) } if enableHTTPServer { - err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{}) + err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache}) if err != nil { log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err) } @@ -493,7 +496,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, } else { if dnsProvider == "" { // using mock server, don't use wildcard certs - err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{}) + err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) if err != nil { log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err) } From 38426c26db6e461a678ff8e344f24a6e7bef38d6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 14:47:33 +0100 Subject: [PATCH 12/41] move upstream into own package --- html/error.go | 23 +++ server/domains.go | 9 +- server/handler.go | 303 +++--------------------------------- server/upstream/2rm.go | 10 ++ server/upstream/const.go | 18 +++ server/upstream/helper.go | 53 +++++++ server/upstream/upstream.go | 193 +++++++++++++++++++++++ 7 files changed, 321 insertions(+), 288 deletions(-) create mode 100644 html/error.go create mode 100644 server/upstream/2rm.go create mode 100644 server/upstream/const.go create mode 100644 server/upstream/helper.go create mode 100644 server/upstream/upstream.go diff --git a/html/error.go b/html/error.go new file mode 100644 index 0000000..f831443 --- /dev/null +++ b/html/error.go @@ -0,0 +1,23 @@ +package html + +import ( + "bytes" + "github.com/valyala/fasthttp" + "strconv" +) + +// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced +// with the provided status code. +func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) { + ctx.Response.SetStatusCode(code) + ctx.Response.Header.SetContentType("text/html; charset=utf-8") + message := fasthttp.StatusMessage(code) + if code == fasthttp.StatusMisdirectedRequest { + message += " - domain not specified in .domains file" + } + if code == fasthttp.StatusFailedDependency { + message += " - target repo/branch doesn't exist or is private" + } + // TODO: use template engine? + ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message))) +} diff --git a/server/domains.go b/server/domains.go index 5673960..5aeae96 100644 --- a/server/domains.go +++ b/server/domains.go @@ -1,11 +1,14 @@ package server import ( - "github.com/OrlovEvgeny/go-mcache" - "github.com/valyala/fasthttp" "net" "strings" "time" + + "github.com/OrlovEvgeny/go-mcache" + "github.com/valyala/fasthttp" + + "codeberg.org/codeberg/pages/server/upstream" ) // DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache. @@ -84,7 +87,7 @@ func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaApiToken) res := fasthttp.AcquireResponse() - err := upstreamClient.Do(req, res) + err := upstream.Client.Do(req, res) if err == nil && res.StatusCode() == fasthttp.StatusOK { for _, domain := range strings.Split(string(res.Body()), "\n") { domain = strings.ToLower(domain) diff --git a/server/handler.go b/server/handler.go index 80c1c79..a6a19f4 100644 --- a/server/handler.go +++ b/server/handler.go @@ -2,21 +2,13 @@ package server import ( "bytes" - "fmt" - "io" - "mime" - "path" - "strconv" "strings" - "time" - - "github.com/OrlovEvgeny/go-mcache" - "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/utils" + "github.com/rs/zerolog/log" + "github.com/valyala/fasthttp" ) // Handler handles a single HTTP request to the web server. @@ -49,7 +41,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp // Block blacklisted paths (like ACME challenges) for _, blacklistedPath := range blacklistedPaths { if bytes.HasPrefix(ctx.Path(), blacklistedPath) { - returnErrorPage(ctx, fasthttp.StatusForbidden) + html.ReturnErrorPage(ctx, fasthttp.StatusForbidden) return } } @@ -74,7 +66,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp // Prepare request information to Gitea var targetOwner, targetRepo, targetBranch, targetPath string - var targetOptions = &upstreamOptions{ + var targetOptions = &upstream.Options{ ForbiddenMimeTypes: map[string]struct{}{}, TryIndexPages: true, } @@ -87,7 +79,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp } // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken) + branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken) if branchTimestampResult == nil { // branch doesn't exist return false @@ -96,9 +88,9 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp // Branch exists, use it targetRepo = repo targetPath = strings.Trim(strings.Join(path, "/"), "/") - targetBranch = branchTimestampResult.branch + targetBranch = branchTimestampResult.Branch - targetOptions.BranchTimestamp = branchTimestampResult.timestamp + targetOptions.BranchTimestamp = branchTimestampResult.Timestamp if canonicalLink != "" { // Hide from search machines & add canonical link @@ -128,8 +120,8 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp } // Try to request the file from the Gitea API - if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions) { - returnErrorPage(ctx, ctx.Response.StatusCode()) + if !upstream.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions) { + html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) } } @@ -163,7 +155,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp return } log.Debug().Msg("missing branch") - returnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return } else { log.Debug().Msg("raw domain preparations, now trying with default branch") @@ -206,7 +198,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) } return } @@ -219,7 +211,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp log.Debug().Msg("tryBranch, now trying upstream") tryUpstream() } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) } return } @@ -244,7 +236,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp } // Couldn't find a valid repo/branch - returnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return } else { trimmedHostStr := string(trimmedHost) @@ -252,7 +244,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp // Serve pages from external domains targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix)) if targetOwner == "" { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return } @@ -269,7 +261,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken) if !valid { - returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) + html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHostStr { // only redirect if the target is also a codeberg page! @@ -278,7 +270,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) return } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return } } @@ -287,268 +279,9 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp tryUpstream() return } else { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return } } } } - -// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced -// with the provided status code. -func returnErrorPage(ctx *fasthttp.RequestCtx, code int) { - ctx.Response.SetStatusCode(code) - ctx.Response.Header.SetContentType("text/html; charset=utf-8") - message := fasthttp.StatusMessage(code) - if code == fasthttp.StatusMisdirectedRequest { - message += " - domain not specified in .domains file" - } - if code == fasthttp.StatusFailedDependency { - message += " - target repo/branch doesn't exist or is private" - } - // TODO: use template engine? - ctx.Response.SetBody(bytes.ReplaceAll(html.NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message))) -} - -// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. -var DefaultBranchCacheTimeout = 15 * time.Minute - -// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter -// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be -// picked up faster, while still allowing the content to be cached longer if nothing changes. -var BranchExistanceCacheTimeout = 5 * time.Minute - -// branchTimestampCache stores branch timestamps for faster cache checking -var branchTimestampCache = mcache.New() - -type branchTimestamp struct { - branch string - timestamp time.Time -} - -// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending -// on your available memory. -var FileCacheTimeout = 5 * time.Minute - -// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. -var FileCacheSizeLimit = 1024 * 1024 - -// fileResponseCache stores responses from the Gitea server -// TODO: make this an MRU cache with a size limit -var fileResponseCache = mcache.New() - -type fileResponse struct { - exists bool - mimeType string - body []byte -} - -// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch -// (or nil if the branch doesn't exist) -func getBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string) *branchTimestamp { - if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { - if result == nil { - return nil - } - return result.(*branchTimestamp) - } - result := &branchTimestamp{} - result.branch = branch - if branch == "" { - // Get default branch - var body = make([]byte, 0) - // TODO: use header for API key? - status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second) - if err != nil || status != 200 { - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout) - return nil - } - result.branch = fastjson.GetString(body, "default_branch") - } - - var body = make([]byte, 0) - status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second) - if err != nil || status != 200 { - return nil - } - - result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp")) - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout) - return result -} - -var upstreamClient = fasthttp.Client{ - ReadTimeout: 10 * time.Second, - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! -} - -// upstreamIndexPages lists pages that may be considered as index pages for directories. -var upstreamIndexPages = []string{ - "index.html", -} - -// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *upstreamOptions) (final bool) { - log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() - - if options.ForbiddenMimeTypes == nil { - options.ForbiddenMimeTypes = map[string]struct{}{} - } - - // Check if the branch exists and when it was modified - if options.BranchTimestamp == (time.Time{}) { - branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken) - - if branch == nil { - returnErrorPage(ctx, fasthttp.StatusFailedDependency) - return true - } - targetBranch = branch.branch - options.BranchTimestamp = branch.timestamp - } - - if targetOwner == "" || targetRepo == "" || targetBranch == "" { - returnErrorPage(ctx, fasthttp.StatusBadRequest) - return true - } - - // Check if the browser has a cached version - if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { - if !ifModifiedSince.Before(options.BranchTimestamp) { - ctx.Response.SetStatusCode(fasthttp.StatusNotModified) - return true - } - } - log.Debug().Msg("preparations") - - // Make a GET request to the upstream URL - uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath - var req *fasthttp.Request - var res *fasthttp.Response - var cachedResponse fileResponse - var err error - if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { - cachedResponse = cachedValue.(fileResponse) - } else { - req = fasthttp.AcquireRequest() - req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaApiToken) - res = fasthttp.AcquireResponse() - res.SetBodyStream(&strings.Reader{}, -1) - err = upstreamClient.Do(req, res) - } - log.Debug().Msg("acquisition") - - // Handle errors - if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { - if options.TryIndexPages { - // copy the options struct & try if an index page exists - optionsForIndexPages := *options - optionsForIndexPages.TryIndexPages = false - optionsForIndexPages.AppendTrailingSlash = true - for _, indexPage := range upstreamIndexPages { - if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, - }, FileCacheTimeout) - return true - } - } - // compatibility fix for GitHub Pages (/example → /example.html) - optionsForIndexPages.AppendTrailingSlash = false - optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" - if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, - }, FileCacheTimeout) - return true - } - } - ctx.Response.SetStatusCode(fasthttp.StatusNotFound) - if res != nil { - // Update cache if the request is fresh - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ - exists: false, - }, FileCacheTimeout) - } - return false - } - if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { - fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode()) - returnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true - } - - // Append trailing slash if missing (for index files), and redirect to fix filenames in general - // options.AppendTrailingSlash is only true when looking for index pages - if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { - ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) - return true - } - if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) { - ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) - return true - } - if options.RedirectIfExists != "" { - ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect) - return true - } - log.Debug().Msg("error handling") - - // Set the MIME type - mimeType := mime.TypeByExtension(path.Ext(targetPath)) - mimeTypeSplit := strings.SplitN(mimeType, ";", 2) - if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { - if options.DefaultMimeType != "" { - mimeType = options.DefaultMimeType - } else { - mimeType = "application/octet-stream" - } - } - ctx.Response.Header.SetContentType(mimeType) - - // Everything's okay so far - ctx.Response.SetStatusCode(fasthttp.StatusOK) - ctx.Response.Header.SetLastModified(options.BranchTimestamp) - - log.Debug().Msg("response preparations") - - // Write the response body to the original request - var cacheBodyWriter bytes.Buffer - if res != nil { - if res.Header.ContentLength() > FileCacheSizeLimit { - err = res.BodyWriteTo(ctx.Response.BodyWriter()) - } else { - // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? - err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) - } - } else { - _, err = ctx.Write(cachedResponse.body) - } - if err != nil { - fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err) - returnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true - } - log.Debug().Msg("response") - - if res != nil && ctx.Err() == nil { - cachedResponse.exists = true - cachedResponse.mimeType = mimeType - cachedResponse.body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout) - } - - return true -} - -// upstreamOptions provides various options for the upstream request. -type upstreamOptions struct { - DefaultMimeType string - ForbiddenMimeTypes map[string]struct{} - TryIndexPages bool - AppendTrailingSlash bool - RedirectIfExists string - BranchTimestamp time.Time -} diff --git a/server/upstream/2rm.go b/server/upstream/2rm.go new file mode 100644 index 0000000..f02b452 --- /dev/null +++ b/server/upstream/2rm.go @@ -0,0 +1,10 @@ +package upstream + +import "github.com/OrlovEvgeny/go-mcache" + +// branchTimestampCache stores branch timestamps for faster cache checking +var branchTimestampCache = mcache.New() + +// fileResponseCache stores responses from the Gitea server +// TODO: make this an MRU cache with a size limit +var fileResponseCache = mcache.New() diff --git a/server/upstream/const.go b/server/upstream/const.go new file mode 100644 index 0000000..f51b152 --- /dev/null +++ b/server/upstream/const.go @@ -0,0 +1,18 @@ +package upstream + +import "time" + +// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. +var DefaultBranchCacheTimeout = 15 * time.Minute + +// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter +// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be +// picked up faster, while still allowing the content to be cached longer if nothing changes. +var BranchExistanceCacheTimeout = 5 * time.Minute + +// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending +// on your available memory. +var FileCacheTimeout = 5 * time.Minute + +// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. +var FileCacheSizeLimit = 1024 * 1024 diff --git a/server/upstream/helper.go b/server/upstream/helper.go new file mode 100644 index 0000000..eeda2c2 --- /dev/null +++ b/server/upstream/helper.go @@ -0,0 +1,53 @@ +package upstream + +import ( + "time" + + "github.com/valyala/fasthttp" + "github.com/valyala/fastjson" +) + +type branchTimestamp struct { + Branch string + Timestamp time.Time +} + +// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch +// (or nil if the branch doesn't exist) +func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string) *branchTimestamp { + if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { + if result == nil { + return nil + } + return result.(*branchTimestamp) + } + result := &branchTimestamp{} + result.Branch = branch + if branch == "" { + // Get default branch + var body = make([]byte, 0) + // TODO: use header for API key? + status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second) + if err != nil || status != 200 { + _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout) + return nil + } + result.Branch = fastjson.GetString(body, "default_branch") + } + + var body = make([]byte, 0) + status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second) + if err != nil || status != 200 { + return nil + } + + result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp")) + _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout) + return result +} + +type fileResponse struct { + exists bool + mimeType string + body []byte +} diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go new file mode 100644 index 0000000..a269e06 --- /dev/null +++ b/server/upstream/upstream.go @@ -0,0 +1,193 @@ +package upstream + +import ( + "bytes" + "fmt" + "io" + "mime" + "path" + "strconv" + "strings" + "time" + + "codeberg.org/codeberg/pages/html" + + "github.com/rs/zerolog/log" + "github.com/valyala/fasthttp" +) + +// upstreamIndexPages lists pages that may be considered as index pages for directories. +var upstreamIndexPages = []string{ + "index.html", +} + +// Options provides various options for the upstream request. +type Options struct { + DefaultMimeType string + ForbiddenMimeTypes map[string]struct{} + TryIndexPages bool + AppendTrailingSlash bool + RedirectIfExists string + BranchTimestamp time.Time +} + +var Client = fasthttp.Client{ + ReadTimeout: 10 * time.Second, + MaxConnDuration: 60 * time.Second, + MaxConnWaitTimeout: 1000 * time.Millisecond, + MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! +} + +// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. +func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *Options) (final bool) { + log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() + + if options.ForbiddenMimeTypes == nil { + options.ForbiddenMimeTypes = map[string]struct{}{} + } + + // Check if the branch exists and when it was modified + if options.BranchTimestamp == (time.Time{}) { + branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken) + + if branch == nil { + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + return true + } + targetBranch = branch.Branch + options.BranchTimestamp = branch.Timestamp + } + + if targetOwner == "" || targetRepo == "" || targetBranch == "" { + html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest) + return true + } + + // Check if the browser has a cached version + if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { + if !ifModifiedSince.Before(options.BranchTimestamp) { + ctx.Response.SetStatusCode(fasthttp.StatusNotModified) + return true + } + } + log.Debug().Msg("preparations") + + // Make a GET request to the upstream URL + uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath + var req *fasthttp.Request + var res *fasthttp.Response + var cachedResponse fileResponse + var err error + if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { + cachedResponse = cachedValue.(fileResponse) + } else { + req = fasthttp.AcquireRequest() + req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaApiToken) + res = fasthttp.AcquireResponse() + res.SetBodyStream(&strings.Reader{}, -1) + err = Client.Do(req, res) + } + log.Debug().Msg("acquisition") + + // Handle errors + if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { + if options.TryIndexPages { + // copy the options struct & try if an index page exists + optionsForIndexPages := *options + optionsForIndexPages.TryIndexPages = false + optionsForIndexPages.AppendTrailingSlash = true + for _, indexPage := range upstreamIndexPages { + if Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages) { + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + exists: false, + }, FileCacheTimeout) + return true + } + } + // compatibility fix for GitHub Pages (/example → /example.html) + optionsForIndexPages.AppendTrailingSlash = false + optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" + if Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages) { + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + exists: false, + }, FileCacheTimeout) + return true + } + } + ctx.Response.SetStatusCode(fasthttp.StatusNotFound) + if res != nil { + // Update cache if the request is fresh + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + exists: false, + }, FileCacheTimeout) + } + return false + } + if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { + fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode()) + html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) + return true + } + + // Append trailing slash if missing (for index files), and redirect to fix filenames in general + // options.AppendTrailingSlash is only true when looking for index pages + if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { + ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) + return true + } + if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) { + ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) + return true + } + if options.RedirectIfExists != "" { + ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect) + return true + } + log.Debug().Msg("error handling") + + // Set the MIME type + mimeType := mime.TypeByExtension(path.Ext(targetPath)) + mimeTypeSplit := strings.SplitN(mimeType, ";", 2) + if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { + if options.DefaultMimeType != "" { + mimeType = options.DefaultMimeType + } else { + mimeType = "application/octet-stream" + } + } + ctx.Response.Header.SetContentType(mimeType) + + // Everything's okay so far + ctx.Response.SetStatusCode(fasthttp.StatusOK) + ctx.Response.Header.SetLastModified(options.BranchTimestamp) + + log.Debug().Msg("response preparations") + + // Write the response body to the original request + var cacheBodyWriter bytes.Buffer + if res != nil { + if res.Header.ContentLength() > FileCacheSizeLimit { + err = res.BodyWriteTo(ctx.Response.BodyWriter()) + } else { + // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? + err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter)) + } + } else { + _, err = ctx.Write(cachedResponse.body) + } + if err != nil { + fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err) + html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) + return true + } + log.Debug().Msg("response") + + if res != nil && ctx.Err() == nil { + cachedResponse.exists = true + cachedResponse.mimeType = mimeType + cachedResponse.body = cacheBodyWriter.Bytes() + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout) + } + + return true +} From 5b2e91a37a93789639ae63e9493810b07d824b0c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Dec 2021 21:15:51 +0100 Subject: [PATCH 13/41] REDIRECT_RAW_INFO -> RAW_INFO_PAGE --- README.md | 2 +- cmd/flags.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 044379e..ec87cc1 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - `RAW_DOMAIN` (default: `raw.codeberg.org`): domain for raw resources. - `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance. - `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos. -- `REDIRECT_RAW_INFO` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided. +- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided. - `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging). ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet. - `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider. diff --git a/cmd/flags.go b/cmd/flags.go index a631492..57cddda 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -41,7 +41,7 @@ var ServeFlags = []cli.Flag{ &cli.StringFlag{ Name: "raw-info-page", Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)", - EnvVars: []string{"REDIRECT_RAW_INFO"}, + EnvVars: []string{"RAW_INFO_PAGE"}, Value: "https://docs.codeberg.org/pages/raw-content/", }, From fdd04610e5d37bfa960503ce06e7407dd40aec43 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Dec 2021 21:16:22 +0100 Subject: [PATCH 14/41] fix .domains and make it redable --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec87cc1..616d69d 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,12 @@ - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. See https://go-acme.github.io/lego/dns/ for available values & additional environment variables. - +``` // Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories. // // Mapping custom domains is not static anymore, but can be done with DNS: // -// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The +// 1) add a ".domains" text file to your repository, containing the allowed domains, separated by new lines. The // first line will be the canonical domain/URL; all other occurrences will be redirected to it. // // 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to @@ -33,3 +33,4 @@ // example.org IN ALIAS codeberg.page. // // Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge. +``` \ No newline at end of file From 97d4ea9d6b50b65a2dbf5673b466c94312e73543 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Dec 2021 21:17:52 +0100 Subject: [PATCH 15/41] main-domain-suffix -> pages-domain --- cmd/flags.go | 2 +- cmd/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 57cddda..944e86e 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -9,7 +9,7 @@ var ServeFlags = []cli.Flag{ // pages, or used for comparison in CNAME lookups. Static pages can be accessed through // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". &cli.StringFlag{ - Name: "main-domain-suffix", + Name: "pages-domain", Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", EnvVars: []string{"PAGES_DOMAIN"}, Value: "codeberg.page", diff --git a/cmd/main.go b/cmd/main.go index 98b65d2..1d1cede 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -37,7 +37,7 @@ func Serve(ctx *cli.Context) error { giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain") - mainDomainSuffix := []byte(ctx.String("main-domain-suffix")) + mainDomainSuffix := []byte(ctx.String("pages-domain")) rawInfoPage := ctx.String("raw-info-page") listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) enableHTTPServer := ctx.Bool("enable-http-server") From b28204a468b23dd36bc956c7ce56cd6d3c90fffc Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Dec 2021 21:21:27 +0100 Subject: [PATCH 16/41] acme-api -> acme-api-endpoint --- cmd/flags.go | 2 +- cmd/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 944e86e..6381ee5 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -66,7 +66,7 @@ var ServeFlags = []cli.Flag{ // ACME &cli.StringFlag{ - Name: "acme-api", + Name: "acme-api-endpoint", EnvVars: []string{"ACME_API"}, Value: "https://acme-v02.api.letsencrypt.org/directory", }, diff --git a/cmd/main.go b/cmd/main.go index 1d1cede..fcc2b83 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,7 +42,7 @@ func Serve(ctx *cli.Context) error { listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) enableHTTPServer := ctx.Bool("enable-http-server") - acmeAPI := ctx.String("acme-api") + acmeAPI := ctx.String("acme-api-endpoint") acmeMail := ctx.String("acme-email") acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") acmeAcceptTerms := ctx.Bool("acme-accept-terms") From 2b49039252755ec1a982374825ab3cbc6a4da806 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 4 Dec 2021 21:59:04 +0100 Subject: [PATCH 17/41] add todo --- cmd/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/main.go b/cmd/main.go index fcc2b83..7fab2d3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -28,6 +28,7 @@ var AllowedCorsDomains = [][]byte{ } // BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. +// TODO: Make it a flag too var BlacklistedPaths = [][]byte{ []byte("/.well-known/acme-challenge/"), } From b3830e979c663a4a9cacbd672cfd86a3f1687ae8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 15:02:44 +0100 Subject: [PATCH 18/41] inject all cache --- cmd/main.go | 22 +++++++++++++++++----- server/certificates.go | 10 +++++++--- server/domains.go | 12 +++--------- server/handler.go | 21 +++++++++++++-------- server/handler_test.go | 4 ++++ 5 files changed, 44 insertions(+), 25 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7fab2d3..e40e385 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -64,8 +64,18 @@ func Serve(ctx *cli.Context) error { mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...) } + keyCache := cache.NewKeyValueCache() + challengeCache := cache.NewKeyValueCache() + // canonicalDomainCache stores canonical domains + var canonicalDomainCache = cache.NewKeyValueCache() + // dnsLookupCache stores DNS lookups for custom domains + var dnsLookupCache = cache.NewKeyValueCache() + // Create handler based on settings - handler := server.Handler(mainDomainSuffix, []byte(rawDomain), giteaRoot, rawInfoPage, giteaAPIToken, BlacklistedPaths, allowedCorsDomains) + handler := server.Handler(mainDomainSuffix, []byte(rawDomain), + giteaRoot, rawInfoPage, giteaAPIToken, + BlacklistedPaths, allowedCorsDomains, + dnsLookupCache, canonicalDomainCache) // Enable compression by wrapping the handler with the compression function provided by FastHTTP compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) @@ -93,11 +103,13 @@ func Serve(ctx *cli.Context) error { if err != nil { return fmt.Errorf("could not create database: %v", err) } - defer keyDatabase.Sync() // database has no close ... sync behave like it + defer keyDatabase.Sync() //nolint:errcheck // database has no close ... sync behave like it - keyCache := cache.NewKeyValueCache() - challengeCache := cache.NewKeyValueCache() - listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, challengeCache, keyDatabase)) + listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, + giteaRoot, giteaAPIToken, dnsProvider, + acmeUseRateLimits, + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, + keyDatabase)) server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase) if enableHTTPServer { diff --git a/server/certificates.go b/server/certificates.go index 12d42a1..1493d03 100644 --- a/server/certificates.go +++ b/server/certificates.go @@ -38,7 +38,11 @@ import ( ) // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. -func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool, keyCache, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) *tls.Config { +func TLSConfig(mainDomainSuffix []byte, + giteaRoot, giteaApiToken, dnsProvider string, + acmeUseRateLimits bool, + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, + keyDatabase database.KeyDB) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -71,14 +75,14 @@ func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider st sni = string(sniBytes) } else { var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni, string(mainDomainSuffix)) + targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs sniBytes = mainDomainSuffix sni = string(sniBytes) } else { _, _ = targetRepo, targetBranch - _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken) + _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) if !valid { sniBytes = mainDomainSuffix sni = string(sniBytes) diff --git a/server/domains.go b/server/domains.go index 5aeae96..cbbdef9 100644 --- a/server/domains.go +++ b/server/domains.go @@ -5,21 +5,18 @@ import ( "strings" "time" - "github.com/OrlovEvgeny/go-mcache" "github.com/valyala/fasthttp" + "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/upstream" ) // DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache. var DnsLookupCacheTimeout = 15 * time.Minute -// dnsLookupCache stores DNS lookups for custom domains -var dnsLookupCache = mcache.New() - // getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. -func getTargetFromDNS(domain, mainDomainSuffix string) (targetOwner, targetRepo, targetBranch string) { +func getTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { // Get CNAME or TXT var cname string var err error @@ -68,11 +65,8 @@ func getTargetFromDNS(domain, mainDomainSuffix string) (targetOwner, targetRepo, // CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. var CanonicalDomainCacheTimeout = 15 * time.Minute -// canonicalDomainCache stores canonical domains -var canonicalDomainCache = mcache.New() - // checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`). -func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string) (canonicalDomain string, valid bool) { +func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string, canonicalDomainCache cache.SetGetKey) (canonicalDomain string, valid bool) { domains := []string{} if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { domains = cachedValue.([]string) diff --git a/server/handler.go b/server/handler.go index a6a19f4..ba6cfdd 100644 --- a/server/handler.go +++ b/server/handler.go @@ -4,15 +4,20 @@ import ( "bytes" "strings" - "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/upstream" - "codeberg.org/codeberg/pages/server/utils" "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/upstream" + "codeberg.org/codeberg/pages/server/utils" ) // Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaApiToken string, blacklistedPaths, allowedCorsDomains [][]byte) func(ctx *fasthttp.RequestCtx) { +func Handler(mainDomainSuffix, rawDomain []byte, + giteaRoot, rawInfoPage, giteaApiToken string, + blacklistedPaths, allowedCorsDomains [][]byte, + dnsLookupCache, canonicalDomainCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) { log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() @@ -108,7 +113,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp var tryUpstream = func() { // check if a canonical domain exists on a request on MainDomain if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken) + canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { canonicalPath := string(ctx.RequestURI()) if targetRepo != "pages" { @@ -242,7 +247,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp trimmedHostStr := string(trimmedHost) // Serve pages from external domains - targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix)) + targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache) if targetOwner == "" { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return @@ -259,13 +264,13 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaAp // Try to use the given repo on the given branch or the default branch log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken) + canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) if !valid { html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHostStr { // only redirect if the target is also a codeberg page! - targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) + targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) if targetOwner != "" { ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) return diff --git a/server/handler_test.go b/server/handler_test.go index c772dfc..3488ae7 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -5,6 +5,8 @@ import ( "github.com/valyala/fasthttp" "testing" "time" + + "codeberg.org/codeberg/pages/server/cache" ) func TestHandlerPerformance(t *testing.T) { @@ -16,6 +18,8 @@ func TestHandlerPerformance(t *testing.T) { "", [][]byte{[]byte("/.well-known/acme-challenge/")}, [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, + cache.NewKeyValueCache(), + cache.NewKeyValueCache(), ) ctx := &fasthttp.RequestCtx{ From bb6f28fe570793872ab2b60a8ff706459f3e1feb Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 15:09:21 +0100 Subject: [PATCH 19/41] move setup of fastServer into own func --- cmd/main.go | 15 +-------------- server/setup.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) create mode 100644 server/setup.go diff --git a/cmd/main.go b/cmd/main.go index e40e385..f4ee205 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,7 +8,6 @@ import ( "net" "net/http" "strings" - "time" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" @@ -77,19 +76,7 @@ func Serve(ctx *cli.Context) error { BlacklistedPaths, allowedCorsDomains, dnsLookupCache, canonicalDomainCache) - // Enable compression by wrapping the handler with the compression function provided by FastHTTP - compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) - - fastServer := &fasthttp.Server{ - Handler: compressedHandler, - DisablePreParseMultipartForm: true, - MaxRequestBodySize: 0, - NoDefaultServerHeader: true, - NoDefaultDate: true, - ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge - Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea! - MaxConnsPerIP: 100, - } + fastServer, err := server.SetupServer(handler) // Setup listener and TLS log.Info().Msgf("Listening on https://%s", listeningAddress) diff --git a/server/setup.go b/server/setup.go new file mode 100644 index 0000000..6986c7c --- /dev/null +++ b/server/setup.go @@ -0,0 +1,25 @@ +package server + +import ( + "time" + + "github.com/valyala/fasthttp" +) + +func SetupServer(handler fasthttp.RequestHandler) (*fasthttp.Server, error) { + // Enable compression by wrapping the handler with the compression function provided by FastHTTP + compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) + + fastServer := &fasthttp.Server{ + Handler: compressedHandler, + DisablePreParseMultipartForm: true, + MaxRequestBodySize: 0, + NoDefaultServerHeader: true, + NoDefaultDate: true, + ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge + Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea! + MaxConnsPerIP: 100, + } + + return fastServer, nil +} From ccada3e6dfd1b306922ce697b9aec1ff834b1a3a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 15:21:05 +0100 Subject: [PATCH 20/41] split cert func to related packages --- cmd/main.go | 5 +- server/{ => certificates}/certificates.go | 8 +- server/dns/const.go | 6 ++ server/dns/dns.go | 56 +++++++++++ server/domains.go | 110 ---------------------- server/handler.go | 9 +- server/upstream/const.go | 3 + server/upstream/domains.go | 53 +++++++++++ 8 files changed, 131 insertions(+), 119 deletions(-) rename server/{ => certificates}/certificates.go (97%) create mode 100644 server/dns/const.go create mode 100644 server/dns/dns.go delete mode 100644 server/domains.go create mode 100644 server/upstream/domains.go diff --git a/cmd/main.go b/cmd/main.go index f4ee205..246d0d7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,7 @@ import ( "codeberg.org/codeberg/pages/server" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/database" "codeberg.org/codeberg/pages/server/utils" ) @@ -92,13 +93,13 @@ func Serve(ctx *cli.Context) error { } defer keyDatabase.Sync() //nolint:errcheck // database has no close ... sync behave like it - listener = tls.NewListener(listener, server.TLSConfig(mainDomainSuffix, + listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, keyDatabase)) - server.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase) + certificates.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase) if enableHTTPServer { go (func() { challengePath := []byte("/.well-known/acme-challenge/") diff --git a/server/certificates.go b/server/certificates/certificates.go similarity index 97% rename from server/certificates.go rename to server/certificates/certificates.go index 1493d03..b63ed0e 100644 --- a/server/certificates.go +++ b/server/certificates/certificates.go @@ -1,4 +1,4 @@ -package server +package certificates import ( "bytes" @@ -35,6 +35,8 @@ import ( "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/database" + dnsutils "codeberg.org/codeberg/pages/server/dns" + "codeberg.org/codeberg/pages/server/upstream" ) // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. @@ -75,14 +77,14 @@ func TLSConfig(mainDomainSuffix []byte, sni = string(sniBytes) } else { var targetRepo, targetBranch string - targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache) + targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs sniBytes = mainDomainSuffix sni = string(sniBytes) } else { _, _ = targetRepo, targetBranch - _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) + _, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) if !valid { sniBytes = mainDomainSuffix sni = string(sniBytes) diff --git a/server/dns/const.go b/server/dns/const.go new file mode 100644 index 0000000..fcf669b --- /dev/null +++ b/server/dns/const.go @@ -0,0 +1,6 @@ +package dns + +import "time" + +// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache. +var DnsLookupCacheTimeout = 15 * time.Minute diff --git a/server/dns/dns.go b/server/dns/dns.go new file mode 100644 index 0000000..c231439 --- /dev/null +++ b/server/dns/dns.go @@ -0,0 +1,56 @@ +package dns + +import ( + "net" + "strings" + + "codeberg.org/codeberg/pages/server/cache" +) + +// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. +// If everything is fine, it returns the target data. +func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { + // Get CNAME or TXT + var cname string + var err error + if cachedName, ok := dnsLookupCache.Get(domain); ok { + cname = cachedName.(string) + } else { + cname, err = net.LookupCNAME(domain) + cname = strings.TrimSuffix(cname, ".") + if err != nil || !strings.HasSuffix(cname, mainDomainSuffix) { + cname = "" + // TODO: check if the A record matches! + names, err := net.LookupTXT(domain) + if err == nil { + for _, name := range names { + name = strings.TrimSuffix(name, ".") + if strings.HasSuffix(name, mainDomainSuffix) { + cname = name + break + } + } + } + } + _ = dnsLookupCache.Set(domain, cname, DnsLookupCacheTimeout) + } + if cname == "" { + return + } + cnameParts := strings.Split(strings.TrimSuffix(cname, mainDomainSuffix), ".") + targetOwner = cnameParts[len(cnameParts)-1] + if len(cnameParts) > 1 { + targetRepo = cnameParts[len(cnameParts)-2] + } + if len(cnameParts) > 2 { + targetBranch = cnameParts[len(cnameParts)-3] + } + if targetRepo == "" { + targetRepo = "pages" + } + if targetBranch == "" && targetRepo != "pages" { + targetBranch = "pages" + } + // if targetBranch is still empty, the caller must find the default branch + return +} diff --git a/server/domains.go b/server/domains.go deleted file mode 100644 index cbbdef9..0000000 --- a/server/domains.go +++ /dev/null @@ -1,110 +0,0 @@ -package server - -import ( - "net" - "strings" - "time" - - "github.com/valyala/fasthttp" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/upstream" -) - -// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache. -var DnsLookupCacheTimeout = 15 * time.Minute - -// getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. -// If everything is fine, it returns the target data. -func getTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { - // Get CNAME or TXT - var cname string - var err error - if cachedName, ok := dnsLookupCache.Get(domain); ok { - cname = cachedName.(string) - } else { - cname, err = net.LookupCNAME(domain) - cname = strings.TrimSuffix(cname, ".") - if err != nil || !strings.HasSuffix(cname, mainDomainSuffix) { - cname = "" - // TODO: check if the A record matches! - names, err := net.LookupTXT(domain) - if err == nil { - for _, name := range names { - name = strings.TrimSuffix(name, ".") - if strings.HasSuffix(name, mainDomainSuffix) { - cname = name - break - } - } - } - } - _ = dnsLookupCache.Set(domain, cname, DnsLookupCacheTimeout) - } - if cname == "" { - return - } - cnameParts := strings.Split(strings.TrimSuffix(cname, mainDomainSuffix), ".") - targetOwner = cnameParts[len(cnameParts)-1] - if len(cnameParts) > 1 { - targetRepo = cnameParts[len(cnameParts)-2] - } - if len(cnameParts) > 2 { - targetBranch = cnameParts[len(cnameParts)-3] - } - if targetRepo == "" { - targetRepo = "pages" - } - if targetBranch == "" && targetRepo != "pages" { - targetBranch = "pages" - } - // if targetBranch is still empty, the caller must find the default branch - return -} - -// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. -var CanonicalDomainCacheTimeout = 15 * time.Minute - -// checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`). -func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string, canonicalDomainCache cache.SetGetKey) (canonicalDomain string, valid bool) { - domains := []string{} - if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { - domains = cachedValue.([]string) - for _, domain := range domains { - if domain == actualDomain { - valid = true - break - } - } - } else { - req := fasthttp.AcquireRequest() - req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaApiToken) - res := fasthttp.AcquireResponse() - - err := upstream.Client.Do(req, res) - if err == nil && res.StatusCode() == fasthttp.StatusOK { - for _, domain := range strings.Split(string(res.Body()), "\n") { - domain = strings.ToLower(domain) - domain = strings.TrimSpace(domain) - domain = strings.TrimPrefix(domain, "http://") - domain = strings.TrimPrefix(domain, "https://") - if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { - domains = append(domains, domain) - } - if domain == actualDomain { - valid = true - } - } - } - domains = append(domains, targetOwner+mainDomainSuffix) - if domains[len(domains)-1] == actualDomain { - valid = true - } - if targetRepo != "" && targetRepo != "pages" { - domains[len(domains)-1] += "/" + targetRepo - } - _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, CanonicalDomainCacheTimeout) - } - canonicalDomain = domains[0] - return -} diff --git a/server/handler.go b/server/handler.go index ba6cfdd..89f3f4a 100644 --- a/server/handler.go +++ b/server/handler.go @@ -9,6 +9,7 @@ import ( "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/dns" "codeberg.org/codeberg/pages/server/upstream" "codeberg.org/codeberg/pages/server/utils" ) @@ -113,7 +114,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, var tryUpstream = func() { // check if a canonical domain exists on a request on MainDomain if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) + canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { canonicalPath := string(ctx.RequestURI()) if targetRepo != "pages" { @@ -247,7 +248,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, trimmedHostStr := string(trimmedHost) // Serve pages from external domains - targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache) + targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache) if targetOwner == "" { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return @@ -264,13 +265,13 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Try to use the given repo on the given branch or the default branch log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) + canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) if !valid { html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) return } else if canonicalDomain != trimmedHostStr { // only redirect if the target is also a codeberg page! - targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) + targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) if targetOwner != "" { ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) return diff --git a/server/upstream/const.go b/server/upstream/const.go index f51b152..8c351df 100644 --- a/server/upstream/const.go +++ b/server/upstream/const.go @@ -16,3 +16,6 @@ var FileCacheTimeout = 5 * time.Minute // FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. var FileCacheSizeLimit = 1024 * 1024 + +// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. +var CanonicalDomainCacheTimeout = 15 * time.Minute diff --git a/server/upstream/domains.go b/server/upstream/domains.go new file mode 100644 index 0000000..87941dd --- /dev/null +++ b/server/upstream/domains.go @@ -0,0 +1,53 @@ +package upstream + +import ( + "strings" + + "github.com/valyala/fasthttp" + + "codeberg.org/codeberg/pages/server/cache" +) + +// CheckCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`). +func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string, canonicalDomainCache cache.SetGetKey) (canonicalDomain string, valid bool) { + domains := []string{} + if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { + domains = cachedValue.([]string) + for _, domain := range domains { + if domain == actualDomain { + valid = true + break + } + } + } else { + req := fasthttp.AcquireRequest() + req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaApiToken) + res := fasthttp.AcquireResponse() + + err := Client.Do(req, res) + if err == nil && res.StatusCode() == fasthttp.StatusOK { + for _, domain := range strings.Split(string(res.Body()), "\n") { + domain = strings.ToLower(domain) + domain = strings.TrimSpace(domain) + domain = strings.TrimPrefix(domain, "http://") + domain = strings.TrimPrefix(domain, "https://") + if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { + domains = append(domains, domain) + } + if domain == actualDomain { + valid = true + } + } + } + domains = append(domains, targetOwner+mainDomainSuffix) + if domains[len(domains)-1] == actualDomain { + valid = true + } + if targetRepo != "" && targetRepo != "pages" { + domains[len(domains)-1] += "/" + targetRepo + } + _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, CanonicalDomainCacheTimeout) + } + canonicalDomain = domains[0] + return +} From b6c4c63fb4aeeb86e9401ccbd4a3fd6d0bc290e8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 15:25:12 +0100 Subject: [PATCH 21/41] own file --- server/certificates/certificates.go | 70 ------------------------ server/certificates/mock.go | 84 +++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 70 deletions(-) create mode 100644 server/certificates/mock.go diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index b63ed0e..afaa4f0 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -6,17 +6,13 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" - "crypto/rsa" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/gob" "encoding/json" - "encoding/pem" "errors" "io/ioutil" "log" - "math/big" "os" "strconv" "strings" @@ -335,72 +331,6 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re return tlsCertificate, nil } -func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.KeyDB) tls.Certificate { - key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048) - if err != nil { - panic(err) - } - - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - CommonName: domain, - Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"}, - OrganizationalUnit: []string{ - "Will not try again for 6 hours to avoid hitting rate limits for your domain.", - "Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " + - "free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n", - "Error message: " + msg, - }, - }, - - // certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours - NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6), - NotBefore: time.Now(), - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - } - certBytes, err := x509.CreateCertificate( - rand.Reader, - &template, - &template, - &key.(*rsa.PrivateKey).PublicKey, - key, - ) - if err != nil { - panic(err) - } - - out := &bytes.Buffer{} - err = pem.Encode(out, &pem.Block{ - Bytes: certBytes, - Type: "CERTIFICATE", - }) - if err != nil { - panic(err) - } - outBytes := out.Bytes() - res := &certificate.Resource{ - PrivateKey: certcrypto.PEMEncode(key), - Certificate: outBytes, - IssuerCertificate: outBytes, - Domain: domain, - } - databaseName := domain - if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] { - databaseName = mainDomainSuffix - } - database.PogrebPut(keyDatabase, []byte(databaseName), res) - - tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) - if err != nil { - panic(err) - } - return tlsCertificate -} - func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer bool, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) { // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits mainCertBytes, err := keyDatabase.Get(mainDomainSuffix) diff --git a/server/certificates/mock.go b/server/certificates/mock.go new file mode 100644 index 0000000..19adb92 --- /dev/null +++ b/server/certificates/mock.go @@ -0,0 +1,84 @@ +package certificates + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "time" + + "github.com/go-acme/lego/v4/certcrypto" + "github.com/go-acme/lego/v4/certificate" + + "codeberg.org/codeberg/pages/server/database" +) + +func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.KeyDB) tls.Certificate { + key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048) + if err != nil { + panic(err) + } + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: domain, + Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"}, + OrganizationalUnit: []string{ + "Will not try again for 6 hours to avoid hitting rate limits for your domain.", + "Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " + + "free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n", + "Error message: " + msg, + }, + }, + + // certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours + NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6), + NotBefore: time.Now(), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + certBytes, err := x509.CreateCertificate( + rand.Reader, + &template, + &template, + &key.(*rsa.PrivateKey).PublicKey, + key, + ) + if err != nil { + panic(err) + } + + out := &bytes.Buffer{} + err = pem.Encode(out, &pem.Block{ + Bytes: certBytes, + Type: "CERTIFICATE", + }) + if err != nil { + panic(err) + } + outBytes := out.Bytes() + res := &certificate.Resource{ + PrivateKey: certcrypto.PEMEncode(key), + Certificate: outBytes, + IssuerCertificate: outBytes, + Domain: domain, + } + databaseName := domain + if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] { + databaseName = mainDomainSuffix + } + database.PogrebPut(keyDatabase, []byte(databaseName), res) + + tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) + if err != nil { + panic(err) + } + return tlsCertificate +} From 76c867cfcaacbe3c2ee900d286e3e68bf00bc84c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 15:45:22 +0100 Subject: [PATCH 22/41] move "http acme server setup" into own func --- cmd/main.go | 29 ++++++++--------------------- server/setup.go | 30 ++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 246d0d7..8aa56e0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,18 +6,15 @@ import ( "errors" "fmt" "net" - "net/http" "strings" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - "github.com/valyala/fasthttp" "codeberg.org/codeberg/pages/server" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/certificates" "codeberg.org/codeberg/pages/server/database" - "codeberg.org/codeberg/pages/server/utils" ) // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. @@ -77,7 +74,8 @@ func Serve(ctx *cli.Context) error { BlacklistedPaths, allowedCorsDomains, dnsLookupCache, canonicalDomainCache) - fastServer, err := server.SetupServer(handler) + fastServer := server.SetupServer(handler) + httpServer := server.SetupHttpACMEChallengeServer(challengeCache) // Setup listener and TLS log.Info().Msgf("Listening on https://%s", listeningAddress) @@ -100,31 +98,20 @@ func Serve(ctx *cli.Context) error { keyDatabase)) certificates.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase) + if enableHTTPServer { - go (func() { - challengePath := []byte("/.well-known/acme-challenge/") - err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) { - if bytes.HasPrefix(ctx.Path(), challengePath) { - challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) - if !ok || challenge == nil { - ctx.SetStatusCode(http.StatusNotFound) - ctx.SetBodyString("no challenge for this token") - } - ctx.SetBodyString(challenge.(string)) - } else { - ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently) - } - }) + go func() { + err := httpServer.ListenAndServe("[::]:80") if err != nil { - log.Fatal().Err(err).Msg("Couldn't start HTTP fastServer") + log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") } - })() + }() } // Start the web fastServer err = fastServer.Serve(listener) if err != nil { - log.Fatal().Err(err).Msg("Couldn't start fastServer") + log.Panic().Err(err).Msg("Couldn't start fastServer") } return nil diff --git a/server/setup.go b/server/setup.go index 6986c7c..546aba1 100644 --- a/server/setup.go +++ b/server/setup.go @@ -1,16 +1,21 @@ package server import ( + "bytes" + "net/http" "time" "github.com/valyala/fasthttp" + + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/utils" ) -func SetupServer(handler fasthttp.RequestHandler) (*fasthttp.Server, error) { +func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server { // Enable compression by wrapping the handler with the compression function provided by FastHTTP compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) - fastServer := &fasthttp.Server{ + return &fasthttp.Server{ Handler: compressedHandler, DisablePreParseMultipartForm: true, MaxRequestBodySize: 0, @@ -20,6 +25,23 @@ func SetupServer(handler fasthttp.RequestHandler) (*fasthttp.Server, error) { Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea! MaxConnsPerIP: 100, } - - return fastServer, nil +} + +func SetupHttpACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server { + challengePath := []byte("/.well-known/acme-challenge/") + + return &fasthttp.Server{ + Handler: func(ctx *fasthttp.RequestCtx) { + if bytes.HasPrefix(ctx.Path(), challengePath) { + challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath))) + if !ok || challenge == nil { + ctx.SetStatusCode(http.StatusNotFound) + ctx.SetBodyString("no challenge for this token") + } + ctx.SetBodyString(challenge.(string)) + } else { + ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently) + } + }, + } } From de4706bf588c93ec03ca9d7a47d0e7c57a6b96f0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 15:53:46 +0100 Subject: [PATCH 23/41] rm 2rm --- cmd/main.go | 7 ++++++- server/handler.go | 6 +++--- server/handler_test.go | 2 ++ server/upstream/2rm.go | 10 ---------- server/upstream/helper.go | 4 +++- server/upstream/upstream.go | 12 +++++++----- 6 files changed, 21 insertions(+), 20 deletions(-) delete mode 100644 server/upstream/2rm.go diff --git a/cmd/main.go b/cmd/main.go index 8aa56e0..ceec3d5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -67,12 +67,17 @@ func Serve(ctx *cli.Context) error { var canonicalDomainCache = cache.NewKeyValueCache() // dnsLookupCache stores DNS lookups for custom domains var dnsLookupCache = cache.NewKeyValueCache() + // branchTimestampCache stores branch timestamps for faster cache checking + var branchTimestampCache = cache.NewKeyValueCache() + // fileResponseCache stores responses from the Gitea server + // TODO: make this an MRU cache with a size limit + var fileResponseCache = cache.NewKeyValueCache() // Create handler based on settings handler := server.Handler(mainDomainSuffix, []byte(rawDomain), giteaRoot, rawInfoPage, giteaAPIToken, BlacklistedPaths, allowedCorsDomains, - dnsLookupCache, canonicalDomainCache) + dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) fastServer := server.SetupServer(handler) httpServer := server.SetupHttpACMEChallengeServer(challengeCache) diff --git a/server/handler.go b/server/handler.go index 89f3f4a..4b3c4ef 100644 --- a/server/handler.go +++ b/server/handler.go @@ -18,7 +18,7 @@ import ( func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, rawInfoPage, giteaApiToken string, blacklistedPaths, allowedCorsDomains [][]byte, - dnsLookupCache, canonicalDomainCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) { + dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) { log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger() @@ -85,7 +85,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, } // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken) + branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken, branchTimestampCache) if branchTimestampResult == nil { // branch doesn't exist return false @@ -126,7 +126,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, } // Try to request the file from the Gitea API - if !upstream.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions) { + if !upstream.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions, branchTimestampCache, fileResponseCache) { html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) } } diff --git a/server/handler_test.go b/server/handler_test.go index 3488ae7..0ec9fcd 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -20,6 +20,8 @@ func TestHandlerPerformance(t *testing.T) { [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")}, cache.NewKeyValueCache(), cache.NewKeyValueCache(), + cache.NewKeyValueCache(), + cache.NewKeyValueCache(), ) ctx := &fasthttp.RequestCtx{ diff --git a/server/upstream/2rm.go b/server/upstream/2rm.go deleted file mode 100644 index f02b452..0000000 --- a/server/upstream/2rm.go +++ /dev/null @@ -1,10 +0,0 @@ -package upstream - -import "github.com/OrlovEvgeny/go-mcache" - -// branchTimestampCache stores branch timestamps for faster cache checking -var branchTimestampCache = mcache.New() - -// fileResponseCache stores responses from the Gitea server -// TODO: make this an MRU cache with a size limit -var fileResponseCache = mcache.New() diff --git a/server/upstream/helper.go b/server/upstream/helper.go index eeda2c2..d8882b6 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -5,6 +5,8 @@ import ( "github.com/valyala/fasthttp" "github.com/valyala/fastjson" + + "codeberg.org/codeberg/pages/server/cache" ) type branchTimestamp struct { @@ -14,7 +16,7 @@ type branchTimestamp struct { // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // (or nil if the branch doesn't exist) -func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string) *branchTimestamp { +func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp { if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result == nil { return nil diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index a269e06..3dfddb7 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -10,10 +10,12 @@ import ( "strings" "time" - "codeberg.org/codeberg/pages/html" "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" ) // upstreamIndexPages lists pages that may be considered as index pages for directories. @@ -39,7 +41,7 @@ var Client = fasthttp.Client{ } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *Options) (final bool) { +func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *Options, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() if options.ForbiddenMimeTypes == nil { @@ -48,7 +50,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t // Check if the branch exists and when it was modified if options.BranchTimestamp == (time.Time{}) { - branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken) + branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken, branchTimestampCache) if branch == nil { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -97,7 +99,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t optionsForIndexPages.TryIndexPages = false optionsForIndexPages.AppendTrailingSlash = true for _, indexPage := range upstreamIndexPages { - if Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages) { + if Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, FileCacheTimeout) @@ -107,7 +109,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.AppendTrailingSlash = false optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" - if Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages) { + if Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, FileCacheTimeout) From e6198e4ddd32c32a9634ca0543445766e98e4848 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 15:59:43 +0100 Subject: [PATCH 24/41] start refactor Upstream func --- server/handler.go | 2 +- server/upstream/domains.go | 2 +- server/upstream/upstream.go | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/server/handler.go b/server/handler.go index 4b3c4ef..e7652dd 100644 --- a/server/handler.go +++ b/server/handler.go @@ -126,7 +126,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, } // Try to request the file from the Gitea API - if !upstream.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, targetOptions, branchTimestampCache, fileResponseCache) { + if !targetOptions.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) } } diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 87941dd..89bc7fb 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -24,7 +24,7 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaApiToken) res := fasthttp.AcquireResponse() - err := Client.Do(req, res) + err := client.Do(req, res) if err == nil && res.StatusCode() == fasthttp.StatusOK { for _, domain := range strings.Split(string(res.Body()), "\n") { domain = strings.ToLower(domain) diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 3dfddb7..396f3c5 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" @@ -33,7 +32,7 @@ type Options struct { BranchTimestamp time.Time } -var Client = fasthttp.Client{ +var client = fasthttp.Client{ ReadTimeout: 10 * time.Second, MaxConnDuration: 60 * time.Second, MaxConnWaitTimeout: 1000 * time.Millisecond, @@ -41,7 +40,7 @@ var Client = fasthttp.Client{ } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, options *Options, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { +func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() if options.ForbiddenMimeTypes == nil { @@ -87,7 +86,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaApiToken) res = fasthttp.AcquireResponse() res.SetBodyStream(&strings.Reader{}, -1) - err = Client.Do(req, res) + err = client.Do(req, res) } log.Debug().Msg("acquisition") @@ -99,7 +98,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t optionsForIndexPages.TryIndexPages = false optionsForIndexPages.AppendTrailingSlash = true for _, indexPage := range upstreamIndexPages { - if Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, &optionsForIndexPages, branchTimestampCache, fileResponseCache) { + if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, FileCacheTimeout) @@ -109,7 +108,7 @@ func Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, t // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.AppendTrailingSlash = false optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" - if Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, &optionsForIndexPages, branchTimestampCache, fileResponseCache) { + if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, FileCacheTimeout) From 77e39b22132696cadb29b894a8a19ec8494d139d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 16:24:26 +0100 Subject: [PATCH 25/41] unexport if posible --- server/certificates/certificates.go | 4 ++-- server/dns/const.go | 4 ++-- server/dns/dns.go | 2 +- server/upstream/const.go | 22 +++++++++++----------- server/upstream/domains.go | 2 +- server/upstream/helper.go | 4 ++-- server/upstream/upstream.go | 10 +++++----- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index afaa4f0..fb61bd8 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -134,7 +134,7 @@ func TLSConfig(mainDomainSuffix []byte, } } -func CheckUserLimit(user string) error { +func checkUserLimit(user string) error { userLimit, ok := acmeClientCertificateLimitPerUser[user] if !ok { // Each Codeberg user can only add 10 new domains per day. @@ -292,7 +292,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re } if res == nil { if user != "" { - if err := CheckUserLimit(user); err != nil { + if err := checkUserLimit(user); err != nil { return tls.Certificate{}, err } } diff --git a/server/dns/const.go b/server/dns/const.go index fcf669b..bb2413b 100644 --- a/server/dns/const.go +++ b/server/dns/const.go @@ -2,5 +2,5 @@ package dns import "time" -// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache. -var DnsLookupCacheTimeout = 15 * time.Minute +// lookupCacheTimeout specifies the timeout for the DNS lookup cache. +var lookupCacheTimeout = 15 * time.Minute diff --git a/server/dns/dns.go b/server/dns/dns.go index c231439..dc759b0 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -32,7 +32,7 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG } } } - _ = dnsLookupCache.Set(domain, cname, DnsLookupCacheTimeout) + _ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout) } if cname == "" { return diff --git a/server/upstream/const.go b/server/upstream/const.go index 8c351df..77f64dd 100644 --- a/server/upstream/const.go +++ b/server/upstream/const.go @@ -2,20 +2,20 @@ package upstream import "time" -// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. -var DefaultBranchCacheTimeout = 15 * time.Minute +// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long. +var defaultBranchCacheTimeout = 15 * time.Minute -// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter -// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be +// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter +// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be // picked up faster, while still allowing the content to be cached longer if nothing changes. -var BranchExistanceCacheTimeout = 5 * time.Minute +var branchExistenceCacheTimeout = 5 * time.Minute -// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending +// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending // on your available memory. -var FileCacheTimeout = 5 * time.Minute +var fileCacheTimeout = 5 * time.Minute -// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. -var FileCacheSizeLimit = 1024 * 1024 +// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. +var fileCacheSizeLimit = 1024 * 1024 -// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. -var CanonicalDomainCacheTimeout = 15 * time.Minute +// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. +var canonicalDomainCacheTimeout = 15 * time.Minute diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 89bc7fb..5971e13 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -46,7 +46,7 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m if targetRepo != "" && targetRepo != "pages" { domains[len(domains)-1] += "/" + targetRepo } - _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, CanonicalDomainCacheTimeout) + _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout) } canonicalDomain = domains[0] return diff --git a/server/upstream/helper.go b/server/upstream/helper.go index d8882b6..b5ee77a 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -31,7 +31,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br // TODO: use header for API key? status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second) if err != nil || status != 200 { - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout) + _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout) return nil } result.Branch = fastjson.GetString(body, "default_branch") @@ -44,7 +44,7 @@ func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, br } result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp")) - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout) + _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout) return result } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 396f3c5..306e1db 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -101,7 +101,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, - }, FileCacheTimeout) + }, fileCacheTimeout) return true } } @@ -111,7 +111,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, - }, FileCacheTimeout) + }, fileCacheTimeout) return true } } @@ -120,7 +120,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe // Update cache if the request is fresh _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, - }, FileCacheTimeout) + }, fileCacheTimeout) } return false } @@ -167,7 +167,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe // Write the response body to the original request var cacheBodyWriter bytes.Buffer if res != nil { - if res.Header.ContentLength() > FileCacheSizeLimit { + if res.Header.ContentLength() > fileCacheSizeLimit { err = res.BodyWriteTo(ctx.Response.BodyWriter()) } else { // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? @@ -187,7 +187,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe cachedResponse.exists = true cachedResponse.mimeType = mimeType cachedResponse.body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout) + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout) } return true From 11fa729686663018fc59edd3aba6b0b5b136f81f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 16:33:56 +0100 Subject: [PATCH 26/41] mv acme config setup into own func --- cmd/main.go | 7 +- server/certificates/acme_account.go | 27 +++++ server/certificates/certificates.go | 156 +++++++++++++--------------- 3 files changed, 106 insertions(+), 84 deletions(-) create mode 100644 server/certificates/acme_account.go diff --git a/cmd/main.go b/cmd/main.go index ceec3d5..222c295 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -102,7 +102,12 @@ func Serve(ctx *cli.Context) error { keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, keyDatabase)) - certificates.SetupCertificates(mainDomainSuffix, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer, challengeCache, keyDatabase) + acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) + if err != nil { + return err + } + + certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, keyDatabase) if enableHTTPServer { go func() { diff --git a/server/certificates/acme_account.go b/server/certificates/acme_account.go new file mode 100644 index 0000000..2ee2e80 --- /dev/null +++ b/server/certificates/acme_account.go @@ -0,0 +1,27 @@ +package certificates + +import ( + "crypto" + + "github.com/go-acme/lego/v4/registration" +) + +type AcmeAccount struct { + Email string + Registration *registration.Resource + Key crypto.PrivateKey `json:"-"` + KeyPEM string `json:"Key"` +} + +// make sure AcmeAccount match User interface +var _ registration.User = &AcmeAccount{} + +func (u *AcmeAccount) GetEmail() string { + return u.Email +} +func (u AcmeAccount) GetRegistration() *registration.Resource { + return u.Registration +} +func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey { + return u.Key +} diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index fb61bd8..b20f623 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -2,7 +2,6 @@ package certificates import ( "bytes" - "crypto" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -12,15 +11,12 @@ import ( "encoding/json" "errors" "io/ioutil" - "log" "os" "strconv" "strings" "sync" "time" - "github.com/reugn/equalizer" - "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/certificate" "github.com/go-acme/lego/v4/challenge" @@ -28,6 +24,8 @@ import ( "github.com/go-acme/lego/v4/lego" "github.com/go-acme/lego/v4/providers/dns" "github.com/go-acme/lego/v4/registration" + "github.com/reugn/equalizer" + "github.com/rs/zerolog/log" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/database" @@ -147,26 +145,6 @@ func checkUserLimit(user string) error { return nil } -var myAcmeAccount AcmeAccount -var myAcmeConfig *lego.Config - -type AcmeAccount struct { - Email string - Registration *registration.Resource - Key crypto.PrivateKey `json:"-"` - KeyPEM string `json:"Key"` -} - -func (u *AcmeAccount) GetEmail() string { - return u.Email -} -func (u AcmeAccount) GetRegistration() *registration.Resource { - return u.Registration -} -func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey { - return u.Key -} - var acmeClient, mainDomainAcmeClient *lego.Client var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{} @@ -331,15 +309,12 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re return tlsCertificate, nil } -func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeUseRateLimits, acmeAcceptTerms, enableHTTPServer bool, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) { - // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits - mainCertBytes, err := keyDatabase.Get(mainDomainSuffix) - if err != nil { - // key database is not working - panic(err) - } +func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { + const configFile = "acme-account.json" + var myAcmeAccount AcmeAccount + var myAcmeConfig *lego.Config - if account, err := ioutil.ReadFile("acme-account.json"); err == nil { + if account, err := ioutil.ReadFile(configFile); err == nil { err = json.Unmarshal(account, &myAcmeAccount) if err != nil { panic(err) @@ -351,66 +326,81 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, myAcmeConfig = lego.NewConfig(&myAcmeAccount) myAcmeConfig.CADirURL = acmeAPI myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 + + // Validate Config _, err := lego.NewClient(myAcmeConfig) if err != nil { + // TODO: should we fail hard instead? log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } - } else if os.IsNotExist(err) { - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - panic(err) - } - myAcmeAccount = AcmeAccount{ - Email: acmeMail, - Key: privateKey, - KeyPEM: string(certcrypto.PEMEncode(privateKey)), - } - myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI - myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 - tempClient, err := lego.NewClient(myAcmeConfig) - if err != nil { - log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) - } else { - // accept terms & log in to EAB - if acmeEabKID == "" || acmeEabHmac == "" { - reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) - if err != nil { - log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) - } else { - myAcmeAccount.Registration = reg - } - } else { - reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ - TermsOfServiceAgreed: acmeAcceptTerms, - Kid: acmeEabKID, - HmacEncoded: acmeEabHmac, - }) - if err != nil { - log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) - } else { - myAcmeAccount.Registration = reg - } - } + return myAcmeConfig, nil + } else if !os.IsNotExist(err) { + return nil, err + } - if myAcmeAccount.Registration != nil { - acmeAccountJson, err := json.Marshal(myAcmeAccount) - if err != nil { - log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err) - select {} - } - err = ioutil.WriteFile("acme-account.json", acmeAccountJson, 0600) - if err != nil { - log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err) - select {} - } + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(err) + } + myAcmeAccount = AcmeAccount{ + Email: acmeMail, + Key: privateKey, + KeyPEM: string(certcrypto.PEMEncode(privateKey)), + } + myAcmeConfig = lego.NewConfig(&myAcmeAccount) + myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 + tempClient, err := lego.NewClient(myAcmeConfig) + if err != nil { + log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) + } else { + // accept terms & log in to EAB + if acmeEabKID == "" || acmeEabHmac == "" { + reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) + if err != nil { + log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) + } else { + myAcmeAccount.Registration = reg + } + } else { + reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ + TermsOfServiceAgreed: acmeAcceptTerms, + Kid: acmeEabKID, + HmacEncoded: acmeEabHmac, + }) + if err != nil { + log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) + } else { + myAcmeAccount.Registration = reg } } - } else { + + if myAcmeAccount.Registration != nil { + acmeAccountJson, err := json.Marshal(myAcmeAccount) + if err != nil { + log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err) + select {} + } + err = ioutil.WriteFile(configFile, acmeAccountJson, 0600) + if err != nil { + log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err) + select {} + } + } + } + + return myAcmeConfig, nil +} + +func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) { + // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits + mainCertBytes, err := keyDatabase.Get(mainDomainSuffix) + if err != nil { + // key database is not working panic(err) } - acmeClient, err = lego.NewClient(myAcmeConfig) + acmeClient, err = lego.NewClient(acmeConfig) if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { @@ -426,7 +416,7 @@ func SetupCertificates(mainDomainSuffix []byte, acmeAPI, acmeMail, acmeEabHmac, } } - mainDomainAcmeClient, err = lego.NewClient(myAcmeConfig) + mainDomainAcmeClient, err = lego.NewClient(acmeConfig) if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { From de439f9becccbb3f9cd5d32ba14a50b19294da8c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 17:42:53 +0100 Subject: [PATCH 27/41] wrap cert db and make sync gracefull --- cmd/certs.go | 2 +- cmd/main.go | 2 +- server/certificates/certificates.go | 8 +-- server/certificates/mock.go | 2 +- server/database/helpers.go | 4 +- server/database/interface.go | 4 +- server/database/setup.go | 87 ++++++++++++++++++++++++++++- 7 files changed, 95 insertions(+), 14 deletions(-) diff --git a/cmd/certs.go b/cmd/certs.go index 89521da..6603e55 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -35,7 +35,7 @@ func certs(ctx *cli.Context) error { panic(err) } } - if err := keyDatabase.Sync(); err != nil { + if err := keyDatabase.Close(); err != nil { panic(err) } os.Exit(0) diff --git a/cmd/main.go b/cmd/main.go index 222c295..7eb3fac 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -94,7 +94,7 @@ func Serve(ctx *cli.Context) error { if err != nil { return fmt.Errorf("could not create database: %v", err) } - defer keyDatabase.Sync() //nolint:errcheck // database has no close ... sync behave like it + defer keyDatabase.Close() //nolint:errcheck // database has no close ... sync behave like it listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index b20f623..d2f7c76 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -38,7 +38,7 @@ func TLSConfig(mainDomainSuffix []byte, giteaRoot, giteaApiToken, dnsProvider string, acmeUseRateLimits bool, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, - keyDatabase database.KeyDB) *tls.Config { + keyDatabase database.CertDB) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -185,7 +185,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.KeyDB) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, bool) { // parse certificate from database res := &certificate.Resource{} if !database.PogrebGet(keyDatabase, sni, res) { @@ -229,7 +229,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs var obtainLocks = sync.Map{} -func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.KeyDB) (tls.Certificate, error) { +func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] @@ -392,7 +392,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce return myAcmeConfig, nil } -func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, keyDatabase database.KeyDB) { +func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, keyDatabase database.CertDB) { // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits mainCertBytes, err := keyDatabase.Get(mainDomainSuffix) if err != nil { diff --git a/server/certificates/mock.go b/server/certificates/mock.go index 19adb92..22d5470 100644 --- a/server/certificates/mock.go +++ b/server/certificates/mock.go @@ -17,7 +17,7 @@ import ( "codeberg.org/codeberg/pages/server/database" ) -func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.KeyDB) tls.Certificate { +func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) tls.Certificate { key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048) if err != nil { panic(err) diff --git a/server/database/helpers.go b/server/database/helpers.go index 98ea3fa..ea3e899 100644 --- a/server/database/helpers.go +++ b/server/database/helpers.go @@ -5,7 +5,7 @@ import ( "encoding/gob" ) -func PogrebPut(db KeyDB, name []byte, obj interface{}) { +func PogrebPut(db CertDB, name []byte, obj interface{}) { var resGob bytes.Buffer resEnc := gob.NewEncoder(&resGob) err := resEnc.Encode(obj) @@ -18,7 +18,7 @@ func PogrebPut(db KeyDB, name []byte, obj interface{}) { } } -func PogrebGet(db KeyDB, name []byte, obj interface{}) bool { +func PogrebGet(db CertDB, name []byte, obj interface{}) bool { resBytes, err := db.Get(name) if err != nil { panic(err) diff --git a/server/database/interface.go b/server/database/interface.go index 2b582ae..80d74d3 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -2,8 +2,8 @@ package database import "github.com/akrylysov/pogreb" -type KeyDB interface { - Sync() error +type CertDB interface { + Close() error Put(key []byte, value []byte) error Get(key []byte) ([]byte, error) Delete(key []byte) error diff --git a/server/database/setup.go b/server/database/setup.go index c16ff36..f7eeafc 100644 --- a/server/database/setup.go +++ b/server/database/setup.go @@ -1,19 +1,100 @@ package database import ( + "context" "fmt" + "time" + + "github.com/rs/zerolog/log" + "github.com/akrylysov/pogreb" "github.com/akrylysov/pogreb/fs" - "time" ) -func New(path string) (KeyDB, error) { +type aDB struct { + ctx context.Context + cancel context.CancelFunc + intern *pogreb.DB + syncInterval time.Duration +} + +func (p aDB) Close() error { + p.cancel() + return p.intern.Sync() +} + +func (p aDB) Put(key []byte, value []byte) error { + return p.intern.Put(key, value) +} + +func (p aDB) Get(key []byte) ([]byte, error) { + return p.intern.Get(key) +} + +func (p aDB) Delete(key []byte) error { + return p.intern.Delete(key) +} + +func (p aDB) Compact() (pogreb.CompactionResult, error) { + return p.intern.Compact() +} + +func (p aDB) Items() *pogreb.ItemIterator { + return p.intern.Items() +} + +var _ CertDB = &aDB{} + +func (p aDB) sync() { + for { + err := p.intern.Sync() + if err != nil { + log.Err(err).Msg("Syncing cert database failed") + } + select { + case <-p.ctx.Done(): + return + case <-time.After(p.syncInterval): + } + } +} + +func (p aDB) compact() { + for { + err := p.intern.Sync() + if err != nil { + log.Err(err).Msg("Syncing cert database failed") + } + select { + case <-p.ctx.Done(): + return + case <-time.After(p.syncInterval): + } + } +} + +func New(path string) (CertDB, error) { if path == "" { return nil, fmt.Errorf("path not set") } - return pogreb.Open(path, &pogreb.Options{ + db, err := pogreb.Open(path, &pogreb.Options{ BackgroundSyncInterval: 30 * time.Second, BackgroundCompactionInterval: 6 * time.Hour, FileSystem: fs.OSMMap, }) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + result := &aDB{ + ctx: ctx, + cancel: cancel, + intern: db, + syncInterval: 5 * time.Minute, + } + + go result.sync() + + return result, nil } From a0e0d2d335e219d6696c2b9d0c9e4f01be5f7c7b Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 17:44:10 +0100 Subject: [PATCH 28/41] make certdb maintain go routine a own func --- cmd/main.go | 3 + server/certificates/certificates.go | 132 +++++++++++++--------------- 2 files changed, 64 insertions(+), 71 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7eb3fac..b026885 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -109,6 +109,9 @@ func Serve(ctx *cli.Context) error { certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, keyDatabase) + // TODO: make it graceful + go certificates.MaintainCertDB(mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) + if enableHTTPServer { go func() { err := httpServer.ListenAndServe("[::]:80") diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index d2f7c76..e484d78 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -444,75 +444,65 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err) } } - - go (func() { - for { - err := keyDatabase.Sync() - if err != nil { - log.Printf("[ERROR] Syncing key database failed: %s", err) - } - time.Sleep(5 * time.Minute) - // TODO: graceful exit - } - })() - go (func() { - for { - // clean up expired certs - now := time.Now() - expiredCertCount := 0 - keyDatabaseIterator := keyDatabase.Items() - key, resBytes, err := keyDatabaseIterator.Next() - for err == nil { - if !bytes.Equal(key, mainDomainSuffix) { - resGob := bytes.NewBuffer(resBytes) - resDec := gob.NewDecoder(resGob) - res := &certificate.Resource{} - err = resDec.Decode(res) - if err != nil { - panic(err) - } - - tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) - if err != nil || !tlsCertificates[0].NotAfter.After(now) { - err := keyDatabase.Delete(key) - if err != nil { - log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) - } else { - expiredCertCount++ - } - } - } - key, resBytes, err = keyDatabaseIterator.Next() - } - log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) - - // compact the database - result, err := keyDatabase.Compact() - if err != nil { - log.Printf("[ERROR] Compacting key database failed: %s", err) - } else { - log.Printf("[INFO] Compacted key database (%+v)", result) - } - - // update main cert - res := &certificate.Resource{} - if !database.PogrebGet(keyDatabase, mainDomainSuffix, res) { - log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") - } else { - tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) - - // renew main certificate 30 days before it expires - if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) { - go (func() { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) - if err != nil { - log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err) - } - })() - } - } - - time.Sleep(12 * time.Hour) - } - })() +} + +func MaintainCertDB(mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.CertDB) { + for { + // clean up expired certs + now := time.Now() + expiredCertCount := 0 + keyDatabaseIterator := keyDatabase.Items() + key, resBytes, err := keyDatabaseIterator.Next() + for err == nil { + if !bytes.Equal(key, mainDomainSuffix) { + resGob := bytes.NewBuffer(resBytes) + resDec := gob.NewDecoder(resGob) + res := &certificate.Resource{} + err = resDec.Decode(res) + if err != nil { + panic(err) + } + + tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) + if err != nil || !tlsCertificates[0].NotAfter.After(now) { + err := keyDatabase.Delete(key) + if err != nil { + log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) + } else { + expiredCertCount++ + } + } + } + key, resBytes, err = keyDatabaseIterator.Next() + } + log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) + + // compact the database + result, err := keyDatabase.Compact() + if err != nil { + log.Printf("[ERROR] Compacting key database failed: %s", err) + } else { + log.Printf("[INFO] Compacted key database (%+v)", result) + } + + // update main cert + res := &certificate.Resource{} + if !database.PogrebGet(keyDatabase, mainDomainSuffix, res) { + log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") + } else { + tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) + + // renew main certificate 30 days before it expires + if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) { + go (func() { + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) + if err != nil { + log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err) + } + })() + } + } + + time.Sleep(12 * time.Hour) + } } From e85f21ed2ef460ba61c632f48b6182ab884a38e5 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 17:57:54 +0100 Subject: [PATCH 29/41] some renames --- cmd/main.go | 2 +- server/setup.go | 2 +- server/upstream/upstream.go | 71 +++++++++++++++++++------------------ 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index b026885..eafadb6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -80,7 +80,7 @@ func Serve(ctx *cli.Context) error { dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) fastServer := server.SetupServer(handler) - httpServer := server.SetupHttpACMEChallengeServer(challengeCache) + httpServer := server.SetupHTTPACMEChallengeServer(challengeCache) // Setup listener and TLS log.Info().Msgf("Listening on https://%s", listeningAddress) diff --git a/server/setup.go b/server/setup.go index 546aba1..67c1c42 100644 --- a/server/setup.go +++ b/server/setup.go @@ -27,7 +27,7 @@ func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server { } } -func SetupHttpACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server { +func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server { challengePath := []byte("/.well-known/acme-challenge/") return &fasthttp.Server{ diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 306e1db..2338ef2 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -24,12 +24,13 @@ var upstreamIndexPages = []string{ // Options provides various options for the upstream request. type Options struct { - DefaultMimeType string - ForbiddenMimeTypes map[string]struct{} - TryIndexPages bool - AppendTrailingSlash bool - RedirectIfExists string - BranchTimestamp time.Time + DefaultMimeType string + ForbiddenMimeTypes map[string]struct{} + TryIndexPages bool + BranchTimestamp time.Time + // internal + appendTrailingSlash bool + redirectIfExists string } var client = fasthttp.Client{ @@ -40,23 +41,23 @@ var client = fasthttp.Client{ } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { +func (o *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() - if options.ForbiddenMimeTypes == nil { - options.ForbiddenMimeTypes = map[string]struct{}{} + if o.ForbiddenMimeTypes == nil { + o.ForbiddenMimeTypes = map[string]struct{}{} } // Check if the branch exists and when it was modified - if options.BranchTimestamp == (time.Time{}) { - branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaApiToken, branchTimestampCache) + if o.BranchTimestamp == (time.Time{}) { + branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaAPIToken, branchTimestampCache) if branch == nil { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return true } targetBranch = branch.Branch - options.BranchTimestamp = branch.Timestamp + o.BranchTimestamp = branch.Timestamp } if targetOwner == "" || targetRepo == "" || targetBranch == "" { @@ -66,7 +67,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe // Check if the browser has a cached version if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { - if !ifModifiedSince.Before(options.BranchTimestamp) { + if !ifModifiedSince.Before(o.BranchTimestamp) { ctx.Response.SetStatusCode(fasthttp.StatusNotModified) return true } @@ -79,11 +80,11 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe var res *fasthttp.Response var cachedResponse fileResponse var err error - if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { + if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 { cachedResponse = cachedValue.(fileResponse) } else { req = fasthttp.AcquireRequest() - req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaApiToken) + req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken) res = fasthttp.AcquireResponse() res.SetBodyStream(&strings.Reader{}, -1) err = client.Do(req, res) @@ -92,24 +93,24 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe // Handle errors if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) { - if options.TryIndexPages { - // copy the options struct & try if an index page exists - optionsForIndexPages := *options + if o.TryIndexPages { + // copy the o struct & try if an index page exists + optionsForIndexPages := *o optionsForIndexPages.TryIndexPages = false - optionsForIndexPages.AppendTrailingSlash = true + optionsForIndexPages.appendTrailingSlash = true for _, indexPage := range upstreamIndexPages { - if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, fileCacheTimeout) return true } } // compatibility fix for GitHub Pages (/example → /example.html) - optionsForIndexPages.AppendTrailingSlash = false - optionsForIndexPages.RedirectIfExists = string(ctx.Request.URI().Path()) + ".html" - if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + optionsForIndexPages.appendTrailingSlash = false + optionsForIndexPages.redirectIfExists = string(ctx.Request.URI().Path()) + ".html" + if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, fileCacheTimeout) return true @@ -118,7 +119,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe ctx.Response.SetStatusCode(fasthttp.StatusNotFound) if res != nil { // Update cache if the request is fresh - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, fileCacheTimeout) } @@ -131,8 +132,8 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe } // Append trailing slash if missing (for index files), and redirect to fix filenames in general - // options.AppendTrailingSlash is only true when looking for index pages - if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { + // o.appendTrailingSlash is only true when looking for index pages + if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) return true } @@ -140,8 +141,8 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) return true } - if options.RedirectIfExists != "" { - ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect) + if o.redirectIfExists != "" { + ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect) return true } log.Debug().Msg("error handling") @@ -149,9 +150,9 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe // Set the MIME type mimeType := mime.TypeByExtension(path.Ext(targetPath)) mimeTypeSplit := strings.SplitN(mimeType, ";", 2) - if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { - if options.DefaultMimeType != "" { - mimeType = options.DefaultMimeType + if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { + if o.DefaultMimeType != "" { + mimeType = o.DefaultMimeType } else { mimeType = "application/octet-stream" } @@ -160,7 +161,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe // Everything's okay so far ctx.Response.SetStatusCode(fasthttp.StatusOK) - ctx.Response.Header.SetLastModified(options.BranchTimestamp) + ctx.Response.Header.SetLastModified(o.BranchTimestamp) log.Debug().Msg("response preparations") @@ -187,7 +188,7 @@ func (options *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRe cachedResponse.exists = true cachedResponse.mimeType = mimeType cachedResponse.body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout) + _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout) } return true From 0374e95d230e39980392006d1647084b09aea99a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 18:17:28 +0100 Subject: [PATCH 30/41] make tryUpstream independent func --- server/handler.go | 57 +++++++++++++++++++++++------------------------ server/try.go | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 29 deletions(-) create mode 100644 server/try.go diff --git a/server/handler.go b/server/handler.go index e7652dd..0aa5f3a 100644 --- a/server/handler.go +++ b/server/handler.go @@ -110,29 +110,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, return true } - // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. - var tryUpstream = func() { - // check if a canonical domain exists on a request on MainDomain - if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) - if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { - canonicalPath := string(ctx.RequestURI()) - if targetRepo != "pages" { - canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2] - } - ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) - return - } - } - - // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaApiToken, branchTimestampCache, fileResponseCache) { - html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) - } - } - log.Debug().Msg("preparations") - if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) { // Serve raw content from RawDomain log.Debug().Msg("raw domain") @@ -157,7 +135,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaApiToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) return } log.Debug().Msg("missing branch") @@ -169,7 +150,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", ) log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaApiToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -202,7 +186,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, "/"+pathElements[0]+"/%p", ) { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaApiToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) } @@ -215,7 +202,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with specified branch") if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaApiToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) } @@ -228,7 +218,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with specified repo") if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaApiToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -237,7 +230,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("main domain preparations, now trying with default repo/branch") if tryBranch("pages", "", pathElements, "") { log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaApiToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -282,7 +278,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, } log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream() + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaApiToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) return } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) diff --git a/server/try.go b/server/try.go new file mode 100644 index 0000000..0636ffc --- /dev/null +++ b/server/try.go @@ -0,0 +1,40 @@ +package server + +import ( + "bytes" + "strings" + + "github.com/valyala/fasthttp" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/upstream" +) + +// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. +func tryUpstream(ctx *fasthttp.RequestCtx, + mainDomainSuffix, trimmedHost []byte, + + targetOptions *upstream.Options, + targetOwner, targetRepo, targetBranch, targetPath, + + giteaRoot, giteaAPIToken string, + canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) { + // check if a canonical domain exists on a request on MainDomain + if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { + canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) + if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { + canonicalPath := string(ctx.RequestURI()) + if targetRepo != "pages" { + canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2] + } + ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) + return + } + } + + // Try to request the file from the Gitea API + if !targetOptions.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) + } +} From 26a199053ba41c69f156238dbe984986a68ffcd7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 18:18:05 +0100 Subject: [PATCH 31/41] lint: rename --- server/handler.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/handler.go b/server/handler.go index 0aa5f3a..1031716 100644 --- a/server/handler.go +++ b/server/handler.go @@ -16,7 +16,7 @@ import ( // Handler handles a single HTTP request to the web server. func Handler(mainDomainSuffix, rawDomain []byte, - giteaRoot, rawInfoPage, giteaApiToken string, + giteaRoot, rawInfoPage, giteaAPIToken string, blacklistedPaths, allowedCorsDomains [][]byte, dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) { return func(ctx *fasthttp.RequestCtx) { @@ -85,7 +85,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, } // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaApiToken, branchTimestampCache) + branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache) if branchTimestampResult == nil { // branch doesn't exist return false @@ -137,7 +137,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("tryBranch, now trying upstream") tryUpstream(ctx, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaApiToken, + giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -152,7 +152,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("tryBranch, now trying upstream") tryUpstream(ctx, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaApiToken, + giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -188,7 +188,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("tryBranch, now trying upstream") tryUpstream(ctx, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaApiToken, + giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -204,7 +204,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("tryBranch, now trying upstream") tryUpstream(ctx, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaApiToken, + giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) } else { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) @@ -220,7 +220,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("tryBranch, now trying upstream") tryUpstream(ctx, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaApiToken, + giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -232,7 +232,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("tryBranch, now trying upstream") tryUpstream(ctx, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaApiToken, + giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } @@ -261,7 +261,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, // Try to use the given repo on the given branch or the default branch log.Debug().Msg("custom domain preparations, now trying with details from DNS") if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) + canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) if !valid { html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) return @@ -280,7 +280,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("tryBranch, now trying upstream") tryUpstream(ctx, mainDomainSuffix, trimmedHost, targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaApiToken, + giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return } else { From a0534f1fde96b77419311e7c0b9ce237575d54b2 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 18:26:54 +0100 Subject: [PATCH 32/41] make MaintainCertDB able to cancel --- cmd/main.go | 8 ++++++-- server/certificates/certificates.go | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index eafadb6..21da71a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,11 +2,13 @@ package cmd import ( "bytes" + "context" "crypto/tls" "errors" "fmt" "net" "strings" + "time" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" @@ -109,8 +111,10 @@ func Serve(ctx *cli.Context) error { certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, keyDatabase) - // TODO: make it graceful - go certificates.MaintainCertDB(mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) + interval := 12 * time.Hour + certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) + defer cancelCertMaintain() + go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) if enableHTTPServer { go func() { diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index e484d78..bb6a3c3 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -2,6 +2,7 @@ package certificates import ( "bytes" + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -446,7 +447,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * } } -func MaintainCertDB(mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.CertDB) { +func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.CertDB) { for { // clean up expired certs now := time.Now() @@ -503,6 +504,10 @@ func MaintainCertDB(mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimi } } - time.Sleep(12 * time.Hour) + select { + case <-ctx.Done(): + return + case <-time.After(interval): + } } } From 5fe51d86214298637838539230cf43058d041533 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 19:00:57 +0100 Subject: [PATCH 33/41] rm certDB helper and build in --- server/certificates/certificates.go | 33 +++++++++++++++---------- server/certificates/mock.go | 4 +++- server/database/helpers.go | 37 ----------------------------- server/database/interface.go | 9 ++++--- server/database/setup.go | 29 +++++++++++++++++----- 5 files changed, 53 insertions(+), 59 deletions(-) delete mode 100644 server/database/helpers.go diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index bb6a3c3..fa76538 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -188,8 +188,11 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, bool) { // parse certificate from database - res := &certificate.Resource{} - if !database.PogrebGet(keyDatabase, sni, res) { + res, err := keyDatabase.Get(sni) + if err != nil { + panic(err) // TODO: no panic + } + if res == nil { return tls.Certificate{}, false } @@ -294,7 +297,9 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) { // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) - database.PogrebPut(keyDatabase, []byte(name), renew) + if err := keyDatabase.Put(name, renew); err != nil { + return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err + } return tlsCertificate, nil } } @@ -302,7 +307,9 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re } log.Printf("Obtained certificate for %v", domains) - database.PogrebPut(keyDatabase, []byte(name), res) + if err := keyDatabase.Put(name, res); err != nil { + return tls.Certificate{}, err + } tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { return tls.Certificate{}, err @@ -447,12 +454,12 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * } } -func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.CertDB) { +func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { for { // clean up expired certs now := time.Now() expiredCertCount := 0 - keyDatabaseIterator := keyDatabase.Items() + keyDatabaseIterator := certDB.Items() key, resBytes, err := keyDatabaseIterator.Next() for err == nil { if !bytes.Equal(key, mainDomainSuffix) { @@ -466,7 +473,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) if err != nil || !tlsCertificates[0].NotAfter.After(now) { - err := keyDatabase.Delete(key) + err := certDB.Delete(key) if err != nil { log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) } else { @@ -479,7 +486,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) // compact the database - result, err := keyDatabase.Compact() + result, err := certDB.Compact() if err != nil { log.Printf("[ERROR] Compacting key database failed: %s", err) } else { @@ -487,16 +494,18 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi } // update main cert - res := &certificate.Resource{} - if !database.PogrebGet(keyDatabase, mainDomainSuffix, res) { - log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") + res, err := certDB.Get(mainDomainSuffix) + if err != nil { + log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix) + } else if res == nil { + log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") } else { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) // renew main certificate 30 days before it expires if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) { go (func() { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err) } diff --git a/server/certificates/mock.go b/server/certificates/mock.go index 22d5470..0e87e6e 100644 --- a/server/certificates/mock.go +++ b/server/certificates/mock.go @@ -74,7 +74,9 @@ func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] { databaseName = mainDomainSuffix } - database.PogrebPut(keyDatabase, []byte(databaseName), res) + if err := keyDatabase.Put(databaseName, res); err != nil { + panic(err) + } tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { diff --git a/server/database/helpers.go b/server/database/helpers.go deleted file mode 100644 index ea3e899..0000000 --- a/server/database/helpers.go +++ /dev/null @@ -1,37 +0,0 @@ -package database - -import ( - "bytes" - "encoding/gob" -) - -func PogrebPut(db CertDB, name []byte, obj interface{}) { - var resGob bytes.Buffer - resEnc := gob.NewEncoder(&resGob) - err := resEnc.Encode(obj) - if err != nil { - panic(err) - } - err = db.Put(name, resGob.Bytes()) - if err != nil { - panic(err) - } -} - -func PogrebGet(db CertDB, name []byte, obj interface{}) bool { - resBytes, err := db.Get(name) - if err != nil { - panic(err) - } - if resBytes == nil { - return false - } - - resGob := bytes.NewBuffer(resBytes) - resDec := gob.NewDecoder(resGob) - err = resDec.Decode(obj) - if err != nil { - panic(err) - } - return true -} diff --git a/server/database/interface.go b/server/database/interface.go index 80d74d3..01b9872 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -1,11 +1,14 @@ package database -import "github.com/akrylysov/pogreb" +import ( + "github.com/akrylysov/pogreb" + "github.com/go-acme/lego/v4/certificate" +) type CertDB interface { Close() error - Put(key []byte, value []byte) error - Get(key []byte) ([]byte, error) + Put(name string, cert *certificate.Resource) error + Get(name []byte) (*certificate.Resource, error) Delete(key []byte) error Compact() (pogreb.CompactionResult, error) Items() *pogreb.ItemIterator diff --git a/server/database/setup.go b/server/database/setup.go index f7eeafc..f3cac16 100644 --- a/server/database/setup.go +++ b/server/database/setup.go @@ -1,14 +1,16 @@ package database import ( + "bytes" "context" + "encoding/gob" "fmt" "time" - "github.com/rs/zerolog/log" - "github.com/akrylysov/pogreb" "github.com/akrylysov/pogreb/fs" + "github.com/go-acme/lego/v4/certificate" + "github.com/rs/zerolog/log" ) type aDB struct { @@ -23,12 +25,27 @@ func (p aDB) Close() error { return p.intern.Sync() } -func (p aDB) Put(key []byte, value []byte) error { - return p.intern.Put(key, value) +func (p aDB) Put(name string, cert *certificate.Resource) error { + var resGob bytes.Buffer + if err := gob.NewEncoder(&resGob).Encode(cert); err != nil { + return err + } + return p.intern.Put([]byte(name), resGob.Bytes()) } -func (p aDB) Get(key []byte) ([]byte, error) { - return p.intern.Get(key) +func (p aDB) Get(name []byte) (*certificate.Resource, error) { + cert := &certificate.Resource{} + resBytes, err := p.intern.Get(name) + if err != nil { + return nil, err + } + if resBytes == nil { + return nil, nil + } + if err = gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil { + return nil, err + } + return cert, nil } func (p aDB) Delete(key []byte) error { From 2f6b280fcebdea91c5e662c31f5c6400c5d772a7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 19:02:26 +0100 Subject: [PATCH 34/41] meaningfull var names --- cmd/main.go | 10 +++++----- server/certificates/certificates.go | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 21da71a..a1bee6a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -92,29 +92,29 @@ func Serve(ctx *cli.Context) error { } // TODO: make "key-database.pogreb" set via flag - keyDatabase, err := database.New("key-database.pogreb") + certDB, err := database.New("key-database.pogreb") if err != nil { return fmt.Errorf("could not create database: %v", err) } - defer keyDatabase.Close() //nolint:errcheck // database has no close ... sync behave like it + defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, giteaRoot, giteaAPIToken, dnsProvider, acmeUseRateLimits, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, - keyDatabase)) + certDB)) acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) if err != nil { return err } - certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, keyDatabase) + certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB) interval := 12 * time.Hour certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) defer cancelCertMaintain() - go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase) + go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB) if enableHTTPServer { go func() { diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index fa76538..105f049 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -36,10 +36,10 @@ import ( // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates. func TLSConfig(mainDomainSuffix []byte, - giteaRoot, giteaApiToken, dnsProvider string, + giteaRoot, giteaAPIToken, dnsProvider string, acmeUseRateLimits bool, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, - keyDatabase database.CertDB) *tls.Config { + certDB database.CertDB) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -79,7 +79,7 @@ func TLSConfig(mainDomainSuffix []byte, sni = string(sniBytes) } else { _, _ = targetRepo, targetBranch - _, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaApiToken, canonicalDomainCache) + _, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) if !valid { sniBytes = mainDomainSuffix sni = string(sniBytes) @@ -95,13 +95,13 @@ func TLSConfig(mainDomainSuffix []byte, var tlsCertificate tls.Certificate var err error var ok bool - if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase); !ok { + if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok { // request a new certificate if bytes.Equal(sniBytes, mainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } - tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) + tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { return nil, err } @@ -186,9 +186,9 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, bool) { +func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) { // parse certificate from database - res, err := keyDatabase.Get(sni) + res, err := certDB.Get(sni) if err != nil { panic(err) // TODO: no panic } @@ -220,7 +220,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs } go (func() { res.CSR = nil // acme client doesn't like CSR to be set - tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) + tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { log.Printf("Couldn't renew certificate for %s: %s", sni, err) } @@ -400,9 +400,9 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce return myAcmeConfig, nil } -func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, keyDatabase database.CertDB) { +func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) { // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits - mainCertBytes, err := keyDatabase.Get(mainDomainSuffix) + mainCertBytes, err := certDB.Get(mainDomainSuffix) if err != nil { // key database is not working panic(err) @@ -447,7 +447,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * } if mainCertBytes == nil { - _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, keyDatabase) + _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB) if err != nil { log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err) } From a7bb3448a41006c3e8dbc8ed16622dca4284095d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 19:53:23 +0100 Subject: [PATCH 35/41] move more args of Upstream() to upstream Options --- server/handler.go | 36 ++++++++++++++++++------------------ server/try.go | 8 +++++++- server/upstream/upstream.go | 29 ++++++++++++++++++----------- server/utils/utils_test.go | 13 +++++++++++++ 4 files changed, 56 insertions(+), 30 deletions(-) create mode 100644 server/utils/utils_test.go diff --git a/server/handler.go b/server/handler.go index 1031716..1aaf476 100644 --- a/server/handler.go +++ b/server/handler.go @@ -144,19 +144,19 @@ func Handler(mainDomainSuffix, rawDomain []byte, log.Debug().Msg("missing branch") html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return - } else { - log.Debug().Msg("raw domain preparations, now trying with default branch") - tryBranch(targetRepo, "", pathElements[2:], - giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", - ) - log.Debug().Msg("tryBranch, now trying upstream") - tryUpstream(ctx, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - giteaRoot, giteaAPIToken, - canonicalDomainCache, branchTimestampCache, fileResponseCache) - return } + log.Debug().Msg("raw domain preparations, now trying with default branch") + tryBranch(targetRepo, "", pathElements[2:], + giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", + ) + log.Debug().Msg("tryBranch, now trying upstream") + tryUpstream(ctx, mainDomainSuffix, trimmedHost, + targetOptions, targetOwner, targetRepo, targetBranch, targetPath, + giteaRoot, giteaAPIToken, + canonicalDomainCache, branchTimestampCache, fileResponseCache) + return + } else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { // Serve pages from subdomains of MainDomainSuffix log.Debug().Msg("main domain suffix") @@ -167,7 +167,7 @@ func Handler(mainDomainSuffix, rawDomain []byte, targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/") if targetOwner == "www" { - // www.codeberg.page redirects to codeberg.page + // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname? ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect) return } @@ -271,10 +271,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, if targetOwner != "" { ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) return - } else { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return } + + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + return } log.Debug().Msg("tryBranch, now trying upstream") @@ -283,10 +283,10 @@ func Handler(mainDomainSuffix, rawDomain []byte, giteaRoot, giteaAPIToken, canonicalDomainCache, branchTimestampCache, fileResponseCache) return - } else { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return } + + html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) + return } } } diff --git a/server/try.go b/server/try.go index 0636ffc..7223dfa 100644 --- a/server/try.go +++ b/server/try.go @@ -20,6 +20,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) { + // check if a canonical domain exists on a request on MainDomain if bytes.HasSuffix(trimmedHost, mainDomainSuffix) { canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache) @@ -33,8 +34,13 @@ func tryUpstream(ctx *fasthttp.RequestCtx, } } + targetOptions.TargetOwner = targetOwner + targetOptions.TargetRepo = targetRepo + targetOptions.TargetBranch = targetBranch + targetOptions.TargetPath = targetPath + // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) } } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 2338ef2..925183a 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -24,7 +24,12 @@ var upstreamIndexPages = []string{ // Options provides various options for the upstream request. type Options struct { - DefaultMimeType string + TargetOwner, + TargetRepo, + TargetBranch, + TargetPath, + + DefaultMimeType string ForbiddenMimeTypes map[string]struct{} TryIndexPages bool BranchTimestamp time.Time @@ -41,26 +46,26 @@ var client = fasthttp.Client{ } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, targetBranch, targetPath, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { - log := log.With().Strs("upstream", []string{targetOwner, targetRepo, targetBranch, targetPath}).Logger() +func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) { + log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.ForbiddenMimeTypes == nil { o.ForbiddenMimeTypes = map[string]struct{}{} } // Check if the branch exists and when it was modified - if o.BranchTimestamp == (time.Time{}) { - branch := GetBranchTimestamp(targetOwner, targetRepo, targetBranch, giteaRoot, giteaAPIToken, branchTimestampCache) + if o.BranchTimestamp.IsZero() { + branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache) if branch == nil { html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) return true } - targetBranch = branch.Branch + o.TargetBranch = branch.Branch o.BranchTimestamp = branch.Timestamp } - if targetOwner == "" || targetRepo == "" || targetBranch == "" { + if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" { html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest) return true } @@ -75,7 +80,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, ta log.Debug().Msg("preparations") // Make a GET request to the upstream URL - uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath + uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath var req *fasthttp.Request var res *fasthttp.Response var cachedResponse fileResponse @@ -99,7 +104,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, ta optionsForIndexPages.TryIndexPages = false optionsForIndexPages.appendTrailingSlash = true for _, indexPage := range upstreamIndexPages { - if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage + if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, fileCacheTimeout) @@ -109,7 +115,8 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, ta // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.appendTrailingSlash = false optionsForIndexPages.redirectIfExists = string(ctx.Request.URI().Path()) + ".html" - if optionsForIndexPages.Upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath+".html", giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { + optionsForIndexPages.TargetPath = o.TargetPath + ".html" + if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) { _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, fileCacheTimeout) @@ -148,7 +155,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, targetOwner, targetRepo, ta log.Debug().Msg("error handling") // Set the MIME type - mimeType := mime.TypeByExtension(path.Ext(targetPath)) + mimeType := mime.TypeByExtension(path.Ext(o.TargetPath)) mimeTypeSplit := strings.SplitN(mimeType, ";", 2) if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" { if o.DefaultMimeType != "" { diff --git a/server/utils/utils_test.go b/server/utils/utils_test.go new file mode 100644 index 0000000..3dc0632 --- /dev/null +++ b/server/utils/utils_test.go @@ -0,0 +1,13 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTrimHostPort(t *testing.T) { + assert.EqualValues(t, "aa", TrimHostPort([]byte("aa"))) + assert.EqualValues(t, "", TrimHostPort([]byte(":"))) + assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80"))) +} From 196482da07af0a12f788a48f2d5468954bf9f541 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 23:20:34 +0100 Subject: [PATCH 36/41] less panic --- go.mod | 1 + server/certificates/certificates.go | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index a2bd8ee..615bb12 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-acme/lego/v4 v4.5.3 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.26.0 + github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 github.com/valyala/fasthttp v1.31.0 github.com/valyala/fastjson v1.6.3 diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 105f049..b40c76d 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -11,6 +11,7 @@ import ( "encoding/gob" "encoding/json" "errors" + "fmt" "io/ioutil" "os" "strconv" @@ -107,9 +108,8 @@ func TLSConfig(mainDomainSuffix []byte, } } - err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute) - if err != nil { - panic(err) + if err := keyCache.Set(sni, &tlsCertificate, 15*time.Minute); err != nil { + return nil, err } return &tlsCertificate, nil }, @@ -323,13 +323,12 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce var myAcmeConfig *lego.Config if account, err := ioutil.ReadFile(configFile); err == nil { - err = json.Unmarshal(account, &myAcmeAccount) - if err != nil { - panic(err) + if err := json.Unmarshal(account, &myAcmeAccount); err != nil { + return nil, err } myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM)) if err != nil { - panic(err) + return nil, err } myAcmeConfig = lego.NewConfig(&myAcmeAccount) myAcmeConfig.CADirURL = acmeAPI @@ -348,7 +347,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - panic(err) + return nil, err } myAcmeAccount = AcmeAccount{ Email: acmeMail, @@ -384,12 +383,12 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce } if myAcmeAccount.Registration != nil { - acmeAccountJson, err := json.Marshal(myAcmeAccount) + acmeAccountJSON, err := json.Marshal(myAcmeAccount) if err != nil { log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err) select {} } - err = ioutil.WriteFile(configFile, acmeAccountJson, 0600) + err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600) if err != nil { log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err) select {} @@ -400,12 +399,11 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce return myAcmeConfig, nil } -func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) { - // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits +func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error { + // getting main cert before ACME account so that we can fail here without hitting rate limits mainCertBytes, err := certDB.Get(mainDomainSuffix) if err != nil { - // key database is not working - panic(err) + return fmt.Errorf("cert database is not working") } acmeClient, err = lego.NewClient(acmeConfig) @@ -452,6 +450,8 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig * log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err) } } + + return nil } func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) { From aa0638903a083e7e037b91831795369350002dc6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 5 Dec 2021 23:56:06 +0100 Subject: [PATCH 37/41] fix argument check and some nits --- cmd/main.go | 6 ++++-- html/error.go | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index a1bee6a..cd1a1c2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -49,7 +49,7 @@ func Serve(ctx *cli.Context) error { acmeEabKID := ctx.String("acme-eab-kid") acmeEabHmac := ctx.String("acme-eab-hmac") dnsProvider := ctx.String("dns-provider") - if acmeAcceptTerms || (dnsProvider == "" && acmeAPI != "https://acme.mock.directory") { + if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory") } @@ -109,7 +109,9 @@ func Serve(ctx *cli.Context) error { return err } - certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB) + if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil { + return err + } interval := 12 * time.Hour certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) diff --git a/html/error.go b/html/error.go index f831443..325dada 100644 --- a/html/error.go +++ b/html/error.go @@ -2,8 +2,9 @@ package html import ( "bytes" - "github.com/valyala/fasthttp" "strconv" + + "github.com/valyala/fasthttp" ) // ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced From 70c7065f764231c40f266ff39e412f15124b9cfa Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 9 Dec 2021 19:32:30 +0100 Subject: [PATCH 38/41] fix #31 --- server/try.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/try.go b/server/try.go index 7223dfa..6dffa44 100644 --- a/server/try.go +++ b/server/try.go @@ -27,7 +27,10 @@ func tryUpstream(ctx *fasthttp.RequestCtx, if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { canonicalPath := string(ctx.RequestURI()) if targetRepo != "pages" { - canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2] + path := strings.SplitN(canonicalPath, "/", 3) + if len(path) >= 3 { + canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2] + } } ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) return From 6af6523a0f83a171e19b5bb9c2b25b28d4916bee Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 9 Dec 2021 20:16:43 +0100 Subject: [PATCH 39/41] code format --- server/try.go | 2 +- server/upstream/domains.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/try.go b/server/try.go index 6dffa44..31cd7f4 100644 --- a/server/try.go +++ b/server/try.go @@ -29,7 +29,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, if targetRepo != "pages" { path := strings.SplitN(canonicalPath, "/", 3) if len(path) >= 3 { - canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2] + canonicalPath = "/" + path[2] } } ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 5971e13..47a5564 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -8,9 +8,10 @@ import ( "codeberg.org/codeberg/pages/server/cache" ) -// CheckCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`). -func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaApiToken string, canonicalDomainCache cache.SetGetKey) (canonicalDomain string, valid bool) { +// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). +func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) { domains := []string{} + valid := false if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { domains = cachedValue.([]string) for _, domain := range domains { @@ -21,7 +22,7 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m } } else { req := fasthttp.AcquireRequest() - req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaApiToken) + req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken) res := fasthttp.AcquireResponse() err := client.Do(req, res) @@ -48,6 +49,5 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m } _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout) } - canonicalDomain = domains[0] - return + return domains[0], valid } From 73fa2da6467e196d9afbec1487401e16f4887842 Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 10 Dec 2021 14:31:58 +0100 Subject: [PATCH 40/41] Update default to raw.codeberg.page & improve documentation on custom domains --- README.md | 9 ++++++--- cmd/flags.go | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 616d69d..087d7ae 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ - `HOST` & `PORT` (default: `[::]` & `443`): listen address. - `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages. -- `RAW_DOMAIN` (default: `raw.codeberg.org`): domain for raw resources. +- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources. - `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance. - `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos. - `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided. @@ -15,6 +15,7 @@ - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80. - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. See https://go-acme.github.io/lego/dns/ for available values & additional environment variables. +- `DEBUG` (default: false): Set this to true to enable debug logging. ``` // Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories. @@ -29,8 +30,10 @@ // www.example.org. IN CNAME main.pages.example.codeberg.page. // // 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record -// for "example.org" (if your provider allows ALIAS or similar records): +// for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT +// record that points to your repo (just like the CNAME record): // example.org IN ALIAS codeberg.page. +// example.org IN TXT main.pages.example.codeberg.page. // // Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge. -``` \ No newline at end of file +``` diff --git a/cmd/flags.go b/cmd/flags.go index 6381ee5..c6eaf4f 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -35,14 +35,14 @@ var ServeFlags = []cli.Flag{ Name: "raw-domain", Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", EnvVars: []string{"RAW_DOMAIN"}, - Value: "raw.codeberg.org", + Value: "raw.codeberg.page", }, // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path). &cli.StringFlag{ Name: "raw-info-page", Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)", EnvVars: []string{"RAW_INFO_PAGE"}, - Value: "https://docs.codeberg.org/pages/raw-content/", + Value: "https://docs.codeberg.org/codeberg-pages/raw-content/", }, // Server From adfc96ab94d3e7a9b8210220ee934c4fb24bfd12 Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 10 Dec 2021 14:32:14 +0100 Subject: [PATCH 41/41] Add --verbose flag and hide debug messages by default --- Justfile | 2 +- cmd/flags.go | 6 ++++++ cmd/main.go | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index a5feb7a..3d23175 100644 --- a/Justfile +++ b/Justfile @@ -6,7 +6,7 @@ dev: export PAGES_DOMAIN=localhost.mock.directory export RAW_DOMAIN=raw.localhost.mock.directory export PORT=4430 - go run . + go run . --verbose build: CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./ diff --git a/cmd/flags.go b/cmd/flags.go index c6eaf4f..e1838c2 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -5,6 +5,12 @@ import ( ) var ServeFlags = []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + // TODO: Usage + EnvVars: []string{"DEBUG"}, + }, + // MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static // pages, or used for comparison in CNAME lookups. Static pages can be accessed through // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages". diff --git a/cmd/main.go b/cmd/main.go index cd1a1c2..fb0c26e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" @@ -34,6 +35,11 @@ var BlacklistedPaths = [][]byte{ // Serve sets up and starts the web server. func Serve(ctx *cli.Context) error { + verbose := ctx.Bool("verbose") + if !verbose { + zerolog.SetGlobalLevel(zerolog.InfoLevel) + } + giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain")