From 4267d54a6302f1acb92ec2e4d382b2ddaae3a520 Mon Sep 17 00:00:00 2001 From: 6543 <6543@noreply.codeberg.org> Date: Wed, 20 Apr 2022 23:42:01 +0200 Subject: [PATCH] refactor (2) (#34) move forward with refactoring: - initial implementation of a smal "gitea client for fasthttp" - move constant into const.go Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/34 Reviewed-by: Otto Richter --- server/upstream/const.go | 2 ++ server/upstream/domains.go | 22 ++++++------ server/upstream/gitea.go | 67 +++++++++++++++++++++++++++++++++++++ server/upstream/helper.go | 30 +++++++---------- server/upstream/upstream.go | 19 ++++++----- 5 files changed, 103 insertions(+), 37 deletions(-) create mode 100644 server/upstream/gitea.go diff --git a/server/upstream/const.go b/server/upstream/const.go index 77f64dd..ce76e21 100644 --- a/server/upstream/const.go +++ b/server/upstream/const.go @@ -19,3 +19,5 @@ var fileCacheSizeLimit = 1024 * 1024 // canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. var canonicalDomainCacheTimeout = 15 * time.Minute + +const canonicalDomainConfig = ".domains" diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 47a5564..28a2d9c 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -3,15 +3,19 @@ package upstream import ( "strings" - "github.com/valyala/fasthttp" - "codeberg.org/codeberg/pages/server/cache" ) // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) { - domains := []string{} - valid := false + return checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken, canonicalDomainCache) +} + +func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) { + var ( + domains []string + valid bool + ) if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok { domains = cachedValue.([]string) for _, domain := range domains { @@ -21,13 +25,9 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m } } } else { - req := fasthttp.AcquireRequest() - req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken) - res := fasthttp.AcquireResponse() - - err := client.Do(req, res) - if err == nil && res.StatusCode() == fasthttp.StatusOK { - for _, domain := range strings.Split(string(res.Body()), "\n") { + body, err := giteaRawContent(giteaRoot, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig) + if err == nil { + for _, domain := range strings.Split(string(body), "\n") { domain = strings.ToLower(domain) domain = strings.TrimSpace(domain) domain = strings.TrimPrefix(domain, "http://") diff --git a/server/upstream/gitea.go b/server/upstream/gitea.go new file mode 100644 index 0000000..eeeb0a6 --- /dev/null +++ b/server/upstream/gitea.go @@ -0,0 +1,67 @@ +package upstream + +import ( + "fmt" + "net/url" + "path" + "time" + + "github.com/valyala/fasthttp" + "github.com/valyala/fastjson" +) + +const giteaAPIRepos = "/api/v1/repos/" + +// TODOs: +// * own client to store token & giteaRoot +// * handle 404 -> page will show 500 atm + +func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) { + req := fasthttp.AcquireRequest() + + req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))) + req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) + res := fasthttp.AcquireResponse() + + if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil { + return nil, err + } + if res.StatusCode() != fasthttp.StatusOK { + return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return res.Body(), nil +} + +func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) { + client := getFastHTTPClient(5 * time.Second) + + req := fasthttp.AcquireRequest() + req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)) + req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) + res := fasthttp.AcquireResponse() + + if err := client.Do(req, res); err != nil { + return time.Time{}, err + } + if res.StatusCode() != fasthttp.StatusOK { + return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) +} + +func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) { + client := getFastHTTPClient(5 * time.Second) + + req := fasthttp.AcquireRequest() + req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName)) + req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) + res := fasthttp.AcquireResponse() + + if err := client.Do(req, res); err != nil { + return "", err + } + if res.StatusCode() != fasthttp.StatusOK { + return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + } + return fastjson.GetString(res.Body(), "default_branch"), nil +} diff --git a/server/upstream/helper.go b/server/upstream/helper.go index f61d746..3b51479 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -3,9 +3,6 @@ package upstream import ( "time" - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" - "codeberg.org/codeberg/pages/server/cache" ) @@ -16,34 +13,31 @@ type branchTimestamp struct { // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch // (or nil if the branch doesn't exist) -func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp { +func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp { if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result == nil { return nil } return result.(*branchTimestamp) } - result := &branchTimestamp{} - result.Branch = branch - if branch == "" { + result := &branchTimestamp{ + Branch: branch, + } + if len(branch) == 0 { // Get default branch - body := make([]byte, 0) - // TODO: use header for API key? - status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second) - if err != nil || status != 200 { - _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout) + defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken) + if err != nil { + _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout) return nil } - result.Branch = fastjson.GetString(body, "default_branch") + result.Branch = defaultBranch } - body := make([]byte, 0) - status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second) - if err != nil || status != 200 { + timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken) + if err != nil { return nil } - - result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp")) + result.Timestamp = timestamp _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout) return result } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 88d3471..33c80e9 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -38,11 +38,13 @@ type Options struct { redirectIfExists string } -var client = fasthttp.Client{ - ReadTimeout: 10 * time.Second, - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! +func getFastHTTPClient(timeout time.Duration) *fasthttp.Client { + return &fasthttp.Client{ + ReadTimeout: timeout, + MaxConnDuration: 60 * time.Second, + MaxConnWaitTimeout: 1000 * time.Millisecond, + MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! + } } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. @@ -80,7 +82,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st log.Debug().Msg("preparations") // Make a GET request to the upstream URL - uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath + uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) var req *fasthttp.Request var res *fasthttp.Response var cachedResponse fileResponse @@ -89,10 +91,11 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st cachedResponse = cachedValue.(fileResponse) } else { req = fasthttp.AcquireRequest() - req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken) + req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri)) + req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken) res = fasthttp.AcquireResponse() res.SetBodyStream(&strings.Reader{}, -1) - err = client.Do(req, res) + err = getFastHTTPClient(10*time.Second).Do(req, res) } log.Debug().Msg("acquisition")