tildeverse
/
tildepages
5
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

refactor (2) (#34) 

move forward with refactoring:
 - initial implementation of a smal "gitea client for fasthttp"
 - move constant into const.go

Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/34
Reviewed-by: Otto Richter <otto@codeberg.org>
This commit is contained in:
6543 2022-04-20 23:42:01 +02:00
parent a2c5376d9a
commit 4267d54a63
5 changed files with 103 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/upstream/const.go
View File

@ -19,3 +19,5 @@ var fileCacheSizeLimit = 1024 * 1024
// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
var canonicalDomainCacheTimeout = 15 * time.Minute
const canonicalDomainConfig = ".domains"

22
server/upstream/domains.go
View File

@ -3,15 +3,19 @@ package upstream
import (
"strings"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
)
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
domains := []string{}
valid := false
return checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken, canonicalDomainCache)
}
func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
var (
domains []string
valid bool
)
if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
domains = cachedValue.([]string)
for _, domain := range domains {
@ -21,13 +25,9 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
}
}
} else {
req := fasthttp.AcquireRequest()
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
res := fasthttp.AcquireResponse()
err := client.Do(req, res)
if err == nil && res.StatusCode() == fasthttp.StatusOK {
for _, domain := range strings.Split(string(res.Body()), "\n") {
body, err := giteaRawContent(giteaRoot, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig)
if err == nil {
for _, domain := range strings.Split(string(body), "\n") {
domain = strings.ToLower(domain)
domain = strings.TrimSpace(domain)
domain = strings.TrimPrefix(domain, "http://")

67
server/upstream/gitea.go Normal file
View File

@ -0,0 +1,67 @@
package upstream
import (
"fmt"
"net/url"
"path"
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
)
const giteaAPIRepos = "/api/v1/repos/"
// TODOs:
// * own client to store token & giteaRoot
// * handle 404 -> page will show 500 atm
func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil {
return nil, err
}
if res.StatusCode() != fasthttp.StatusOK {
return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return res.Body(), nil
}
func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) {
client := getFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := client.Do(req, res); err != nil {
return time.Time{}, err
}
if res.StatusCode() != fasthttp.StatusOK {
return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
}
func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) {
client := getFastHTTPClient(5 * time.Second)
req := fasthttp.AcquireRequest()
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res := fasthttp.AcquireResponse()
if err := client.Do(req, res); err != nil {
return "", err
}
if res.StatusCode() != fasthttp.StatusOK {
return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return fastjson.GetString(res.Body(), "default_branch"), nil
}

30
server/upstream/helper.go
View File

@ -3,9 +3,6 @@ package upstream
import (
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
"codeberg.org/codeberg/pages/server/cache"
)
@ -16,34 +13,31 @@ type branchTimestamp struct {
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist)
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
if result == nil {
return nil
}
return result.(*branchTimestamp)
}
result := &branchTimestamp{}
result.Branch = branch
if branch == "" {
result := &branchTimestamp{
Branch: branch,
}
if len(branch) == 0 {
// Get default branch
body := make([]byte, 0)
// TODO: use header for API key?
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken)
if err != nil {
_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
return nil
}
result.Branch = fastjson.GetString(body, "default_branch")
result.Branch = defaultBranch
}
body := make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken)
if err != nil {
return nil
}
result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
result.Timestamp = timestamp
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
return result
}

19
server/upstream/upstream.go
View File

@ -38,11 +38,13 @@ type Options struct {
redirectIfExists string
}
var client = fasthttp.Client{
ReadTimeout: 10 * time.Second,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
return &fasthttp.Client{
ReadTimeout: timeout,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
@ -80,7 +82,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
log.Debug().Msg("preparations")
// Make a GET request to the upstream URL
uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
var req *fasthttp.Request
var res *fasthttp.Response
var cachedResponse fileResponse
@ -89,10 +91,11 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
cachedResponse = cachedValue.(fileResponse)
} else {
req = fasthttp.AcquireRequest()
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri))
req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
res = fasthttp.AcquireResponse()
res.SetBodyStream(&strings.Reader{}, -1)
err = client.Do(req, res)
err = getFastHTTPClient(10*time.Second).Do(req, res)
}
log.Debug().Msg("acquisition")