tildeverse
/
tildepages
5
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request 'Refactor: restructure in packages and dont use golbal vars' (#18) from 6543/codeberg-pages:refactoring into main 

Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/18
This commit is contained in:
Moritz Marquardt 2021-12-10 14:33:18 +01:00
parent 67a190f68a adfc96ab94
commit e73c79da77
37 changed files with 2013 additions and 1531 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Justfile
View File

@ -6,7 +6,7 @@ dev:
export PAGES_DOMAIN=localhost.mock.directory
export RAW_DOMAIN=raw.localhost.mock.directory
export PORT=4430
go run .
go run . --verbose
build:
CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./

26
README.md
View File

@ -2,10 +2,10 @@
- `HOST` & `PORT` (default: `[::]` & `443`): listen address.
- `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
- `RAW_DOMAIN` (default: `raw.codeberg.org`): domain for raw resources.
- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources.
- `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
- `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
- `REDIRECT_RAW_INFO` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).
ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
- `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider.
@ -15,3 +15,25 @@
- `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
- `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.
See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
- `DEBUG` (default: false): Set this to true to enable debug logging.
```
// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
//
// Mapping custom domains is not static anymore, but can be done with DNS:
//
// 1) add a ".domains" text file to your repository, containing the allowed domains, separated by new lines. The
// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
//
// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
// www.example.org. IN CNAME main.pages.example.codeberg.page.
//
// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
// for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
// record that points to your repo (just like the CNAME record):
// example.org IN ALIAS codeberg.page.
// example.org IN TXT main.pages.example.codeberg.page.
//
// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
```

599
certificates.go
View File

@ -1,599 +0,0 @@
package main
import (
"bytes"
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/gob"
"encoding/json"
"encoding/pem"
"errors"
"github.com/OrlovEvgeny/go-mcache"
"github.com/akrylysov/pogreb/fs"
"github.com/go-acme/lego/v4/certificate"
"github.com/go-acme/lego/v4/challenge"
"github.com/go-acme/lego/v4/challenge/tlsalpn01"
"github.com/go-acme/lego/v4/providers/dns"
"io/ioutil"
"log"
"math/big"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/akrylysov/pogreb"
"github.com/reugn/equalizer"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/registration"
)
// tlsConfig contains the configuration for generating, serving and cleaning up Let's Encrypt certificates.
var tlsConfig = &tls.Config{
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
sniBytes := []byte(sni)
if len(sni) < 1 {
return nil, errors.New("missing sni")
}
if info.SupportedProtos != nil {
for _, proto := range info.SupportedProtos {
if proto == tlsalpn01.ACMETLS1Protocol {
challenge, ok := challengeCache.Get(sni)
if !ok {
return nil, errors.New("no challenge for this domain")
}
cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
if err != nil {
return nil, err
}
return cert, nil
}
}
}
targetOwner := ""
if bytes.HasSuffix(sniBytes, MainDomainSuffix) || bytes.Equal(sniBytes, MainDomainSuffix[1:]) {
// deliver default certificate for the main domain (*.codeberg.page)
sniBytes = MainDomainSuffix
sni = string(sniBytes)
} else {
var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni)
if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs
sniBytes = MainDomainSuffix
sni = string(sniBytes)
} else {
_, _ = targetRepo, targetBranch
_, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni)
if !valid {
sniBytes = MainDomainSuffix
sni = string(sniBytes)
}
}
}
if tlsCertificate, ok := keyCache.Get(sni); ok {
// we can use an existing certificate object
return tlsCertificate.(*tls.Certificate), nil
}
var tlsCertificate tls.Certificate
var err error
var ok bool
if tlsCertificate, ok = retrieveCertFromDB(sniBytes); !ok {
// request a new certificate
if bytes.Equal(sniBytes, MainDomainSuffix) {
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
}
tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner)
if err != nil {
return nil, err
}
}
err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute)
if err != nil {
panic(err)
}
return &tlsCertificate, nil
},
PreferServerCipherSuites: true,
NextProtos: []string{
"http/1.1",
tlsalpn01.ACMETLS1Protocol,
},
// generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
// https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
},
}
var keyCache = mcache.New()
var keyDatabase, keyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{
BackgroundSyncInterval: 30 * time.Second,
BackgroundCompactionInterval: 6 * time.Hour,
FileSystem: fs.OSMMap,
})
func CheckUserLimit(user string) error {
userLimit, ok := acmeClientCertificateLimitPerUser[user]
if !ok {
// Each Codeberg user can only add 10 new domains per day.
userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
acmeClientCertificateLimitPerUser[user] = userLimit
}
if !userLimit.Ask() {
return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
}
return nil
}
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
type AcmeAccount struct {
Email string
Registration *registration.Resource
Key crypto.PrivateKey `json:"-"`
KeyPEM string `json:"Key"`
}
func (u *AcmeAccount) GetEmail() string {
return u.Email
}
func (u AcmeAccount) GetRegistration() *registration.Resource {
return u.Registration
}
func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
return u.Key
}
var acmeClient, mainDomainAcmeClient *lego.Client
var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
var challengeCache = mcache.New()
type AcmeTLSChallengeProvider struct{}
var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return challengeCache.Set(domain, keyAuth, 1*time.Hour)
}
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
challengeCache.Remove(domain)
return nil
}
type AcmeHTTPChallengeProvider struct{}
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
}
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
challengeCache.Remove(domain + "/" + token)
return nil
}
func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) {
// parse certificate from database
res := &certificate.Resource{}
if !PogrebGet(keyDatabase, sni, res) {
return tls.Certificate{}, false
}
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
panic(err)
}
if !bytes.Equal(sni, MainDomainSuffix) {
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
if err != nil {
panic(err)
}
// renew certificates 7 days before they expire
if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
if res.CSR != nil && len(res.CSR) > 0 {
// CSR stores the time when the renewal shall be tried again
nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
return tlsCertificate, true
}
}
go (func() {
res.CSR = nil // acme client doesn't like CSR to be set
tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "")
if err != nil {
log.Printf("Couldn't renew certificate for %s: %s", sni, err)
}
})()
}
}
return tlsCertificate, true
}
var obtainLocks = sync.Map{}
func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string) (tls.Certificate, error) {
name := strings.TrimPrefix(domains[0], "*")
if os.Getenv("DNS_PROVIDER") == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
domains = domains[1:]
}
// lock to avoid simultaneous requests
_, working := obtainLocks.LoadOrStore(name, struct{}{})
if working {
for working {
time.Sleep(100 * time.Millisecond)
_, working = obtainLocks.Load(name)
}
cert, ok := retrieveCertFromDB([]byte(name))
if !ok {
return tls.Certificate{}, errors.New("certificate failed in synchronous request")
}
return cert, nil
}
defer obtainLocks.Delete(name)
if acmeClient == nil {
return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!"), nil
}
// request actual cert
var res *certificate.Resource
var err error
if renew != nil && renew.CertURL != "" {
if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
acmeClientRequestLimit.Take()
}
log.Printf("Renewing certificate for %v", domains)
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
if err != nil {
log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
res = nil
}
}
if res == nil {
if user != "" {
if err := CheckUserLimit(user); err != nil {
return tls.Certificate{}, err
}
}
if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
acmeClientOrderLimit.Take()
acmeClientRequestLimit.Take()
}
log.Printf("Requesting new certificate for %v", domains)
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
Domains: domains,
Bundle: true,
MustStaple: false,
})
}
if err != nil {
log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
if renew != nil && renew.CertURL != "" {
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
PogrebPut(keyDatabase, []byte(name), renew)
return tlsCertificate, nil
}
} else {
return mockCert(domains[0], err.Error()), err
}
}
log.Printf("Obtained certificate for %v", domains)
PogrebPut(keyDatabase, []byte(name), res)
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
return tls.Certificate{}, err
}
return tlsCertificate, nil
}
func mockCert(domain string, msg string) tls.Certificate {
key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
if err != nil {
panic(err)
}
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: domain,
Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
OrganizationalUnit: []string{
"Will not try again for 6 hours to avoid hitting rate limits for your domain.",
"Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
"free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
"Error message: " + msg,
},
},
// certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6),
NotBefore: time.Now(),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
certBytes, err := x509.CreateCertificate(
rand.Reader,
&template,
&template,
&key.(*rsa.PrivateKey).PublicKey,
key,
)
if err != nil {
panic(err)
}
out := &bytes.Buffer{}
err = pem.Encode(out, &pem.Block{
Bytes: certBytes,
Type: "CERTIFICATE",
})
if err != nil {
panic(err)
}
outBytes := out.Bytes()
res := &certificate.Resource{
PrivateKey: certcrypto.PEMEncode(key),
Certificate: outBytes,
IssuerCertificate: outBytes,
Domain: domain,
}
databaseName := domain
if domain == "*"+string(MainDomainSuffix) || domain == string(MainDomainSuffix[1:]) {
databaseName = string(MainDomainSuffix)
}
PogrebPut(keyDatabase, []byte(databaseName), res)
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
panic(err)
}
return tlsCertificate
}
func setupCertificates() {
if keyDatabaseErr != nil {
panic(keyDatabaseErr)
}
if os.Getenv("ACME_ACCEPT_TERMS") != "true" || (os.Getenv("DNS_PROVIDER") == "" && os.Getenv("ACME_API") != "https://acme.mock.directory") {
panic(errors.New("you must set ACME_ACCEPT_TERMS and DNS_PROVIDER, unless ACME_API is set to https://acme.mock.directory"))
}
// getting main cert before ACME account so that we can panic here on database failure without hitting rate limits
mainCertBytes, err := keyDatabase.Get(MainDomainSuffix)
if err != nil {
// key database is not working
panic(err)
}
if account, err := ioutil.ReadFile("acme-account.json"); err == nil {
err = json.Unmarshal(account, &myAcmeAccount)
if err != nil {
panic(err)
}
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
if err != nil {
panic(err)
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
}
} else if os.IsNotExist(err) {
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
panic(err)
}
myAcmeAccount = AcmeAccount{
Email: envOr("ACME_EMAIL", "noreply@example.email"),
Key: privateKey,
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
// accept terms & log in to EAB
if os.Getenv("ACME_EAB_KID") == "" || os.Getenv("ACME_EAB_HMAC") == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true"})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
} else {
myAcmeAccount.Registration = reg
}
} else {
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true",
Kid: os.Getenv("ACME_EAB_KID"),
HmacEncoded: os.Getenv("ACME_EAB_HMAC"),
})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
} else {
myAcmeAccount.Registration = reg
}
}
if myAcmeAccount.Registration != nil {
acmeAccountJson, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
err = ioutil.WriteFile("acme-account.json", acmeAccountJson, 0600)
if err != nil {
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
}
}
} else {
panic(err)
}
acmeClient, err = lego.NewClient(myAcmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
}
if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{})
if err != nil {
log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
}
}
}
mainDomainAcmeClient, err = lego.NewClient(myAcmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
if os.Getenv("DNS_PROVIDER") == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
}
} else {
provider, err := dns.NewDNSChallengeProviderByName(os.Getenv("DNS_PROVIDER"))
if err != nil {
log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
}
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
if err != nil {
log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
}
}
}
if mainCertBytes == nil {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, nil, "")
if err != nil {
log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
}
}
go (func() {
for {
err := keyDatabase.Sync()
if err != nil {
log.Printf("[ERROR] Syncinc key database failed: %s", err)
}
time.Sleep(5 * time.Minute)
}
})()
go (func() {
for {
// clean up expired certs
now := time.Now()
expiredCertCount := 0
keyDatabaseIterator := keyDatabase.Items()
key, resBytes, err := keyDatabaseIterator.Next()
for err == nil {
if !bytes.Equal(key, MainDomainSuffix) {
resGob := bytes.NewBuffer(resBytes)
resDec := gob.NewDecoder(resGob)
res := &certificate.Resource{}
err = resDec.Decode(res)
if err != nil {
panic(err)
}
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
if err != nil || !tlsCertificates[0].NotAfter.After(now) {
err := keyDatabase.Delete(key)
if err != nil {
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
} else {
expiredCertCount++
}
}
}
key, resBytes, err = keyDatabaseIterator.Next()
}
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
// compact the database
result, err := keyDatabase.Compact()
if err != nil {
log.Printf("[ERROR] Compacting key database failed: %s", err)
} else {
log.Printf("[INFO] Compacted key database (%+v)", result)
}
// update main cert
res := &certificate.Resource{}
if !PogrebGet(keyDatabase, MainDomainSuffix, res) {
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
} else {
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
// renew main certificate 30 days before it expires
if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
go (func() {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, res, "")
if err != nil {
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
}
})()
}
}
time.Sleep(12 * time.Hour)
}
})()
}

44
cmd/certs.go Normal file
View File

@ -0,0 +1,44 @@
package cmd
import (
"fmt"
"os"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/database"
)
var Certs = &cli.Command{
Name: "certs",
Usage: "manage certs manually",
Action: certs,
}
func certs(ctx *cli.Context) error {
if ctx.Args().Len() >= 1 && ctx.Args().First() == "--remove-certificate" {
if ctx.Args().Len() == 1 {
println("--remove-certificate requires at least one domain as an argument")
os.Exit(1)
}
domains := ctx.Args().Slice()[2:]
// TODO: make "key-database.pogreb" set via flag
keyDatabase, err := database.New("key-database.pogreb")
if err != nil {
return fmt.Errorf("could not create database: %v", err)
}
for _, domain := range domains {
if err := keyDatabase.Delete([]byte(domain)); err != nil {
panic(err)
}
}
if err := keyDatabase.Close(); err != nil {
panic(err)
}
os.Exit(0)
}
return nil
}

110
cmd/flags.go Normal file
View File

@ -0,0 +1,110 @@
package cmd
import (
"github.com/urfave/cli/v2"
)
var ServeFlags = []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
// TODO: Usage
EnvVars: []string{"DEBUG"},
},
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
&cli.StringFlag{
Name: "pages-domain",
Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
EnvVars: []string{"PAGES_DOMAIN"},
Value: "codeberg.page",
},
// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
&cli.StringFlag{
Name: "gitea-root",
Usage: "specifies the root URL of the Gitea instance, without a trailing slash.",
EnvVars: []string{"GITEA_ROOT"},
Value: "https://codeberg.org",
},
// GiteaApiToken specifies an api token for the Gitea instance
&cli.StringFlag{
Name: "gitea-api-token",
Usage: "specifies an api token for the Gitea instance",
EnvVars: []string{"GITEA_API_TOKEN"},
Value: "",
},
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
// (set to []byte(nil) to disable raw content hosting)
&cli.StringFlag{
Name: "raw-domain",
Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting",
EnvVars: []string{"RAW_DOMAIN"},
Value: "raw.codeberg.page",
},
// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
&cli.StringFlag{
Name: "raw-info-page",
Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)",
EnvVars: []string{"RAW_INFO_PAGE"},
Value: "https://docs.codeberg.org/codeberg-pages/raw-content/",
},
// Server
&cli.StringFlag{
Name: "host",
Usage: "specifies host of listening address",
EnvVars: []string{"HOST"},
Value: "[::]",
},
&cli.StringFlag{
Name: "port",
Usage: "specifies port of listening address",
EnvVars: []string{"PORT"},
Value: "443",
},
&cli.BoolFlag{
Name: "enable-http-server",
// TODO: desc
EnvVars: []string{"ENABLE_HTTP_SERVER"},
},
// ACME
&cli.StringFlag{
Name: "acme-api-endpoint",
EnvVars: []string{"ACME_API"},
Value: "https://acme-v02.api.letsencrypt.org/directory",
},
&cli.StringFlag{
Name: "acme-email",
EnvVars: []string{"ACME_EMAIL"},
Value: "noreply@example.email",
},
&cli.BoolFlag{
Name: "acme-use-rate-limits",
// TODO: Usage
EnvVars: []string{"ACME_USE_RATE_LIMITS"},
Value: true,
},
&cli.BoolFlag{
Name: "acme-accept-terms",
// TODO: Usage
EnvVars: []string{"ACME_ACCEPT_TERMS"},
},
&cli.StringFlag{
Name: "acme-eab-kid",
// TODO: Usage
EnvVars: []string{"ACME_EAB_KID"},
},
&cli.StringFlag{
Name: "acme-eab-hmac",
// TODO: Usage
EnvVars: []string{"ACME_EAB_HMAC"},
},
&cli.StringFlag{
Name: "dns-provider",
// TODO: Usage
EnvVars: []string{"DNS_PROVIDER"},
},
}

143
cmd/main.go Normal file
View File

@ -0,0 +1,143 @@
package cmd
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
)
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
// TODO: make it a flag
var AllowedCorsDomains = [][]byte{
[]byte("fonts.codeberg.org"),
[]byte("design.codeberg.org"),
}
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
// TODO: Make it a flag too
var BlacklistedPaths = [][]byte{
[]byte("/.well-known/acme-challenge/"),
}
// Serve sets up and starts the web server.
func Serve(ctx *cli.Context) error {
verbose := ctx.Bool("verbose")
if !verbose {
zerolog.SetGlobalLevel(zerolog.InfoLevel)
}
giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/")
giteaAPIToken := ctx.String("gitea-api-token")
rawDomain := ctx.String("raw-domain")
mainDomainSuffix := []byte(ctx.String("pages-domain"))
rawInfoPage := ctx.String("raw-info-page")
listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
enableHTTPServer := ctx.Bool("enable-http-server")
acmeAPI := ctx.String("acme-api-endpoint")
acmeMail := ctx.String("acme-email")
acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
acmeAcceptTerms := ctx.Bool("acme-accept-terms")
acmeEabKID := ctx.String("acme-eab-kid")
acmeEabHmac := ctx.String("acme-eab-hmac")
dnsProvider := ctx.String("dns-provider")
if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory")
}
allowedCorsDomains := AllowedCorsDomains
if len(rawDomain) != 0 {
allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain))
}
// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) {
mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...)
}
keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache()
// canonicalDomainCache stores canonical domains
var canonicalDomainCache = cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains
var dnsLookupCache = cache.NewKeyValueCache()
// branchTimestampCache stores branch timestamps for faster cache checking
var branchTimestampCache = cache.NewKeyValueCache()
// fileResponseCache stores responses from the Gitea server
// TODO: make this an MRU cache with a size limit
var fileResponseCache = cache.NewKeyValueCache()
// Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
giteaRoot, rawInfoPage, giteaAPIToken,
BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
fastServer := server.SetupServer(handler)
httpServer := server.SetupHTTPACMEChallengeServer(challengeCache)
// Setup listener and TLS
log.Info().Msgf("Listening on https://%s", listeningAddress)
listener, err := net.Listen("tcp", listeningAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %s", err)
}
// TODO: make "key-database.pogreb" set via flag
certDB, err := database.New("key-database.pogreb")
if err != nil {
return fmt.Errorf("could not create database: %v", err)
}
defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaRoot, giteaAPIToken, dnsProvider,
acmeUseRateLimits,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB))
acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
if err != nil {
return err
}
if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil {
return err
}
interval := 12 * time.Hour
certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
defer cancelCertMaintain()
go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB)
if enableHTTPServer {
go func() {
err := httpServer.ListenAndServe("[::]:80")
if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
}
}()
}
// Start the web fastServer
err = fastServer.Serve(listener)
if err != nil {
log.Panic().Err(err).Msg("Couldn't start fastServer")
}
return nil
}

68
debug-stepper/stepper.go
View File

@ -1,68 +0,0 @@
package debug_stepper
import (
"fmt"
"os"
"strings"
"time"
)
var Enabled = strings.HasSuffix(os.Args[0], ".test") || os.Getenv("DEBUG") == "1"
var Logger = func(s string, i ...interface{}) {
fmt.Printf(s, i...)
}
type Stepper struct {
Name string
Start time.Time
LastStep time.Time
Completion time.Time
}
func Start(name string) *Stepper {
if !Enabled {
return nil
}
t := time.Now()
Logger("%s: started at %s\n", name, t.Format(time.RFC3339))
return &Stepper{
Name: name,
Start: t,
LastStep: t,
}
}
func (s *Stepper) Debug(text string) {
if !Enabled {
return
}
t := time.Now()
Logger("%s: %s (at %s, %s since last step, %s since start)\n", s.Name, text, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
}
func (s *Stepper) Step(description string) {
if !Enabled {
return
}
if s.Completion != (time.Time{}) {
Logger("%s: already completed all tasks.\n")
return
}
t := time.Now()
Logger("%s: completed %s at %s (%s)\n", s.Name, description, t.Format(time.RFC3339), t.Sub(s.LastStep).String())
s.LastStep = t
}
func (s *Stepper) Complete() {
if !Enabled {
return
}
if s.Completion != (time.Time{}) {
Logger("%s: already completed all tasks.\n")
return
}
t := time.Now()
Logger("%s: completed all tasks at %s (%s since last step; total time: %s)\n", s.Name, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
s.Completion = t
}

113
domains.go
View File

@ -1,113 +0,0 @@
package main
import (
"github.com/OrlovEvgeny/go-mcache"
"github.com/valyala/fasthttp"
"net"
"strings"
"time"
)
// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache.
var DnsLookupCacheTimeout = 15 * time.Minute
// dnsLookupCache stores DNS lookups for custom domains
var dnsLookupCache = mcache.New()
// getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data.
func getTargetFromDNS(domain string) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT
var cname string
var err error
if cachedName, ok := dnsLookupCache.Get(domain); ok {
cname = cachedName.(string)
} else {
cname, err = net.LookupCNAME(domain)
cname = strings.TrimSuffix(cname, ".")
if err != nil || !strings.HasSuffix(cname, string(MainDomainSuffix)) {
cname = ""
// TODO: check if the A record matches!
names, err := net.LookupTXT(domain)
if err == nil {
for _, name := range names {
name = strings.TrimSuffix(name, ".")
if strings.HasSuffix(name, string(MainDomainSuffix)) {
cname = name
break
}
}
}
}
_ = dnsLookupCache.Set(domain, cname, DnsLookupCacheTimeout)
}
if cname == "" {
return
}
cnameParts := strings.Split(strings.TrimSuffix(cname, string(MainDomainSuffix)), ".")
targetOwner = cnameParts[len(cnameParts)-1]
if len(cnameParts) > 1 {
targetRepo = cnameParts[len(cnameParts)-2]
}
if len(cnameParts) > 2 {
targetBranch = cnameParts[len(cnameParts)-3]
}
if targetRepo == "" {
targetRepo = "pages"
}
if targetBranch == "" && targetRepo != "pages" {
targetBranch = "pages"
}
// if targetBranch is still empty, the caller must find the default branch
return
}
// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
var CanonicalDomainCacheTimeout = 15 * time.Minute
// canonicalDomainCache stores canonical domains
var canonicalDomainCache = mcache.New()
// checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`).
func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain string) (canonicalDomain string, valid bool) {
domains := []string{}
if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
domains = cachedValue.([]string)
for _, domain := range domains {
if domain == actualDomain {
valid = true
break
}
}
} else {
req := fasthttp.AcquireRequest()
req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + GiteaApiToken)
res := fasthttp.AcquireResponse()
err := upstreamClient.Do(req, res)
if err == nil && res.StatusCode() == fasthttp.StatusOK {
for _, domain := range strings.Split(string(res.Body()), "\n") {
domain = strings.ToLower(domain)
domain = strings.TrimSpace(domain)
domain = strings.TrimPrefix(domain, "http://")
domain = strings.TrimPrefix(domain, "https://")
if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
domains = append(domains, domain)
}
if domain == actualDomain {
valid = true
}
}
}
domains = append(domains, targetOwner+string(MainDomainSuffix))
if domains[len(domains)-1] == actualDomain {
valid = true
}
if targetRepo != "" && targetRepo != "pages" {
domains[len(domains)-1] += "/" + targetRepo
}
_ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, CanonicalDomainCacheTimeout)
}
canonicalDomain = domains[0]
return
}

3
go.mod
View File

@ -7,6 +7,9 @@ require (
github.com/akrylysov/pogreb v0.10.1
github.com/go-acme/lego/v4 v4.5.3
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
github.com/rs/zerolog v1.26.0
github.com/stretchr/testify v1.7.0
github.com/urfave/cli/v2 v2.3.0
github.com/valyala/fasthttp v1.31.0
github.com/valyala/fastjson v1.6.3
)

19
go.sum
View File

@ -95,10 +95,12 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -148,6 +150,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b h1:/vQ+oYKu+JoyaMPDsv5FzwuL2wwWBgBbtj/YLCi4LuA=
github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -422,6 +425,10 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad h1:WtSUHi5zthjudjI
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe2Y+6iHTjIq/9HzUq7NII/Nffp0HkFrsAKq4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI=
@ -429,6 +436,7 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f h1:WSnaD0/cvbKJgSTYbjAPf4RJXVvNNDAwVm+W8wEmnGE=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@ -478,7 +486,9 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@ -499,6 +509,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@ -557,6 +568,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -589,8 +601,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -653,8 +666,9 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -708,6 +722,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

542
handler.go
View File

@ -1,542 +0,0 @@
package main
import (
"bytes"
debug_stepper "codeberg.org/codeberg/pages/debug-stepper"
"fmt"
"github.com/OrlovEvgeny/go-mcache"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
"io"
"mime"
"path"
"strconv"
"strings"
"time"
)
// handler handles a single HTTP request to the web server.
func handler(ctx *fasthttp.RequestCtx) {
s := debug_stepper.Start("handler")
defer s.Complete()
ctx.Response.Header.Set("Server", "Codeberg Pages")
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
// Enable browser caching for up to 10 minutes
ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
trimmedHost := TrimHostPort(ctx.Request.Host())
// Add HSTS for RawDomain and MainDomainSuffix
if hsts := GetHSTSHeader(trimmedHost); hsts != "" {
ctx.Response.Header.Set("Strict-Transport-Security", hsts)
}
// Block all methods not required for static pages
if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
return
}
// Block blacklisted paths (like ACME challenges)
for _, blacklistedPath := range BlacklistedPaths {
if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
returnErrorPage(ctx, fasthttp.StatusForbidden)
return
}
}
// Allow CORS for specified domains
if ctx.IsOptions() {
allowCors := false
for _, allowedCorsDomain := range AllowedCorsDomains {
if bytes.Equal(trimmedHost, allowedCorsDomain) {
allowCors = true
break
}
}
if allowCors {
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
}
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
return
}
// Prepare request information to Gitea
var targetOwner, targetRepo, targetBranch, targetPath string
var targetOptions = &upstreamOptions{
ForbiddenMimeTypes: map[string]struct{}{},
TryIndexPages: true,
}
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL.
var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
if repo == "" {
return false
}
// Check if the branch exists, otherwise treat it as a file path
branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch)
if branchTimestampResult == nil {
// branch doesn't exist
return false
}
// Branch exists, use it
targetRepo = repo
targetPath = strings.Trim(strings.Join(path, "/"), "/")
targetBranch = branchTimestampResult.branch
targetOptions.BranchTimestamp = branchTimestampResult.timestamp
if canonicalLink != "" {
// Hide from search machines & add canonical link
ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
ctx.Response.Header.Set("Link",
strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
"; rel=\"canonical\"",
)
}
return true
}
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
var tryUpstream = func() {
// check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "")
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(MainDomainSuffix)) {
canonicalPath := string(ctx.RequestURI())
if targetRepo != "pages" {
canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2]
}
ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
return
}
}
// Try to request the file from the Gitea API
if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, targetOptions) {
returnErrorPage(ctx, ctx.Response.StatusCode())
}
}
s.Step("preparations")
if RawDomain != nil && bytes.Equal(trimmedHost, RawDomain) {
// Serve raw content from RawDomain
s.Debug("raw domain")
targetOptions.TryIndexPages = false
targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
if len(pathElements) < 2 {
// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
ctx.Redirect(RawInfoPage, fasthttp.StatusTemporaryRedirect)
return
}
targetOwner = pathElements[0]
targetRepo = pathElements[1]
// raw.codeberg.org/example/myrepo/@main/index.html
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
s.Step("raw domain preparations, now trying with specified branch")
if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) {
s.Step("tryBranch, now trying upstream")
tryUpstream()
return
}
s.Debug("missing branch")
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
} else {
s.Step("raw domain preparations, now trying with default branch")
tryBranch(targetRepo, "", pathElements[2:],
string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
)
s.Step("tryBranch, now trying upstream")
tryUpstream()
return
}
} else if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
// Serve pages from subdomains of MainDomainSuffix
s.Debug("main domain suffix")
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
targetOwner = string(bytes.TrimSuffix(trimmedHost, MainDomainSuffix))
targetRepo = pathElements[0]
targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
if targetOwner == "www" {
// www.codeberg.page redirects to codeberg.page
ctx.Redirect("https://" + string(MainDomainSuffix[1:]) + string(ctx.Path()), fasthttp.StatusPermanentRedirect)
return
}
// Check if the first directory is a repo with the second directory as a branch
// example.codeberg.page/myrepo/@main/index.html
if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
if targetRepo == "pages" {
// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
return
}
s.Step("main domain preparations, now trying with specified repo & branch")
if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
"/"+pathElements[0]+"/%p",
) {
s.Step("tryBranch, now trying upstream")
tryUpstream()
} else {
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
}
// Check if the first directory is a branch for the "pages" repo
// example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") {
s.Step("main domain preparations, now trying with specified branch")
if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
s.Step("tryBranch, now trying upstream")
tryUpstream()
} else {
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
}
// Check if the first directory is a repo with a "pages" branch
// example.codeberg.page/myrepo/index.html
// example.codeberg.page/pages/... is not allowed here.
s.Step("main domain preparations, now trying with specified repo")
if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
s.Step("tryBranch, now trying upstream")
tryUpstream()
return
}
// Try to use the "pages" repo on its default branch
// example.codeberg.page/index.html
s.Step("main domain preparations, now trying with default repo/branch")
if tryBranch("pages", "", pathElements, "") {
s.Step("tryBranch, now trying upstream")
tryUpstream()
return
}
// Couldn't find a valid repo/branch
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
} else {
trimmedHostStr := string(trimmedHost)
// Serve pages from external domains
targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr)
if targetOwner == "" {
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
canonicalLink := ""
if strings.HasPrefix(pathElements[0], "@") {
targetBranch = pathElements[0][1:]
pathElements = pathElements[1:]
canonicalLink = "/%p"
}
// Try to use the given repo on the given branch or the default branch
s.Step("custom domain preparations, now trying with details from DNS")
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr)
if !valid {
returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHostStr {
// only redirect if the target is also a codeberg page!
targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0])
if targetOwner != "" {
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
return
} else {
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
}
s.Step("tryBranch, now trying upstream")
tryUpstream()
return
} else {
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
}
}
// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
// with the provided status code.
func returnErrorPage(ctx *fasthttp.RequestCtx, code int) {
ctx.Response.SetStatusCode(code)
ctx.Response.Header.SetContentType("text/html; charset=utf-8")
message := fasthttp.StatusMessage(code)
if code == fasthttp.StatusMisdirectedRequest {
message += " - domain not specified in <code>.domains</code> file"
}
if code == fasthttp.StatusFailedDependency {
message += " - target repo/branch doesn't exist or is private"
}
ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
}
// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
var DefaultBranchCacheTimeout = 15 * time.Minute
// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter
// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
// picked up faster, while still allowing the content to be cached longer if nothing changes.
var BranchExistanceCacheTimeout = 5 * time.Minute
// branchTimestampCache stores branch timestamps for faster cache checking
var branchTimestampCache = mcache.New()
type branchTimestamp struct {
branch string
timestamp time.Time
}
// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
// on your available memory.
var FileCacheTimeout = 5 * time.Minute
// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
var FileCacheSizeLimit = 1024 * 1024
// fileResponseCache stores responses from the Gitea server
// TODO: make this an MRU cache with a size limit
var fileResponseCache = mcache.New()
type fileResponse struct {
exists bool
mimeType string
body []byte
}
// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist)
func getBranchTimestamp(owner, repo, branch string) *branchTimestamp {
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
if result == nil {
return nil
}
return result.(*branchTimestamp)
}
result := &branchTimestamp{}
result.branch = branch
if branch == "" {
// Get default branch
var body = make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+GiteaApiToken, 5*time.Second)
if err != nil || status != 200 {
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout)
return nil
}
result.branch = fastjson.GetString(body, "default_branch")
}
var body = make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+GiteaApiToken, 5*time.Second)
if err != nil || status != 200 {
return nil
}
result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout)
return result
}
var upstreamClient = fasthttp.Client{
ReadTimeout: 10 * time.Second,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, targetBranch string, targetPath string, options *upstreamOptions) (final bool) {
s := debug_stepper.Start("upstream")
defer s.Complete()
if options.ForbiddenMimeTypes == nil {
options.ForbiddenMimeTypes = map[string]struct{}{}
}
// Check if the branch exists and when it was modified
if options.BranchTimestamp == (time.Time{}) {
branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch)
if branch == nil {
returnErrorPage(ctx, fasthttp.StatusFailedDependency)
return true
}
targetBranch = branch.branch
options.BranchTimestamp = branch.timestamp
}
if targetOwner == "" || targetRepo == "" || targetBranch == "" {
returnErrorPage(ctx, fasthttp.StatusBadRequest)
return true
}
// Check if the browser has a cached version
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
if !ifModifiedSince.Before(options.BranchTimestamp) {
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
return true
}
}
s.Step("preparations")
// Make a GET request to the upstream URL
uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath
var req *fasthttp.Request
var res *fasthttp.Response
var cachedResponse fileResponse
var err error
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
cachedResponse = cachedValue.(fileResponse)
} else {
req = fasthttp.AcquireRequest()
req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + uri + "?access_token=" + GiteaApiToken)
res = fasthttp.AcquireResponse()
res.SetBodyStream(&strings.Reader{}, -1)
err = upstreamClient.Do(req, res)
}
s.Step("acquisition")
// Handle errors
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
if options.TryIndexPages {
// copy the options struct & try if an index page exists
optionsForIndexPages := *options
optionsForIndexPages.TryIndexPages = false
optionsForIndexPages.AppendTrailingSlash = true
for _, indexPage := range IndexPages {
if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, &optionsForIndexPages) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, FileCacheTimeout)
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.AppendTrailingSlash = false
optionsForIndexPages.RedirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath + ".html", &optionsForIndexPages) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, FileCacheTimeout)
return true
}
}
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if res != nil {
// Update cache if the request is fresh
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, FileCacheTimeout)
}
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
returnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// options.AppendTrailingSlash is only true when looking for index pages
if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
return true
}
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
return true
}
if options.RedirectIfExists != "" {
ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect)
return true
}
s.Step("error handling")
// Set the MIME type
mimeType := mime.TypeByExtension(path.Ext(targetPath))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if options.DefaultMimeType != "" {
mimeType = options.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
ctx.Response.Header.SetContentType(mimeType)
// Everything's okay so far
ctx.Response.SetStatusCode(fasthttp.StatusOK)
ctx.Response.Header.SetLastModified(options.BranchTimestamp)
s.Step("response preparations")
// Write the response body to the original request
var cacheBodyWriter bytes.Buffer
if res != nil {
if res.Header.ContentLength() > FileCacheSizeLimit {
err = res.BodyWriteTo(ctx.Response.BodyWriter())
} else {
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
}
} else {
_, err = ctx.Write(cachedResponse.body)
}
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
returnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
s.Step("response")
if res != nil && ctx.Err() == nil {
cachedResponse.exists = true
cachedResponse.mimeType = mimeType
cachedResponse.body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout)
}
return true
}
// upstreamOptions provides various options for the upstream request.
type upstreamOptions struct {
DefaultMimeType string
ForbiddenMimeTypes map[string]struct{}
TryIndexPages bool
AppendTrailingSlash bool
RedirectIfExists string
BranchTimestamp time.Time
}

1
haproxy-sni/haproxy.cfg
View File

@ -51,6 +51,7 @@ frontend https_sni_frontend
###################################################
acl use_http_backend req.ssl_sni -i "codeberg.org"
acl use_http_backend req.ssl_sni -i "join.codeberg.org"
# TODO: use this if no SNI exists
use_backend https_termination_backend if use_http_backend
############################

56
helpers.go
View File

@ -1,56 +0,0 @@
package main
import (
"bytes"
"encoding/gob"
"github.com/akrylysov/pogreb"
)
// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
// string for custom domains.
func GetHSTSHeader(host []byte) string {
if bytes.HasSuffix(host, MainDomainSuffix) || bytes.Equal(host, RawDomain) {
return "max-age=63072000; includeSubdomains; preload"
} else {
return ""
}
}
func TrimHostPort(host []byte) []byte {
i := bytes.IndexByte(host, ':')
if i >= 0 {
return host[:i]
}
return host
}
func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) {
var resGob bytes.Buffer
resEnc := gob.NewEncoder(&resGob)
err := resEnc.Encode(obj)
if err != nil {
panic(err)
}
err = db.Put(name, resGob.Bytes())
if err != nil {
panic(err)
}
}
func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool {
resBytes, err := db.Get(name)
if err != nil {
panic(err)
}
if resBytes == nil {
return false
}
resGob := bytes.NewBuffer(resBytes)
resDec := gob.NewDecoder(resGob)
err = resDec.Decode(obj)
if err != nil {
panic(err)
}
return true
}

0
404.html → html/404.html
View File

24
html/error.go Normal file
View File

@ -0,0 +1,24 @@
package html
import (
"bytes"
"strconv"
"github.com/valyala/fasthttp"
)
// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
// with the provided status code.
func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
ctx.Response.SetStatusCode(code)
ctx.Response.Header.SetContentType("text/html; charset=utf-8")
message := fasthttp.StatusMessage(code)
if code == fasthttp.StatusMisdirectedRequest {
message += " - domain not specified in <code>.domains</code> file"
}
if code == fasthttp.StatusFailedDependency {
message += " - target repo/branch doesn't exist or is private"
}
// TODO: use template engine?
ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
}

6
html/html.go Normal file
View File

@ -0,0 +1,6 @@
package html
import _ "embed"
//go:embed 404.html
var NotFoundPage []byte

161
main.go
View File

@ -1,159 +1,32 @@
// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
//
// Mapping custom domains is not static anymore, but can be done with DNS:
//
// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The
// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
//
// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
// www.example.org. IN CNAME main.pages.example.codeberg.page.
//
// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
// for "example.org" (if your provider allows ALIAS or similar records):
// example.org IN ALIAS codeberg.page.
//
// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
package main
import (
"bytes"
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
"os"
"time"
_ "embed"
"github.com/urfave/cli/v2"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/cmd"
)
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
var MainDomainSuffix = []byte("." + envOr("PAGES_DOMAIN", "codeberg.page"))
var (
// can be changed with -X on compile
version = "dev"
)
// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
var GiteaRoot = []byte(envOr("GITEA_ROOT", "https://codeberg.org"))
var GiteaApiToken = envOr("GITEA_API_TOKEN", "")
//go:embed 404.html
var NotFoundPage []byte
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
// (set to []byte(nil) to disable raw content hosting)
var RawDomain = []byte(envOr("RAW_DOMAIN", "raw.codeberg.org"))
// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
var RawInfoPage = envOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/")
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
var AllowedCorsDomains = [][]byte{
RawDomain,
[]byte("fonts.codeberg.org"),
[]byte("design.codeberg.org"),
}
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
var BlacklistedPaths = [][]byte{
[]byte("/.well-known/acme-challenge/"),
}
// IndexPages lists pages that may be considered as index pages for directories.
var IndexPages = []string{
"index.html",
}
// main sets up and starts the web server.
func main() {
if len(os.Args) > 1 && os.Args[1] == "--remove-certificate" {
if len(os.Args) < 2 {
println("--remove-certificate requires at least one domain as an argument")
os.Exit(1)
}
if keyDatabaseErr != nil {
panic(keyDatabaseErr)
}
for _, domain := range os.Args[2:] {
if err := keyDatabase.Delete([]byte(domain)); err != nil {
panic(err)
}
}
if err := keyDatabase.Sync(); err != nil {
panic(err)
}
os.Exit(0)
app := cli.NewApp()
app.Name = "pages-server"
app.Version = version
app.Usage = "pages server"
app.Action = cmd.Serve
app.Flags = cmd.ServeFlags
app.Commands = []*cli.Command{
cmd.Certs,
}
// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
if !bytes.HasPrefix(MainDomainSuffix, []byte{'.'}) {
MainDomainSuffix = append([]byte{'.'}, MainDomainSuffix...)
}
GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'})
// Use HOST and PORT environment variables to determine listening address
address := fmt.Sprintf("%s:%s", envOr("HOST", "[::]"), envOr("PORT", "443"))
log.Printf("Listening on https://%s", address)
// Enable compression by wrapping the handler() method with the compression function provided by FastHTTP
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
server := &fasthttp.Server{
Handler: compressedHandler,
DisablePreParseMultipartForm: false,
MaxRequestBodySize: 0,
NoDefaultServerHeader: true,
NoDefaultDate: true,
ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
MaxConnsPerIP: 100,
}
// Setup listener and TLS
listener, err := net.Listen("tcp", address)
if err != nil {
log.Fatalf("Couldn't create listener: %s", err)
}
listener = tls.NewListener(listener, tlsConfig)
setupCertificates()
if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
go (func() {
challengePath := []byte("/.well-known/acme-challenge/")
err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) {
if bytes.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := challengeCache.Get(string(TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
if !ok || challenge == nil {
ctx.SetStatusCode(http.StatusNotFound)
ctx.SetBodyString("no challenge for this token")
}
ctx.SetBodyString(challenge.(string))
} else {
ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
}
})
if err != nil {
log.Fatalf("Couldn't start HTTP server: %s", err)
}
})()
}
// Start the web server
err = server.Serve(listener)
if err != nil {
log.Fatalf("Couldn't start server: %s", err)
if err := app.Run(os.Args); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
// envOr reads an environment variable and returns a default value if it's empty.
func envOr(env string, or string) string {
if v := os.Getenv(env); v != "" {
return v
}
return or
}

9
server/cache/interface.go vendored Normal file
View File

@ -0,0 +1,9 @@
package cache
import "time"
type SetGetKey interface {
Set(key string, value interface{}, ttl time.Duration) error
Get(key string) (interface{}, bool)
Remove(key string)
}

7
server/cache/setup.go vendored Normal file
View File

@ -0,0 +1,7 @@
package cache
import "github.com/OrlovEvgeny/go-mcache"
func NewKeyValueCache() SetGetKey {
return mcache.New()
}

27
server/certificates/acme_account.go Normal file
View File

@ -0,0 +1,27 @@
package certificates
import (
"crypto"
"github.com/go-acme/lego/v4/registration"
)
type AcmeAccount struct {
Email string
Registration *registration.Resource
Key crypto.PrivateKey `json:"-"`
KeyPEM string `json:"Key"`
}
// make sure AcmeAccount match User interface
var _ registration.User = &AcmeAccount{}
func (u *AcmeAccount) GetEmail() string {
return u.Email
}
func (u AcmeAccount) GetRegistration() *registration.Resource {
return u.Registration
}
func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
return u.Key
}

522
server/certificates/certificates.go Normal file
View File

@ -0,0 +1,522 @@
package certificates
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/gob"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/certificate"
"github.com/go-acme/lego/v4/challenge"
"github.com/go-acme/lego/v4/challenge/tlsalpn01"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/providers/dns"
"github.com/go-acme/lego/v4/registration"
"github.com/reugn/equalizer"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/database"
dnsutils "codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/upstream"
)
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
func TLSConfig(mainDomainSuffix []byte,
giteaRoot, giteaAPIToken, dnsProvider string,
acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
certDB database.CertDB) *tls.Config {
return &tls.Config{
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
sniBytes := []byte(sni)
if len(sni) < 1 {
return nil, errors.New("missing sni")
}
if info.SupportedProtos != nil {
for _, proto := range info.SupportedProtos {
if proto == tlsalpn01.ACMETLS1Protocol {
challenge, ok := challengeCache.Get(sni)
if !ok {
return nil, errors.New("no challenge for this domain")
}
cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
if err != nil {
return nil, err
}
return cert, nil
}
}
}
targetOwner := ""
if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) {
// deliver default certificate for the main domain (*.codeberg.page)
sniBytes = mainDomainSuffix
sni = string(sniBytes)
} else {
var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache)
if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs
sniBytes = mainDomainSuffix
sni = string(sniBytes)
} else {
_, _ = targetRepo, targetBranch
_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
if !valid {
sniBytes = mainDomainSuffix
sni = string(sniBytes)
}
}
}
if tlsCertificate, ok := keyCache.Get(sni); ok {
// we can use an existing certificate object
return tlsCertificate.(*tls.Certificate), nil
}
var tlsCertificate tls.Certificate
var err error
var ok bool
if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
// request a new certificate
if bytes.Equal(sniBytes, mainDomainSuffix) {
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
}
tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
return nil, err
}
}
if err := keyCache.Set(sni, &tlsCertificate, 15*time.Minute); err != nil {
return nil, err
}
return &tlsCertificate, nil
},
PreferServerCipherSuites: true,
NextProtos: []string{
"http/1.1",
tlsalpn01.ACMETLS1Protocol,
},
// generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
// https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
},
}
}
func checkUserLimit(user string) error {
userLimit, ok := acmeClientCertificateLimitPerUser[user]
if !ok {
// Each Codeberg user can only add 10 new domains per day.
userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
acmeClientCertificateLimitPerUser[user] = userLimit
}
if !userLimit.Ask() {
return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
}
return nil
}
var acmeClient, mainDomainAcmeClient *lego.Client
var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
type AcmeTLSChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeTLSChallengeProvider match Provider interface
var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
}
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
a.challengeCache.Remove(domain)
return nil
}
type AcmeHTTPChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeHTTPChallengeProvider match Provider interface
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
}
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token)
return nil
}
func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
// parse certificate from database
res, err := certDB.Get(sni)
if err != nil {
panic(err) // TODO: no panic
}
if res == nil {
return tls.Certificate{}, false
}
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
panic(err)
}
// TODO: document & put into own function
if !bytes.Equal(sni, mainDomainSuffix) {
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
if err != nil {
panic(err)
}
// renew certificates 7 days before they expire
if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
// TODO: add ValidUntil to custom res struct
if res.CSR != nil && len(res.CSR) > 0 {
// CSR stores the time when the renewal shall be tried again
nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
return tlsCertificate, true
}
}
go (func() {
res.CSR = nil // acme client doesn't like CSR to be set
tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("Couldn't renew certificate for %s: %s", sni, err)
}
})()
}
}
return tlsCertificate, true
}
var obtainLocks = sync.Map{}
func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
name := strings.TrimPrefix(domains[0], "*")
if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
domains = domains[1:]
}
// lock to avoid simultaneous requests
_, working := obtainLocks.LoadOrStore(name, struct{}{})
if working {
for working {
time.Sleep(100 * time.Millisecond)
_, working = obtainLocks.Load(name)
}
cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
if !ok {
return tls.Certificate{}, errors.New("certificate failed in synchronous request")
}
return cert, nil
}
defer obtainLocks.Delete(name)
if acmeClient == nil {
return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix), keyDatabase), nil
}
// request actual cert
var res *certificate.Resource
var err error
if renew != nil && renew.CertURL != "" {
if acmeUseRateLimits {
acmeClientRequestLimit.Take()
}
log.Printf("Renewing certificate for %v", domains)
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
if err != nil {
log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
res = nil
}
}
if res == nil {
if user != "" {
if err := checkUserLimit(user); err != nil {
return tls.Certificate{}, err
}
}
if acmeUseRateLimits {
acmeClientOrderLimit.Take()
acmeClientRequestLimit.Take()
}
log.Printf("Requesting new certificate for %v", domains)
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
Domains: domains,
Bundle: true,
MustStaple: false,
})
}
if err != nil {
log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
if renew != nil && renew.CertURL != "" {
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
if err := keyDatabase.Put(name, renew); err != nil {
return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
}
return tlsCertificate, nil
}
}
return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
}
log.Printf("Obtained certificate for %v", domains)
if err := keyDatabase.Put(name, res); err != nil {
return tls.Certificate{}, err
}
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
return tls.Certificate{}, err
}
return tlsCertificate, nil
}
func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
const configFile = "acme-account.json"
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
if account, err := ioutil.ReadFile(configFile); err == nil {
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
if err != nil {
return nil, err
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
// Validate Config
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
// TODO: should we fail hard instead?
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
return nil, err
}
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, err
}
myAcmeAccount = AcmeAccount{
Email: acmeMail,
Key: privateKey,
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
} else {
myAcmeAccount.Registration = reg
}
} else {
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
TermsOfServiceAgreed: acmeAcceptTerms,
Kid: acmeEabKID,
HmacEncoded: acmeEabHmac,
})
if err != nil {
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
} else {
myAcmeAccount.Registration = reg
}
}
if myAcmeAccount.Registration != nil {
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600)
if err != nil {
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
}
}
return myAcmeConfig, nil
}
func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
// getting main cert before ACME account so that we can fail here without hitting rate limits
mainCertBytes, err := certDB.Get(mainDomainSuffix)
if err != nil {
return fmt.Errorf("cert database is not working")
}
acmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
}
if enableHTTPServer {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
}
}
}
mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
if dnsProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
}
} else {
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
if err != nil {
log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
}
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
if err != nil {
log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
}
}
}
if mainCertBytes == nil {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
}
}
return nil
}
func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
for {
// clean up expired certs
now := time.Now()
expiredCertCount := 0
keyDatabaseIterator := certDB.Items()
key, resBytes, err := keyDatabaseIterator.Next()
for err == nil {
if !bytes.Equal(key, mainDomainSuffix) {
resGob := bytes.NewBuffer(resBytes)
resDec := gob.NewDecoder(resGob)
res := &certificate.Resource{}
err = resDec.Decode(res)
if err != nil {
panic(err)
}
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
if err != nil || !tlsCertificates[0].NotAfter.After(now) {
err := certDB.Delete(key)
if err != nil {
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
} else {
expiredCertCount++
}
}
}
key, resBytes, err = keyDatabaseIterator.Next()
}
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
// compact the database
result, err := certDB.Compact()
if err != nil {
log.Printf("[ERROR] Compacting key database failed: %s", err)
} else {
log.Printf("[INFO] Compacted key database (%+v)", result)
}
// update main cert
res, err := certDB.Get(mainDomainSuffix)
if err != nil {
log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
} else if res == nil {
log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
} else {
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
// renew main certificate 30 days before it expires
if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
go (func() {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
}
})()
}
}
select {
case <-ctx.Done():
return
case <-time.After(interval):
}
}
}

86
server/certificates/mock.go Normal file
View File

@ -0,0 +1,86 @@
package certificates
import (
"bytes"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"time"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/certificate"
"codeberg.org/codeberg/pages/server/database"
)
func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) tls.Certificate {
key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
if err != nil {
panic(err)
}
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: domain,
Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
OrganizationalUnit: []string{
"Will not try again for 6 hours to avoid hitting rate limits for your domain.",
"Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
"free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
"Error message: " + msg,
},
},
// certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6),
NotBefore: time.Now(),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
certBytes, err := x509.CreateCertificate(
rand.Reader,
&template,
&template,
&key.(*rsa.PrivateKey).PublicKey,
key,
)
if err != nil {
panic(err)
}
out := &bytes.Buffer{}
err = pem.Encode(out, &pem.Block{
Bytes: certBytes,
Type: "CERTIFICATE",
})
if err != nil {
panic(err)
}
outBytes := out.Bytes()
res := &certificate.Resource{
PrivateKey: certcrypto.PEMEncode(key),
Certificate: outBytes,
IssuerCertificate: outBytes,
Domain: domain,
}
databaseName := domain
if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] {
databaseName = mainDomainSuffix
}
if err := keyDatabase.Put(databaseName, res); err != nil {
panic(err)
}
tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
if err != nil {
panic(err)
}
return tlsCertificate
}

15
server/database/interface.go Normal file
View File

@ -0,0 +1,15 @@
package database
import (
"github.com/akrylysov/pogreb"
"github.com/go-acme/lego/v4/certificate"
)
type CertDB interface {
Close() error
Put(name string, cert *certificate.Resource) error
Get(name []byte) (*certificate.Resource, error)
Delete(key []byte) error
Compact() (pogreb.CompactionResult, error)
Items() *pogreb.ItemIterator
}

117
server/database/setup.go Normal file
View File

@ -0,0 +1,117 @@
package database
import (
"bytes"
"context"
"encoding/gob"
"fmt"
"time"
"github.com/akrylysov/pogreb"
"github.com/akrylysov/pogreb/fs"
"github.com/go-acme/lego/v4/certificate"
"github.com/rs/zerolog/log"
)
type aDB struct {
ctx context.Context
cancel context.CancelFunc
intern *pogreb.DB
syncInterval time.Duration
}
func (p aDB) Close() error {
p.cancel()
return p.intern.Sync()
}
func (p aDB) Put(name string, cert *certificate.Resource) error {
var resGob bytes.Buffer
if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
return err
}
return p.intern.Put([]byte(name), resGob.Bytes())
}
func (p aDB) Get(name []byte) (*certificate.Resource, error) {
cert := &certificate.Resource{}
resBytes, err := p.intern.Get(name)
if err != nil {
return nil, err
}
if resBytes == nil {
return nil, nil
}
if err = gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
return nil, err
}
return cert, nil
}
func (p aDB) Delete(key []byte) error {
return p.intern.Delete(key)
}
func (p aDB) Compact() (pogreb.CompactionResult, error) {
return p.intern.Compact()
}
func (p aDB) Items() *pogreb.ItemIterator {
return p.intern.Items()
}
var _ CertDB = &aDB{}
func (p aDB) sync() {
for {
err := p.intern.Sync()
if err != nil {
log.Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():
return
case <-time.After(p.syncInterval):
}
}
}
func (p aDB) compact() {
for {
err := p.intern.Sync()
if err != nil {
log.Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():
return
case <-time.After(p.syncInterval):
}
}
}
func New(path string) (CertDB, error) {
if path == "" {
return nil, fmt.Errorf("path not set")
}
db, err := pogreb.Open(path, &pogreb.Options{
BackgroundSyncInterval: 30 * time.Second,
BackgroundCompactionInterval: 6 * time.Hour,
FileSystem: fs.OSMMap,
})
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
result := &aDB{
ctx: ctx,
cancel: cancel,
intern: db,
syncInterval: 5 * time.Minute,
}
go result.sync()
return result, nil
}

6
server/dns/const.go Normal file
View File

@ -0,0 +1,6 @@
package dns
import "time"
// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
var lookupCacheTimeout = 15 * time.Minute

56
server/dns/dns.go Normal file
View File

@ -0,0 +1,56 @@
package dns
import (
"net"
"strings"
"codeberg.org/codeberg/pages/server/cache"
)
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data.
func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT
var cname string
var err error
if cachedName, ok := dnsLookupCache.Get(domain); ok {
cname = cachedName.(string)
} else {
cname, err = net.LookupCNAME(domain)
cname = strings.TrimSuffix(cname, ".")
if err != nil || !strings.HasSuffix(cname, mainDomainSuffix) {
cname = ""
// TODO: check if the A record matches!
names, err := net.LookupTXT(domain)
if err == nil {
for _, name := range names {
name = strings.TrimSuffix(name, ".")
if strings.HasSuffix(name, mainDomainSuffix) {
cname = name
break
}
}
}
}
_ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout)
}
if cname == "" {
return
}
cnameParts := strings.Split(strings.TrimSuffix(cname, mainDomainSuffix), ".")
targetOwner = cnameParts[len(cnameParts)-1]
if len(cnameParts) > 1 {
targetRepo = cnameParts[len(cnameParts)-2]
}
if len(cnameParts) > 2 {
targetBranch = cnameParts[len(cnameParts)-3]
}
if targetRepo == "" {
targetRepo = "pages"
}
if targetBranch == "" && targetRepo != "pages" {
targetBranch = "pages"
}
// if targetBranch is still empty, the caller must find the default branch
return
}

292
server/handler.go Normal file
View File

@ -0,0 +1,292 @@
package server
import (
"bytes"
"strings"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/upstream"
"codeberg.org/codeberg/pages/server/utils"
)
// Handler handles a single HTTP request to the web server.
func Handler(mainDomainSuffix, rawDomain []byte,
giteaRoot, rawInfoPage, giteaAPIToken string,
blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) {
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
ctx.Response.Header.Set("Server", "Codeberg Pages")
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
// Enable browser caching for up to 10 minutes
ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
trimmedHost := utils.TrimHostPort(ctx.Request.Host())
// Add HSTS for RawDomain and MainDomainSuffix
if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
ctx.Response.Header.Set("Strict-Transport-Security", hsts)
}
// Block all methods not required for static pages
if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
return
}
// Block blacklisted paths (like ACME challenges)
for _, blacklistedPath := range blacklistedPaths {
if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
html.ReturnErrorPage(ctx, fasthttp.StatusForbidden)
return
}
}
// Allow CORS for specified domains
if ctx.IsOptions() {
allowCors := false
for _, allowedCorsDomain := range allowedCorsDomains {
if bytes.Equal(trimmedHost, allowedCorsDomain) {
allowCors = true
break
}
}
if allowCors {
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
}
ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
return
}
// Prepare request information to Gitea
var targetOwner, targetRepo, targetBranch, targetPath string
var targetOptions = &upstream.Options{
ForbiddenMimeTypes: map[string]struct{}{},
TryIndexPages: true,
}
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL.
var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
if repo == "" {
return false
}
// Check if the branch exists, otherwise treat it as a file path
branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
if branchTimestampResult == nil {
// branch doesn't exist
return false
}
// Branch exists, use it
targetRepo = repo
targetPath = strings.Trim(strings.Join(path, "/"), "/")
targetBranch = branchTimestampResult.Branch
targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
if canonicalLink != "" {
// Hide from search machines & add canonical link
ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
ctx.Response.Header.Set("Link",
strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
"; rel=\"canonical\"",
)
}
return true
}
log.Debug().Msg("preparations")
if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
// Serve raw content from RawDomain
log.Debug().Msg("raw domain")
targetOptions.TryIndexPages = false
targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
if len(pathElements) < 2 {
// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect)
return
}
targetOwner = pathElements[0]
targetRepo = pathElements[1]
// raw.codeberg.org/example/myrepo/@main/index.html
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
log.Debug().Msg("raw domain preparations, now trying with specified branch")
if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
log.Debug().Msg("missing branch")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
log.Debug().Msg("raw domain preparations, now trying with default branch")
tryBranch(targetRepo, "", pathElements[2:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
)
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
// Serve pages from subdomains of MainDomainSuffix
log.Debug().Msg("main domain suffix")
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
targetRepo = pathElements[0]
targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
if targetOwner == "www" {
// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect)
return
}
// Check if the first directory is a repo with the second directory as a branch
// example.codeberg.page/myrepo/@main/index.html
if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
if targetRepo == "pages" {
// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
return
}
log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
"/"+pathElements[0]+"/%p",
) {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
}
// Check if the first directory is a branch for the "pages" repo
// example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") {
log.Debug().Msg("main domain preparations, now trying with specified branch")
if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
}
// Check if the first directory is a repo with a "pages" branch
// example.codeberg.page/myrepo/index.html
// example.codeberg.page/pages/... is not allowed here.
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
// Try to use the "pages" repo on its default branch
// example.codeberg.page/index.html
log.Debug().Msg("main domain preparations, now trying with default repo/branch")
if tryBranch("pages", "", pathElements, "") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
// Couldn't find a valid repo/branch
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
} else {
trimmedHostStr := string(trimmedHost)
// Serve pages from external domains
targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
if targetOwner == "" {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
canonicalLink := ""
if strings.HasPrefix(pathElements[0], "@") {
targetBranch = pathElements[0][1:]
pathElements = pathElements[1:]
canonicalLink = "/%p"
}
// Try to use the given repo on the given branch or the default branch
log.Debug().Msg("custom domain preparations, now trying with details from DNS")
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
if !valid {
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHostStr {
// only redirect if the target is also a codeberg page!
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
if targetOwner != "" {
ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
return
}
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
}
}

24
handler_test.go → server/handler_test.go
View File

@ -1,13 +1,29 @@
package main
package server
import (
"fmt"
"github.com/valyala/fasthttp"
"testing"
"time"
"codeberg.org/codeberg/pages/server/cache"
)
func TestHandlerPerformance(t *testing.T) {
testHandler := Handler(
[]byte("codeberg.page"),
[]byte("raw.codeberg.org"),
"https://codeberg.org",
"https://docs.codeberg.org/pages/raw-content/",
"",
[][]byte{[]byte("/.well-known/acme-challenge/")},
[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
)
ctx := &fasthttp.RequestCtx{
Request: *fasthttp.AcquireRequest(),
Response: *fasthttp.AcquireResponse(),
@ -15,7 +31,7 @@ func TestHandlerPerformance(t *testing.T) {
ctx.Request.SetRequestURI("http://mondstern.codeberg.page/")
fmt.Printf("Start: %v\n", time.Now())
start := time.Now()
handler(ctx)
testHandler(ctx)
end := time.Now()
fmt.Printf("Done: %v\n", time.Now())
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
@ -28,7 +44,7 @@ func TestHandlerPerformance(t *testing.T) {
ctx.Response.ResetBody()
fmt.Printf("Start: %v\n", time.Now())
start = time.Now()
handler(ctx)
testHandler(ctx)
end = time.Now()
fmt.Printf("Done: %v\n", time.Now())
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
@ -42,7 +58,7 @@ func TestHandlerPerformance(t *testing.T) {
ctx.Request.SetRequestURI("http://example.momar.xyz/")
fmt.Printf("Start: %v\n", time.Now())
start = time.Now()
handler(ctx)
testHandler(ctx)
end = time.Now()
fmt.Printf("Done: %v\n", time.Now())
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 1 {

15
server/helpers.go Normal file
View File

@ -0,0 +1,15 @@
package server
import (
"bytes"
)
// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
// string for custom domains.
func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string {
if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) {
return "max-age=63072000; includeSubdomains; preload"
} else {
return ""
}
}

47
server/setup.go Normal file
View File

@ -0,0 +1,47 @@
package server
import (
"bytes"
"net/http"
"time"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/utils"
)
func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
// Enable compression by wrapping the handler with the compression function provided by FastHTTP
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
return &fasthttp.Server{
Handler: compressedHandler,
DisablePreParseMultipartForm: true,
MaxRequestBodySize: 0,
NoDefaultServerHeader: true,
NoDefaultDate: true,
ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
MaxConnsPerIP: 100,
}
}
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server {
challengePath := []byte("/.well-known/acme-challenge/")
return &fasthttp.Server{
Handler: func(ctx *fasthttp.RequestCtx) {
if bytes.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
if !ok || challenge == nil {
ctx.SetStatusCode(http.StatusNotFound)
ctx.SetBodyString("no challenge for this token")
}
ctx.SetBodyString(challenge.(string))
} else {
ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
}
},
}
}

49
server/try.go Normal file
View File

@ -0,0 +1,49 @@
package server
import (
"bytes"
"strings"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/upstream"
)
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
func tryUpstream(ctx *fasthttp.RequestCtx,
mainDomainSuffix, trimmedHost []byte,
targetOptions *upstream.Options,
targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken string,
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) {
// check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
canonicalPath := string(ctx.RequestURI())
if targetRepo != "pages" {
path := strings.SplitN(canonicalPath, "/", 3)
if len(path) >= 3 {
canonicalPath = "/" + path[2]
}
}
ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
return
}
}
targetOptions.TargetOwner = targetOwner
targetOptions.TargetRepo = targetRepo
targetOptions.TargetBranch = targetBranch
targetOptions.TargetPath = targetPath
// Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
}
}

21
server/upstream/const.go Normal file
View File

@ -0,0 +1,21 @@
package upstream
import "time"
// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
var defaultBranchCacheTimeout = 15 * time.Minute
// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
// picked up faster, while still allowing the content to be cached longer if nothing changes.
var branchExistenceCacheTimeout = 5 * time.Minute
// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
// on your available memory.
var fileCacheTimeout = 5 * time.Minute
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
var fileCacheSizeLimit = 1024 * 1024
// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
var canonicalDomainCacheTimeout = 15 * time.Minute

53
server/upstream/domains.go Normal file
View File

@ -0,0 +1,53 @@
package upstream
import (
"strings"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
)
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
domains := []string{}
valid := false
if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
domains = cachedValue.([]string)
for _, domain := range domains {
if domain == actualDomain {
valid = true
break
}
}
} else {
req := fasthttp.AcquireRequest()
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
res := fasthttp.AcquireResponse()
err := client.Do(req, res)
if err == nil && res.StatusCode() == fasthttp.StatusOK {
for _, domain := range strings.Split(string(res.Body()), "\n") {
domain = strings.ToLower(domain)
domain = strings.TrimSpace(domain)
domain = strings.TrimPrefix(domain, "http://")
domain = strings.TrimPrefix(domain, "https://")
if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
domains = append(domains, domain)
}
if domain == actualDomain {
valid = true
}
}
}
domains = append(domains, targetOwner+mainDomainSuffix)
if domains[len(domains)-1] == actualDomain {
valid = true
}
if targetRepo != "" && targetRepo != "pages" {
domains[len(domains)-1] += "/" + targetRepo
}
_ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout)
}
return domains[0], valid
}

55
server/upstream/helper.go Normal file
View File

@ -0,0 +1,55 @@
package upstream
import (
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
"codeberg.org/codeberg/pages/server/cache"
)
type branchTimestamp struct {
Branch string
Timestamp time.Time
}
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist)
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
if result == nil {
return nil
}
return result.(*branchTimestamp)
}
result := &branchTimestamp{}
result.Branch = branch
if branch == "" {
// Get default branch
var body = make([]byte, 0)
// TODO: use header for API key?
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
return nil
}
result.Branch = fastjson.GetString(body, "default_branch")
}
var body = make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
return nil
}
result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
return result
}
type fileResponse struct {
exists bool
mimeType string
body []byte
}

202
server/upstream/upstream.go Normal file
View File

@ -0,0 +1,202 @@
package upstream
import (
"bytes"
"fmt"
"io"
"mime"
"path"
"strconv"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
)
// upstreamIndexPages lists pages that may be considered as index pages for directories.
var upstreamIndexPages = []string{
"index.html",
}
// Options provides various options for the upstream request.
type Options struct {
TargetOwner,
TargetRepo,
TargetBranch,
TargetPath,
DefaultMimeType string
ForbiddenMimeTypes map[string]struct{}
TryIndexPages bool
BranchTimestamp time.Time
// internal
appendTrailingSlash bool
redirectIfExists string
}
var client = fasthttp.Client{
ReadTimeout: 10 * time.Second,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
if o.ForbiddenMimeTypes == nil {
o.ForbiddenMimeTypes = map[string]struct{}{}
}
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return true
}
o.TargetBranch = branch.Branch
o.BranchTimestamp = branch.Timestamp
}
if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
return true
}
// Check if the browser has a cached version
if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
if !ifModifiedSince.Before(o.BranchTimestamp) {
ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
return true
}
}
log.Debug().Msg("preparations")
// Make a GET request to the upstream URL
uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
var req *fasthttp.Request
var res *fasthttp.Response
var cachedResponse fileResponse
var err error
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
cachedResponse = cachedValue.(fileResponse)
} else {
req = fasthttp.AcquireRequest()
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
res = fasthttp.AcquireResponse()
res.SetBodyStream(&strings.Reader{}, -1)
err = client.Do(req, res)
}
log.Debug().Msg("acquisition")
// Handle errors
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
optionsForIndexPages.TryIndexPages = false
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)
return true
}
}
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if res != nil {
// Update cache if the request is fresh
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)
}
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// o.appendTrailingSlash is only true when looking for index pages
if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
return true
}
if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
return true
}
if o.redirectIfExists != "" {
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
return true
}
log.Debug().Msg("error handling")
// Set the MIME type
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
ctx.Response.Header.SetContentType(mimeType)
// Everything's okay so far
ctx.Response.SetStatusCode(fasthttp.StatusOK)
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
log.Debug().Msg("response preparations")
// Write the response body to the original request
var cacheBodyWriter bytes.Buffer
if res != nil {
if res.Header.ContentLength() > fileCacheSizeLimit {
err = res.BodyWriteTo(ctx.Response.BodyWriter())
} else {
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
}
} else {
_, err = ctx.Write(cachedResponse.body)
}
if err != nil {
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
log.Debug().Msg("response")
if res != nil && ctx.Err() == nil {
cachedResponse.exists = true
cachedResponse.mimeType = mimeType
cachedResponse.body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
}
return true
}

11
server/utils/utils.go Normal file
View File

@ -0,0 +1,11 @@
package utils
import "bytes"
func TrimHostPort(host []byte) []byte {
i := bytes.IndexByte(host, ':')
if i >= 0 {
return host[:i]
}
return host
}

13
server/utils/utils_test.go Normal file
View File

@ -0,0 +1,13 @@
package utils
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTrimHostPort(t *testing.T) {
assert.EqualValues(t, "aa", TrimHostPort([]byte("aa")))
assert.EqualValues(t, "", TrimHostPort([]byte(":")))
assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80")))
}