diff --git a/Justfile b/Justfile
index a5feb7a..3d23175 100644
--- a/Justfile
+++ b/Justfile
@@ -6,7 +6,7 @@ dev:
export PAGES_DOMAIN=localhost.mock.directory
export RAW_DOMAIN=raw.localhost.mock.directory
export PORT=4430
- go run .
+ go run . --verbose
build:
CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
diff --git a/README.md b/README.md
index 35c230e..087d7ae 100644
--- a/README.md
+++ b/README.md
@@ -2,10 +2,10 @@
- `HOST` & `PORT` (default: `[::]` & `443`): listen address.
- `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
-- `RAW_DOMAIN` (default: `raw.codeberg.org`): domain for raw resources.
+- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources.
- `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
- `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
-- `REDIRECT_RAW_INFO` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
+- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).
ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
- `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider.
@@ -15,3 +15,25 @@
- `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
- `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.
See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
+- `DEBUG` (default: false): Set this to true to enable debug logging.
+
+```
+// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
+//
+// Mapping custom domains is not static anymore, but can be done with DNS:
+//
+// 1) add a ".domains" text file to your repository, containing the allowed domains, separated by new lines. The
+// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
+//
+// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
+// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
+// www.example.org. IN CNAME main.pages.example.codeberg.page.
+//
+// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
+// for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
+// record that points to your repo (just like the CNAME record):
+// example.org IN ALIAS codeberg.page.
+// example.org IN TXT main.pages.example.codeberg.page.
+//
+// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
+```
diff --git a/certificates.go b/certificates.go
deleted file mode 100644
index db51020..0000000
--- a/certificates.go
+++ /dev/null
@@ -1,599 +0,0 @@
-package main
-
-import (
- "bytes"
- "crypto"
- "crypto/ecdsa"
- "crypto/elliptic"
- "crypto/rand"
- "crypto/rsa"
- "crypto/tls"
- "crypto/x509"
- "crypto/x509/pkix"
- "encoding/gob"
- "encoding/json"
- "encoding/pem"
- "errors"
- "github.com/OrlovEvgeny/go-mcache"
- "github.com/akrylysov/pogreb/fs"
- "github.com/go-acme/lego/v4/certificate"
- "github.com/go-acme/lego/v4/challenge"
- "github.com/go-acme/lego/v4/challenge/tlsalpn01"
- "github.com/go-acme/lego/v4/providers/dns"
- "io/ioutil"
- "log"
- "math/big"
- "os"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/akrylysov/pogreb"
- "github.com/reugn/equalizer"
-
- "github.com/go-acme/lego/v4/certcrypto"
- "github.com/go-acme/lego/v4/lego"
- "github.com/go-acme/lego/v4/registration"
-)
-
-// tlsConfig contains the configuration for generating, serving and cleaning up Let's Encrypt certificates.
-var tlsConfig = &tls.Config{
- // check DNS name & get certificate from Let's Encrypt
- GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
- sni := strings.ToLower(strings.TrimSpace(info.ServerName))
- sniBytes := []byte(sni)
- if len(sni) < 1 {
- return nil, errors.New("missing sni")
- }
-
- if info.SupportedProtos != nil {
- for _, proto := range info.SupportedProtos {
- if proto == tlsalpn01.ACMETLS1Protocol {
- challenge, ok := challengeCache.Get(sni)
- if !ok {
- return nil, errors.New("no challenge for this domain")
- }
- cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
- if err != nil {
- return nil, err
- }
- return cert, nil
- }
- }
- }
-
- targetOwner := ""
- if bytes.HasSuffix(sniBytes, MainDomainSuffix) || bytes.Equal(sniBytes, MainDomainSuffix[1:]) {
- // deliver default certificate for the main domain (*.codeberg.page)
- sniBytes = MainDomainSuffix
- sni = string(sniBytes)
- } else {
- var targetRepo, targetBranch string
- targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni)
- if targetOwner == "" {
- // DNS not set up, return main certificate to redirect to the docs
- sniBytes = MainDomainSuffix
- sni = string(sniBytes)
- } else {
- _, _ = targetRepo, targetBranch
- _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni)
- if !valid {
- sniBytes = MainDomainSuffix
- sni = string(sniBytes)
- }
- }
- }
-
- if tlsCertificate, ok := keyCache.Get(sni); ok {
- // we can use an existing certificate object
- return tlsCertificate.(*tls.Certificate), nil
- }
-
- var tlsCertificate tls.Certificate
- var err error
- var ok bool
- if tlsCertificate, ok = retrieveCertFromDB(sniBytes); !ok {
- // request a new certificate
- if bytes.Equal(sniBytes, MainDomainSuffix) {
- return nil, errors.New("won't request certificate for main domain, something really bad has happened")
- }
-
- tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner)
- if err != nil {
- return nil, err
- }
- }
-
- err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute)
- if err != nil {
- panic(err)
- }
- return &tlsCertificate, nil
- },
- PreferServerCipherSuites: true,
- NextProtos: []string{
- "http/1.1",
- tlsalpn01.ACMETLS1Protocol,
- },
-
- // generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
- // https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
- MinVersion: tls.VersionTLS12,
- CipherSuites: []uint16{
- tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
- tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
- tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
- tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
- tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
- },
-}
-
-var keyCache = mcache.New()
-var keyDatabase, keyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{
- BackgroundSyncInterval: 30 * time.Second,
- BackgroundCompactionInterval: 6 * time.Hour,
- FileSystem: fs.OSMMap,
-})
-
-func CheckUserLimit(user string) error {
- userLimit, ok := acmeClientCertificateLimitPerUser[user]
- if !ok {
- // Each Codeberg user can only add 10 new domains per day.
- userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
- acmeClientCertificateLimitPerUser[user] = userLimit
- }
- if !userLimit.Ask() {
- return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
- }
- return nil
-}
-
-var myAcmeAccount AcmeAccount
-var myAcmeConfig *lego.Config
-
-type AcmeAccount struct {
- Email string
- Registration *registration.Resource
- Key crypto.PrivateKey `json:"-"`
- KeyPEM string `json:"Key"`
-}
-
-func (u *AcmeAccount) GetEmail() string {
- return u.Email
-}
-func (u AcmeAccount) GetRegistration() *registration.Resource {
- return u.Registration
-}
-func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
- return u.Key
-}
-
-var acmeClient, mainDomainAcmeClient *lego.Client
-var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
-
-// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
-// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
-var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
-
-// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
-var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
-
-var challengeCache = mcache.New()
-
-type AcmeTLSChallengeProvider struct{}
-
-var _ challenge.Provider = AcmeTLSChallengeProvider{}
-
-func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
- return challengeCache.Set(domain, keyAuth, 1*time.Hour)
-}
-func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
- challengeCache.Remove(domain)
- return nil
-}
-
-type AcmeHTTPChallengeProvider struct{}
-
-var _ challenge.Provider = AcmeHTTPChallengeProvider{}
-
-func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
- return challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
-}
-func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
- challengeCache.Remove(domain + "/" + token)
- return nil
-}
-
-func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) {
- // parse certificate from database
- res := &certificate.Resource{}
- if !PogrebGet(keyDatabase, sni, res) {
- return tls.Certificate{}, false
- }
-
- tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
- if err != nil {
- panic(err)
- }
-
- if !bytes.Equal(sni, MainDomainSuffix) {
- tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
- if err != nil {
- panic(err)
- }
-
- // renew certificates 7 days before they expire
- if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
- if res.CSR != nil && len(res.CSR) > 0 {
- // CSR stores the time when the renewal shall be tried again
- nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
- if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
- return tlsCertificate, true
- }
- }
- go (func() {
- res.CSR = nil // acme client doesn't like CSR to be set
- tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "")
- if err != nil {
- log.Printf("Couldn't renew certificate for %s: %s", sni, err)
- }
- })()
- }
- }
-
- return tlsCertificate, true
-}
-
-var obtainLocks = sync.Map{}
-
-func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string) (tls.Certificate, error) {
- name := strings.TrimPrefix(domains[0], "*")
- if os.Getenv("DNS_PROVIDER") == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
- domains = domains[1:]
- }
-
- // lock to avoid simultaneous requests
- _, working := obtainLocks.LoadOrStore(name, struct{}{})
- if working {
- for working {
- time.Sleep(100 * time.Millisecond)
- _, working = obtainLocks.Load(name)
- }
- cert, ok := retrieveCertFromDB([]byte(name))
- if !ok {
- return tls.Certificate{}, errors.New("certificate failed in synchronous request")
- }
- return cert, nil
- }
- defer obtainLocks.Delete(name)
-
- if acmeClient == nil {
- return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!"), nil
- }
-
- // request actual cert
- var res *certificate.Resource
- var err error
- if renew != nil && renew.CertURL != "" {
- if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
- acmeClientRequestLimit.Take()
- }
- log.Printf("Renewing certificate for %v", domains)
- res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
- if err != nil {
- log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
- res = nil
- }
- }
- if res == nil {
- if user != "" {
- if err := CheckUserLimit(user); err != nil {
- return tls.Certificate{}, err
- }
- }
-
- if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
- acmeClientOrderLimit.Take()
- acmeClientRequestLimit.Take()
- }
- log.Printf("Requesting new certificate for %v", domains)
- res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
- Domains: domains,
- Bundle: true,
- MustStaple: false,
- })
- }
- if err != nil {
- log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
- if renew != nil && renew.CertURL != "" {
- tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
- if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
- // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
- renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
- PogrebPut(keyDatabase, []byte(name), renew)
- return tlsCertificate, nil
- }
- } else {
- return mockCert(domains[0], err.Error()), err
- }
- }
- log.Printf("Obtained certificate for %v", domains)
-
- PogrebPut(keyDatabase, []byte(name), res)
- tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
- if err != nil {
- return tls.Certificate{}, err
- }
- return tlsCertificate, nil
-}
-
-func mockCert(domain string, msg string) tls.Certificate {
- key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
- if err != nil {
- panic(err)
- }
-
- template := x509.Certificate{
- SerialNumber: big.NewInt(1),
- Subject: pkix.Name{
- CommonName: domain,
- Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
- OrganizationalUnit: []string{
- "Will not try again for 6 hours to avoid hitting rate limits for your domain.",
- "Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
- "free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
- "Error message: " + msg,
- },
- },
-
- // certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
- NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6),
- NotBefore: time.Now(),
-
- KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
- BasicConstraintsValid: true,
- }
- certBytes, err := x509.CreateCertificate(
- rand.Reader,
- &template,
- &template,
- &key.(*rsa.PrivateKey).PublicKey,
- key,
- )
- if err != nil {
- panic(err)
- }
-
- out := &bytes.Buffer{}
- err = pem.Encode(out, &pem.Block{
- Bytes: certBytes,
- Type: "CERTIFICATE",
- })
- if err != nil {
- panic(err)
- }
- outBytes := out.Bytes()
- res := &certificate.Resource{
- PrivateKey: certcrypto.PEMEncode(key),
- Certificate: outBytes,
- IssuerCertificate: outBytes,
- Domain: domain,
- }
- databaseName := domain
- if domain == "*"+string(MainDomainSuffix) || domain == string(MainDomainSuffix[1:]) {
- databaseName = string(MainDomainSuffix)
- }
- PogrebPut(keyDatabase, []byte(databaseName), res)
-
- tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
- if err != nil {
- panic(err)
- }
- return tlsCertificate
-}
-
-func setupCertificates() {
- if keyDatabaseErr != nil {
- panic(keyDatabaseErr)
- }
-
- if os.Getenv("ACME_ACCEPT_TERMS") != "true" || (os.Getenv("DNS_PROVIDER") == "" && os.Getenv("ACME_API") != "https://acme.mock.directory") {
- panic(errors.New("you must set ACME_ACCEPT_TERMS and DNS_PROVIDER, unless ACME_API is set to https://acme.mock.directory"))
- }
-
- // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits
- mainCertBytes, err := keyDatabase.Get(MainDomainSuffix)
- if err != nil {
- // key database is not working
- panic(err)
- }
-
- if account, err := ioutil.ReadFile("acme-account.json"); err == nil {
- err = json.Unmarshal(account, &myAcmeAccount)
- if err != nil {
- panic(err)
- }
- myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
- if err != nil {
- panic(err)
- }
- myAcmeConfig = lego.NewConfig(&myAcmeAccount)
- myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
- myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
- _, err := lego.NewClient(myAcmeConfig)
- if err != nil {
- log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
- }
- } else if os.IsNotExist(err) {
- privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
- if err != nil {
- panic(err)
- }
- myAcmeAccount = AcmeAccount{
- Email: envOr("ACME_EMAIL", "noreply@example.email"),
- Key: privateKey,
- KeyPEM: string(certcrypto.PEMEncode(privateKey)),
- }
- myAcmeConfig = lego.NewConfig(&myAcmeAccount)
- myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
- myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
- tempClient, err := lego.NewClient(myAcmeConfig)
- if err != nil {
- log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
- } else {
- // accept terms & log in to EAB
- if os.Getenv("ACME_EAB_KID") == "" || os.Getenv("ACME_EAB_HMAC") == "" {
- reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true"})
- if err != nil {
- log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
- } else {
- myAcmeAccount.Registration = reg
- }
- } else {
- reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
- TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true",
- Kid: os.Getenv("ACME_EAB_KID"),
- HmacEncoded: os.Getenv("ACME_EAB_HMAC"),
- })
- if err != nil {
- log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
- } else {
- myAcmeAccount.Registration = reg
- }
- }
-
- if myAcmeAccount.Registration != nil {
- acmeAccountJson, err := json.Marshal(myAcmeAccount)
- if err != nil {
- log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
- select {}
- }
- err = ioutil.WriteFile("acme-account.json", acmeAccountJson, 0600)
- if err != nil {
- log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
- select {}
- }
- }
- }
- } else {
- panic(err)
- }
-
- acmeClient, err = lego.NewClient(myAcmeConfig)
- if err != nil {
- log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
- } else {
- err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
- if err != nil {
- log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
- }
- if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
- err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{})
- if err != nil {
- log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
- }
- }
- }
-
- mainDomainAcmeClient, err = lego.NewClient(myAcmeConfig)
- if err != nil {
- log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
- } else {
- if os.Getenv("DNS_PROVIDER") == "" {
- // using mock server, don't use wildcard certs
- err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
- if err != nil {
- log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
- }
- } else {
- provider, err := dns.NewDNSChallengeProviderByName(os.Getenv("DNS_PROVIDER"))
- if err != nil {
- log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
- }
- err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
- if err != nil {
- log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
- }
- }
- }
-
- if mainCertBytes == nil {
- _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, nil, "")
- if err != nil {
- log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
- }
- }
-
- go (func() {
- for {
- err := keyDatabase.Sync()
- if err != nil {
- log.Printf("[ERROR] Syncinc key database failed: %s", err)
- }
- time.Sleep(5 * time.Minute)
- }
- })()
- go (func() {
- for {
- // clean up expired certs
- now := time.Now()
- expiredCertCount := 0
- keyDatabaseIterator := keyDatabase.Items()
- key, resBytes, err := keyDatabaseIterator.Next()
- for err == nil {
- if !bytes.Equal(key, MainDomainSuffix) {
- resGob := bytes.NewBuffer(resBytes)
- resDec := gob.NewDecoder(resGob)
- res := &certificate.Resource{}
- err = resDec.Decode(res)
- if err != nil {
- panic(err)
- }
-
- tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
- if err != nil || !tlsCertificates[0].NotAfter.After(now) {
- err := keyDatabase.Delete(key)
- if err != nil {
- log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
- } else {
- expiredCertCount++
- }
- }
- }
- key, resBytes, err = keyDatabaseIterator.Next()
- }
- log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
-
- // compact the database
- result, err := keyDatabase.Compact()
- if err != nil {
- log.Printf("[ERROR] Compacting key database failed: %s", err)
- } else {
- log.Printf("[INFO] Compacted key database (%+v)", result)
- }
-
- // update main cert
- res := &certificate.Resource{}
- if !PogrebGet(keyDatabase, MainDomainSuffix, res) {
- log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
- } else {
- tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
-
- // renew main certificate 30 days before it expires
- if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
- go (func() {
- _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, res, "")
- if err != nil {
- log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
- }
- })()
- }
- }
-
- time.Sleep(12 * time.Hour)
- }
- })()
-}
diff --git a/cmd/certs.go b/cmd/certs.go
new file mode 100644
index 0000000..6603e55
--- /dev/null
+++ b/cmd/certs.go
@@ -0,0 +1,44 @@
+package cmd
+
+import (
+ "fmt"
+ "os"
+
+ "github.com/urfave/cli/v2"
+
+ "codeberg.org/codeberg/pages/server/database"
+)
+
+var Certs = &cli.Command{
+ Name: "certs",
+ Usage: "manage certs manually",
+ Action: certs,
+}
+
+func certs(ctx *cli.Context) error {
+ if ctx.Args().Len() >= 1 && ctx.Args().First() == "--remove-certificate" {
+ if ctx.Args().Len() == 1 {
+ println("--remove-certificate requires at least one domain as an argument")
+ os.Exit(1)
+ }
+
+ domains := ctx.Args().Slice()[2:]
+
+ // TODO: make "key-database.pogreb" set via flag
+ keyDatabase, err := database.New("key-database.pogreb")
+ if err != nil {
+ return fmt.Errorf("could not create database: %v", err)
+ }
+
+ for _, domain := range domains {
+ if err := keyDatabase.Delete([]byte(domain)); err != nil {
+ panic(err)
+ }
+ }
+ if err := keyDatabase.Close(); err != nil {
+ panic(err)
+ }
+ os.Exit(0)
+ }
+ return nil
+}
diff --git a/cmd/flags.go b/cmd/flags.go
new file mode 100644
index 0000000..e1838c2
--- /dev/null
+++ b/cmd/flags.go
@@ -0,0 +1,110 @@
+package cmd
+
+import (
+ "github.com/urfave/cli/v2"
+)
+
+var ServeFlags = []cli.Flag{
+ &cli.BoolFlag{
+ Name: "verbose",
+ // TODO: Usage
+ EnvVars: []string{"DEBUG"},
+ },
+
+ // MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
+ // pages, or used for comparison in CNAME lookups. Static pages can be accessed through
+ // https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
+ &cli.StringFlag{
+ Name: "pages-domain",
+ Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
+ EnvVars: []string{"PAGES_DOMAIN"},
+ Value: "codeberg.page",
+ },
+ // GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
+ &cli.StringFlag{
+ Name: "gitea-root",
+ Usage: "specifies the root URL of the Gitea instance, without a trailing slash.",
+ EnvVars: []string{"GITEA_ROOT"},
+ Value: "https://codeberg.org",
+ },
+ // GiteaApiToken specifies an api token for the Gitea instance
+ &cli.StringFlag{
+ Name: "gitea-api-token",
+ Usage: "specifies an api token for the Gitea instance",
+ EnvVars: []string{"GITEA_API_TOKEN"},
+ Value: "",
+ },
+ // RawDomain specifies the domain from which raw repository content shall be served in the following format:
+ // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
+ // (set to []byte(nil) to disable raw content hosting)
+ &cli.StringFlag{
+ Name: "raw-domain",
+ Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting",
+ EnvVars: []string{"RAW_DOMAIN"},
+ Value: "raw.codeberg.page",
+ },
+ // RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
+ &cli.StringFlag{
+ Name: "raw-info-page",
+ Usage: "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)",
+ EnvVars: []string{"RAW_INFO_PAGE"},
+ Value: "https://docs.codeberg.org/codeberg-pages/raw-content/",
+ },
+
+ // Server
+ &cli.StringFlag{
+ Name: "host",
+ Usage: "specifies host of listening address",
+ EnvVars: []string{"HOST"},
+ Value: "[::]",
+ },
+ &cli.StringFlag{
+ Name: "port",
+ Usage: "specifies port of listening address",
+ EnvVars: []string{"PORT"},
+ Value: "443",
+ },
+ &cli.BoolFlag{
+ Name: "enable-http-server",
+ // TODO: desc
+ EnvVars: []string{"ENABLE_HTTP_SERVER"},
+ },
+
+ // ACME
+ &cli.StringFlag{
+ Name: "acme-api-endpoint",
+ EnvVars: []string{"ACME_API"},
+ Value: "https://acme-v02.api.letsencrypt.org/directory",
+ },
+ &cli.StringFlag{
+ Name: "acme-email",
+ EnvVars: []string{"ACME_EMAIL"},
+ Value: "noreply@example.email",
+ },
+ &cli.BoolFlag{
+ Name: "acme-use-rate-limits",
+ // TODO: Usage
+ EnvVars: []string{"ACME_USE_RATE_LIMITS"},
+ Value: true,
+ },
+ &cli.BoolFlag{
+ Name: "acme-accept-terms",
+ // TODO: Usage
+ EnvVars: []string{"ACME_ACCEPT_TERMS"},
+ },
+ &cli.StringFlag{
+ Name: "acme-eab-kid",
+ // TODO: Usage
+ EnvVars: []string{"ACME_EAB_KID"},
+ },
+ &cli.StringFlag{
+ Name: "acme-eab-hmac",
+ // TODO: Usage
+ EnvVars: []string{"ACME_EAB_HMAC"},
+ },
+ &cli.StringFlag{
+ Name: "dns-provider",
+ // TODO: Usage
+ EnvVars: []string{"DNS_PROVIDER"},
+ },
+}
diff --git a/cmd/main.go b/cmd/main.go
new file mode 100644
index 0000000..fb0c26e
--- /dev/null
+++ b/cmd/main.go
@@ -0,0 +1,143 @@
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "net"
+ "strings"
+ "time"
+
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
+ "github.com/urfave/cli/v2"
+
+ "codeberg.org/codeberg/pages/server"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/certificates"
+ "codeberg.org/codeberg/pages/server/database"
+)
+
+// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
+// TODO: make it a flag
+var AllowedCorsDomains = [][]byte{
+ []byte("fonts.codeberg.org"),
+ []byte("design.codeberg.org"),
+}
+
+// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
+// TODO: Make it a flag too
+var BlacklistedPaths = [][]byte{
+ []byte("/.well-known/acme-challenge/"),
+}
+
+// Serve sets up and starts the web server.
+func Serve(ctx *cli.Context) error {
+ verbose := ctx.Bool("verbose")
+ if !verbose {
+ zerolog.SetGlobalLevel(zerolog.InfoLevel)
+ }
+
+ giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/")
+ giteaAPIToken := ctx.String("gitea-api-token")
+ rawDomain := ctx.String("raw-domain")
+ mainDomainSuffix := []byte(ctx.String("pages-domain"))
+ rawInfoPage := ctx.String("raw-info-page")
+ listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
+ enableHTTPServer := ctx.Bool("enable-http-server")
+
+ acmeAPI := ctx.String("acme-api-endpoint")
+ acmeMail := ctx.String("acme-email")
+ acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
+ acmeAcceptTerms := ctx.Bool("acme-accept-terms")
+ acmeEabKID := ctx.String("acme-eab-kid")
+ acmeEabHmac := ctx.String("acme-eab-hmac")
+ dnsProvider := ctx.String("dns-provider")
+ if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
+ return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory")
+ }
+
+ allowedCorsDomains := AllowedCorsDomains
+ if len(rawDomain) != 0 {
+ allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain))
+ }
+
+ // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
+ if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) {
+ mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...)
+ }
+
+ keyCache := cache.NewKeyValueCache()
+ challengeCache := cache.NewKeyValueCache()
+ // canonicalDomainCache stores canonical domains
+ var canonicalDomainCache = cache.NewKeyValueCache()
+ // dnsLookupCache stores DNS lookups for custom domains
+ var dnsLookupCache = cache.NewKeyValueCache()
+ // branchTimestampCache stores branch timestamps for faster cache checking
+ var branchTimestampCache = cache.NewKeyValueCache()
+ // fileResponseCache stores responses from the Gitea server
+ // TODO: make this an MRU cache with a size limit
+ var fileResponseCache = cache.NewKeyValueCache()
+
+ // Create handler based on settings
+ handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
+ giteaRoot, rawInfoPage, giteaAPIToken,
+ BlacklistedPaths, allowedCorsDomains,
+ dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
+
+ fastServer := server.SetupServer(handler)
+ httpServer := server.SetupHTTPACMEChallengeServer(challengeCache)
+
+ // Setup listener and TLS
+ log.Info().Msgf("Listening on https://%s", listeningAddress)
+ listener, err := net.Listen("tcp", listeningAddress)
+ if err != nil {
+ return fmt.Errorf("couldn't create listener: %s", err)
+ }
+
+ // TODO: make "key-database.pogreb" set via flag
+ certDB, err := database.New("key-database.pogreb")
+ if err != nil {
+ return fmt.Errorf("could not create database: %v", err)
+ }
+ defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it
+
+ listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
+ giteaRoot, giteaAPIToken, dnsProvider,
+ acmeUseRateLimits,
+ keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
+ certDB))
+
+ acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
+ if err != nil {
+ return err
+ }
+
+ if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil {
+ return err
+ }
+
+ interval := 12 * time.Hour
+ certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
+ defer cancelCertMaintain()
+ go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB)
+
+ if enableHTTPServer {
+ go func() {
+ err := httpServer.ListenAndServe("[::]:80")
+ if err != nil {
+ log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
+ }
+ }()
+ }
+
+ // Start the web fastServer
+ err = fastServer.Serve(listener)
+ if err != nil {
+ log.Panic().Err(err).Msg("Couldn't start fastServer")
+ }
+
+ return nil
+}
diff --git a/debug-stepper/stepper.go b/debug-stepper/stepper.go
deleted file mode 100644
index 05506b6..0000000
--- a/debug-stepper/stepper.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package debug_stepper
-
-import (
- "fmt"
- "os"
- "strings"
- "time"
-)
-
-var Enabled = strings.HasSuffix(os.Args[0], ".test") || os.Getenv("DEBUG") == "1"
-
-var Logger = func(s string, i ...interface{}) {
- fmt.Printf(s, i...)
-}
-
-type Stepper struct {
- Name string
- Start time.Time
- LastStep time.Time
- Completion time.Time
-}
-
-func Start(name string) *Stepper {
- if !Enabled {
- return nil
- }
- t := time.Now()
- Logger("%s: started at %s\n", name, t.Format(time.RFC3339))
- return &Stepper{
- Name: name,
- Start: t,
- LastStep: t,
- }
-}
-
-func (s *Stepper) Debug(text string) {
- if !Enabled {
- return
- }
- t := time.Now()
- Logger("%s: %s (at %s, %s since last step, %s since start)\n", s.Name, text, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
-}
-
-func (s *Stepper) Step(description string) {
- if !Enabled {
- return
- }
- if s.Completion != (time.Time{}) {
- Logger("%s: already completed all tasks.\n")
- return
- }
- t := time.Now()
- Logger("%s: completed %s at %s (%s)\n", s.Name, description, t.Format(time.RFC3339), t.Sub(s.LastStep).String())
- s.LastStep = t
-}
-
-func (s *Stepper) Complete() {
- if !Enabled {
- return
- }
- if s.Completion != (time.Time{}) {
- Logger("%s: already completed all tasks.\n")
- return
- }
- t := time.Now()
- Logger("%s: completed all tasks at %s (%s since last step; total time: %s)\n", s.Name, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
- s.Completion = t
-}
diff --git a/domains.go b/domains.go
deleted file mode 100644
index 0a5abc1..0000000
--- a/domains.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package main
-
-import (
- "github.com/OrlovEvgeny/go-mcache"
- "github.com/valyala/fasthttp"
- "net"
- "strings"
- "time"
-)
-
-// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache.
-var DnsLookupCacheTimeout = 15 * time.Minute
-
-// dnsLookupCache stores DNS lookups for custom domains
-var dnsLookupCache = mcache.New()
-
-// getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
-// If everything is fine, it returns the target data.
-func getTargetFromDNS(domain string) (targetOwner, targetRepo, targetBranch string) {
- // Get CNAME or TXT
- var cname string
- var err error
- if cachedName, ok := dnsLookupCache.Get(domain); ok {
- cname = cachedName.(string)
- } else {
- cname, err = net.LookupCNAME(domain)
- cname = strings.TrimSuffix(cname, ".")
- if err != nil || !strings.HasSuffix(cname, string(MainDomainSuffix)) {
- cname = ""
- // TODO: check if the A record matches!
- names, err := net.LookupTXT(domain)
- if err == nil {
- for _, name := range names {
- name = strings.TrimSuffix(name, ".")
- if strings.HasSuffix(name, string(MainDomainSuffix)) {
- cname = name
- break
- }
- }
- }
- }
- _ = dnsLookupCache.Set(domain, cname, DnsLookupCacheTimeout)
- }
- if cname == "" {
- return
- }
- cnameParts := strings.Split(strings.TrimSuffix(cname, string(MainDomainSuffix)), ".")
- targetOwner = cnameParts[len(cnameParts)-1]
- if len(cnameParts) > 1 {
- targetRepo = cnameParts[len(cnameParts)-2]
- }
- if len(cnameParts) > 2 {
- targetBranch = cnameParts[len(cnameParts)-3]
- }
- if targetRepo == "" {
- targetRepo = "pages"
- }
- if targetBranch == "" && targetRepo != "pages" {
- targetBranch = "pages"
- }
- // if targetBranch is still empty, the caller must find the default branch
- return
-}
-
-// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
-var CanonicalDomainCacheTimeout = 15 * time.Minute
-
-// canonicalDomainCache stores canonical domains
-var canonicalDomainCache = mcache.New()
-
-// checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`).
-func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain string) (canonicalDomain string, valid bool) {
- domains := []string{}
- if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
- domains = cachedValue.([]string)
- for _, domain := range domains {
- if domain == actualDomain {
- valid = true
- break
- }
- }
- } else {
- req := fasthttp.AcquireRequest()
- req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + GiteaApiToken)
- res := fasthttp.AcquireResponse()
-
- err := upstreamClient.Do(req, res)
- if err == nil && res.StatusCode() == fasthttp.StatusOK {
- for _, domain := range strings.Split(string(res.Body()), "\n") {
- domain = strings.ToLower(domain)
- domain = strings.TrimSpace(domain)
- domain = strings.TrimPrefix(domain, "http://")
- domain = strings.TrimPrefix(domain, "https://")
- if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
- domains = append(domains, domain)
- }
- if domain == actualDomain {
- valid = true
- }
- }
- }
- domains = append(domains, targetOwner+string(MainDomainSuffix))
- if domains[len(domains)-1] == actualDomain {
- valid = true
- }
- if targetRepo != "" && targetRepo != "pages" {
- domains[len(domains)-1] += "/" + targetRepo
- }
- _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, CanonicalDomainCacheTimeout)
- }
- canonicalDomain = domains[0]
- return
-}
diff --git a/go.mod b/go.mod
index fd52ef1..615bb12 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,9 @@ require (
github.com/akrylysov/pogreb v0.10.1
github.com/go-acme/lego/v4 v4.5.3
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
+ github.com/rs/zerolog v1.26.0
+ github.com/stretchr/testify v1.7.0
+ github.com/urfave/cli/v2 v2.3.0
github.com/valyala/fasthttp v1.31.0
github.com/valyala/fastjson v1.6.3
)
diff --git a/go.sum b/go.sum
index c93f985..d04f727 100644
--- a/go.sum
+++ b/go.sum
@@ -95,10 +95,12 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -148,6 +150,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b h1:/vQ+oYKu+JoyaMPDsv5FzwuL2wwWBgBbtj/YLCi4LuA=
github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -422,6 +425,10 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad h1:WtSUHi5zthjudjI
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe2Y+6iHTjIq/9HzUq7NII/Nffp0HkFrsAKq4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
+github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI=
@@ -429,6 +436,7 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f h1:WSnaD0/cvbKJgSTYbjAPf4RJXVvNNDAwVm+W8wEmnGE=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@@ -478,7 +486,9 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
+github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@@ -499,6 +509,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -557,6 +568,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -589,8 +601,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -653,8 +666,9 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -708,6 +722,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/handler.go b/handler.go
deleted file mode 100644
index d5b0448..0000000
--- a/handler.go
+++ /dev/null
@@ -1,542 +0,0 @@
-package main
-
-import (
- "bytes"
- debug_stepper "codeberg.org/codeberg/pages/debug-stepper"
- "fmt"
- "github.com/OrlovEvgeny/go-mcache"
- "github.com/valyala/fasthttp"
- "github.com/valyala/fastjson"
- "io"
- "mime"
- "path"
- "strconv"
- "strings"
- "time"
-)
-
-// handler handles a single HTTP request to the web server.
-func handler(ctx *fasthttp.RequestCtx) {
- s := debug_stepper.Start("handler")
- defer s.Complete()
-
- ctx.Response.Header.Set("Server", "Codeberg Pages")
-
- // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
- ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
-
- // Enable browser caching for up to 10 minutes
- ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
-
- trimmedHost := TrimHostPort(ctx.Request.Host())
-
- // Add HSTS for RawDomain and MainDomainSuffix
- if hsts := GetHSTSHeader(trimmedHost); hsts != "" {
- ctx.Response.Header.Set("Strict-Transport-Security", hsts)
- }
-
- // Block all methods not required for static pages
- if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
- ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
- ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
- return
- }
-
- // Block blacklisted paths (like ACME challenges)
- for _, blacklistedPath := range BlacklistedPaths {
- if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
- returnErrorPage(ctx, fasthttp.StatusForbidden)
- return
- }
- }
-
- // Allow CORS for specified domains
- if ctx.IsOptions() {
- allowCors := false
- for _, allowedCorsDomain := range AllowedCorsDomains {
- if bytes.Equal(trimmedHost, allowedCorsDomain) {
- allowCors = true
- break
- }
- }
- if allowCors {
- ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
- ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
- }
- ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
- ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
- return
- }
-
- // Prepare request information to Gitea
- var targetOwner, targetRepo, targetBranch, targetPath string
- var targetOptions = &upstreamOptions{
- ForbiddenMimeTypes: map[string]struct{}{},
- TryIndexPages: true,
- }
-
- // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
- // also disallow search indexing and add a Link header to the canonical URL.
- var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
- if repo == "" {
- return false
- }
-
- // Check if the branch exists, otherwise treat it as a file path
- branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch)
- if branchTimestampResult == nil {
- // branch doesn't exist
- return false
- }
-
- // Branch exists, use it
- targetRepo = repo
- targetPath = strings.Trim(strings.Join(path, "/"), "/")
- targetBranch = branchTimestampResult.branch
-
- targetOptions.BranchTimestamp = branchTimestampResult.timestamp
-
- if canonicalLink != "" {
- // Hide from search machines & add canonical link
- ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
- ctx.Response.Header.Set("Link",
- strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
- "; rel=\"canonical\"",
- )
- }
-
- return true
- }
-
- // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
- var tryUpstream = func() {
- // check if a canonical domain exists on a request on MainDomain
- if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
- canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "")
- if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(MainDomainSuffix)) {
- canonicalPath := string(ctx.RequestURI())
- if targetRepo != "pages" {
- canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2]
- }
- ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
- return
- }
- }
-
- // Try to request the file from the Gitea API
- if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, targetOptions) {
- returnErrorPage(ctx, ctx.Response.StatusCode())
- }
- }
-
- s.Step("preparations")
-
- if RawDomain != nil && bytes.Equal(trimmedHost, RawDomain) {
- // Serve raw content from RawDomain
- s.Debug("raw domain")
-
- targetOptions.TryIndexPages = false
- targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
- targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
-
- pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
- if len(pathElements) < 2 {
- // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
- ctx.Redirect(RawInfoPage, fasthttp.StatusTemporaryRedirect)
- return
- }
- targetOwner = pathElements[0]
- targetRepo = pathElements[1]
-
- // raw.codeberg.org/example/myrepo/@main/index.html
- if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
- s.Step("raw domain preparations, now trying with specified branch")
- if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
- string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
- ) {
- s.Step("tryBranch, now trying upstream")
- tryUpstream()
- return
- }
- s.Debug("missing branch")
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- return
- } else {
- s.Step("raw domain preparations, now trying with default branch")
- tryBranch(targetRepo, "", pathElements[2:],
- string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
- )
- s.Step("tryBranch, now trying upstream")
- tryUpstream()
- return
- }
-
- } else if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
- // Serve pages from subdomains of MainDomainSuffix
- s.Debug("main domain suffix")
-
- pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
- targetOwner = string(bytes.TrimSuffix(trimmedHost, MainDomainSuffix))
- targetRepo = pathElements[0]
- targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
-
- if targetOwner == "www" {
- // www.codeberg.page redirects to codeberg.page
- ctx.Redirect("https://" + string(MainDomainSuffix[1:]) + string(ctx.Path()), fasthttp.StatusPermanentRedirect)
- return
- }
-
- // Check if the first directory is a repo with the second directory as a branch
- // example.codeberg.page/myrepo/@main/index.html
- if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
- if targetRepo == "pages" {
- // example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
- ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
- return
- }
-
- s.Step("main domain preparations, now trying with specified repo & branch")
- if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
- "/"+pathElements[0]+"/%p",
- ) {
- s.Step("tryBranch, now trying upstream")
- tryUpstream()
- } else {
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- }
- return
- }
-
- // Check if the first directory is a branch for the "pages" repo
- // example.codeberg.page/@main/index.html
- if strings.HasPrefix(pathElements[0], "@") {
- s.Step("main domain preparations, now trying with specified branch")
- if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
- s.Step("tryBranch, now trying upstream")
- tryUpstream()
- } else {
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- }
- return
- }
-
- // Check if the first directory is a repo with a "pages" branch
- // example.codeberg.page/myrepo/index.html
- // example.codeberg.page/pages/... is not allowed here.
- s.Step("main domain preparations, now trying with specified repo")
- if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
- s.Step("tryBranch, now trying upstream")
- tryUpstream()
- return
- }
-
- // Try to use the "pages" repo on its default branch
- // example.codeberg.page/index.html
- s.Step("main domain preparations, now trying with default repo/branch")
- if tryBranch("pages", "", pathElements, "") {
- s.Step("tryBranch, now trying upstream")
- tryUpstream()
- return
- }
-
- // Couldn't find a valid repo/branch
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- return
- } else {
- trimmedHostStr := string(trimmedHost)
-
- // Serve pages from external domains
- targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr)
- if targetOwner == "" {
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- return
- }
-
- pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
- canonicalLink := ""
- if strings.HasPrefix(pathElements[0], "@") {
- targetBranch = pathElements[0][1:]
- pathElements = pathElements[1:]
- canonicalLink = "/%p"
- }
-
- // Try to use the given repo on the given branch or the default branch
- s.Step("custom domain preparations, now trying with details from DNS")
- if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
- canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr)
- if !valid {
- returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
- return
- } else if canonicalDomain != trimmedHostStr {
- // only redirect if the target is also a codeberg page!
- targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0])
- if targetOwner != "" {
- ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
- return
- } else {
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- return
- }
- }
-
- s.Step("tryBranch, now trying upstream")
- tryUpstream()
- return
- } else {
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- return
- }
- }
-}
-
-// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
-// with the provided status code.
-func returnErrorPage(ctx *fasthttp.RequestCtx, code int) {
- ctx.Response.SetStatusCode(code)
- ctx.Response.Header.SetContentType("text/html; charset=utf-8")
- message := fasthttp.StatusMessage(code)
- if code == fasthttp.StatusMisdirectedRequest {
- message += " - domain not specified in .domains file"
- }
- if code == fasthttp.StatusFailedDependency {
- message += " - target repo/branch doesn't exist or is private"
- }
- ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
-}
-
-// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
-var DefaultBranchCacheTimeout = 15 * time.Minute
-
-// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter
-// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
-// picked up faster, while still allowing the content to be cached longer if nothing changes.
-var BranchExistanceCacheTimeout = 5 * time.Minute
-
-// branchTimestampCache stores branch timestamps for faster cache checking
-var branchTimestampCache = mcache.New()
-
-type branchTimestamp struct {
- branch string
- timestamp time.Time
-}
-
-// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
-// on your available memory.
-var FileCacheTimeout = 5 * time.Minute
-
-// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
-var FileCacheSizeLimit = 1024 * 1024
-
-// fileResponseCache stores responses from the Gitea server
-// TODO: make this an MRU cache with a size limit
-var fileResponseCache = mcache.New()
-
-type fileResponse struct {
- exists bool
- mimeType string
- body []byte
-}
-
-// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
-// (or nil if the branch doesn't exist)
-func getBranchTimestamp(owner, repo, branch string) *branchTimestamp {
- if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
- if result == nil {
- return nil
- }
- return result.(*branchTimestamp)
- }
- result := &branchTimestamp{}
- result.branch = branch
- if branch == "" {
- // Get default branch
- var body = make([]byte, 0)
- status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+GiteaApiToken, 5*time.Second)
- if err != nil || status != 200 {
- _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout)
- return nil
- }
- result.branch = fastjson.GetString(body, "default_branch")
- }
-
- var body = make([]byte, 0)
- status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+GiteaApiToken, 5*time.Second)
- if err != nil || status != 200 {
- return nil
- }
-
- result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
- _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout)
- return result
-}
-
-var upstreamClient = fasthttp.Client{
- ReadTimeout: 10 * time.Second,
- MaxConnDuration: 60 * time.Second,
- MaxConnWaitTimeout: 1000 * time.Millisecond,
- MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-}
-
-// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, targetBranch string, targetPath string, options *upstreamOptions) (final bool) {
- s := debug_stepper.Start("upstream")
- defer s.Complete()
-
- if options.ForbiddenMimeTypes == nil {
- options.ForbiddenMimeTypes = map[string]struct{}{}
- }
-
- // Check if the branch exists and when it was modified
- if options.BranchTimestamp == (time.Time{}) {
- branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch)
-
- if branch == nil {
- returnErrorPage(ctx, fasthttp.StatusFailedDependency)
- return true
- }
- targetBranch = branch.branch
- options.BranchTimestamp = branch.timestamp
- }
-
- if targetOwner == "" || targetRepo == "" || targetBranch == "" {
- returnErrorPage(ctx, fasthttp.StatusBadRequest)
- return true
- }
-
- // Check if the browser has a cached version
- if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
- if !ifModifiedSince.Before(options.BranchTimestamp) {
- ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
- return true
- }
- }
- s.Step("preparations")
-
- // Make a GET request to the upstream URL
- uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath
- var req *fasthttp.Request
- var res *fasthttp.Response
- var cachedResponse fileResponse
- var err error
- if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
- cachedResponse = cachedValue.(fileResponse)
- } else {
- req = fasthttp.AcquireRequest()
- req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + uri + "?access_token=" + GiteaApiToken)
- res = fasthttp.AcquireResponse()
- res.SetBodyStream(&strings.Reader{}, -1)
- err = upstreamClient.Do(req, res)
- }
- s.Step("acquisition")
-
- // Handle errors
- if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
- if options.TryIndexPages {
- // copy the options struct & try if an index page exists
- optionsForIndexPages := *options
- optionsForIndexPages.TryIndexPages = false
- optionsForIndexPages.AppendTrailingSlash = true
- for _, indexPage := range IndexPages {
- if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, &optionsForIndexPages) {
- _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
- exists: false,
- }, FileCacheTimeout)
- return true
- }
- }
- // compatibility fix for GitHub Pages (/example → /example.html)
- optionsForIndexPages.AppendTrailingSlash = false
- optionsForIndexPages.RedirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
- if upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath + ".html", &optionsForIndexPages) {
- _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
- exists: false,
- }, FileCacheTimeout)
- return true
- }
- }
- ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
- if res != nil {
- // Update cache if the request is fresh
- _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
- exists: false,
- }, FileCacheTimeout)
- }
- return false
- }
- if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
- fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
- returnErrorPage(ctx, fasthttp.StatusInternalServerError)
- return true
- }
-
- // Append trailing slash if missing (for index files), and redirect to fix filenames in general
- // options.AppendTrailingSlash is only true when looking for index pages
- if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
- ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
- return true
- }
- if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
- ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
- return true
- }
- if options.RedirectIfExists != "" {
- ctx.Redirect(options.RedirectIfExists, fasthttp.StatusTemporaryRedirect)
- return true
- }
- s.Step("error handling")
-
- // Set the MIME type
- mimeType := mime.TypeByExtension(path.Ext(targetPath))
- mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
- if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
- if options.DefaultMimeType != "" {
- mimeType = options.DefaultMimeType
- } else {
- mimeType = "application/octet-stream"
- }
- }
- ctx.Response.Header.SetContentType(mimeType)
-
- // Everything's okay so far
- ctx.Response.SetStatusCode(fasthttp.StatusOK)
- ctx.Response.Header.SetLastModified(options.BranchTimestamp)
-
- s.Step("response preparations")
-
- // Write the response body to the original request
- var cacheBodyWriter bytes.Buffer
- if res != nil {
- if res.Header.ContentLength() > FileCacheSizeLimit {
- err = res.BodyWriteTo(ctx.Response.BodyWriter())
- } else {
- err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
- }
- } else {
- _, err = ctx.Write(cachedResponse.body)
- }
- if err != nil {
- fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
- returnErrorPage(ctx, fasthttp.StatusInternalServerError)
- return true
- }
- s.Step("response")
-
- if res != nil && ctx.Err() == nil {
- cachedResponse.exists = true
- cachedResponse.mimeType = mimeType
- cachedResponse.body = cacheBodyWriter.Bytes()
- _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout)
- }
-
- return true
-}
-
-// upstreamOptions provides various options for the upstream request.
-type upstreamOptions struct {
- DefaultMimeType string
- ForbiddenMimeTypes map[string]struct{}
- TryIndexPages bool
- AppendTrailingSlash bool
- RedirectIfExists string
- BranchTimestamp time.Time
-}
diff --git a/haproxy-sni/haproxy.cfg b/haproxy-sni/haproxy.cfg
index 869bae3..c8f3610 100644
--- a/haproxy-sni/haproxy.cfg
+++ b/haproxy-sni/haproxy.cfg
@@ -51,6 +51,7 @@ frontend https_sni_frontend
###################################################
acl use_http_backend req.ssl_sni -i "codeberg.org"
acl use_http_backend req.ssl_sni -i "join.codeberg.org"
+ # TODO: use this if no SNI exists
use_backend https_termination_backend if use_http_backend
############################
diff --git a/helpers.go b/helpers.go
deleted file mode 100644
index 46a1492..0000000
--- a/helpers.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package main
-
-import (
- "bytes"
- "encoding/gob"
- "github.com/akrylysov/pogreb"
-)
-
-// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
-// string for custom domains.
-func GetHSTSHeader(host []byte) string {
- if bytes.HasSuffix(host, MainDomainSuffix) || bytes.Equal(host, RawDomain) {
- return "max-age=63072000; includeSubdomains; preload"
- } else {
- return ""
- }
-}
-
-func TrimHostPort(host []byte) []byte {
- i := bytes.IndexByte(host, ':')
- if i >= 0 {
- return host[:i]
- }
- return host
-}
-
-func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) {
- var resGob bytes.Buffer
- resEnc := gob.NewEncoder(&resGob)
- err := resEnc.Encode(obj)
- if err != nil {
- panic(err)
- }
- err = db.Put(name, resGob.Bytes())
- if err != nil {
- panic(err)
- }
-}
-
-func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool {
- resBytes, err := db.Get(name)
- if err != nil {
- panic(err)
- }
- if resBytes == nil {
- return false
- }
-
- resGob := bytes.NewBuffer(resBytes)
- resDec := gob.NewDecoder(resGob)
- err = resDec.Decode(obj)
- if err != nil {
- panic(err)
- }
- return true
-}
diff --git a/404.html b/html/404.html
similarity index 100%
rename from 404.html
rename to html/404.html
diff --git a/html/error.go b/html/error.go
new file mode 100644
index 0000000..325dada
--- /dev/null
+++ b/html/error.go
@@ -0,0 +1,24 @@
+package html
+
+import (
+ "bytes"
+ "strconv"
+
+ "github.com/valyala/fasthttp"
+)
+
+// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
+// with the provided status code.
+func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
+ ctx.Response.SetStatusCode(code)
+ ctx.Response.Header.SetContentType("text/html; charset=utf-8")
+ message := fasthttp.StatusMessage(code)
+ if code == fasthttp.StatusMisdirectedRequest {
+ message += " - domain not specified in .domains file"
+ }
+ if code == fasthttp.StatusFailedDependency {
+ message += " - target repo/branch doesn't exist or is private"
+ }
+ // TODO: use template engine?
+ ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
+}
diff --git a/html/html.go b/html/html.go
new file mode 100644
index 0000000..d223e15
--- /dev/null
+++ b/html/html.go
@@ -0,0 +1,6 @@
+package html
+
+import _ "embed"
+
+//go:embed 404.html
+var NotFoundPage []byte
diff --git a/main.go b/main.go
index 44cec0f..41aba22 100644
--- a/main.go
+++ b/main.go
@@ -1,159 +1,32 @@
-// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
-//
-// Mapping custom domains is not static anymore, but can be done with DNS:
-//
-// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The
-// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
-//
-// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
-// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
-// www.example.org. IN CNAME main.pages.example.codeberg.page.
-//
-// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
-// for "example.org" (if your provider allows ALIAS or similar records):
-// example.org IN ALIAS codeberg.page.
-//
-// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
package main
import (
- "bytes"
- "crypto/tls"
"fmt"
- "log"
- "net"
- "net/http"
"os"
- "time"
- _ "embed"
+ "github.com/urfave/cli/v2"
- "github.com/valyala/fasthttp"
+ "codeberg.org/codeberg/pages/cmd"
)
-// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
-// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
-// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
-var MainDomainSuffix = []byte("." + envOr("PAGES_DOMAIN", "codeberg.page"))
+var (
+ // can be changed with -X on compile
+ version = "dev"
+)
-// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
-var GiteaRoot = []byte(envOr("GITEA_ROOT", "https://codeberg.org"))
-
-var GiteaApiToken = envOr("GITEA_API_TOKEN", "")
-
-//go:embed 404.html
-var NotFoundPage []byte
-
-// RawDomain specifies the domain from which raw repository content shall be served in the following format:
-// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
-// (set to []byte(nil) to disable raw content hosting)
-var RawDomain = []byte(envOr("RAW_DOMAIN", "raw.codeberg.org"))
-
-// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
-var RawInfoPage = envOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/")
-
-// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
-var AllowedCorsDomains = [][]byte{
- RawDomain,
- []byte("fonts.codeberg.org"),
- []byte("design.codeberg.org"),
-}
-
-// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
-var BlacklistedPaths = [][]byte{
- []byte("/.well-known/acme-challenge/"),
-}
-
-// IndexPages lists pages that may be considered as index pages for directories.
-var IndexPages = []string{
- "index.html",
-}
-
-// main sets up and starts the web server.
func main() {
- if len(os.Args) > 1 && os.Args[1] == "--remove-certificate" {
- if len(os.Args) < 2 {
- println("--remove-certificate requires at least one domain as an argument")
- os.Exit(1)
- }
- if keyDatabaseErr != nil {
- panic(keyDatabaseErr)
- }
- for _, domain := range os.Args[2:] {
- if err := keyDatabase.Delete([]byte(domain)); err != nil {
- panic(err)
- }
- }
- if err := keyDatabase.Sync(); err != nil {
- panic(err)
- }
- os.Exit(0)
+ app := cli.NewApp()
+ app.Name = "pages-server"
+ app.Version = version
+ app.Usage = "pages server"
+ app.Action = cmd.Serve
+ app.Flags = cmd.ServeFlags
+ app.Commands = []*cli.Command{
+ cmd.Certs,
}
- // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
- if !bytes.HasPrefix(MainDomainSuffix, []byte{'.'}) {
- MainDomainSuffix = append([]byte{'.'}, MainDomainSuffix...)
- }
- GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'})
-
- // Use HOST and PORT environment variables to determine listening address
- address := fmt.Sprintf("%s:%s", envOr("HOST", "[::]"), envOr("PORT", "443"))
- log.Printf("Listening on https://%s", address)
-
- // Enable compression by wrapping the handler() method with the compression function provided by FastHTTP
- compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
-
- server := &fasthttp.Server{
- Handler: compressedHandler,
- DisablePreParseMultipartForm: false,
- MaxRequestBodySize: 0,
- NoDefaultServerHeader: true,
- NoDefaultDate: true,
- ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
- Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
- MaxConnsPerIP: 100,
- }
-
- // Setup listener and TLS
- listener, err := net.Listen("tcp", address)
- if err != nil {
- log.Fatalf("Couldn't create listener: %s", err)
- }
- listener = tls.NewListener(listener, tlsConfig)
-
- setupCertificates()
- if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
- go (func() {
- challengePath := []byte("/.well-known/acme-challenge/")
- err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) {
- if bytes.HasPrefix(ctx.Path(), challengePath) {
- challenge, ok := challengeCache.Get(string(TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
- if !ok || challenge == nil {
- ctx.SetStatusCode(http.StatusNotFound)
- ctx.SetBodyString("no challenge for this token")
- }
- ctx.SetBodyString(challenge.(string))
- } else {
- ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
- }
- })
- if err != nil {
- log.Fatalf("Couldn't start HTTP server: %s", err)
- }
- })()
- }
-
- // Start the web server
- err = server.Serve(listener)
- if err != nil {
- log.Fatalf("Couldn't start server: %s", err)
+ if err := app.Run(os.Args); err != nil {
+ _, _ = fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
}
}
-
-// envOr reads an environment variable and returns a default value if it's empty.
-func envOr(env string, or string) string {
- if v := os.Getenv(env); v != "" {
- return v
- }
- return or
-}
diff --git a/server/cache/interface.go b/server/cache/interface.go
new file mode 100644
index 0000000..2952b29
--- /dev/null
+++ b/server/cache/interface.go
@@ -0,0 +1,9 @@
+package cache
+
+import "time"
+
+type SetGetKey interface {
+ Set(key string, value interface{}, ttl time.Duration) error
+ Get(key string) (interface{}, bool)
+ Remove(key string)
+}
diff --git a/server/cache/setup.go b/server/cache/setup.go
new file mode 100644
index 0000000..a5928b0
--- /dev/null
+++ b/server/cache/setup.go
@@ -0,0 +1,7 @@
+package cache
+
+import "github.com/OrlovEvgeny/go-mcache"
+
+func NewKeyValueCache() SetGetKey {
+ return mcache.New()
+}
diff --git a/server/certificates/acme_account.go b/server/certificates/acme_account.go
new file mode 100644
index 0000000..2ee2e80
--- /dev/null
+++ b/server/certificates/acme_account.go
@@ -0,0 +1,27 @@
+package certificates
+
+import (
+ "crypto"
+
+ "github.com/go-acme/lego/v4/registration"
+)
+
+type AcmeAccount struct {
+ Email string
+ Registration *registration.Resource
+ Key crypto.PrivateKey `json:"-"`
+ KeyPEM string `json:"Key"`
+}
+
+// make sure AcmeAccount match User interface
+var _ registration.User = &AcmeAccount{}
+
+func (u *AcmeAccount) GetEmail() string {
+ return u.Email
+}
+func (u AcmeAccount) GetRegistration() *registration.Resource {
+ return u.Registration
+}
+func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
+ return u.Key
+}
diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go
new file mode 100644
index 0000000..b40c76d
--- /dev/null
+++ b/server/certificates/certificates.go
@@ -0,0 +1,522 @@
+package certificates
+
+import (
+ "bytes"
+ "context"
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/tls"
+ "crypto/x509"
+ "encoding/gob"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/go-acme/lego/v4/certcrypto"
+ "github.com/go-acme/lego/v4/certificate"
+ "github.com/go-acme/lego/v4/challenge"
+ "github.com/go-acme/lego/v4/challenge/tlsalpn01"
+ "github.com/go-acme/lego/v4/lego"
+ "github.com/go-acme/lego/v4/providers/dns"
+ "github.com/go-acme/lego/v4/registration"
+ "github.com/reugn/equalizer"
+ "github.com/rs/zerolog/log"
+
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/database"
+ dnsutils "codeberg.org/codeberg/pages/server/dns"
+ "codeberg.org/codeberg/pages/server/upstream"
+)
+
+// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
+func TLSConfig(mainDomainSuffix []byte,
+ giteaRoot, giteaAPIToken, dnsProvider string,
+ acmeUseRateLimits bool,
+ keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+ certDB database.CertDB) *tls.Config {
+ return &tls.Config{
+ // check DNS name & get certificate from Let's Encrypt
+ GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+ sni := strings.ToLower(strings.TrimSpace(info.ServerName))
+ sniBytes := []byte(sni)
+ if len(sni) < 1 {
+ return nil, errors.New("missing sni")
+ }
+
+ if info.SupportedProtos != nil {
+ for _, proto := range info.SupportedProtos {
+ if proto == tlsalpn01.ACMETLS1Protocol {
+ challenge, ok := challengeCache.Get(sni)
+ if !ok {
+ return nil, errors.New("no challenge for this domain")
+ }
+ cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
+ if err != nil {
+ return nil, err
+ }
+ return cert, nil
+ }
+ }
+ }
+
+ targetOwner := ""
+ if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) {
+ // deliver default certificate for the main domain (*.codeberg.page)
+ sniBytes = mainDomainSuffix
+ sni = string(sniBytes)
+ } else {
+ var targetRepo, targetBranch string
+ targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache)
+ if targetOwner == "" {
+ // DNS not set up, return main certificate to redirect to the docs
+ sniBytes = mainDomainSuffix
+ sni = string(sniBytes)
+ } else {
+ _, _ = targetRepo, targetBranch
+ _, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+ if !valid {
+ sniBytes = mainDomainSuffix
+ sni = string(sniBytes)
+ }
+ }
+ }
+
+ if tlsCertificate, ok := keyCache.Get(sni); ok {
+ // we can use an existing certificate object
+ return tlsCertificate.(*tls.Certificate), nil
+ }
+
+ var tlsCertificate tls.Certificate
+ var err error
+ var ok bool
+ if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
+ // request a new certificate
+ if bytes.Equal(sniBytes, mainDomainSuffix) {
+ return nil, errors.New("won't request certificate for main domain, something really bad has happened")
+ }
+
+ tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ if err := keyCache.Set(sni, &tlsCertificate, 15*time.Minute); err != nil {
+ return nil, err
+ }
+ return &tlsCertificate, nil
+ },
+ PreferServerCipherSuites: true,
+ NextProtos: []string{
+ "http/1.1",
+ tlsalpn01.ACMETLS1Protocol,
+ },
+
+ // generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
+ // https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
+ MinVersion: tls.VersionTLS12,
+ CipherSuites: []uint16{
+ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+ tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+ tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+ tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+ },
+ }
+}
+
+func checkUserLimit(user string) error {
+ userLimit, ok := acmeClientCertificateLimitPerUser[user]
+ if !ok {
+ // Each Codeberg user can only add 10 new domains per day.
+ userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
+ acmeClientCertificateLimitPerUser[user] = userLimit
+ }
+ if !userLimit.Ask() {
+ return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
+ }
+ return nil
+}
+
+var acmeClient, mainDomainAcmeClient *lego.Client
+var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+
+// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
+// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
+var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
+
+// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
+var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
+
+type AcmeTLSChallengeProvider struct {
+ challengeCache cache.SetGetKey
+}
+
+// make sure AcmeTLSChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeTLSChallengeProvider{}
+
+func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
+ return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
+}
+func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
+ a.challengeCache.Remove(domain)
+ return nil
+}
+
+type AcmeHTTPChallengeProvider struct {
+ challengeCache cache.SetGetKey
+}
+
+// make sure AcmeHTTPChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeHTTPChallengeProvider{}
+
+func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
+ return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
+}
+func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
+ a.challengeCache.Remove(domain + "/" + token)
+ return nil
+}
+
+func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
+ // parse certificate from database
+ res, err := certDB.Get(sni)
+ if err != nil {
+ panic(err) // TODO: no panic
+ }
+ if res == nil {
+ return tls.Certificate{}, false
+ }
+
+ tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
+ if err != nil {
+ panic(err)
+ }
+
+ // TODO: document & put into own function
+ if !bytes.Equal(sni, mainDomainSuffix) {
+ tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
+ if err != nil {
+ panic(err)
+ }
+
+ // renew certificates 7 days before they expire
+ if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
+ // TODO: add ValidUntil to custom res struct
+ if res.CSR != nil && len(res.CSR) > 0 {
+ // CSR stores the time when the renewal shall be tried again
+ nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
+ if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
+ return tlsCertificate, true
+ }
+ }
+ go (func() {
+ res.CSR = nil // acme client doesn't like CSR to be set
+ tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+ if err != nil {
+ log.Printf("Couldn't renew certificate for %s: %s", sni, err)
+ }
+ })()
+ }
+ }
+
+ return tlsCertificate, true
+}
+
+var obtainLocks = sync.Map{}
+
+func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
+ name := strings.TrimPrefix(domains[0], "*")
+ if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
+ domains = domains[1:]
+ }
+
+ // lock to avoid simultaneous requests
+ _, working := obtainLocks.LoadOrStore(name, struct{}{})
+ if working {
+ for working {
+ time.Sleep(100 * time.Millisecond)
+ _, working = obtainLocks.Load(name)
+ }
+ cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
+ if !ok {
+ return tls.Certificate{}, errors.New("certificate failed in synchronous request")
+ }
+ return cert, nil
+ }
+ defer obtainLocks.Delete(name)
+
+ if acmeClient == nil {
+ return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix), keyDatabase), nil
+ }
+
+ // request actual cert
+ var res *certificate.Resource
+ var err error
+ if renew != nil && renew.CertURL != "" {
+ if acmeUseRateLimits {
+ acmeClientRequestLimit.Take()
+ }
+ log.Printf("Renewing certificate for %v", domains)
+ res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
+ if err != nil {
+ log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
+ res = nil
+ }
+ }
+ if res == nil {
+ if user != "" {
+ if err := checkUserLimit(user); err != nil {
+ return tls.Certificate{}, err
+ }
+ }
+
+ if acmeUseRateLimits {
+ acmeClientOrderLimit.Take()
+ acmeClientRequestLimit.Take()
+ }
+ log.Printf("Requesting new certificate for %v", domains)
+ res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
+ Domains: domains,
+ Bundle: true,
+ MustStaple: false,
+ })
+ }
+ if err != nil {
+ log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
+ if renew != nil && renew.CertURL != "" {
+ tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
+ if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
+ // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
+ renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
+ if err := keyDatabase.Put(name, renew); err != nil {
+ return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
+ }
+ return tlsCertificate, nil
+ }
+ }
+ return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
+ }
+ log.Printf("Obtained certificate for %v", domains)
+
+ if err := keyDatabase.Put(name, res); err != nil {
+ return tls.Certificate{}, err
+ }
+ tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
+ if err != nil {
+ return tls.Certificate{}, err
+ }
+ return tlsCertificate, nil
+}
+
+func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
+ const configFile = "acme-account.json"
+ var myAcmeAccount AcmeAccount
+ var myAcmeConfig *lego.Config
+
+ if account, err := ioutil.ReadFile(configFile); err == nil {
+ if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
+ return nil, err
+ }
+ myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
+ if err != nil {
+ return nil, err
+ }
+ myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+ myAcmeConfig.CADirURL = acmeAPI
+ myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+
+ // Validate Config
+ _, err := lego.NewClient(myAcmeConfig)
+ if err != nil {
+ // TODO: should we fail hard instead?
+ log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+ }
+ return myAcmeConfig, nil
+ } else if !os.IsNotExist(err) {
+ return nil, err
+ }
+
+ privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ if err != nil {
+ return nil, err
+ }
+ myAcmeAccount = AcmeAccount{
+ Email: acmeMail,
+ Key: privateKey,
+ KeyPEM: string(certcrypto.PEMEncode(privateKey)),
+ }
+ myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+ myAcmeConfig.CADirURL = acmeAPI
+ myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+ tempClient, err := lego.NewClient(myAcmeConfig)
+ if err != nil {
+ log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+ } else {
+ // accept terms & log in to EAB
+ if acmeEabKID == "" || acmeEabHmac == "" {
+ reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
+ if err != nil {
+ log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
+ } else {
+ myAcmeAccount.Registration = reg
+ }
+ } else {
+ reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
+ TermsOfServiceAgreed: acmeAcceptTerms,
+ Kid: acmeEabKID,
+ HmacEncoded: acmeEabHmac,
+ })
+ if err != nil {
+ log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
+ } else {
+ myAcmeAccount.Registration = reg
+ }
+ }
+
+ if myAcmeAccount.Registration != nil {
+ acmeAccountJSON, err := json.Marshal(myAcmeAccount)
+ if err != nil {
+ log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
+ select {}
+ }
+ err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600)
+ if err != nil {
+ log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
+ select {}
+ }
+ }
+ }
+
+ return myAcmeConfig, nil
+}
+
+func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
+ // getting main cert before ACME account so that we can fail here without hitting rate limits
+ mainCertBytes, err := certDB.Get(mainDomainSuffix)
+ if err != nil {
+ return fmt.Errorf("cert database is not working")
+ }
+
+ acmeClient, err = lego.NewClient(acmeConfig)
+ if err != nil {
+ log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+ } else {
+ err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
+ if err != nil {
+ log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
+ }
+ if enableHTTPServer {
+ err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
+ if err != nil {
+ log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
+ }
+ }
+ }
+
+ mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
+ if err != nil {
+ log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+ } else {
+ if dnsProvider == "" {
+ // using mock server, don't use wildcard certs
+ err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
+ if err != nil {
+ log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
+ }
+ } else {
+ provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
+ if err != nil {
+ log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
+ }
+ err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
+ if err != nil {
+ log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
+ }
+ }
+ }
+
+ if mainCertBytes == nil {
+ _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+ if err != nil {
+ log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
+ }
+ }
+
+ return nil
+}
+
+func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
+ for {
+ // clean up expired certs
+ now := time.Now()
+ expiredCertCount := 0
+ keyDatabaseIterator := certDB.Items()
+ key, resBytes, err := keyDatabaseIterator.Next()
+ for err == nil {
+ if !bytes.Equal(key, mainDomainSuffix) {
+ resGob := bytes.NewBuffer(resBytes)
+ resDec := gob.NewDecoder(resGob)
+ res := &certificate.Resource{}
+ err = resDec.Decode(res)
+ if err != nil {
+ panic(err)
+ }
+
+ tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
+ if err != nil || !tlsCertificates[0].NotAfter.After(now) {
+ err := certDB.Delete(key)
+ if err != nil {
+ log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
+ } else {
+ expiredCertCount++
+ }
+ }
+ }
+ key, resBytes, err = keyDatabaseIterator.Next()
+ }
+ log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
+
+ // compact the database
+ result, err := certDB.Compact()
+ if err != nil {
+ log.Printf("[ERROR] Compacting key database failed: %s", err)
+ } else {
+ log.Printf("[INFO] Compacted key database (%+v)", result)
+ }
+
+ // update main cert
+ res, err := certDB.Get(mainDomainSuffix)
+ if err != nil {
+ log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
+ } else if res == nil {
+ log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
+ } else {
+ tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
+
+ // renew main certificate 30 days before it expires
+ if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
+ go (func() {
+ _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+ if err != nil {
+ log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
+ }
+ })()
+ }
+ }
+
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(interval):
+ }
+ }
+}
diff --git a/server/certificates/mock.go b/server/certificates/mock.go
new file mode 100644
index 0000000..0e87e6e
--- /dev/null
+++ b/server/certificates/mock.go
@@ -0,0 +1,86 @@
+package certificates
+
+import (
+ "bytes"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/pem"
+ "math/big"
+ "time"
+
+ "github.com/go-acme/lego/v4/certcrypto"
+ "github.com/go-acme/lego/v4/certificate"
+
+ "codeberg.org/codeberg/pages/server/database"
+)
+
+func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) tls.Certificate {
+ key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
+ if err != nil {
+ panic(err)
+ }
+
+ template := x509.Certificate{
+ SerialNumber: big.NewInt(1),
+ Subject: pkix.Name{
+ CommonName: domain,
+ Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
+ OrganizationalUnit: []string{
+ "Will not try again for 6 hours to avoid hitting rate limits for your domain.",
+ "Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
+ "free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
+ "Error message: " + msg,
+ },
+ },
+
+ // certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
+ NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6),
+ NotBefore: time.Now(),
+
+ KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ BasicConstraintsValid: true,
+ }
+ certBytes, err := x509.CreateCertificate(
+ rand.Reader,
+ &template,
+ &template,
+ &key.(*rsa.PrivateKey).PublicKey,
+ key,
+ )
+ if err != nil {
+ panic(err)
+ }
+
+ out := &bytes.Buffer{}
+ err = pem.Encode(out, &pem.Block{
+ Bytes: certBytes,
+ Type: "CERTIFICATE",
+ })
+ if err != nil {
+ panic(err)
+ }
+ outBytes := out.Bytes()
+ res := &certificate.Resource{
+ PrivateKey: certcrypto.PEMEncode(key),
+ Certificate: outBytes,
+ IssuerCertificate: outBytes,
+ Domain: domain,
+ }
+ databaseName := domain
+ if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] {
+ databaseName = mainDomainSuffix
+ }
+ if err := keyDatabase.Put(databaseName, res); err != nil {
+ panic(err)
+ }
+
+ tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
+ if err != nil {
+ panic(err)
+ }
+ return tlsCertificate
+}
diff --git a/server/database/interface.go b/server/database/interface.go
new file mode 100644
index 0000000..01b9872
--- /dev/null
+++ b/server/database/interface.go
@@ -0,0 +1,15 @@
+package database
+
+import (
+ "github.com/akrylysov/pogreb"
+ "github.com/go-acme/lego/v4/certificate"
+)
+
+type CertDB interface {
+ Close() error
+ Put(name string, cert *certificate.Resource) error
+ Get(name []byte) (*certificate.Resource, error)
+ Delete(key []byte) error
+ Compact() (pogreb.CompactionResult, error)
+ Items() *pogreb.ItemIterator
+}
diff --git a/server/database/setup.go b/server/database/setup.go
new file mode 100644
index 0000000..f3cac16
--- /dev/null
+++ b/server/database/setup.go
@@ -0,0 +1,117 @@
+package database
+
+import (
+ "bytes"
+ "context"
+ "encoding/gob"
+ "fmt"
+ "time"
+
+ "github.com/akrylysov/pogreb"
+ "github.com/akrylysov/pogreb/fs"
+ "github.com/go-acme/lego/v4/certificate"
+ "github.com/rs/zerolog/log"
+)
+
+type aDB struct {
+ ctx context.Context
+ cancel context.CancelFunc
+ intern *pogreb.DB
+ syncInterval time.Duration
+}
+
+func (p aDB) Close() error {
+ p.cancel()
+ return p.intern.Sync()
+}
+
+func (p aDB) Put(name string, cert *certificate.Resource) error {
+ var resGob bytes.Buffer
+ if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
+ return err
+ }
+ return p.intern.Put([]byte(name), resGob.Bytes())
+}
+
+func (p aDB) Get(name []byte) (*certificate.Resource, error) {
+ cert := &certificate.Resource{}
+ resBytes, err := p.intern.Get(name)
+ if err != nil {
+ return nil, err
+ }
+ if resBytes == nil {
+ return nil, nil
+ }
+ if err = gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
+ return nil, err
+ }
+ return cert, nil
+}
+
+func (p aDB) Delete(key []byte) error {
+ return p.intern.Delete(key)
+}
+
+func (p aDB) Compact() (pogreb.CompactionResult, error) {
+ return p.intern.Compact()
+}
+
+func (p aDB) Items() *pogreb.ItemIterator {
+ return p.intern.Items()
+}
+
+var _ CertDB = &aDB{}
+
+func (p aDB) sync() {
+ for {
+ err := p.intern.Sync()
+ if err != nil {
+ log.Err(err).Msg("Syncing cert database failed")
+ }
+ select {
+ case <-p.ctx.Done():
+ return
+ case <-time.After(p.syncInterval):
+ }
+ }
+}
+
+func (p aDB) compact() {
+ for {
+ err := p.intern.Sync()
+ if err != nil {
+ log.Err(err).Msg("Syncing cert database failed")
+ }
+ select {
+ case <-p.ctx.Done():
+ return
+ case <-time.After(p.syncInterval):
+ }
+ }
+}
+
+func New(path string) (CertDB, error) {
+ if path == "" {
+ return nil, fmt.Errorf("path not set")
+ }
+ db, err := pogreb.Open(path, &pogreb.Options{
+ BackgroundSyncInterval: 30 * time.Second,
+ BackgroundCompactionInterval: 6 * time.Hour,
+ FileSystem: fs.OSMMap,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ result := &aDB{
+ ctx: ctx,
+ cancel: cancel,
+ intern: db,
+ syncInterval: 5 * time.Minute,
+ }
+
+ go result.sync()
+
+ return result, nil
+}
diff --git a/server/dns/const.go b/server/dns/const.go
new file mode 100644
index 0000000..bb2413b
--- /dev/null
+++ b/server/dns/const.go
@@ -0,0 +1,6 @@
+package dns
+
+import "time"
+
+// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
+var lookupCacheTimeout = 15 * time.Minute
diff --git a/server/dns/dns.go b/server/dns/dns.go
new file mode 100644
index 0000000..dc759b0
--- /dev/null
+++ b/server/dns/dns.go
@@ -0,0 +1,56 @@
+package dns
+
+import (
+ "net"
+ "strings"
+
+ "codeberg.org/codeberg/pages/server/cache"
+)
+
+// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
+// If everything is fine, it returns the target data.
+func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
+ // Get CNAME or TXT
+ var cname string
+ var err error
+ if cachedName, ok := dnsLookupCache.Get(domain); ok {
+ cname = cachedName.(string)
+ } else {
+ cname, err = net.LookupCNAME(domain)
+ cname = strings.TrimSuffix(cname, ".")
+ if err != nil || !strings.HasSuffix(cname, mainDomainSuffix) {
+ cname = ""
+ // TODO: check if the A record matches!
+ names, err := net.LookupTXT(domain)
+ if err == nil {
+ for _, name := range names {
+ name = strings.TrimSuffix(name, ".")
+ if strings.HasSuffix(name, mainDomainSuffix) {
+ cname = name
+ break
+ }
+ }
+ }
+ }
+ _ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout)
+ }
+ if cname == "" {
+ return
+ }
+ cnameParts := strings.Split(strings.TrimSuffix(cname, mainDomainSuffix), ".")
+ targetOwner = cnameParts[len(cnameParts)-1]
+ if len(cnameParts) > 1 {
+ targetRepo = cnameParts[len(cnameParts)-2]
+ }
+ if len(cnameParts) > 2 {
+ targetBranch = cnameParts[len(cnameParts)-3]
+ }
+ if targetRepo == "" {
+ targetRepo = "pages"
+ }
+ if targetBranch == "" && targetRepo != "pages" {
+ targetBranch = "pages"
+ }
+ // if targetBranch is still empty, the caller must find the default branch
+ return
+}
diff --git a/server/handler.go b/server/handler.go
new file mode 100644
index 0000000..1aaf476
--- /dev/null
+++ b/server/handler.go
@@ -0,0 +1,292 @@
+package server
+
+import (
+ "bytes"
+ "strings"
+
+ "github.com/rs/zerolog/log"
+ "github.com/valyala/fasthttp"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/dns"
+ "codeberg.org/codeberg/pages/server/upstream"
+ "codeberg.org/codeberg/pages/server/utils"
+)
+
+// Handler handles a single HTTP request to the web server.
+func Handler(mainDomainSuffix, rawDomain []byte,
+ giteaRoot, rawInfoPage, giteaAPIToken string,
+ blacklistedPaths, allowedCorsDomains [][]byte,
+ dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
+ return func(ctx *fasthttp.RequestCtx) {
+ log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
+
+ ctx.Response.Header.Set("Server", "Codeberg Pages")
+
+ // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
+ ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
+
+ // Enable browser caching for up to 10 minutes
+ ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
+
+ trimmedHost := utils.TrimHostPort(ctx.Request.Host())
+
+ // Add HSTS for RawDomain and MainDomainSuffix
+ if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
+ ctx.Response.Header.Set("Strict-Transport-Security", hsts)
+ }
+
+ // Block all methods not required for static pages
+ if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
+ ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
+ ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
+ return
+ }
+
+ // Block blacklisted paths (like ACME challenges)
+ for _, blacklistedPath := range blacklistedPaths {
+ if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
+ html.ReturnErrorPage(ctx, fasthttp.StatusForbidden)
+ return
+ }
+ }
+
+ // Allow CORS for specified domains
+ if ctx.IsOptions() {
+ allowCors := false
+ for _, allowedCorsDomain := range allowedCorsDomains {
+ if bytes.Equal(trimmedHost, allowedCorsDomain) {
+ allowCors = true
+ break
+ }
+ }
+ if allowCors {
+ ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
+ ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
+ }
+ ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
+ ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
+ return
+ }
+
+ // Prepare request information to Gitea
+ var targetOwner, targetRepo, targetBranch, targetPath string
+ var targetOptions = &upstream.Options{
+ ForbiddenMimeTypes: map[string]struct{}{},
+ TryIndexPages: true,
+ }
+
+ // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
+ // also disallow search indexing and add a Link header to the canonical URL.
+ var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
+ if repo == "" {
+ return false
+ }
+
+ // Check if the branch exists, otherwise treat it as a file path
+ branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
+ if branchTimestampResult == nil {
+ // branch doesn't exist
+ return false
+ }
+
+ // Branch exists, use it
+ targetRepo = repo
+ targetPath = strings.Trim(strings.Join(path, "/"), "/")
+ targetBranch = branchTimestampResult.Branch
+
+ targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
+
+ if canonicalLink != "" {
+ // Hide from search machines & add canonical link
+ ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
+ ctx.Response.Header.Set("Link",
+ strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
+ "; rel=\"canonical\"",
+ )
+ }
+
+ return true
+ }
+
+ log.Debug().Msg("preparations")
+ if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
+ // Serve raw content from RawDomain
+ log.Debug().Msg("raw domain")
+
+ targetOptions.TryIndexPages = false
+ targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
+ targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
+
+ pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+ if len(pathElements) < 2 {
+ // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
+ ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect)
+ return
+ }
+ targetOwner = pathElements[0]
+ targetRepo = pathElements[1]
+
+ // raw.codeberg.org/example/myrepo/@main/index.html
+ if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
+ log.Debug().Msg("raw domain preparations, now trying with specified branch")
+ if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
+ giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
+ ) {
+ log.Debug().Msg("tryBranch, now trying upstream")
+ tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+ targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+ giteaRoot, giteaAPIToken,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ return
+ }
+ log.Debug().Msg("missing branch")
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ return
+ }
+
+ log.Debug().Msg("raw domain preparations, now trying with default branch")
+ tryBranch(targetRepo, "", pathElements[2:],
+ giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
+ )
+ log.Debug().Msg("tryBranch, now trying upstream")
+ tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+ targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+ giteaRoot, giteaAPIToken,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ return
+
+ } else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
+ // Serve pages from subdomains of MainDomainSuffix
+ log.Debug().Msg("main domain suffix")
+
+ pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+ targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
+ targetRepo = pathElements[0]
+ targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
+
+ if targetOwner == "www" {
+ // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
+ ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect)
+ return
+ }
+
+ // Check if the first directory is a repo with the second directory as a branch
+ // example.codeberg.page/myrepo/@main/index.html
+ if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
+ if targetRepo == "pages" {
+ // example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
+ ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
+ return
+ }
+
+ log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
+ if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
+ "/"+pathElements[0]+"/%p",
+ ) {
+ log.Debug().Msg("tryBranch, now trying upstream")
+ tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+ targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+ giteaRoot, giteaAPIToken,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ } else {
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ }
+ return
+ }
+
+ // Check if the first directory is a branch for the "pages" repo
+ // example.codeberg.page/@main/index.html
+ if strings.HasPrefix(pathElements[0], "@") {
+ log.Debug().Msg("main domain preparations, now trying with specified branch")
+ if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
+ log.Debug().Msg("tryBranch, now trying upstream")
+ tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+ targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+ giteaRoot, giteaAPIToken,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ } else {
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ }
+ return
+ }
+
+ // Check if the first directory is a repo with a "pages" branch
+ // example.codeberg.page/myrepo/index.html
+ // example.codeberg.page/pages/... is not allowed here.
+ log.Debug().Msg("main domain preparations, now trying with specified repo")
+ if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
+ log.Debug().Msg("tryBranch, now trying upstream")
+ tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+ targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+ giteaRoot, giteaAPIToken,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ return
+ }
+
+ // Try to use the "pages" repo on its default branch
+ // example.codeberg.page/index.html
+ log.Debug().Msg("main domain preparations, now trying with default repo/branch")
+ if tryBranch("pages", "", pathElements, "") {
+ log.Debug().Msg("tryBranch, now trying upstream")
+ tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+ targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+ giteaRoot, giteaAPIToken,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ return
+ }
+
+ // Couldn't find a valid repo/branch
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ return
+ } else {
+ trimmedHostStr := string(trimmedHost)
+
+ // Serve pages from external domains
+ targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
+ if targetOwner == "" {
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ return
+ }
+
+ pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+ canonicalLink := ""
+ if strings.HasPrefix(pathElements[0], "@") {
+ targetBranch = pathElements[0][1:]
+ pathElements = pathElements[1:]
+ canonicalLink = "/%p"
+ }
+
+ // Try to use the given repo on the given branch or the default branch
+ log.Debug().Msg("custom domain preparations, now trying with details from DNS")
+ if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
+ canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+ if !valid {
+ html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
+ return
+ } else if canonicalDomain != trimmedHostStr {
+ // only redirect if the target is also a codeberg page!
+ targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
+ if targetOwner != "" {
+ ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
+ return
+ }
+
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ return
+ }
+
+ log.Debug().Msg("tryBranch, now trying upstream")
+ tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+ targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+ giteaRoot, giteaAPIToken,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ return
+ }
+
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ return
+ }
+ }
+}
diff --git a/handler_test.go b/server/handler_test.go
similarity index 74%
rename from handler_test.go
rename to server/handler_test.go
index 70b655e..0ec9fcd 100644
--- a/handler_test.go
+++ b/server/handler_test.go
@@ -1,13 +1,29 @@
-package main
+package server
import (
"fmt"
"github.com/valyala/fasthttp"
"testing"
"time"
+
+ "codeberg.org/codeberg/pages/server/cache"
)
func TestHandlerPerformance(t *testing.T) {
+ testHandler := Handler(
+ []byte("codeberg.page"),
+ []byte("raw.codeberg.org"),
+ "https://codeberg.org",
+ "https://docs.codeberg.org/pages/raw-content/",
+ "",
+ [][]byte{[]byte("/.well-known/acme-challenge/")},
+ [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
+ cache.NewKeyValueCache(),
+ cache.NewKeyValueCache(),
+ cache.NewKeyValueCache(),
+ cache.NewKeyValueCache(),
+ )
+
ctx := &fasthttp.RequestCtx{
Request: *fasthttp.AcquireRequest(),
Response: *fasthttp.AcquireResponse(),
@@ -15,7 +31,7 @@ func TestHandlerPerformance(t *testing.T) {
ctx.Request.SetRequestURI("http://mondstern.codeberg.page/")
fmt.Printf("Start: %v\n", time.Now())
start := time.Now()
- handler(ctx)
+ testHandler(ctx)
end := time.Now()
fmt.Printf("Done: %v\n", time.Now())
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
@@ -28,7 +44,7 @@ func TestHandlerPerformance(t *testing.T) {
ctx.Response.ResetBody()
fmt.Printf("Start: %v\n", time.Now())
start = time.Now()
- handler(ctx)
+ testHandler(ctx)
end = time.Now()
fmt.Printf("Done: %v\n", time.Now())
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
@@ -42,7 +58,7 @@ func TestHandlerPerformance(t *testing.T) {
ctx.Request.SetRequestURI("http://example.momar.xyz/")
fmt.Printf("Start: %v\n", time.Now())
start = time.Now()
- handler(ctx)
+ testHandler(ctx)
end = time.Now()
fmt.Printf("Done: %v\n", time.Now())
if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 1 {
diff --git a/server/helpers.go b/server/helpers.go
new file mode 100644
index 0000000..6d55ddf
--- /dev/null
+++ b/server/helpers.go
@@ -0,0 +1,15 @@
+package server
+
+import (
+ "bytes"
+)
+
+// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
+// string for custom domains.
+func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string {
+ if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) {
+ return "max-age=63072000; includeSubdomains; preload"
+ } else {
+ return ""
+ }
+}
diff --git a/server/setup.go b/server/setup.go
new file mode 100644
index 0000000..67c1c42
--- /dev/null
+++ b/server/setup.go
@@ -0,0 +1,47 @@
+package server
+
+import (
+ "bytes"
+ "net/http"
+ "time"
+
+ "github.com/valyala/fasthttp"
+
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/utils"
+)
+
+func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
+ // Enable compression by wrapping the handler with the compression function provided by FastHTTP
+ compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
+
+ return &fasthttp.Server{
+ Handler: compressedHandler,
+ DisablePreParseMultipartForm: true,
+ MaxRequestBodySize: 0,
+ NoDefaultServerHeader: true,
+ NoDefaultDate: true,
+ ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
+ Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
+ MaxConnsPerIP: 100,
+ }
+}
+
+func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server {
+ challengePath := []byte("/.well-known/acme-challenge/")
+
+ return &fasthttp.Server{
+ Handler: func(ctx *fasthttp.RequestCtx) {
+ if bytes.HasPrefix(ctx.Path(), challengePath) {
+ challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
+ if !ok || challenge == nil {
+ ctx.SetStatusCode(http.StatusNotFound)
+ ctx.SetBodyString("no challenge for this token")
+ }
+ ctx.SetBodyString(challenge.(string))
+ } else {
+ ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
+ }
+ },
+ }
+}
diff --git a/server/try.go b/server/try.go
new file mode 100644
index 0000000..31cd7f4
--- /dev/null
+++ b/server/try.go
@@ -0,0 +1,49 @@
+package server
+
+import (
+ "bytes"
+ "strings"
+
+ "github.com/valyala/fasthttp"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/upstream"
+)
+
+// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
+func tryUpstream(ctx *fasthttp.RequestCtx,
+ mainDomainSuffix, trimmedHost []byte,
+
+ targetOptions *upstream.Options,
+ targetOwner, targetRepo, targetBranch, targetPath,
+
+ giteaRoot, giteaAPIToken string,
+ canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) {
+
+ // check if a canonical domain exists on a request on MainDomain
+ if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
+ canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+ if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
+ canonicalPath := string(ctx.RequestURI())
+ if targetRepo != "pages" {
+ path := strings.SplitN(canonicalPath, "/", 3)
+ if len(path) >= 3 {
+ canonicalPath = "/" + path[2]
+ }
+ }
+ ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
+ return
+ }
+ }
+
+ targetOptions.TargetOwner = targetOwner
+ targetOptions.TargetRepo = targetRepo
+ targetOptions.TargetBranch = targetBranch
+ targetOptions.TargetPath = targetPath
+
+ // Try to request the file from the Gitea API
+ if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+ html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
+ }
+}
diff --git a/server/upstream/const.go b/server/upstream/const.go
new file mode 100644
index 0000000..77f64dd
--- /dev/null
+++ b/server/upstream/const.go
@@ -0,0 +1,21 @@
+package upstream
+
+import "time"
+
+// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
+var defaultBranchCacheTimeout = 15 * time.Minute
+
+// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
+// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
+// picked up faster, while still allowing the content to be cached longer if nothing changes.
+var branchExistenceCacheTimeout = 5 * time.Minute
+
+// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
+// on your available memory.
+var fileCacheTimeout = 5 * time.Minute
+
+// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
+var fileCacheSizeLimit = 1024 * 1024
+
+// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
+var canonicalDomainCacheTimeout = 15 * time.Minute
diff --git a/server/upstream/domains.go b/server/upstream/domains.go
new file mode 100644
index 0000000..47a5564
--- /dev/null
+++ b/server/upstream/domains.go
@@ -0,0 +1,53 @@
+package upstream
+
+import (
+ "strings"
+
+ "github.com/valyala/fasthttp"
+
+ "codeberg.org/codeberg/pages/server/cache"
+)
+
+// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
+func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+ domains := []string{}
+ valid := false
+ if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
+ domains = cachedValue.([]string)
+ for _, domain := range domains {
+ if domain == actualDomain {
+ valid = true
+ break
+ }
+ }
+ } else {
+ req := fasthttp.AcquireRequest()
+ req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
+ res := fasthttp.AcquireResponse()
+
+ err := client.Do(req, res)
+ if err == nil && res.StatusCode() == fasthttp.StatusOK {
+ for _, domain := range strings.Split(string(res.Body()), "\n") {
+ domain = strings.ToLower(domain)
+ domain = strings.TrimSpace(domain)
+ domain = strings.TrimPrefix(domain, "http://")
+ domain = strings.TrimPrefix(domain, "https://")
+ if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
+ domains = append(domains, domain)
+ }
+ if domain == actualDomain {
+ valid = true
+ }
+ }
+ }
+ domains = append(domains, targetOwner+mainDomainSuffix)
+ if domains[len(domains)-1] == actualDomain {
+ valid = true
+ }
+ if targetRepo != "" && targetRepo != "pages" {
+ domains[len(domains)-1] += "/" + targetRepo
+ }
+ _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout)
+ }
+ return domains[0], valid
+}
diff --git a/server/upstream/helper.go b/server/upstream/helper.go
new file mode 100644
index 0000000..b5ee77a
--- /dev/null
+++ b/server/upstream/helper.go
@@ -0,0 +1,55 @@
+package upstream
+
+import (
+ "time"
+
+ "github.com/valyala/fasthttp"
+ "github.com/valyala/fastjson"
+
+ "codeberg.org/codeberg/pages/server/cache"
+)
+
+type branchTimestamp struct {
+ Branch string
+ Timestamp time.Time
+}
+
+// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
+// (or nil if the branch doesn't exist)
+func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+ if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
+ if result == nil {
+ return nil
+ }
+ return result.(*branchTimestamp)
+ }
+ result := &branchTimestamp{}
+ result.Branch = branch
+ if branch == "" {
+ // Get default branch
+ var body = make([]byte, 0)
+ // TODO: use header for API key?
+ status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
+ if err != nil || status != 200 {
+ _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
+ return nil
+ }
+ result.Branch = fastjson.GetString(body, "default_branch")
+ }
+
+ var body = make([]byte, 0)
+ status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
+ if err != nil || status != 200 {
+ return nil
+ }
+
+ result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
+ _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
+ return result
+}
+
+type fileResponse struct {
+ exists bool
+ mimeType string
+ body []byte
+}
diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go
new file mode 100644
index 0000000..88d3471
--- /dev/null
+++ b/server/upstream/upstream.go
@@ -0,0 +1,202 @@
+package upstream
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "mime"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/rs/zerolog/log"
+ "github.com/valyala/fasthttp"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+)
+
+// upstreamIndexPages lists pages that may be considered as index pages for directories.
+var upstreamIndexPages = []string{
+ "index.html",
+}
+
+// Options provides various options for the upstream request.
+type Options struct {
+ TargetOwner,
+ TargetRepo,
+ TargetBranch,
+ TargetPath,
+
+ DefaultMimeType string
+ ForbiddenMimeTypes map[string]struct{}
+ TryIndexPages bool
+ BranchTimestamp time.Time
+ // internal
+ appendTrailingSlash bool
+ redirectIfExists string
+}
+
+var client = fasthttp.Client{
+ ReadTimeout: 10 * time.Second,
+ MaxConnDuration: 60 * time.Second,
+ MaxConnWaitTimeout: 1000 * time.Millisecond,
+ MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+}
+
+// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
+func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
+ log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
+
+ if o.ForbiddenMimeTypes == nil {
+ o.ForbiddenMimeTypes = map[string]struct{}{}
+ }
+
+ // Check if the branch exists and when it was modified
+ if o.BranchTimestamp.IsZero() {
+ branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
+
+ if branch == nil {
+ html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ return true
+ }
+ o.TargetBranch = branch.Branch
+ o.BranchTimestamp = branch.Timestamp
+ }
+
+ if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
+ html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
+ return true
+ }
+
+ // Check if the browser has a cached version
+ if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
+ if !ifModifiedSince.Before(o.BranchTimestamp) {
+ ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
+ return true
+ }
+ }
+ log.Debug().Msg("preparations")
+
+ // Make a GET request to the upstream URL
+ uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
+ var req *fasthttp.Request
+ var res *fasthttp.Response
+ var cachedResponse fileResponse
+ var err error
+ if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
+ cachedResponse = cachedValue.(fileResponse)
+ } else {
+ req = fasthttp.AcquireRequest()
+ req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
+ res = fasthttp.AcquireResponse()
+ res.SetBodyStream(&strings.Reader{}, -1)
+ err = client.Do(req, res)
+ }
+ log.Debug().Msg("acquisition")
+
+ // Handle errors
+ if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
+ if o.TryIndexPages {
+ // copy the o struct & try if an index page exists
+ optionsForIndexPages := *o
+ optionsForIndexPages.TryIndexPages = false
+ optionsForIndexPages.appendTrailingSlash = true
+ for _, indexPage := range upstreamIndexPages {
+ optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
+ if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+ _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+ exists: false,
+ }, fileCacheTimeout)
+ return true
+ }
+ }
+ // compatibility fix for GitHub Pages (/example → /example.html)
+ optionsForIndexPages.appendTrailingSlash = false
+ optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
+ optionsForIndexPages.TargetPath = o.TargetPath + ".html"
+ if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+ _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+ exists: false,
+ }, fileCacheTimeout)
+ return true
+ }
+ }
+ ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+ if res != nil {
+ // Update cache if the request is fresh
+ _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+ exists: false,
+ }, fileCacheTimeout)
+ }
+ return false
+ }
+ if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
+ fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
+ html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
+ return true
+ }
+
+ // Append trailing slash if missing (for index files), and redirect to fix filenames in general
+ // o.appendTrailingSlash is only true when looking for index pages
+ if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
+ ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
+ return true
+ }
+ if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
+ ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
+ return true
+ }
+ if o.redirectIfExists != "" {
+ ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
+ return true
+ }
+ log.Debug().Msg("error handling")
+
+ // Set the MIME type
+ mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+ mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
+ if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
+ if o.DefaultMimeType != "" {
+ mimeType = o.DefaultMimeType
+ } else {
+ mimeType = "application/octet-stream"
+ }
+ }
+ ctx.Response.Header.SetContentType(mimeType)
+
+ // Everything's okay so far
+ ctx.Response.SetStatusCode(fasthttp.StatusOK)
+ ctx.Response.Header.SetLastModified(o.BranchTimestamp)
+
+ log.Debug().Msg("response preparations")
+
+ // Write the response body to the original request
+ var cacheBodyWriter bytes.Buffer
+ if res != nil {
+ if res.Header.ContentLength() > fileCacheSizeLimit {
+ err = res.BodyWriteTo(ctx.Response.BodyWriter())
+ } else {
+ // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
+ err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
+ }
+ } else {
+ _, err = ctx.Write(cachedResponse.body)
+ }
+ if err != nil {
+ fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
+ html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
+ return true
+ }
+ log.Debug().Msg("response")
+
+ if res != nil && ctx.Err() == nil {
+ cachedResponse.exists = true
+ cachedResponse.mimeType = mimeType
+ cachedResponse.body = cacheBodyWriter.Bytes()
+ _ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
+ }
+
+ return true
+}
diff --git a/server/utils/utils.go b/server/utils/utils.go
new file mode 100644
index 0000000..7be330f
--- /dev/null
+++ b/server/utils/utils.go
@@ -0,0 +1,11 @@
+package utils
+
+import "bytes"
+
+func TrimHostPort(host []byte) []byte {
+ i := bytes.IndexByte(host, ':')
+ if i >= 0 {
+ return host[:i]
+ }
+ return host
+}
diff --git a/server/utils/utils_test.go b/server/utils/utils_test.go
new file mode 100644
index 0000000..3dc0632
--- /dev/null
+++ b/server/utils/utils_test.go
@@ -0,0 +1,13 @@
+package utils
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestTrimHostPort(t *testing.T) {
+ assert.EqualValues(t, "aa", TrimHostPort([]byte("aa")))
+ assert.EqualValues(t, "", TrimHostPort([]byte(":")))
+ assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80")))
+}