package main import ( "bytes" "crypto" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/gob" "encoding/json" "encoding/pem" "errors" "github.com/OrlovEvgeny/go-mcache" "github.com/akrylysov/pogreb/fs" "github.com/go-acme/lego/v4/certificate" "github.com/go-acme/lego/v4/challenge" "github.com/go-acme/lego/v4/challenge/tlsalpn01" "github.com/go-acme/lego/v4/providers/dns" "io/ioutil" "log" "math/big" "os" "strconv" "strings" "sync" "time" "github.com/akrylysov/pogreb" "github.com/reugn/equalizer" "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/lego" "github.com/go-acme/lego/v4/registration" ) // tlsConfig contains the configuration for generating, serving and cleaning up Let's Encrypt certificates. var tlsConfig = &tls.Config{ // check DNS name & get certificate from Let's Encrypt GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { sni := strings.ToLower(strings.TrimSpace(info.ServerName)) sniBytes := []byte(sni) if len(sni) < 1 { return nil, errors.New("missing sni") } if info.SupportedProtos != nil { for _, proto := range info.SupportedProtos { if proto == tlsalpn01.ACMETLS1Protocol { challenge, ok := challengeCache.Get(sni) if !ok { return nil, errors.New("no challenge for this domain") } cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string)) if err != nil { return nil, err } return cert, nil } } } targetOwner := "" if bytes.HasSuffix(sniBytes, MainDomainSuffix) || bytes.Equal(sniBytes, MainDomainSuffix[1:]) { // deliver default certificate for the main domain (*.codeberg.page) sniBytes = MainDomainSuffix sni = string(sniBytes) } else { var targetRepo, targetBranch string targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni) if targetOwner == "" { // DNS not set up, return main certificate to redirect to the docs sniBytes = MainDomainSuffix sni = string(sniBytes) } else { _, _ = targetRepo, targetBranch _, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni) if !valid { sniBytes = MainDomainSuffix sni = string(sniBytes) } } } if tlsCertificate, ok := keyCache.Get(sni); ok { // we can use an existing certificate object return tlsCertificate.(*tls.Certificate), nil } var tlsCertificate tls.Certificate var err error var ok bool if tlsCertificate, ok = retrieveCertFromDB(sniBytes); !ok { // request a new certificate if bytes.Equal(sniBytes, MainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner) if err != nil { return nil, err } } err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute) if err != nil { panic(err) } return &tlsCertificate, nil }, PreferServerCipherSuites: true, NextProtos: []string{ "http/1.1", tlsalpn01.ACMETLS1Protocol, }, // generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration // https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6 MinVersion: tls.VersionTLS12, CipherSuites: []uint16{ tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, }, } // TODO: clean up & move to init var keyCache = mcache.New() var keyDatabase, keyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{ BackgroundSyncInterval: 30 * time.Second, BackgroundCompactionInterval: 6 * time.Hour, FileSystem: fs.OSMMap, }) func CheckUserLimit(user string) error { userLimit, ok := acmeClientCertificateLimitPerUser[user] if !ok { // Each Codeberg user can only add 10 new domains per day. userLimit = equalizer.NewTokenBucket(10, time.Hour*24) acmeClientCertificateLimitPerUser[user] = userLimit } if !userLimit.Ask() { return errors.New("rate limit exceeded: 10 certificates per user per 24 hours") } return nil } var myAcmeAccount AcmeAccount var myAcmeConfig *lego.Config type AcmeAccount struct { Email string Registration *registration.Resource Key crypto.PrivateKey `json:"-"` KeyPEM string `json:"Key"` } func (u *AcmeAccount) GetEmail() string { return u.Email } func (u AcmeAccount) GetRegistration() *registration.Resource { return u.Registration } func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey { return u.Key } var acmeClient, mainDomainAcmeClient *lego.Client var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{} // rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes // TODO: when this is used a lot, we probably have to think of a somewhat better solution? var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute) // rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests) var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second) var challengeCache = mcache.New() type AcmeTLSChallengeProvider struct{} var _ challenge.Provider = AcmeTLSChallengeProvider{} func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { return challengeCache.Set(domain, keyAuth, 1*time.Hour) } func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { challengeCache.Remove(domain) return nil } type AcmeHTTPChallengeProvider struct{} var _ challenge.Provider = AcmeHTTPChallengeProvider{} func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { return challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) } func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { challengeCache.Remove(domain + "/" + token) return nil } func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) { // parse certificate from database res := &certificate.Resource{} if !PogrebGet(keyDatabase, sni, res) { return tls.Certificate{}, false } tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { panic(err) } // TODO: document & put into own function if !bytes.Equal(sni, MainDomainSuffix) { tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0]) if err != nil { panic(err) } // renew certificates 7 days before they expire if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) { // TODO: add ValidUntil to custom res struct if res.CSR != nil && len(res.CSR) > 0 { // CSR stores the time when the renewal shall be tried again nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64) if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) { return tlsCertificate, true } } go (func() { res.CSR = nil // acme client doesn't like CSR to be set tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "") if err != nil { log.Printf("Couldn't renew certificate for %s: %s", sni, err) } })() } } return tlsCertificate, true } var obtainLocks = sync.Map{} func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string) (tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") if os.Getenv("DNS_PROVIDER") == "" && len(domains[0]) > 0 && domains[0][0] == '*' { domains = domains[1:] } // lock to avoid simultaneous requests _, working := obtainLocks.LoadOrStore(name, struct{}{}) if working { for working { time.Sleep(100 * time.Millisecond) _, working = obtainLocks.Load(name) } cert, ok := retrieveCertFromDB([]byte(name)) if !ok { return tls.Certificate{}, errors.New("certificate failed in synchronous request") } return cert, nil } defer obtainLocks.Delete(name) if acmeClient == nil { return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!"), nil } // request actual cert var res *certificate.Resource var err error if renew != nil && renew.CertURL != "" { if os.Getenv("ACME_USE_RATE_LIMITS") != "false" { acmeClientRequestLimit.Take() } log.Printf("Renewing certificate for %v", domains) res, err = acmeClient.Certificate.Renew(*renew, true, false, "") if err != nil { log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err) res = nil } } if res == nil { if user != "" { if err := CheckUserLimit(user); err != nil { return tls.Certificate{}, err } } if os.Getenv("ACME_USE_RATE_LIMITS") != "false" { acmeClientOrderLimit.Take() acmeClientRequestLimit.Take() } log.Printf("Requesting new certificate for %v", domains) res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{ Domains: domains, Bundle: true, MustStaple: false, }) } if err != nil { log.Printf("Couldn't obtain certificate for %v: %s", domains, err) if renew != nil && renew.CertURL != "" { tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey) if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) { // avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)) PogrebPut(keyDatabase, []byte(name), renew) return tlsCertificate, nil } } return mockCert(domains[0], err.Error()), err } log.Printf("Obtained certificate for %v", domains) PogrebPut(keyDatabase, []byte(name), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { return tls.Certificate{}, err } return tlsCertificate, nil } func mockCert(domain string, msg string) tls.Certificate { key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048) if err != nil { panic(err) } template := x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{ CommonName: domain, Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"}, OrganizationalUnit: []string{ "Will not try again for 6 hours to avoid hitting rate limits for your domain.", "Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " + "free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n", "Error message: " + msg, }, }, // certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours NotAfter: time.Now().Add(time.Hour*24*7 + time.Hour*6), NotBefore: time.Now(), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, } certBytes, err := x509.CreateCertificate( rand.Reader, &template, &template, &key.(*rsa.PrivateKey).PublicKey, key, ) if err != nil { panic(err) } out := &bytes.Buffer{} err = pem.Encode(out, &pem.Block{ Bytes: certBytes, Type: "CERTIFICATE", }) if err != nil { panic(err) } outBytes := out.Bytes() res := &certificate.Resource{ PrivateKey: certcrypto.PEMEncode(key), Certificate: outBytes, IssuerCertificate: outBytes, Domain: domain, } databaseName := domain if domain == "*"+string(MainDomainSuffix) || domain == string(MainDomainSuffix[1:]) { databaseName = string(MainDomainSuffix) } PogrebPut(keyDatabase, []byte(databaseName), res) tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey) if err != nil { panic(err) } return tlsCertificate } func setupCertificates() { if keyDatabaseErr != nil { panic(keyDatabaseErr) } if os.Getenv("ACME_ACCEPT_TERMS") != "true" || (os.Getenv("DNS_PROVIDER") == "" && os.Getenv("ACME_API") != "https://acme.mock.directory") { panic(errors.New("you must set ACME_ACCEPT_TERMS and DNS_PROVIDER, unless ACME_API is set to https://acme.mock.directory")) } // getting main cert before ACME account so that we can panic here on database failure without hitting rate limits mainCertBytes, err := keyDatabase.Get(MainDomainSuffix) if err != nil { // key database is not working panic(err) } if account, err := ioutil.ReadFile("acme-account.json"); err == nil { err = json.Unmarshal(account, &myAcmeAccount) if err != nil { panic(err) } myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM)) if err != nil { panic(err) } myAcmeConfig = lego.NewConfig(&myAcmeAccount) myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 _, err := lego.NewClient(myAcmeConfig) if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } } else if os.IsNotExist(err) { privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { panic(err) } myAcmeAccount = AcmeAccount{ Email: envOr("ACME_EMAIL", "noreply@example.email"), Key: privateKey, KeyPEM: string(certcrypto.PEMEncode(privateKey)), } myAcmeConfig = lego.NewConfig(&myAcmeAccount) myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 tempClient, err := lego.NewClient(myAcmeConfig) if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { // accept terms & log in to EAB if os.Getenv("ACME_EAB_KID") == "" || os.Getenv("ACME_EAB_HMAC") == "" { reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true"}) if err != nil { log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) } else { myAcmeAccount.Registration = reg } } else { reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true", Kid: os.Getenv("ACME_EAB_KID"), HmacEncoded: os.Getenv("ACME_EAB_HMAC"), }) if err != nil { log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err) } else { myAcmeAccount.Registration = reg } } if myAcmeAccount.Registration != nil { acmeAccountJson, err := json.Marshal(myAcmeAccount) if err != nil { log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err) select {} } err = ioutil.WriteFile("acme-account.json", acmeAccountJson, 0600) if err != nil { log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err) select {} } } } } else { panic(err) } acmeClient, err = lego.NewClient(myAcmeConfig) if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{}) if err != nil { log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err) } if os.Getenv("ENABLE_HTTP_SERVER") == "true" { err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{}) if err != nil { log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err) } } } mainDomainAcmeClient, err = lego.NewClient(myAcmeConfig) if err != nil { log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err) } else { if os.Getenv("DNS_PROVIDER") == "" { // using mock server, don't use wildcard certs err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{}) if err != nil { log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err) } } else { provider, err := dns.NewDNSChallengeProviderByName(os.Getenv("DNS_PROVIDER")) if err != nil { log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err) } err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider) if err != nil { log.Printf("[ERROR] Can't create DNS-01 provider: %s", err) } } } if mainCertBytes == nil { _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, nil, "") if err != nil { log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err) } } go (func() { for { err := keyDatabase.Sync() if err != nil { log.Printf("[ERROR] Syncing key database failed: %s", err) } time.Sleep(5 * time.Minute) // TODO: graceful exit } })() go (func() { for { // clean up expired certs now := time.Now() expiredCertCount := 0 keyDatabaseIterator := keyDatabase.Items() key, resBytes, err := keyDatabaseIterator.Next() for err == nil { if !bytes.Equal(key, MainDomainSuffix) { resGob := bytes.NewBuffer(resBytes) resDec := gob.NewDecoder(resGob) res := &certificate.Resource{} err = resDec.Decode(res) if err != nil { panic(err) } tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) if err != nil || !tlsCertificates[0].NotAfter.After(now) { err := keyDatabase.Delete(key) if err != nil { log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err) } else { expiredCertCount++ } } } key, resBytes, err = keyDatabaseIterator.Next() } log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount) // compact the database result, err := keyDatabase.Compact() if err != nil { log.Printf("[ERROR] Compacting key database failed: %s", err) } else { log.Printf("[INFO] Compacted key database (%+v)", result) } // update main cert res := &certificate.Resource{} if !PogrebGet(keyDatabase, MainDomainSuffix, res) { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted") } else { tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate) // renew main certificate 30 days before it expires if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) { go (func() { _, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, res, "") if err != nil { log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err) } })() } } time.Sleep(12 * time.Hour) } })() }