start using urfave/cli

This commit is contained in:
6543 2021-12-03 02:12:51 +01:00
parent bdc2d0c259
commit ac93a5661c
No known key found for this signature in database
GPG Key ID: C99B82E40B027BAE
7 changed files with 216 additions and 132 deletions

View File

@ -15,3 +15,21 @@
- `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
- `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.
See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
//
// Mapping custom domains is not static anymore, but can be done with DNS:
//
// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The
// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
//
// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
// www.example.org. IN CNAME main.pages.example.codeberg.page.
//
// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
// for "example.org" (if your provider allows ALIAS or similar records):
// example.org IN ALIAS codeberg.page.
//
// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.

40
cmd/certs.go Normal file
View File

@ -0,0 +1,40 @@
package cmd
import (
"os"
"github.com/urfave/cli/v2"
pages_server "codeberg.org/codeberg/pages/server"
)
var Certs = &cli.Command{
Name: "certs",
Usage: "manage certs manually",
Action: certs,
}
func certs(ctx *cli.Context) error {
if ctx.Args().Len() >= 1 && ctx.Args().First() == "--remove-certificate" {
if ctx.Args().Len() == 1 {
println("--remove-certificate requires at least one domain as an argument")
os.Exit(1)
}
domains := ctx.Args().Slice()[2:]
if pages_server.KeyDatabaseErr != nil {
panic(pages_server.KeyDatabaseErr)
}
for _, domain := range domains {
if err := pages_server.KeyDatabase.Delete([]byte(domain)); err != nil {
panic(err)
}
}
if err := pages_server.KeyDatabase.Sync(); err != nil {
panic(err)
}
os.Exit(0)
}
return nil
}

37
cmd/flags.go Normal file
View File

@ -0,0 +1,37 @@
package cmd
import (
"codeberg.org/codeberg/pages/server"
"github.com/urfave/cli/v2"
)
// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
var GiteaRoot = []byte(server.EnvOr("GITEA_ROOT", "https://codeberg.org"))
var GiteaApiToken = server.EnvOr("GITEA_API_TOKEN", "")
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
// (set to []byte(nil) to disable raw content hosting)
var RawDomain = []byte(server.EnvOr("RAW_DOMAIN", "raw.codeberg.org"))
// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
var RawInfoPage = server.EnvOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/")
var ServeFlags = []cli.Flag{
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
// var MainDomainSuffix = []byte("." + server.EnvOr("PAGES_DOMAIN", "codeberg.page"))
&cli.StringFlag{
Name: "main-domain-suffix",
Aliases: nil,
Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
EnvVars: []string{"PAGES_DOMAIN"},
FilePath: "",
Required: false,
Hidden: false,
TakesFile: false,
Value: "codeberg.page",
},
}

98
cmd/main.go Normal file
View File

@ -0,0 +1,98 @@
package cmd
import (
"bytes"
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
"os"
"time"
"github.com/urfave/cli/v2"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server"
)
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
var AllowedCorsDomains = [][]byte{
RawDomain,
[]byte("fonts.codeberg.org"),
[]byte("design.codeberg.org"),
}
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
var BlacklistedPaths = [][]byte{
[]byte("/.well-known/acme-challenge/"),
}
// Serve sets up and starts the web server.
func Serve(ctx *cli.Context) error {
mainDomainSuffix := []byte(ctx.String("main-domain-suffix"))
// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) {
mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...)
}
GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'})
// Use HOST and PORT environment variables to determine listening address
address := fmt.Sprintf("%s:%s", server.EnvOr("HOST", "[::]"), server.EnvOr("PORT", "443"))
log.Printf("Listening on https://%s", address)
// Create handler based on settings
handler := server.Handler(mainDomainSuffix, RawDomain, GiteaRoot, RawInfoPage, GiteaApiToken, BlacklistedPaths, AllowedCorsDomains)
// Enable compression by wrapping the handler with the compression function provided by FastHTTP
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
fastServer := &fasthttp.Server{
Handler: compressedHandler,
DisablePreParseMultipartForm: true,
MaxRequestBodySize: 0,
NoDefaultServerHeader: true,
NoDefaultDate: true,
ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
MaxConnsPerIP: 100,
}
// Setup listener and TLS
listener, err := net.Listen("tcp", address)
if err != nil {
log.Fatalf("Couldn't create listener: %s", err)
}
listener = tls.NewListener(listener, server.TlsConfig(mainDomainSuffix, string(GiteaRoot), GiteaApiToken))
server.SetupCertificates(mainDomainSuffix)
if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
go (func() {
challengePath := []byte("/.well-known/acme-challenge/")
err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) {
if bytes.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := server.ChallengeCache.Get(string(server.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
if !ok || challenge == nil {
ctx.SetStatusCode(http.StatusNotFound)
ctx.SetBodyString("no challenge for this token")
}
ctx.SetBodyString(challenge.(string))
} else {
ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
}
})
if err != nil {
log.Fatalf("Couldn't start HTTP fastServer: %s", err)
}
})()
}
// Start the web fastServer
err = fastServer.Serve(listener)
if err != nil {
log.Fatalf("Couldn't start fastServer: %s", err)
}
return nil
}

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/go-acme/lego/v4 v4.5.3
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
github.com/rs/zerolog v1.26.0
github.com/urfave/cli/v2 v2.3.0
github.com/valyala/fasthttp v1.31.0
github.com/valyala/fastjson v1.6.3
)

5
go.sum
View File

@ -100,6 +100,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -427,6 +428,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI=
@ -434,6 +436,7 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f h1:WSnaD0/cvbKJgSTYbjAPf4RJXVvNNDAwVm+W8wEmnGE=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@ -483,7 +486,9 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=

151
main.go
View File

@ -1,147 +1,32 @@
// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
//
// Mapping custom domains is not static anymore, but can be done with DNS:
//
// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The
// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
//
// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
// www.example.org. IN CNAME main.pages.example.codeberg.page.
//
// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
// for "example.org" (if your provider allows ALIAS or similar records):
// example.org IN ALIAS codeberg.page.
//
// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
package main
import (
"bytes"
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
"os"
"time"
"github.com/valyala/fasthttp"
"github.com/urfave/cli/v2"
pages_server "codeberg.org/codeberg/pages/server"
"codeberg.org/codeberg/pages/cmd"
)
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
var MainDomainSuffix = []byte("." + pages_server.EnvOr("PAGES_DOMAIN", "codeberg.page"))
var (
// can be changed with -X on compile
version = "dev"
)
// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
var GiteaRoot = []byte(pages_server.EnvOr("GITEA_ROOT", "https://codeberg.org"))
var GiteaApiToken = pages_server.EnvOr("GITEA_API_TOKEN", "")
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
// (set to []byte(nil) to disable raw content hosting)
var RawDomain = []byte(pages_server.EnvOr("RAW_DOMAIN", "raw.codeberg.org"))
// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
var RawInfoPage = pages_server.EnvOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/")
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
var AllowedCorsDomains = [][]byte{
RawDomain,
[]byte("fonts.codeberg.org"),
[]byte("design.codeberg.org"),
}
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
var BlacklistedPaths = [][]byte{
[]byte("/.well-known/acme-challenge/"),
}
// main sets up and starts the web server.
func main() {
// TODO: CLI Library
if len(os.Args) > 1 && os.Args[1] == "--remove-certificate" {
if len(os.Args) < 2 {
println("--remove-certificate requires at least one domain as an argument")
app := cli.NewApp()
app.Name = "pages-server"
app.Version = version
app.Usage = "pages server"
app.Action = cmd.Serve
app.Flags = cmd.ServeFlags
app.Commands = []*cli.Command{
cmd.Certs,
}
if err := app.Run(os.Args); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if pages_server.KeyDatabaseErr != nil {
panic(pages_server.KeyDatabaseErr)
}
for _, domain := range os.Args[2:] {
if err := pages_server.KeyDatabase.Delete([]byte(domain)); err != nil {
panic(err)
}
}
if err := pages_server.KeyDatabase.Sync(); err != nil {
panic(err)
}
os.Exit(0)
}
// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
if !bytes.HasPrefix(MainDomainSuffix, []byte{'.'}) {
MainDomainSuffix = append([]byte{'.'}, MainDomainSuffix...)
}
GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'})
// Use HOST and PORT environment variables to determine listening address
address := fmt.Sprintf("%s:%s", pages_server.EnvOr("HOST", "[::]"), pages_server.EnvOr("PORT", "443"))
log.Printf("Listening on https://%s", address)
// Create handler based on settings
handler := pages_server.Handler(MainDomainSuffix, RawDomain, GiteaRoot, RawInfoPage, GiteaApiToken, BlacklistedPaths, AllowedCorsDomains)
// Enable compression by wrapping the handler with the compression function provided by FastHTTP
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
server := &fasthttp.Server{
Handler: compressedHandler,
DisablePreParseMultipartForm: true,
MaxRequestBodySize: 0,
NoDefaultServerHeader: true,
NoDefaultDate: true,
ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
MaxConnsPerIP: 100,
}
// Setup listener and TLS
listener, err := net.Listen("tcp", address)
if err != nil {
log.Fatalf("Couldn't create listener: %s", err)
}
listener = tls.NewListener(listener, pages_server.TlsConfig(MainDomainSuffix, string(GiteaRoot), GiteaApiToken))
pages_server.SetupCertificates(MainDomainSuffix)
if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
go (func() {
challengePath := []byte("/.well-known/acme-challenge/")
err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) {
if bytes.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := pages_server.ChallengeCache.Get(string(pages_server.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
if !ok || challenge == nil {
ctx.SetStatusCode(http.StatusNotFound)
ctx.SetBodyString("no challenge for this token")
}
ctx.SetBodyString(challenge.(string))
} else {
ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
}
})
if err != nil {
log.Fatalf("Couldn't start HTTP server: %s", err)
}
})()
}
// Start the web server
err = server.Serve(listener)
if err != nil {
log.Fatalf("Couldn't start server: %s", err)
}
}