From 2fc33807077cd25bf91f2298bf1a8ace126881a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Remko=20Tron=C3=A7on?= Date: Thu, 2 Jul 2020 18:16:32 +0200 Subject: [PATCH] Add js.Build asset bundling Fixes #7321 --- go.mod | 1 + go.sum | 5 + hugolib/js_test.go | 87 ++++++++++ resources/resource_transformers/js/build.go | 166 ++++++++++++++++++++ tpl/js/init.go | 36 +++++ tpl/js/js.go | 90 +++++++++++ tpl/tplimpl/template_funcs.go | 1 + 7 files changed, 386 insertions(+) create mode 100644 hugolib/js_test.go create mode 100644 resources/resource_transformers/js/build.go create mode 100644 tpl/js/init.go create mode 100644 tpl/js/js.go diff --git a/go.mod b/go.mod index 3969f67f..fc84e013 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/bep/tmc v0.5.1 github.com/disintegration/gift v1.2.1 github.com/dustin/go-humanize v1.0.0 + github.com/evanw/esbuild v0.6.1 github.com/fortytw2/leaktest v1.3.0 github.com/frankban/quicktest v1.7.2 github.com/fsnotify/fsnotify v1.4.7 diff --git a/go.sum b/go.sum index 7549b182..a4ffd884 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/evanw/esbuild v0.6.1 h1:XkoACQJCiqUmwySWssu0/iUj7J6IbNMR9dqbSbh1/vk= +github.com/evanw/esbuild v0.6.1/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw+Q= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -226,6 +228,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyokomi/emoji v2.2.1+incompatible h1:uP/6J5y5U0XxPh6fv8YximpVD1uMrshXG78I1+uF5SA= github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -485,6 +488,8 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= diff --git a/hugolib/js_test.go b/hugolib/js_test.go new file mode 100644 index 00000000..1e59927c --- /dev/null +++ b/hugolib/js_test.go @@ -0,0 +1,87 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "os" + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/htesting" + + "github.com/spf13/viper" + + qt "github.com/frankban/quicktest" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/common/loggers" +) + +func TestJS_Build(t *testing.T) { + if !isCI() { + t.Skip("skip (relative) long running modules test when running locally") + } + + wd, _ := os.Getwd() + defer func() { + os.Chdir(wd) + }() + + c := qt.New(t) + + mainJS := ` + import "./included"; + console.log("main"); + ` + includedJS := ` + console.log("included"); + ` + + workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-babel") + c.Assert(err, qt.IsNil) + defer clean() + + v := viper.New() + v.Set("workingDir", workDir) + v.Set("disableKinds", []string{"taxonomy", "term", "page"}) + b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger()) + + // Need to use OS fs for this. + b.Fs = hugofs.NewDefault(v) + b.WithWorkingDir(workDir) + b.WithViper(v) + b.WithContent("p1.md", "") + + b.WithTemplates("index.html", ` + {{ $options := dict "minify" true }} + {{ $transpiled := resources.Get "js/main.js" | js.Build $options }} + Built: {{ $transpiled.Content | safeJS }} + `) + + jsDir := filepath.Join(workDir, "assets", "js") + b.Assert(os.MkdirAll(jsDir, 0777), qt.IsNil) + b.WithSourceFile("assets/js/main.js", mainJS) + b.WithSourceFile("assets/js/included.js", includedJS) + + _, err = captureStdout(func() error { + return b.BuildE(BuildCfg{}) + }) + b.Assert(err, qt.IsNil) + + b.AssertFileContent("public/index.html", ` + Built: (()=>{console.log("included");console.log("main");})(); + `) + +} diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go new file mode 100644 index 00000000..6224ee17 --- /dev/null +++ b/resources/resource_transformers/js/build.go @@ -0,0 +1,166 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "fmt" + "io/ioutil" + "path" + + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/resources/internal" + + "github.com/mitchellh/mapstructure" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" +) + +type Options struct { + Minify bool + Externals []string + Target string + Loader string + Defines map[string]string + JSXFactory string + JSXFragment string + TSConfig string +} + +func DecodeOptions(m map[string]interface{}) (opts Options, err error) { + if m == nil { + return + } + err = mapstructure.WeakDecode(m, &opts) + return +} + +type Client struct { + rs *resources.Spec + sfs *filesystems.SourceFilesystem +} + +func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) *Client { + return &Client{rs: rs, sfs: fs} +} + +type buildTransformation struct { + options Options + rs *resources.Spec + sfs *filesystems.SourceFilesystem +} + +func (t *buildTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("jsbuild", t.options) +} + +func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { + var target api.Target + switch t.options.Target { + case "", "esnext": + target = api.ESNext + case "es6", "es2015": + target = api.ES2015 + case "es2016": + target = api.ES2016 + case "es2017": + target = api.ES2017 + case "es2018": + target = api.ES2018 + case "es2019": + target = api.ES2019 + case "es2020": + target = api.ES2020 + default: + return fmt.Errorf("invalid target: %q", t.options.Target) + } + + var loader api.Loader + switch t.options.Loader { + case "", "js": + loader = api.LoaderJS + case "jsx": + loader = api.LoaderJSX + case "ts": + loader = api.LoaderTS + case "tsx": + loader = api.LoaderTSX + case "json": + loader = api.LoaderJSON + case "text": + loader = api.LoaderText + case "base64": + loader = api.LoaderBase64 + case "dataURL": + loader = api.LoaderDataURL + case "file": + loader = api.LoaderFile + case "binary": + loader = api.LoaderBinary + default: + return fmt.Errorf("invalid loader: %q", t.options.Loader) + } + + src, err := ioutil.ReadAll(ctx.From) + if err != nil { + return err + } + + sdir, sfile := path.Split(ctx.SourcePath) + sdir = t.sfs.RealFilename(sdir) + + buildOptions := api.BuildOptions{ + Outfile: "", + Bundle: true, + + Target: target, + + MinifyWhitespace: t.options.Minify, + MinifyIdentifiers: t.options.Minify, + MinifySyntax: t.options.Minify, + + Defines: t.options.Defines, + + Externals: t.options.Externals, + + JSXFactory: t.options.JSXFactory, + JSXFragment: t.options.JSXFragment, + + Tsconfig: t.options.TSConfig, + + Stdin: &api.StdinOptions{ + Contents: string(src), + Sourcefile: sfile, + ResolveDir: sdir, + Loader: loader, + }, + } + result := api.Build(buildOptions) + if len(result.Errors) > 0 { + return fmt.Errorf("%s", result.Errors[0].Text) + } + if len(result.OutputFiles) != 1 { + return fmt.Errorf("unexpected output count: %d", len(result.OutputFiles)) + } + + ctx.To.Write(result.OutputFiles[0].Contents) + return nil +} + +func (c *Client) Process(res resources.ResourceTransformer, options Options) (resource.Resource, error) { + return res.Transform( + &buildTransformation{rs: c.rs, sfs: c.sfs, options: options}, + ) +} diff --git a/tpl/js/init.go b/tpl/js/init.go new file mode 100644 index 00000000..0af10bb1 --- /dev/null +++ b/tpl/js/init.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/tpl/internal" +) + +const name = "js" + +func init() { + f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { + ctx := New(d) + + ns := &internal.TemplateFuncsNamespace{ + Name: name, + Context: func(args ...interface{}) interface{} { return ctx }, + } + + return ns + } + + internal.AddTemplateFuncsNamespace(f) +} diff --git a/tpl/js/js.go b/tpl/js/js.go new file mode 100644 index 00000000..d8ba35a7 --- /dev/null +++ b/tpl/js/js.go @@ -0,0 +1,90 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package js + +import ( + "errors" + "fmt" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_transformers/js" + _errors "github.com/pkg/errors" +) + +// New returns a new instance of the js-namespaced template functions. +func New(deps *deps.Deps) *Namespace { + return &Namespace{ + client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), + } +} + +// Namespace provides template functions for the "js" namespace. +type Namespace struct { + deps *deps.Deps + client *js.Client +} + +// Build processes the given Resource with ESBuild. +func (ns *Namespace) Build(args ...interface{}) (resource.Resource, error) { + r, m, err := ns.resolveArgs(args) + if err != nil { + return nil, err + } + var options js.Options + if m != nil { + options, err = js.DecodeOptions(m) + + if err != nil { + return nil, err + } + } + + return ns.client.Process(r, options) + +} + +// This roundabout way of doing it is needed to get both pipeline behaviour and options as arguments. +// This is a copy of tpl/resources/resolveArgs +func (ns *Namespace) resolveArgs(args []interface{}) (resources.ResourceTransformer, map[string]interface{}, error) { + if len(args) == 0 { + return nil, nil, errors.New("no Resource provided in transformation") + } + + if len(args) == 1 { + r, ok := args[0].(resources.ResourceTransformer) + if !ok { + return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) + } + return r, nil, nil + } + + r, ok := args[1].(resources.ResourceTransformer) + if !ok { + if _, ok := args[1].(map[string]interface{}); !ok { + return nil, nil, fmt.Errorf("no Resource provided in transformation") + } + return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) + } + + m, err := maps.ToStringMapE(args[0]) + if err != nil { + return nil, nil, _errors.Wrap(err, "invalid options type") + } + + return r, m, nil +} diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index 9141de3f..a688abb7 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -42,6 +42,7 @@ import ( _ "github.com/gohugoio/hugo/tpl/hugo" _ "github.com/gohugoio/hugo/tpl/images" _ "github.com/gohugoio/hugo/tpl/inflect" + _ "github.com/gohugoio/hugo/tpl/js" _ "github.com/gohugoio/hugo/tpl/lang" _ "github.com/gohugoio/hugo/tpl/math" _ "github.com/gohugoio/hugo/tpl/openapi/openapi3"