From 9185e11effa682ea1ef7dc98f2943743671023a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 13 Oct 2021 08:12:06 +0200 Subject: [PATCH] Reimplement archetypes The old implementation had some issues, mostly related to the context (e.g. name, file paths) passed to the template. This new implementation is using the exact same code path for evaluating the pages as in a regular build. This also makes it more robust and easier to reason about in a multilingual setup. Now, if you are explicit about the target path, Hugo will now always pick the correct mount and language: ```bash hugo new content/en/posts/my-first-post.md ``` Fixes #9032 Fixes #7589 Fixes #9043 Fixes #9046 Fixes #9047 --- commands/hugo.go | 2 +- commands/new.go | 12 +- commands/new_site.go | 2 +- create/content.go | 432 ++++++++++++++--------------- create/content_template_handler.go | 149 ---------- create/content_test.go | 205 +++++++++++--- deps/deps.go | 2 +- hugofs/glob/glob.go | 168 +++++++++-- hugofs/glob/glob_test.go | 35 +++ hugolib/content_factory.go | 181 ++++++++++++ hugolib/content_factory_test.go | 60 ++++ hugolib/filesystems/basefs.go | 36 +++ hugolib/hugo_sites.go | 7 +- hugolib/pages_capture_test.go | 2 +- hugolib/site.go | 8 +- langs/i18n/translationProvider.go | 2 +- source/content_directory_test.go | 2 +- source/filesystem_test.go | 2 +- source/sourceSpec.go | 36 +-- 19 files changed, 877 insertions(+), 466 deletions(-) delete mode 100644 create/content_template_handler.go create mode 100644 hugolib/content_factory.go create mode 100644 hugolib/content_factory_test.go diff --git a/commands/hugo.go b/commands/hugo.go index 4fd20a0f..c1975600 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -870,7 +870,7 @@ func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*wat livereload.ForceRefresh() } case err := <-watcher.Errors(): - if err != nil { + if err != nil && !os.IsNotExist(err) { c.logger.Errorln("Error while watching:", err) } } diff --git a/commands/new.go b/commands/new.go index 7affd354..c5b5cd18 100644 --- a/commands/new.go +++ b/commands/new.go @@ -80,17 +80,7 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error { return newUserError("path needs to be provided") } - createPath := args[0] - - var kind string - - createPath, kind = newContentPathSection(c.hugo(), createPath) - - if n.contentType != "" { - kind = n.contentType - } - - return create.NewContent(c.hugo(), kind, createPath) + return create.NewContent(c.hugo(), n.contentType, args[0]) } func mkdir(x ...string) { diff --git a/commands/new_site.go b/commands/new_site.go index 71097b8f..11e9ce40 100644 --- a/commands/new_site.go +++ b/commands/new_site.go @@ -102,7 +102,7 @@ func (n *newSiteCmd) doNewSite(fs *hugofs.Fs, basepath string, force bool) error // Create a default archetype file. helpers.SafeWriteToDisk(filepath.Join(archeTypePath, "default.md"), - strings.NewReader(create.ArchetypeTemplateTemplate), fs.Source) + strings.NewReader(create.DefaultArchetypeTemplateTemplate), fs.Source) jww.FEEDBACK.Printf("Congratulations! Your new Hugo site is created in %s.\n\n", basepath) jww.FEEDBACK.Println(nextStepsText()) diff --git a/create/content.go b/create/content.go index ea065423..714939f4 100644 --- a/create/content.go +++ b/create/content.go @@ -16,11 +16,14 @@ package create import ( "bytes" + "fmt" "io" "os" "path/filepath" "strings" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/common/paths" "github.com/pkg/errors" @@ -33,125 +36,136 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugolib" "github.com/spf13/afero" - jww "github.com/spf13/jwalterweatherman" ) -// NewContent creates a new content file in the content directory based upon the -// given kind, which is used to lookup an archetype. -func NewContent( - sites *hugolib.HugoSites, kind, targetPath string) error { - targetPath = filepath.Clean(targetPath) +const ( + // DefaultArchetypeTemplateTemplate is the template used in 'hugo new site' + // and the template we use as a fall back. + DefaultArchetypeTemplateTemplate = `--- +title: "{{ replace .Name "-" " " | title }}" +date: {{ .Date }} +draft: true +--- + +` +) + +// NewContent creates a new content file in h (or a full bundle if the archetype is a directory) +// in targetPath. +func NewContent(h *hugolib.HugoSites, kind, targetPath string) error { + cf := hugolib.NewContentFactory(h) + + if kind == "" { + kind = cf.SectionFromFilename(targetPath) + } + + b := &contentBuilder{ + archeTypeFs: h.PathSpec.BaseFs.Archetypes.Fs, + sourceFs: h.PathSpec.Fs.Source, + ps: h.PathSpec, + h: h, + cf: cf, + + kind: kind, + targetPath: targetPath, + } + ext := paths.Ext(targetPath) - ps := sites.PathSpec - archetypeFs := ps.BaseFs.SourceFilesystems.Archetypes.Fs - sourceFs := ps.Fs.Source - jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext) + b.setArcheTypeFilenameToUse(ext) - archetypeFilename, isDir := findArchetype(ps, kind, ext) - contentPath, s := resolveContentPath(sites, sourceFs, targetPath) + if b.isDir { + return b.buildDir() + } - if isDir { + if ext == "" { + return errors.Errorf("failed to resolve %q to a archetype template", targetPath) + } - langFs, err := hugofs.NewLanguageFs(sites.LanguageSet(), archetypeFs) + return b.buildFile() + +} + +type contentBuilder struct { + archeTypeFs afero.Fs + sourceFs afero.Fs + + ps *helpers.PathSpec + h *hugolib.HugoSites + cf hugolib.ContentFactory + + // Builder state + archetypeFilename string + targetPath string + kind string + isDir bool + dirMap archetypeMap +} + +func (b *contentBuilder) buildDir() error { + // Split the dir into content files and the rest. + if err := b.mapArcheTypeDir(); err != nil { + return err + } + + var contentTargetFilenames []string + var baseDir string + + for _, fi := range b.dirMap.contentFiles { + targetFilename := filepath.Join(b.targetPath, strings.TrimPrefix(fi.Meta().Path, b.archetypeFilename)) + abs, err := b.cf.CreateContentPlaceHolder(targetFilename) if err != nil { return err } - - cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename) - if err != nil { - return err + if baseDir == "" { + baseDir = strings.TrimSuffix(abs, targetFilename) } - if cm.siteUsed { - if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { - return err + contentTargetFilenames = append(contentTargetFilenames, abs) + } + + var contentInclusionFilter *glob.FilenameFilter + if !b.dirMap.siteUsed { + // We don't need to build everything. + contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { + for _, cn := range contentTargetFilenames { + if strings.HasPrefix(cn, filename) { + return true + } } - } + return false + }) - name := filepath.Base(targetPath) - return newContentFromDir(archetypeFilename, sites, sourceFs, cm, name, contentPath) } - // Building the sites can be expensive, so only do it if really needed. - siteUsed := false - - if archetypeFilename != "" { - - var err error - siteUsed, err = usesSiteVar(archetypeFs, archetypeFilename) - if err != nil { - return err - } - } - - if siteUsed { - if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { - return err - } - } - - content, err := executeArcheTypeAsTemplate(s, "", kind, targetPath, archetypeFilename) - if err != nil { + if err := b.h.Build(hugolib.BuildCfg{SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil { return err } - if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil { - return err - } - - jww.FEEDBACK.Println(contentPath, "created") - - editor := s.Cfg.GetString("newContentEditor") - if editor != "" { - jww.FEEDBACK.Printf("Editing %s with %q ...\n", targetPath, editor) - - editorCmd := append(strings.Fields(editor), contentPath) - cmd, err := hexec.SafeCommand(editorCmd[0], editorCmd[1:]...) - if err != nil { + for i, filename := range contentTargetFilenames { + if err := b.applyArcheType(filename, b.dirMap.contentFiles[i].Meta().Path); err != nil { return err } - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - return cmd.Run() } - return nil -} - -func targetSite(sites *hugolib.HugoSites, fi hugofs.FileMetaInfo) *hugolib.Site { - for _, s := range sites.Sites { - if fi.Meta().Lang == s.Language().Lang { - return s - } - } - return sites.Sites[0] -} - -func newContentFromDir( - archetypeDir string, - sites *hugolib.HugoSites, - targetFs afero.Fs, - cm archetypeMap, name, targetPath string) error { - for _, f := range cm.otherFiles { + // Copy the rest as is. + for _, f := range b.dirMap.otherFiles { meta := f.Meta() filename := meta.Path - // Just copy the file to destination. + in, err := meta.Open() if err != nil { return errors.Wrap(err, "failed to open non-content file") } - targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir)) - + targetFilename := filepath.Join(baseDir, b.targetPath, strings.TrimPrefix(filename, b.archetypeFilename)) targetDir := filepath.Dir(targetFilename) - if err := targetFs.MkdirAll(targetDir, 0777); err != nil && !os.IsExist(err) { - return errors.Wrapf(err, "failed to create target directory for %s:", targetDir) + + if err := b.sourceFs.MkdirAll(targetDir, 0o777); err != nil && !os.IsExist(err) { + return errors.Wrapf(err, "failed to create target directory for %q", targetDir) } - out, err := targetFs.Create(targetFilename) + out, err := b.sourceFs.Create(targetFilename) if err != nil { return err } @@ -164,41 +178,81 @@ func newContentFromDir( in.Close() out.Close() } - - for _, f := range cm.contentFiles { - filename := f.Meta().Path - s := targetSite(sites, f) - targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir)) - - content, err := executeArcheTypeAsTemplate(s, name, archetypeDir, targetFilename, filename) - if err != nil { - return errors.Wrap(err, "failed to execute archetype template") - } - - if err := helpers.SafeWriteToDisk(targetFilename, bytes.NewReader(content), targetFs); err != nil { - return errors.Wrap(err, "failed to save results") - } - } - - jww.FEEDBACK.Println(targetPath, "created") - return nil } -type archetypeMap struct { - // These needs to be parsed and executed as Go templates. - contentFiles []hugofs.FileMetaInfo - // These are just copied to destination. - otherFiles []hugofs.FileMetaInfo - // If the templates needs a fully built site. This can potentially be - // expensive, so only do when needed. - siteUsed bool +func (b *contentBuilder) buildFile() error { + contentPlaceholderAbsFilename, err := b.cf.CreateContentPlaceHolder(b.targetPath) + if err != nil { + return err + } + + usesSite, err := b.usesSiteVar(b.archetypeFilename) + if err != nil { + return err + } + + var contentInclusionFilter *glob.FilenameFilter + if !usesSite { + // We don't need to build everything. + contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { + return strings.HasPrefix(contentPlaceholderAbsFilename, filename) + }) + } + + if err := b.h.Build(hugolib.BuildCfg{SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil { + return err + } + + if err := b.applyArcheType(contentPlaceholderAbsFilename, b.archetypeFilename); err != nil { + return err + } + + b.h.Log.Infof("Content %q created", contentPlaceholderAbsFilename) + + return b.openInEditorIfConfigured(contentPlaceholderAbsFilename) } -func mapArcheTypeDir( - ps *helpers.PathSpec, - fs afero.Fs, - archetypeDir string) (archetypeMap, error) { +func (b *contentBuilder) setArcheTypeFilenameToUse(ext string) { + var pathsToCheck []string + + if b.kind != "" { + pathsToCheck = append(pathsToCheck, b.kind+ext) + } + pathsToCheck = append(pathsToCheck, "default"+ext, "default") + + for _, p := range pathsToCheck { + fi, err := b.archeTypeFs.Stat(p) + if err == nil { + b.archetypeFilename = p + b.isDir = fi.IsDir() + return + } + } + +} + +func (b *contentBuilder) applyArcheType(contentFilename, archetypeFilename string) error { + p := b.h.GetContentPage(contentFilename) + if p == nil { + panic(fmt.Sprintf("[BUG] no Page found for %q", contentFilename)) + } + + f, err := b.sourceFs.Create(contentFilename) + if err != nil { + return err + } + defer f.Close() + + if archetypeFilename == "" { + return b.cf.AppplyArchetypeTemplate(f, p, b.kind, DefaultArchetypeTemplateTemplate) + } + + return b.cf.AppplyArchetypeFilename(f, p, b.kind, archetypeFilename) + +} + +func (b *contentBuilder) mapArcheTypeDir() error { var m archetypeMap walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { @@ -215,7 +269,7 @@ func mapArcheTypeDir( if files.IsContentFile(path) { m.contentFiles = append(m.contentFiles, fil) if !m.siteUsed { - m.siteUsed, err = usesSiteVar(fs, path) + m.siteUsed, err = b.usesSiteVar(path) if err != nil { return err } @@ -230,120 +284,60 @@ func mapArcheTypeDir( walkCfg := hugofs.WalkwayConfig{ WalkFn: walkFn, - Fs: fs, - Root: archetypeDir, + Fs: b.archeTypeFs, + Root: b.archetypeFilename, } w := hugofs.NewWalkway(walkCfg) if err := w.Walk(); err != nil { - return m, errors.Wrapf(err, "failed to walk archetype dir %q", archetypeDir) + return errors.Wrapf(err, "failed to walk archetype dir %q", b.archetypeFilename) } - return m, nil + b.dirMap = m + + return nil } -func usesSiteVar(fs afero.Fs, filename string) (bool, error) { - f, err := fs.Open(filename) +func (b *contentBuilder) openInEditorIfConfigured(filename string) error { + editor := b.h.Cfg.GetString("newContentEditor") + if editor == "" { + return nil + } + + b.h.Log.Infof("Editing %q with %q ...\n", filename, editor) + + cmd, err := hexec.SafeCommand(editor, filename) + if err != nil { + return err + } + + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} + +func (b *contentBuilder) usesSiteVar(filename string) (bool, error) { + if filename == "" { + return false, nil + } + bb, err := afero.ReadFile(b.archeTypeFs, filename) if err != nil { return false, errors.Wrap(err, "failed to open archetype file") } - defer f.Close() - return helpers.ReaderContains(f, []byte(".Site")), nil + + return bytes.Contains(bb, []byte(".Site")) || bytes.Contains(bb, []byte("site.")), nil + } -// Resolve the target content path. -func resolveContentPath(sites *hugolib.HugoSites, fs afero.Fs, targetPath string) (string, *hugolib.Site) { - targetDir := filepath.Dir(targetPath) - first := sites.Sites[0] - - var ( - s *hugolib.Site - siteContentDir string - ) - - // Try the filename: my-post.en.md - for _, ss := range sites.Sites { - if strings.Contains(targetPath, "."+ss.Language().Lang+".") { - s = ss - break - } - } - - var dirLang string - - for _, dir := range sites.BaseFs.Content.Dirs { - meta := dir.Meta() - contentDir := meta.Filename - - if !strings.HasSuffix(contentDir, helpers.FilePathSeparator) { - contentDir += helpers.FilePathSeparator - } - - if strings.HasPrefix(targetPath, contentDir) { - siteContentDir = contentDir - dirLang = meta.Lang - break - } - } - - if s == nil && dirLang != "" { - for _, ss := range sites.Sites { - if ss.Lang() == dirLang { - s = ss - break - } - } - } - - if s == nil { - s = first - } - - if targetDir != "" && targetDir != "." { - exists, _ := helpers.Exists(targetDir, fs) - - if exists { - return targetPath, s - } - } - - if siteContentDir == "" { - } - - if siteContentDir != "" { - pp := filepath.Join(siteContentDir, strings.TrimPrefix(targetPath, siteContentDir)) - return s.PathSpec.AbsPathify(pp), s - } else { - var contentDir string - for _, dir := range sites.BaseFs.Content.Dirs { - contentDir = dir.Meta().Filename - if dir.Meta().Lang == s.Lang() { - break - } - } - return s.PathSpec.AbsPathify(filepath.Join(contentDir, targetPath)), s - } -} - -// FindArchetype takes a given kind/archetype of content and returns the path -// to the archetype in the archetype filesystem, blank if none found. -func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string, isDir bool) { - fs := ps.BaseFs.Archetypes.Fs - - var pathsToCheck []string - - if kind != "" { - pathsToCheck = append(pathsToCheck, kind+ext) - } - pathsToCheck = append(pathsToCheck, "default"+ext, "default") - - for _, p := range pathsToCheck { - fi, err := fs.Stat(p) - if err == nil { - return p, fi.IsDir() - } - } - - return "", false +type archetypeMap struct { + // These needs to be parsed and executed as Go templates. + contentFiles []hugofs.FileMetaInfo + // These are just copied to destination. + otherFiles []hugofs.FileMetaInfo + // If the templates needs a fully built site. This can potentially be + // expensive, so only do when needed. + siteUsed bool } diff --git a/create/content_template_handler.go b/create/content_template_handler.go deleted file mode 100644 index 09cf4c0a..00000000 --- a/create/content_template_handler.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package create - -import ( - "bytes" - "fmt" - "path/filepath" - "strings" - "time" - - "github.com/gohugoio/hugo/common/paths" - - "github.com/pkg/errors" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/source" - - "github.com/gohugoio/hugo/hugolib" - "github.com/gohugoio/hugo/tpl" - "github.com/spf13/afero" -) - -// ArchetypeFileData represents the data available to an archetype template. -type ArchetypeFileData struct { - // The archetype content type, either given as --kind option or extracted - // from the target path's section, i.e. "blog/mypost.md" will resolve to - // "blog". - Type string - - // The current date and time as a RFC3339 formatted string, suitable for use in front matter. - Date string - - // The Site, fully equipped with all the pages etc. Note: This will only be set if it is actually - // used in the archetype template. Also, if this is a multilingual setup, - // this site is the site that best matches the target content file, based - // on the presence of language code in the filename. - Site *hugolib.SiteInfo - - // Name will in most cases be the same as TranslationBaseName, e.g. "my-post". - // But if that value is "index" (bundles), the Name is instead the owning folder. - // This is the value you in most cases would want to use to construct the title in your - // archetype template. - Name string - - // The target content file. Note that the .Content will be empty, as that - // has not been created yet. - source.File -} - -const ( - // ArchetypeTemplateTemplate is used as initial template when adding an archetype template. - ArchetypeTemplateTemplate = `--- -title: "{{ replace .Name "-" " " | title }}" -date: {{ .Date }} -draft: true ---- - -` -) - -var ( - archetypeShortcodeReplacementsPre = strings.NewReplacer( - "{{<", "{x{<", - "{{%", "{x{%", - ">}}", ">}x}", - "%}}", "%}x}") - - archetypeShortcodeReplacementsPost = strings.NewReplacer( - "{x{<", "{{<", - "{x{%", "{{%", - ">}x}", ">}}", - "%}x}", "%}}") -) - -func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archetypeFilename string) ([]byte, error) { - var ( - archetypeContent []byte - archetypeTemplate []byte - err error - ) - - f, err := s.SourceSpec.NewFileInfoFrom(targetPath, targetPath) - if err != nil { - return nil, err - } - - if name == "" { - name = f.TranslationBaseName() - - if name == "index" || name == "_index" { - // Page bundles; the directory name will hopefully have a better name. - dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator) - _, name = filepath.Split(dir) - } - } - - data := ArchetypeFileData{ - Type: kind, - Date: time.Now().Format(time.RFC3339), - Name: name, - File: f, - Site: s.Info, - } - - if archetypeFilename == "" { - // TODO(bep) archetype revive the issue about wrong tpl funcs arg order - archetypeTemplate = []byte(ArchetypeTemplateTemplate) - } else { - archetypeTemplate, err = afero.ReadFile(s.BaseFs.Archetypes.Fs, archetypeFilename) - if err != nil { - return nil, fmt.Errorf("failed to read archetype file %s", err) - } - - } - - // The archetype template may contain shortcodes, and these does not play well - // with the Go templates. Need to set some temporary delimiters. - archetypeTemplate = []byte(archetypeShortcodeReplacementsPre.Replace(string(archetypeTemplate))) - - // Reuse the Hugo template setup to get the template funcs properly set up. - templateHandler := s.Deps.Tmpl().(tpl.TemplateManager) - templateName := paths.Filename(archetypeFilename) - if err := templateHandler.AddTemplate("_text/"+templateName, string(archetypeTemplate)); err != nil { - return nil, errors.Wrapf(err, "Failed to parse archetype file %q:", archetypeFilename) - } - - templ, _ := templateHandler.Lookup(templateName) - - var buff bytes.Buffer - if err := templateHandler.Execute(templ, &buff, data); err != nil { - return nil, errors.Wrapf(err, "Failed to process archetype file %q:", archetypeFilename) - } - - archetypeContent = []byte(archetypeShortcodeReplacementsPost.Replace(buff.String())) - - return archetypeContent, nil -} diff --git a/create/content_test.go b/create/content_test.go index 38ff7de8..d4063408 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -34,27 +34,31 @@ import ( "github.com/spf13/afero" ) +// TODO(bep) clean this up. Export the test site builder in Hugolib or something. func TestNewContent(t *testing.T) { cases := []struct { + name string kind string path string expected []string }{ - {"post", "post/sample-1.md", []string{`title = "Post Arch title"`, `test = "test1"`, "date = \"2015-01-12T19:20:04-07:00\""}}, - {"post", "post/org-1.org", []string{`#+title: ORG-1`}}, - {"emptydate", "post/sample-ed.md", []string{`title = "Empty Date Arch title"`, `test = "test1"`}}, - {"stump", "stump/sample-2.md", []string{`title: "Sample 2"`}}, // no archetype file - {"", "sample-3.md", []string{`title: "Sample 3"`}}, // no archetype - {"product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter - {"lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}}, - {"lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}}, - {"lang", "content/post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}}, - {"lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}}, - {"lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}}, - {"lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, - {"lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, - {"lang", "content/post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}}, - {"shortcodes", "shortcodes/go.md", []string{ + {"Post", "post", "post/sample-1.md", []string{`title = "Post Arch title"`, `test = "test1"`, "date = \"2015-01-12T19:20:04-07:00\""}}, + {"Post org-mode", "post", "post/org-1.org", []string{`#+title: ORG-1`}}, + {"Empty date", "emptydate", "post/sample-ed.md", []string{`title = "Empty Date Arch title"`, `test = "test1"`}}, + {"Archetype file not found", "stump", "stump/sample-2.md", []string{`title: "Sample 2"`}}, // no archetype file + {"No archetype", "", "sample-3.md", []string{`title: "Sample 3"`}}, // no archetype + {"Empty archetype", "product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter + {"Filenames", "filenames", "content/mypage/index.md", []string{"title = \"INDEX\"\n+++\n\n\nContentBaseName: mypage"}}, + {"Lang 1", "lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}}, + {"Lang 2", "lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}}, + {"Lang nn file", "lang", "content/post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}}, + {"Lang nn dir", "lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}}, + {"Lang en in nn dir", "lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}}, + {"Lang en default", "lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, + {"Lang en file", "lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, + {"Lang nn bundle", "lang", "content/post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}}, + {"Site", "site", "content/mypage/index.md", []string{"RegularPages .Site: 10", "RegularPages site: 10"}}, + {"Shortcodes", "shortcodes", "shortcodes/go.md", []string{ `title = "GO"`, "{{< myshortcode >}}", "{{% myshortcode %}}", @@ -62,11 +66,14 @@ func TestNewContent(t *testing.T) { }}, // shortcodes } + c := qt.New(t) + for i, cas := range cases { cas := cas - t.Run(fmt.Sprintf("%s-%d", cas.kind, i), func(t *testing.T) { - t.Parallel() - c := qt.New(t) + + c.Run(cas.name, func(c *qt.C) { + c.Parallel() + mm := afero.NewMemMapFs() c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) @@ -79,11 +86,11 @@ func TestNewContent(t *testing.T) { if !strings.HasPrefix(fname, "content") { fname = filepath.Join("content", fname) } - content := readFileFromFs(t, fs.Source, fname) + content := readFileFromFs(c, fs.Source, fname) for _, v := range cas.expected { found := strings.Contains(content, v) if !found { - t.Fatalf("[%d] %q missing from output:\n%q", i, v, content) + c.Fatalf("[%d] %q missing from output:\n%q", i, v, content) } } }) @@ -96,10 +103,10 @@ func TestNewContentFromDir(t *testing.T) { c := qt.New(t) archetypeDir := filepath.Join("archetypes", "my-bundle") - c.Assert(mm.MkdirAll(archetypeDir, 0755), qt.IsNil) + c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil) archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle") - c.Assert(mm.MkdirAll(archetypeThemeDir, 0755), qt.IsNil) + c.Assert(mm.MkdirAll(archetypeThemeDir, 0o755), qt.IsNil) contentFile := ` File: %s @@ -108,15 +115,15 @@ Name: {{ replace .Name "-" " " | title }} i18n: {{ T "hugo" }} ` - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0o755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) @@ -135,15 +142,90 @@ i18n: {{ T "hugo" }} cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Post`, `i18n: Hugo Rocks!`) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Site Lang: nn`, `Name: My Post`, `i18n: Hugo Rokkar!`) - cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: My Post`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: Bio`) c.Assert(create.NewContent(h, "my-theme-bundle", "post/my-theme-post"), qt.IsNil) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Theme Post`, `i18n: Hugo Rocks!`) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`) } +func TestNewContentFromDirSiteFunction(t *testing.T) { + mm := afero.NewMemMapFs() + c := qt.New(t) + + archetypeDir := filepath.Join("archetypes", "my-bundle") + c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil) + + contentFile := ` +File: %s +site RegularPages: {{ len site.RegularPages }} + +` + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + + c.Assert(initFs(mm), qt.IsNil) + cfg, fs := newTestCfg(c, mm) + + h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + c.Assert(err, qt.IsNil) + c.Assert(len(h.Sites), qt.Equals, 2) + + c.Assert(create.NewContent(h, "my-bundle", "post/my-post"), qt.IsNil) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `site RegularPages: 10`) +} + +func TestNewContentFromDirNoSite(t *testing.T) { + mm := afero.NewMemMapFs() + c := qt.New(t) + + archetypeDir := filepath.Join("archetypes", "my-bundle") + c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil) + + archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle") + c.Assert(mm.MkdirAll(archetypeThemeDir, 0o755), qt.IsNil) + + contentFile := ` +File: %s +Name: {{ replace .Name "-" " " | title }} +i18n: {{ T "hugo" }} +` + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0o755), qt.IsNil) + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + + c.Assert(initFs(mm), qt.IsNil) + cfg, fs := newTestCfg(c, mm) + + h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + c.Assert(err, qt.IsNil) + c.Assert(len(h.Sites), qt.Equals, 2) + + c.Assert(create.NewContent(h, "my-bundle", "post/my-post"), qt.IsNil) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo2.xml")), `hugo2: {{ printf "no template handling in here" }}`) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Name: My Post`, `i18n: Hugo Rocks!`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Name: My Post`, `i18n: Hugo Rokkar!`) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Name: Bio`) + + c.Assert(create.NewContent(h, "my-theme-bundle", "post/my-theme-post"), qt.IsNil) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/index.md")), `File: index.md`, `Name: My Theme Post`, `i18n: Hugo Rocks!`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`) +} + func initFs(fs afero.Fs) error { - perm := os.FileMode(0755) + perm := os.FileMode(0o755) var err error // create directories @@ -159,7 +241,16 @@ func initFs(fs afero.Fs) error { } } - // create files + // create some dummy content + for i := 1; i <= 10; i++ { + filename := filepath.Join("content", fmt.Sprintf("page%d.md", i)) + afero.WriteFile(fs, filename, []byte(`--- +title: Test +--- +`), 0666) + } + + // create archetype files for _, v := range []struct { path string content string @@ -177,6 +268,29 @@ func initFs(fs afero.Fs) error { content: `+++ title = "{{ .BaseFileName | upper }}" +++`, + }, + { + path: filepath.Join("archetypes", "filenames.md"), + content: `... +title = "{{ .BaseFileName | upper }}" ++++ + + +ContentBaseName: {{ .File.ContentBaseName }} + +`, + }, + { + path: filepath.Join("archetypes", "site.md"), + content: `... +title = "{{ .BaseFileName | upper }}" ++++ + +Len RegularPages .Site: {{ len .Site.RegularPages }} +Len RegularPages site: {{ len site.RegularPages }} + + +`, }, { path: filepath.Join("archetypes", "emptydate.md"), @@ -184,7 +298,7 @@ title = "{{ .BaseFileName | upper }}" }, { path: filepath.Join("archetypes", "lang.md"), - content: `Site Lang: {{ .Site.Language.Lang }}|Name: {{ replace .Name "-" " " | title }}|i18n: {{ T "hugo" }}`, + content: `Site Lang: {{ site.Language.Lang }}|Name: {{ replace .Name "-" " " | title }}|i18n: {{ T "hugo" }}`, }, // #3623x { @@ -227,7 +341,7 @@ func cContains(c *qt.C, v interface{}, matches ...string) { } // TODO(bep) extract common testing package with this and some others -func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { +func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { t.Helper() filename = filepath.FromSlash(filename) b, err := afero.ReadFile(fs, filename) @@ -257,23 +371,34 @@ languageName = "English" [languages.nn] weight = 2 languageName = "Nynorsk" -contentDir = "content_nn" +[module] +[[module.mounts]] + source = 'archetypes' + target = 'archetypes' +[[module.mounts]] + source = 'content' + target = 'content' + lang = 'en' +[[module.mounts]] + source = 'content_nn' + target = 'content' + lang = 'nn' ` if mm == nil { mm = afero.NewMemMapFs() } - mm.MkdirAll(filepath.FromSlash("content_nn"), 0777) + mm.MkdirAll(filepath.FromSlash("content_nn"), 0o777) - mm.MkdirAll(filepath.FromSlash("themes/mytheme"), 0777) + mm.MkdirAll(filepath.FromSlash("themes/mytheme"), 0o777) c.Assert(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo] -other = "Hugo Rocks!"`), 0755), qt.IsNil) +other = "Hugo Rocks!"`), 0o755), qt.IsNil) c.Assert(afero.WriteFile(mm, filepath.Join("i18n", "nn.toml"), []byte(`[hugo] -other = "Hugo Rokkar!"`), 0755), qt.IsNil) +other = "Hugo Rokkar!"`), 0o755), qt.IsNil) - c.Assert(afero.WriteFile(mm, "config.toml", []byte(cfg), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, "config.toml", []byte(cfg), 0o755), qt.IsNil) v, _, err := hugolib.LoadConfig(hugolib.ConfigSourceDescriptor{Fs: mm, Filename: "config.toml"}) c.Assert(err, qt.IsNil) diff --git a/deps/deps.go b/deps/deps.go index c0546db7..6b9da21f 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -253,7 +253,7 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } - sp := source.NewSourceSpec(ps, fs.Source) + sp := source.NewSourceSpec(ps, nil, fs.Source) timeoutms := cfg.Language.GetInt("timeout") if timeoutms <= 0 { diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go index 57115ddf..6dd0df5e 100644 --- a/hugofs/glob/glob.go +++ b/hugofs/glob/glob.go @@ -16,6 +16,7 @@ package glob import ( "path" "path/filepath" + "runtime" "strings" "sync" @@ -23,46 +24,100 @@ import ( "github.com/gobwas/glob/syntax" ) +var ( + isWindows = runtime.GOOS == "windows" + defaultGlobCache = &globCache{ + isCaseSensitive: false, + isWindows: isWindows, + cache: make(map[string]globErr), + } + + filenamesGlobCache = &globCache{ + isCaseSensitive: true, // TODO(bep) bench + isWindows: isWindows, + cache: make(map[string]globErr), + } +) + type globErr struct { glob glob.Glob err error } -var ( - globCache = make(map[string]globErr) - globMu sync.RWMutex -) +type globCache struct { + // Config + isCaseSensitive bool + isWindows bool -type caseInsensitiveGlob struct { - g glob.Glob + // Cache + sync.RWMutex + cache map[string]globErr } -func (g caseInsensitiveGlob) Match(s string) bool { - return g.g.Match(strings.ToLower(s)) - -} -func GetGlob(pattern string) (glob.Glob, error) { +func (gc *globCache) GetGlob(pattern string) (glob.Glob, error) { var eg globErr - globMu.RLock() + gc.RLock() var found bool - eg, found = globCache[pattern] - globMu.RUnlock() + eg, found = gc.cache[pattern] + gc.RUnlock() if found { return eg.glob, eg.err } + var g glob.Glob var err error - g, err := glob.Compile(strings.ToLower(pattern), '/') - eg = globErr{caseInsensitiveGlob{g: g}, err} - globMu.Lock() - globCache[pattern] = eg - globMu.Unlock() + pattern = filepath.ToSlash(pattern) + + if gc.isCaseSensitive { + g, err = glob.Compile(pattern, '/') + } else { + g, err = glob.Compile(strings.ToLower(pattern), '/') + + } + + eg = globErr{ + globDecorator{ + g: g, + isCaseSensitive: gc.isCaseSensitive, + isWindows: gc.isWindows}, + err, + } + + gc.Lock() + gc.cache[pattern] = eg + gc.Unlock() return eg.glob, eg.err } +type globDecorator struct { + // Whether both pattern and the strings to match will be matched + // by their original case. + isCaseSensitive bool + + // On Windows we may get filenames with Windows slashes to match, + // which wee need to normalize. + isWindows bool + + g glob.Glob +} + +func (g globDecorator) Match(s string) bool { + if g.isWindows { + s = filepath.ToSlash(s) + } + if !g.isCaseSensitive { + s = strings.ToLower(s) + } + return g.g.Match(s) +} + +func GetGlob(pattern string) (glob.Glob, error) { + return defaultGlobCache.GetGlob(pattern) +} + func NormalizePath(p string) string { return strings.Trim(path.Clean(filepath.ToSlash(strings.ToLower(p))), "/.") } @@ -106,3 +161,78 @@ func HasGlobChar(s string) bool { } return false } + +type FilenameFilter struct { + shouldInclude func(filename string) bool + inclusions []glob.Glob + exclusions []glob.Glob + isWindows bool +} + +// NewFilenameFilter creates a new Glob where the Match method will +// return true if the file should be exluded. +// Note that the inclusions will be checked first. +func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { + filter := &FilenameFilter{isWindows: isWindows} + + for _, include := range inclusions { + g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(include)) + if err != nil { + return nil, err + } + filter.inclusions = append(filter.inclusions, g) + } + for _, exclude := range exclusions { + g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(exclude)) + if err != nil { + return nil, err + } + filter.exclusions = append(filter.exclusions, g) + } + + return filter, nil +} + +// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. +func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { + return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} +} + +// Match returns whether filename should be included. +func (f *FilenameFilter) Match(filename string) bool { + if f == nil { + return true + } + + if f.shouldInclude != nil { + if f.shouldInclude(filename) { + return true + } + if f.isWindows { + // The Glob matchers below handles this by themselves, + // for the shouldInclude we need to take some extra steps + // to make this robust. + winFilename := filepath.FromSlash(filename) + if filename != winFilename { + if f.shouldInclude(winFilename) { + return true + } + } + } + + } + + for _, inclusion := range f.inclusions { + if inclusion.Match(filename) { + return true + } + } + + for _, exclusion := range f.exclusions { + if exclusion.Match(filename) { + return false + } + } + + return f.inclusions == nil && f.shouldInclude == nil +} diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go index cd64ba11..7ef3fbbe 100644 --- a/hugofs/glob/glob_test.go +++ b/hugofs/glob/glob_test.go @@ -15,6 +15,7 @@ package glob import ( "path/filepath" + "strings" "testing" qt "github.com/frankban/quicktest" @@ -72,6 +73,40 @@ func TestGetGlob(t *testing.T) { c.Assert(g.Match("data/my.json"), qt.Equals, true) } +func TestFilenameFilter(t *testing.T) { + c := qt.New(t) + + excludeAlmostAllJSON, err := NewFilenameFilter([]string{"a/b/c/foo.json"}, []string{"**.json"}) + c.Assert(err, qt.IsNil) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("data/my.json")), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.json")), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.bar")), qt.Equals, false) + + nopFilter, err := NewFilenameFilter(nil, nil) + c.Assert(err, qt.IsNil) + c.Assert(nopFilter.Match("ab.txt"), qt.Equals, true) + + includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) + c.Assert(err, qt.IsNil) + c.Assert(includeOnlyFilter.Match("ab.json"), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.jpg"), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.gif"), qt.Equals, false) + + exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) + c.Assert(err, qt.IsNil) + c.Assert(exlcudeOnlyFilter.Match("ab.json"), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.jpg"), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.gif"), qt.Equals, true) + + var nilFilter *FilenameFilter + c.Assert(nilFilter.Match("ab.gif"), qt.Equals, true) + + funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) + c.Assert(funcFilter.Match("ab.json"), qt.Equals, true) + c.Assert(funcFilter.Match("ab.bson"), qt.Equals, false) + +} + func BenchmarkGetGlob(b *testing.B) { for i := 0; i < b.N; i++ { _, err := GetGlob("**/foo") diff --git a/hugolib/content_factory.go b/hugolib/content_factory.go new file mode 100644 index 00000000..b94608e8 --- /dev/null +++ b/hugolib/content_factory.go @@ -0,0 +1,181 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "io" + "path/filepath" + "strings" + "time" + + "github.com/gohugoio/hugo/helpers" + + "github.com/gohugoio/hugo/source" + + "github.com/gohugoio/hugo/resources/page" + + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +// ContentFactory creates content files from archetype templates. +type ContentFactory struct { + h *HugoSites + + // We parse the archetype templates as Go templates, so we need + // to replace any shortcode with a temporary placeholder. + shortocdeReplacerPre *strings.Replacer + shortocdeReplacerPost *strings.Replacer +} + +// AppplyArchetypeFilename archetypeFilename to w as a template using the given Page p as the foundation for the data context. +func (f ContentFactory) AppplyArchetypeFilename(w io.Writer, p page.Page, archetypeKind, archetypeFilename string) error { + + fi, err := f.h.SourceFilesystems.Archetypes.Fs.Stat(archetypeFilename) + if err != nil { + return err + } + + if fi.IsDir() { + return errors.Errorf("archetype directory (%q) not supported", archetypeFilename) + } + + templateSource, err := afero.ReadFile(f.h.SourceFilesystems.Archetypes.Fs, archetypeFilename) + if err != nil { + return errors.Wrapf(err, "failed to read archetype file %q: %s", archetypeFilename, err) + + } + + return f.AppplyArchetypeTemplate(w, p, archetypeKind, string(templateSource)) + +} + +// AppplyArchetypeFilename templateSource to w as a template using the given Page p as the foundation for the data context. +func (f ContentFactory) AppplyArchetypeTemplate(w io.Writer, p page.Page, archetypeKind, templateSource string) error { + ps := p.(*pageState) + if archetypeKind == "" { + archetypeKind = p.Type() + } + + d := &archetypeFileData{ + Type: archetypeKind, + Date: time.Now().Format(time.RFC3339), + Page: p, + File: p.File(), + } + + templateSource = f.shortocdeReplacerPre.Replace(templateSource) + + templ, err := ps.s.TextTmpl().Parse("archetype.md", string(templateSource)) + if err != nil { + return errors.Wrapf(err, "failed to parse archetype template: %s", err) + } + + result, err := executeToString(ps.s.Tmpl(), templ, d) + if err != nil { + return errors.Wrapf(err, "failed to execute archetype template: %s", err) + } + + _, err = io.WriteString(w, f.shortocdeReplacerPost.Replace(result)) + + return err + +} + +func (f ContentFactory) SectionFromFilename(filename string) string { + filename = filepath.Clean(filename) + rel, _ := f.h.AbsProjectContentDir(filename) + if rel == "" { + return "" + } + + parts := strings.Split(helpers.ToSlashTrimLeading(rel), "/") + if len(parts) < 2 { + return "" + } + return parts[0] +} + +// CreateContentPlaceHolder creates a content placeholder file inside the +// best matching content directory. +func (f ContentFactory) CreateContentPlaceHolder(filename string) (string, error) { + filename = filepath.Clean(filename) + _, abs := f.h.AbsProjectContentDir(filename) + + contentDir := filepath.Dir(abs) + if err := f.h.Fs.Source.MkdirAll(contentDir, 0777); err != nil { + return "", err + } + + // This will be overwritten later, just write a placholder to get + // the paths correct. + placeholder := `--- +title: "Content Placeholder" +_build: + render: never + list: never + publishResources: false +--- + +` + + if err := afero.WriteFile(f.h.Fs.Source, abs, []byte(placeholder), 0777); err != nil { + return "", err + } + + return abs, nil +} + +// NewContentFactory creates a new ContentFactory for h. +func NewContentFactory(h *HugoSites) ContentFactory { + return ContentFactory{ + h: h, + shortocdeReplacerPre: strings.NewReplacer( + "{{<", "{x{<", + "{{%", "{x{%", + ">}}", ">}x}", + "%}}", "%}x}"), + shortocdeReplacerPost: strings.NewReplacer( + "{x{<", "{{<", + "{x{%", "{{%", + ">}x}", ">}}", + "%}x}", "%}}"), + } +} + +// archetypeFileData represents the data available to an archetype template. +type archetypeFileData struct { + // The archetype content type, either given as --kind option or extracted + // from the target path's section, i.e. "blog/mypost.md" will resolve to + // "blog". + Type string + + // The current date and time as a RFC3339 formatted string, suitable for use in front matter. + Date string + + // The temporary page. Note that only the file path information is relevant at this stage. + Page page.Page + + // File is the same as Page.File, embedded here for historic reasons. + // TODO(bep) make this a method. + source.File +} + +func (f *archetypeFileData) Site() page.Site { + return f.Page.Site() +} + +func (f *archetypeFileData) Name() string { + return f.Page.File().ContentBaseName() +} diff --git a/hugolib/content_factory_test.go b/hugolib/content_factory_test.go new file mode 100644 index 00000000..50cc783f --- /dev/null +++ b/hugolib/content_factory_test.go @@ -0,0 +1,60 @@ +package hugolib + +import ( + "bytes" + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestContentFactory(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + c.Run("Simple", func(c *qt.C) { + workingDir := "/my/work" + b := newTestSitesBuilder(c) + b.WithWorkingDir(workingDir).WithConfigFile("toml", ` + +workingDir="/my/work" + +[module] +[[module.mounts]] +source = 'mcontent/en' +target = 'content' +lang = 'en' +[[module.mounts]] +source = 'archetypes' +target = 'archetypes' + +`) + + b.WithSourceFile(filepath.Join("mcontent/en/bundle", "index.md"), "") + + b.WithSourceFile(filepath.Join("archetypes", "post.md"), `--- +title: "{{ replace .Name "-" " " | title }}" +date: {{ .Date }} +draft: true +--- + +Hello World. +`) + b.CreateSites() + cf := NewContentFactory(b.H) + abs, err := cf.CreateContentPlaceHolder(filepath.FromSlash("mcontent/en/blog/mypage.md")) + b.Assert(err, qt.IsNil) + b.Assert(abs, qt.Equals, filepath.FromSlash("/my/work/mcontent/en/blog/mypage.md")) + b.Build(BuildCfg{SkipRender: true}) + + p := b.H.GetContentPage(abs) + b.Assert(p, qt.Not(qt.IsNil)) + + var buf bytes.Buffer + b.Assert(cf.AppplyArchetypeFilename(&buf, p, "", "post.md"), qt.IsNil) + + b.Assert(buf.String(), qt.Contains, `title: "Mypage"`) + }) + +} diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index d238d2e0..dcfee34f 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -102,6 +102,42 @@ func (b *BaseFs) RelContentDir(filename string) string { return filename } +// AbsProjectContentDir tries to create a TODO1 +func (b *BaseFs) AbsProjectContentDir(filename string) (string, string) { + isAbs := filepath.IsAbs(filename) + for _, dir := range b.SourceFilesystems.Content.Dirs { + meta := dir.Meta() + if meta.Module != "project" { + continue + } + if isAbs { + if strings.HasPrefix(filename, meta.Filename) { + return strings.TrimPrefix(filename, meta.Filename), filename + } + } else { + contentDir := strings.TrimPrefix(strings.TrimPrefix(meta.Filename, meta.BaseDir), filePathSeparator) + if strings.HasPrefix(filename, contentDir) { + relFilename := strings.TrimPrefix(filename, contentDir) + absFilename := filepath.Join(meta.Filename, relFilename) + return relFilename, absFilename + } + } + + } + + if !isAbs { + // A filename on the form "posts/mypage.md", put it inside + // the first content folder, usually /content. + // The Dirs are ordered with the most important last, so pick that. + contentDirs := b.SourceFilesystems.Content.Dirs + firstContentDir := contentDirs[len(contentDirs)-1].Meta().Filename + return filename, filepath.Join(firstContentDir, filename) + + } + + return "", "" +} + // ResolveJSConfigFile resolves the JS-related config file to a absolute // filename. One example of such would be postcss.config.js. func (fs *BaseFs) ResolveJSConfigFile(name string) string { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 27c490cc..141019a8 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -22,6 +22,8 @@ import ( "sync" "sync/atomic" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/identity" @@ -677,6 +679,9 @@ type BuildCfg struct { // Recently visited URLs. This is used for partial re-rendering. RecentlyVisited map[string]bool + // Can be set to build only with a sub set of the content source. + ContentInclusionFilter *glob.FilenameFilter + testCounters *testCounters } @@ -819,7 +824,7 @@ func (h *HugoSites) Pages() page.Pages { } func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) { - spec := source.NewSourceSpec(h.PathSpec, nil) + spec := source.NewSourceSpec(h.PathSpec, nil, nil) h.data = make(map[string]interface{}) for _, fi := range fis { diff --git a/hugolib/pages_capture_test.go b/hugolib/pages_capture_test.go index 0fdc73e7..4b2979a0 100644 --- a/hugolib/pages_capture_test.go +++ b/hugolib/pages_capture_test.go @@ -51,7 +51,7 @@ func TestPagesCapture(t *testing.T) { ps, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, loggers.NewErrorLogger()) c.Assert(err, qt.IsNil) - sourceSpec := source.NewSourceSpec(ps, fs) + sourceSpec := source.NewSourceSpec(ps, nil, fs) t.Run("Collect", func(t *testing.T) { c := qt.New(t) diff --git a/hugolib/site.go b/hugolib/site.go index 18c9bfc8..96cf0b93 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1193,7 +1193,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro filenamesChanged = helpers.UniqueStringsReuse(filenamesChanged) - if err := s.readAndProcessContent(filenamesChanged...); err != nil { + if err := s.readAndProcessContent(*config, filenamesChanged...); err != nil { return err } @@ -1207,7 +1207,7 @@ func (s *Site) process(config BuildCfg) (err error) { err = errors.Wrap(err, "initialize") return } - if err = s.readAndProcessContent(); err != nil { + if err = s.readAndProcessContent(config); err != nil { err = errors.Wrap(err, "readAndProcessContent") return } @@ -1376,8 +1376,8 @@ func (s *Site) eventToIdentity(e fsnotify.Event) (identity.PathIdentity, bool) { return identity.PathIdentity{}, false } -func (s *Site) readAndProcessContent(filenames ...string) error { - sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) +func (s *Site) readAndProcessContent(buildConfig BuildCfg, filenames ...string) error { + sourceSpec := source.NewSourceSpec(s.PathSpec, buildConfig.ContentInclusionFilter, s.BaseFs.Content.Fs) proc := newPagesProcessor(s.h, sourceSpec) diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go index e8bfbf6f..9e82b18d 100644 --- a/langs/i18n/translationProvider.go +++ b/langs/i18n/translationProvider.go @@ -46,7 +46,7 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { - spec := source.NewSourceSpec(d.PathSpec, nil) + spec := source.NewSourceSpec(d.PathSpec, nil, nil) bundle := i18n.NewBundle(language.English) bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal) diff --git a/source/content_directory_test.go b/source/content_directory_test.go index d3723c6b..8f1d0df4 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -57,7 +57,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { ps, err := helpers.NewPathSpec(fs, v, nil) c.Assert(err, qt.IsNil) - s := NewSourceSpec(ps, fs.Source) + s := NewSourceSpec(ps, nil, fs.Source) if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored { t.Errorf("[%d] File not ignored", i) diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 0b8c1d39..6343c6a4 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -106,5 +106,5 @@ func newTestSourceSpec() *SourceSpec { if err != nil { panic(err) } - return NewSourceSpec(ps, fs.Source) + return NewSourceSpec(ps, nil, fs.Source) } diff --git a/source/sourceSpec.go b/source/sourceSpec.go index e8407a14..3640c83d 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -19,6 +19,8 @@ import ( "regexp" "runtime" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" @@ -33,8 +35,7 @@ type SourceSpec struct { SourceFs afero.Fs - // This is set if the ignoreFiles config is set. - ignoreFilesRe []*regexp.Regexp + shouldInclude func(filename string) bool Languages map[string]interface{} DefaultContentLanguage string @@ -42,7 +43,7 @@ type SourceSpec struct { } // NewSourceSpec initializes SourceSpec using languages the given filesystem and PathSpec. -func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { +func NewSourceSpec(ps *helpers.PathSpec, inclusionFilter *glob.FilenameFilter, fs afero.Fs) *SourceSpec { cfg := ps.Cfg defaultLang := cfg.GetString("defaultContentLanguage") languages := cfg.GetStringMap("languages") @@ -72,8 +73,19 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { } } + shouldInclude := func(filename string) bool { + if !inclusionFilter.Match(filename) { + return false + } + for _, r := range regexps { + if r.MatchString(filename) { + return false + } + } + return true + } - return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} + return &SourceSpec{shouldInclude: shouldInclude, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} } // IgnoreFile returns whether a given file should be ignored. @@ -97,24 +109,16 @@ func (s *SourceSpec) IgnoreFile(filename string) bool { } } - if len(s.ignoreFilesRe) == 0 { - return false - } - - for _, re := range s.ignoreFilesRe { - if re.MatchString(filename) { - return true - } + if !s.shouldInclude(filename) { + return true } if runtime.GOOS == "windows" { // Also check the forward slash variant if different. unixFilename := filepath.ToSlash(filename) if unixFilename != filename { - for _, re := range s.ignoreFilesRe { - if re.MatchString(unixFilename) { - return true - } + if !s.shouldInclude(unixFilename) { + return true } } }