diff --git a/commands/hugo.go b/commands/hugo.go index 545daa83..d319dda8 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -667,7 +667,7 @@ func (c *commandeer) timeTrack(start time.Time, name string) { // getDirList provides NewWatcher() with a list of directories to watch for changes. func (c *commandeer) getDirList() ([]string, error) { - var dirnames []string + var filenames []string walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { @@ -681,25 +681,29 @@ func (c *commandeer) getDirList() ([]string, error) { return filepath.SkipDir } - dirnames = append(dirnames, fi.Meta().Filename()) + filenames = append(filenames, fi.Meta().Filename()) } return nil } - watchDirs := c.hugo().PathSpec.BaseFs.WatchDirs() - for _, watchDir := range watchDirs { + watchFiles := c.hugo().PathSpec.BaseFs.WatchDirs() + for _, fi := range watchFiles { + if !fi.IsDir() { + filenames = append(filenames, fi.Meta().Filename()) + continue + } - w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: watchDir, WalkFn: walkFn}) + w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: fi, WalkFn: walkFn}) if err := w.Walk(); err != nil { c.logger.ERROR.Println("walker: ", err) } } - dirnames = helpers.UniqueStringsSorted(dirnames) + filenames = helpers.UniqueStringsSorted(filenames) - return dirnames, nil + return filenames, nil } func (c *commandeer) buildSites() (err error) { diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 74beb05d..893436df 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -35,6 +35,9 @@ import ( const ( metaKeyFilename = "filename" + metaKeyPathFile = "pathFile" // Path of filename relative to a root. + metaKeyIsFileMount = "isFileMount" // Whether the source mount was a file. + metaKeyMountRoot = "mountRoot" metaKeyOriginalFilename = "originalFilename" metaKeyName = "name" metaKeyPath = "path" @@ -108,10 +111,34 @@ func (f FileMeta) Lang() string { return f.stringV(metaKeyLang) } +// Path returns the relative file path to where this file is mounted. func (f FileMeta) Path() string { return f.stringV(metaKeyPath) } +// PathFile returns the relative file path for the file source. This +// will in most cases be the same as Path. +func (f FileMeta) PathFile() string { + pf := f.stringV(metaKeyPathFile) + if f.isFileMount() { + return pf + } + mountRoot := f.mountRoot() + if mountRoot == pf { + return f.Path() + } + + return pf + (strings.TrimPrefix(f.Path(), mountRoot)) +} + +func (f FileMeta) mountRoot() string { + return f.stringV(metaKeyMountRoot) +} + +func (f FileMeta) isFileMount() bool { + return f.GetBool(metaKeyIsFileMount) +} + func (f FileMeta) Weight() int { return f.GetInt(metaKeyWeight) } @@ -129,10 +156,6 @@ func (f FileMeta) IsSymlink() bool { return f.GetBool(metaKeyIsSymlink) } -func (f FileMeta) String() string { - return f.Filename() -} - func (f FileMeta) Watch() bool { if v, found := f["watch"]; found { return v.(bool) @@ -210,6 +233,14 @@ func NewFileMetaInfo(fi os.FileInfo, m FileMeta) FileMetaInfo { return &fileInfoMeta{FileInfo: fi, m: m} } +func copyFileMeta(m FileMeta) FileMeta { + c := make(FileMeta) + for k, v := range m { + c[k] = v + } + return c +} + // Merge metadata, last entry wins. func mergeFileMeta(from, to FileMeta) { if from == nil { diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 0df49cd0..dd60452f 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -35,7 +35,7 @@ var filepathSeparator = string(filepath.Separator) func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rootMapToReal := radix.New() - for _, rm := range rms { + for i, rm := range rms { (&rm).clean() fromBase := files.ResolveComponentFolder(rm.From) @@ -47,16 +47,32 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To)) } - _, err := fs.Stat(rm.To) + fi, err := fs.Stat(rm.To) if err != nil { if os.IsNotExist(err) { continue } return nil, err } - // Extract "blog" from "content/blog" rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) + if rm.Meta != nil { + rm.Meta[metaKeyIsFileMount] = !fi.IsDir() + rm.Meta[metaKeyMountRoot] = rm.path + if rm.ToBasedir != "" { + pathFile := strings.TrimPrefix(strings.TrimPrefix(rm.To, rm.ToBasedir), filepathSeparator) + rm.Meta[metaKeyPathFile] = pathFile + } + } + + meta := copyFileMeta(rm.Meta) + + if !fi.IsDir() { + _, name := filepath.Split(rm.From) + meta[metaKeyName] = name + } + + rm.fi = NewFileMetaInfo(fi, meta) key := rm.rootKey() var mappings []RootMapping @@ -67,6 +83,8 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { } mappings = append(mappings, rm) rootMapToReal.Insert(key, mappings) + + rms[i] = rm } rfs := &RootMappingFs{Fs: fs, @@ -91,11 +109,14 @@ func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, } type RootMapping struct { - From string - To string + From string // The virtual mount. + To string // The source directory or file. + ToBasedir string // The base of To. May be empty if an absolute path was provided. + Meta FileMeta // File metadata (lang etc.) + + fi FileMetaInfo + path string // The virtual mount point, e.g. "blog". - path string // The virtual mount point, e.g. "blog". - Meta FileMeta // File metadata (lang etc.) } func (rm *RootMapping) clean() { @@ -148,6 +169,11 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { if err != nil { return nil, errors.Wrap(err, "RootMappingFs.Dirs") } + + if !fi.IsDir() { + mergeFileMeta(r.Meta, fi.(FileMetaInfo).Meta()) + } + fss[i] = fi.(FileMetaInfo) } @@ -168,7 +194,6 @@ func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afer } func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, []FileMetaInfo, bool, error) { - if fs.isRoot(name) { return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, nil, false, nil } @@ -210,10 +235,12 @@ func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInf return nil, nil, false, err } fim := fi.(FileMetaInfo) + fis = append(fis, fim) } for _, root = range rootsInDir { + fi, _, err := fs.statRoot(root, "") if err != nil { if os.IsNotExist(err) { @@ -500,9 +527,9 @@ func (f *rootMappingFile) Name() string { func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { if f.File == nil { - dirsn := make([]os.FileInfo, 0) + filesn := make([]os.FileInfo, 0) roots := f.fs.getRootsWithPrefix(f.name) - seen := make(map[string]bool) + seen := make(map[string]bool) // Do not return duplicate directories j := 0 for _, rm := range roots { @@ -510,13 +537,16 @@ func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { break } - opener := func() (afero.File, error) { - return f.fs.Open(rm.From) + if !rm.fi.IsDir() { + // A single file mount + filesn = append(filesn, rm.fi) + continue } - name := rm.From + from := rm.From + name := from if !f.isRoot { - _, name = filepath.Split(rm.From) + _, name = filepath.Split(from) } if seen[name] { @@ -524,16 +554,21 @@ func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { } seen[name] = true + opener := func() (afero.File, error) { + return f.fs.Open(from) + } + j++ fi := newDirNameOnlyFileInfo(name, false, opener) + if rm.Meta != nil { mergeFileMeta(rm.Meta, fi.Meta()) } - dirsn = append(dirsn, fi) + filesn = append(filesn, fi) } - return dirsn, nil + return filesn, nil } if f.File == nil { diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index 548224c1..7d685c77 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -186,21 +186,40 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", testfile), []byte("some en content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", testfile), []byte("some sv content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "other.txt"), []byte("some sv content"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/singlefiles", "no.txt"), []byte("no text"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/singlefiles", "sv.txt"), []byte("sv text"), 0755), qt.IsNil) bfs := afero.NewBasePathFs(fs, "themes/a").(*afero.BasePathFs) rm := []RootMapping{ - RootMapping{From: "content/blog", + // Directories + RootMapping{ + From: "content/blog", To: "mynoblogcontent", Meta: FileMeta{"lang": "no"}, }, - RootMapping{From: "content/blog", + RootMapping{ + From: "content/blog", To: "myenblogcontent", Meta: FileMeta{"lang": "en"}, }, - RootMapping{From: "content/blog", + RootMapping{ + From: "content/blog", To: "mysvblogcontent", Meta: FileMeta{"lang": "sv"}, }, + // Files + RootMapping{ + From: "content/singles/p1.md", + To: "singlefiles/no.txt", + ToBasedir: "singlefiles", + Meta: FileMeta{"lang": "no"}, + }, + RootMapping{ + From: "content/singles/p1.md", + To: "singlefiles/sv.txt", + ToBasedir: "singlefiles", + Meta: FileMeta{"lang": "sv"}, + }, } rfs, err := NewRootMappingFs(bfs, rm...) @@ -208,6 +227,7 @@ func TestRootMappingFsMount(t *testing.T) { blog, err := rfs.Stat(filepath.FromSlash("content/blog")) c.Assert(err, qt.IsNil) + c.Assert(blog.IsDir(), qt.Equals, true) blogm := blog.(FileMetaInfo).Meta() c.Assert(blogm.Lang(), qt.Equals, "no") // First match @@ -236,6 +256,25 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(string(b), qt.Equals, "some no content") + // Check file mappings + single, err := rfs.Stat(filepath.FromSlash("content/singles/p1.md")) + c.Assert(err, qt.IsNil) + c.Assert(single.IsDir(), qt.Equals, false) + singlem := single.(FileMetaInfo).Meta() + c.Assert(singlem.Lang(), qt.Equals, "no") // First match + + singlesDir, err := rfs.Open(filepath.FromSlash("content/singles")) + c.Assert(err, qt.IsNil) + defer singlesDir.Close() + singles, err := singlesDir.Readdir(-1) + c.Assert(err, qt.IsNil) + c.Assert(singles, qt.HasLen, 2) + for i, lang := range []string{"no", "sv"} { + fi := singles[i].(FileMetaInfo) + c.Assert(fi.Meta().PathFile(), qt.Equals, lang+".txt") + c.Assert(fi.Meta().Lang(), qt.Equals, lang) + c.Assert(fi.Name(), qt.Equals, "p1.md") + } } func TestRootMappingFsMountOverlap(t *testing.T) { diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index d931db4d..cf9ff3c3 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -18,6 +18,7 @@ package filesystems import ( "io" "os" + "path" "path/filepath" "strings" "sync" @@ -55,6 +56,8 @@ type BaseFs struct { theBigFs *filesystemsCollector } +// TODO(bep) we can get regular files in here and that is fine, but +// we need to clean up the naming. func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo { var dirs []hugofs.FileMetaInfo for _, dir := range fs.AllDirs() { @@ -62,7 +65,6 @@ func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo { dirs = append(dirs, dir) } } - return dirs } @@ -90,7 +92,7 @@ func (b *BaseFs) RelContentDir(filename string) string { for _, dir := range b.SourceFilesystems.Content.Dirs { dirname := dir.Meta().Filename() if strings.HasPrefix(filename, dirname) { - rel := strings.TrimPrefix(filename, dirname) + rel := path.Join(dir.Meta().Path(), strings.TrimPrefix(filename, dirname)) return strings.TrimPrefix(rel, filePathSeparator) } } @@ -298,8 +300,16 @@ func (d *SourceFilesystem) Contains(filename string) bool { func (d *SourceFilesystem) Path(filename string) string { for _, dir := range d.Dirs { meta := dir.Meta() + if !dir.IsDir() { + if filename == meta.Filename() { + return meta.PathFile() + } + continue + } + if strings.HasPrefix(filename, meta.Filename()) { p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator) + p = path.Join(meta.PathFile(), p) return p } } @@ -530,11 +540,11 @@ func (b *sourceFilesystemsBuilder) createModFs( fromToStatic []hugofs.RootMapping ) - absPathify := func(path string) string { + absPathify := func(path string) (string, string) { if filepath.IsAbs(path) { - return path + return "", path } - return paths.AbsPathify(md.dir, path) + return md.dir, paths.AbsPathify(md.dir, path) } for _, mount := range md.Mounts() { @@ -544,9 +554,12 @@ func (b *sourceFilesystemsBuilder) createModFs( mountWeight++ } + base, filename := absPathify(mount.Source) + rm := hugofs.RootMapping{ - From: mount.Target, - To: absPathify(mount.Source), + From: mount.Target, + To: filename, + ToBasedir: base, Meta: hugofs.FileMeta{ "watch": md.Watch(), "mountWeight": mountWeight, @@ -621,7 +634,8 @@ func (b *sourceFilesystemsBuilder) createModFs( if md.isMainProject { return b.p.AbsResourcesDir } - return absPathify(files.FolderResources) + _, filename := absPathify(files.FolderResources) + return filename } if collector.overlayMounts == nil { diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 929bd7ab..1c665e6e 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -545,6 +545,85 @@ title: "My Page" b.AssertFileContent("public/mypage/index.html", "Permalink: https://example.org/mypage/") } +// https://github.com/gohugoio/hugo/issues/6684 +func TestMountsContentFile(t *testing.T) { + c := qt.New(t) + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-content-file") + c.Assert(err, qt.IsNil) + defer clean() + + configTemplate := ` +baseURL = "https://example.com" +title = "My Modular Site" +workingDir = %q + +[module] + [[module.mounts]] + source = "README.md" + target = "content/_index.md" + [[module.mounts]] + source = "mycontent" + target = "content/blog" + +` + + config := fmt.Sprintf(configTemplate, workingDir) + + b := newTestSitesBuilder(t).Running() + + b.Fs = hugofs.NewDefault(viper.New()) + + b.WithWorkingDir(workingDir).WithConfigFile("toml", config) + b.WithTemplatesAdded("index.html", ` +{{ .Title }} +{{ .Content }} + +{{ $readme := .Site.GetPage "/README.md" }} +{{ with $readme }}README: {{ .Title }}|Filename: {{ path.Join .File.Filename }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }} + + +{{ $mypage := .Site.GetPage "/blog/mypage.md" }} +{{ with $mypage }}MYPAGE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }} + +`) + + os.Mkdir(filepath.Join(workingDir, "mycontent"), 0777) + + b.WithSourceFile("README.md", `--- +title: "Readme Title" +--- + +Readme Content. +`, + filepath.Join("mycontent", "mypage.md"), ` +--- +title: "My Page" +--- + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +README: Readme Title +/README.md|Path: _index.md|FilePath: README.md +Readme Content. +MYPAGE: My Page|Path: blog/mypage.md|FilePath: mycontent/mypage.md| +`) + b.AssertFileContent("public/blog/mypage/index.html", "Single: My Page") + + b.EditFiles("README.md", `--- +title: "Readme Edit" +--- +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +Readme Edit +`) +} + // https://github.com/gohugoio/hugo/issues/6299 func TestSiteWithGoModButNoModules(t *testing.T) { t.Parallel() diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 8c29e2a8..4e1623b2 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -895,7 +895,7 @@ func (m *contentChangeMap) add(dirname string, tp bundleDirType) { m.mu.Unlock() } -func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) { +func (m *contentChangeMap) resolveAndRemove(filename string) (string, bundleDirType) { m.mu.RLock() defer m.mu.RUnlock() @@ -908,22 +908,22 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu if _, found := m.branchBundles[dir]; found { delete(m.branchBundles, dir) - return dir, dir, bundleBranch + return dir, bundleBranch } if key, _, found := m.leafBundles.LongestPrefix(dir); found { m.leafBundles.Delete(key) dir = string(key) - return dir, dir, bundleLeaf + return dir, bundleLeaf } fileTp, isContent := classifyBundledFile(name) if isContent && fileTp != bundleNot { // A new bundle. - return dir, dir, fileTp + return dir, fileTp } - return dir, filename, bundleNot + return dir, bundleNot } diff --git a/hugolib/page.go b/hugolib/page.go index af3deb59..f5031332 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -946,6 +946,24 @@ func (p *pageState) sourceRef() string { return "" } +func (p *pageState) sourceRefs() []string { + refs := []string{p.sourceRef()} + + if !p.File().IsZero() { + meta := p.File().FileInfo().Meta() + path := meta.PathFile() + + if path != "" { + ref := "/" + path + if ref != refs[0] { + refs = append(refs, ref) + } + + } + } + return refs +} + type pageStatePages []*pageState // Implement sorting. diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go index adcbbcce..57878881 100644 --- a/hugolib/pagecollections.go +++ b/hugolib/pagecollections.go @@ -151,12 +151,11 @@ func newPageCollectionsFromPages(pages pageStatePages) *PageCollections { for _, pageCollection := range []pageStatePages{c.workAllPages, c.headlessPages} { for _, p := range pageCollection { if p.IsPage() { - sourceRef := p.sourceRef() - if sourceRef != "" { - // index the canonical ref - // e.g. /section/article.md - add(sourceRef, p) + sourceRefs := p.sourceRefs() + for _, ref := range sourceRefs { + add(ref, p) } + sourceRef := sourceRefs[0] // Ref/Relref supports this potentially ambiguous lookup. add(p.File().LogicalName(), p) @@ -177,11 +176,9 @@ func newPageCollectionsFromPages(pages pageStatePages) *PageCollections { pathWithNoExtensions := path.Join(dir, translationBaseName) add(pathWithNoExtensions, p) } else { - // index the canonical, unambiguous ref for any backing file - // e.g. /section/_index.md - sourceRef := p.sourceRef() - if sourceRef != "" { - add(sourceRef, p) + sourceRefs := p.sourceRefs() + for _, ref := range sourceRefs { + add(ref, p) } ref := p.SectionsPath() diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 591b8e31..58d65268 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -116,7 +116,7 @@ func (c *pagesCollector) Collect() error { } else { dirs := make(map[contentDirKey]bool) for _, filename := range c.filenames { - dir, filename, btype := c.tracker.resolveAndRemove(filename) + dir, btype := c.tracker.resolveAndRemove(filename) dirs[contentDirKey{dir, filename, btype}] = true } @@ -127,7 +127,7 @@ func (c *pagesCollector) Collect() error { default: // We always start from a directory. collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool { - return strings.HasSuffix(dir.filename, fim.Meta().Path()) + return dir.filename == fim.Meta().Filename() }) } @@ -211,6 +211,7 @@ func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func( for _, fi := range readdir { if filter(fi) { filtered = append(filtered, fi) + if c.tracker != nil { // Track symlinks. c.tracker.addSymbolicLinkMapping(fi)