Add directory based archetypes

Given this content:

```bash
archetypes
├── default.md
└── post-bundle
    ├── bio.md
    ├── images
    │   └── featured.jpg
    └── index.md
```

```bash
hugo new --kind post-bundle post/my-post
```

Will create a new folder in `/content/post/my-post` with the same set of files as in the `post-bundle` archetypes folder.

This commit also improves the archetype language detection, so, if you use template code in your content files, the `.Site` you get is for the correct language. This also means that it is now possible to translate strings defined in  the `i18n` bundles,  e.g. `{{ i18n "hello" }}`.

Fixes #4535
This commit is contained in:
Bjørn Erik Pedersen 2018-09-19 07:48:17 +02:00
parent ef525b15d4
commit 2650fa772b
9 changed files with 347 additions and 139 deletions

View File

@ -85,45 +85,13 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error {
var kind string
createPath, kind = newContentPathSection(createPath)
createPath, kind = newContentPathSection(c.hugo, createPath)
if n.contentType != "" {
kind = n.contentType
}
cfg := c.DepsCfg
ps, err := helpers.NewPathSpec(cfg.Fs, cfg.Cfg)
if err != nil {
return err
}
// If a site isn't in use in the archetype template, we can skip the build.
siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) {
if !siteUsed {
return hugolib.NewSite(*cfg)
}
var s *hugolib.Site
if err := c.hugo.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
return nil, err
}
s = c.hugo.Sites[0]
if len(c.hugo.Sites) > 1 {
// Find the best match.
for _, ss := range c.hugo.Sites {
if strings.Contains(createPath, "."+ss.Language.Lang) {
s = ss
break
}
}
}
return s, nil
}
return create.NewContent(ps, siteFactory, kind, createPath)
return create.NewContent(c.hugo, kind, createPath)
}
func mkdir(x ...string) {
@ -144,10 +112,17 @@ func touchFile(fs afero.Fs, x ...string) {
}
}
func newContentPathSection(path string) (string, string) {
func newContentPathSection(h *hugolib.HugoSites, path string) (string, string) {
// Forward slashes is used in all examples. Convert if needed.
// Issue #1133
createpath := filepath.FromSlash(path)
if h != nil {
for _, s := range h.Sites {
createpath = strings.TrimPrefix(createpath, s.PathSpec.ContentDir)
}
}
var section string
// assume the first directory is the section (kind)
if strings.Contains(createpath[1:], helpers.FilePathSeparator) {

View File

@ -25,7 +25,7 @@ import (
// Issue #1133
func TestNewContentPathSectionWithForwardSlashes(t *testing.T) {
p, s := newContentPathSection("/post/new.md")
p, s := newContentPathSection(nil, "/post/new.md")
assert.Equal(t, filepath.FromSlash("/post/new.md"), p)
assert.Equal(t, "post", s)
}

View File

@ -17,71 +17,76 @@ package create
import (
"bytes"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugolib"
"github.com/spf13/afero"
jww "github.com/spf13/jwalterweatherman"
)
// NewContent creates a new content file in the content directory based upon the
// given kind, which is used to lookup an archetype.
func NewContent(
ps *helpers.PathSpec,
siteFactory func(filename string, siteUsed bool) (*hugolib.Site, error), kind, targetPath string) error {
sites *hugolib.HugoSites, kind, targetPath string) error {
targetPath = filepath.Clean(targetPath)
ext := helpers.Ext(targetPath)
fs := ps.BaseFs.SourceFilesystems.Archetypes.Fs
ps := sites.PathSpec
archetypeFs := ps.BaseFs.SourceFilesystems.Archetypes.Fs
sourceFs := ps.Fs.Source
jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext)
archetypeFilename := findArchetype(ps, kind, ext)
archetypeFilename, isDir := findArchetype(ps, kind, ext)
contentPath, s := resolveContentPath(sites, sourceFs, targetPath)
if isDir {
langFs := hugofs.NewLanguageFs(s.Language.Lang, sites.LanguageSet(), archetypeFs)
cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename)
if err != nil {
return err
}
if cm.siteUsed {
if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
return err
}
}
name := filepath.Base(targetPath)
return newContentFromDir(archetypeFilename, sites, archetypeFs, sourceFs, cm, name, contentPath)
}
// Building the sites can be expensive, so only do it if really needed.
siteUsed := false
if archetypeFilename != "" {
f, err := fs.Open(archetypeFilename)
var err error
siteUsed, err = usesSiteVar(archetypeFs, archetypeFilename)
if err != nil {
return fmt.Errorf("failed to open archetype file: %s", err)
}
defer f.Close()
if helpers.ReaderContains(f, []byte(".Site")) {
siteUsed = true
return err
}
}
s, err := siteFactory(targetPath, siteUsed)
if siteUsed {
if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
return err
}
}
content, err := executeArcheTypeAsTemplate(s, "", kind, targetPath, archetypeFilename)
if err != nil {
return err
}
var content []byte
content, err = executeArcheTypeAsTemplate(s, kind, targetPath, archetypeFilename)
if err != nil {
return err
}
// The site may have multiple content dirs, and we currently do not know which contentDir the
// user wants to create this content in. We should improve on this, but we start by testing if the
// provided path points to an existing dir. If so, use it as is.
var contentPath string
var exists bool
targetDir := filepath.Dir(targetPath)
if targetDir != "" && targetDir != "." {
exists, _ = helpers.Exists(targetDir, fs)
}
if exists {
contentPath = targetPath
} else {
contentPath = s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
}
if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil {
return err
}
@ -103,29 +108,199 @@ func NewContent(
return nil
}
func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib.Site {
for _, s := range sites.Sites {
if fi.Lang() == s.Language.Lang {
return s
}
}
return sites.Sites[0]
}
func newContentFromDir(
archetypeDir string,
sites *hugolib.HugoSites,
sourceFs, targetFs afero.Fs,
cm archetypeMap, name, targetPath string) error {
for _, f := range cm.otherFiles {
filename := f.Filename()
// Just copy the file to destination.
in, err := sourceFs.Open(filename)
if err != nil {
return err
}
targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir))
targetDir := filepath.Dir(targetFilename)
if err := targetFs.MkdirAll(targetDir, 0777); err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to create target directory for %s: %s", targetDir, err)
}
out, err := targetFs.Create(targetFilename)
_, err = io.Copy(out, in)
if err != nil {
return err
}
in.Close()
out.Close()
}
for _, f := range cm.contentFiles {
filename := f.Filename()
s := targetSite(sites, f)
targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir))
content, err := executeArcheTypeAsTemplate(s, name, archetypeDir, targetFilename, filename)
if err != nil {
return err
}
if err := helpers.SafeWriteToDisk(targetFilename, bytes.NewReader(content), targetFs); err != nil {
return err
}
}
jww.FEEDBACK.Println(targetPath, "created")
return nil
}
type archetypeMap struct {
// These needs to be parsed and executed as Go templates.
contentFiles []*hugofs.LanguageFileInfo
// These are just copied to destination.
otherFiles []*hugofs.LanguageFileInfo
// If the templates needs a fully built site. This can potentially be
// expensive, so only do when needed.
siteUsed bool
}
func mapArcheTypeDir(
ps *helpers.PathSpec,
fs afero.Fs,
archetypeDir string) (archetypeMap, error) {
var m archetypeMap
walkFn := func(filename string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if fi.IsDir() {
return nil
}
fil := fi.(*hugofs.LanguageFileInfo)
if hugolib.IsContentFile(filename) {
m.contentFiles = append(m.contentFiles, fil)
if !m.siteUsed {
m.siteUsed, err = usesSiteVar(fs, filename)
if err != nil {
return err
}
}
return nil
}
m.otherFiles = append(m.otherFiles, fil)
return nil
}
if err := helpers.SymbolicWalk(fs, archetypeDir, walkFn); err != nil {
return m, err
}
return m, nil
}
func usesSiteVar(fs afero.Fs, filename string) (bool, error) {
f, err := fs.Open(filename)
if err != nil {
return false, fmt.Errorf("failed to open archetype file: %s", err)
}
defer f.Close()
return helpers.ReaderContains(f, []byte(".Site")), nil
}
// Resolve the target content path.
func resolveContentPath(sites *hugolib.HugoSites, fs afero.Fs, targetPath string) (string, *hugolib.Site) {
targetDir := filepath.Dir(targetPath)
first := sites.Sites[0]
var (
s *hugolib.Site
siteContentDir string
)
// Try the filename: my-post.en.md
for _, ss := range sites.Sites {
if strings.Contains(targetPath, "."+ss.Language.Lang+".") {
s = ss
break
}
}
for _, ss := range sites.Sites {
contentDir := ss.PathSpec.ContentDir
if !strings.HasSuffix(contentDir, helpers.FilePathSeparator) {
contentDir += helpers.FilePathSeparator
}
if strings.HasPrefix(targetPath, contentDir) {
siteContentDir = ss.PathSpec.ContentDir
if s == nil {
s = ss
}
break
}
}
if s == nil {
s = first
}
if targetDir != "" && targetDir != "." {
exists, _ := helpers.Exists(targetDir, fs)
if exists {
return targetPath, s
}
}
if siteContentDir != "" {
pp := filepath.Join(siteContentDir, strings.TrimPrefix(targetPath, siteContentDir))
return s.PathSpec.AbsPathify(pp), s
} else {
return s.PathSpec.AbsPathify(filepath.Join(first.PathSpec.ContentDir, targetPath)), s
}
}
// FindArchetype takes a given kind/archetype of content and returns the path
// to the archetype in the archetype filesystem, blank if none found.
func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string) {
func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string, isDir bool) {
fs := ps.BaseFs.Archetypes.Fs
// If the new content isn't in a subdirectory, kind == "".
// Therefore it should be excluded otherwise `is a directory`
// error will occur. github.com/gohugoio/hugo/issues/411
var pathsToCheck = []string{"default"}
var pathsToCheck []string
if ext != "" {
if kind != "" {
pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...)
} else {
pathsToCheck = append([]string{"default" + ext}, pathsToCheck...)
}
if kind != "" {
pathsToCheck = append(pathsToCheck, kind+ext)
}
pathsToCheck = append(pathsToCheck, "default"+ext, "default")
for _, p := range pathsToCheck {
if exists, _ := helpers.Exists(p, fs); exists {
return p
fi, err := fs.Stat(p)
if err == nil {
return p, fi.IsDir()
}
}
return ""
return "", false
}

View File

@ -80,7 +80,7 @@ var (
"%}x}", "%}}")
)
func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFilename string) ([]byte, error) {
func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archetypeFilename string) ([]byte, error) {
var (
archetypeContent []byte
@ -88,20 +88,16 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
err error
)
ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg)
if err != nil {
return nil, err
}
sp := source.NewSourceSpec(ps, ps.Fs.Source)
f := s.SourceSpec.NewFileInfo("", targetPath, false, nil)
f := sp.NewFileInfo("", targetPath, false, nil)
if name == "" {
name = f.TranslationBaseName()
name := f.TranslationBaseName()
if name == "index" || name == "_index" {
// Page bundles; the directory name will hopefully have a better name.
dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)
_, name = filepath.Split(dir)
if name == "index" || name == "_index" {
// Page bundles; the directory name will hopefully have a better name.
dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)
_, name = filepath.Split(dir)
}
}
data := ArchetypeFileData{

View File

@ -35,8 +35,7 @@ import (
)
func TestNewContent(t *testing.T) {
v := viper.New()
initViper(v)
assert := require.New(t)
cases := []struct {
kind string
@ -49,6 +48,14 @@ func TestNewContent(t *testing.T) {
{"stump", "stump/sample-2.md", []string{`title: "Sample 2"`}}, // no archetype file
{"", "sample-3.md", []string{`title: "Sample 3"`}}, // no archetype
{"product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter
{"lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}},
{"lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}},
{"lang", "post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}},
{"lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}},
{"lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}},
{"lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},
{"lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},
{"lang", "post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}},
{"shortcodes", "shortcodes/go.md", []string{
`title = "GO"`,
"{{< myshortcode >}}",
@ -56,21 +63,20 @@ func TestNewContent(t *testing.T) {
"{{</* comment */>}}\n{{%/* comment */%}}"}}, // shortcodes
}
for _, c := range cases {
cfg, fs := newTestCfg()
require.NoError(t, initFs(fs))
for i, c := range cases {
cfg, fs := newTestCfg(assert)
assert.NoError(initFs(fs))
h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
require.NoError(t, err)
assert.NoError(err)
siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) {
return h.Sites[0], nil
assert.NoError(create.NewContent(h, c.kind, c.path))
fname := filepath.FromSlash(c.path)
if !strings.HasPrefix(fname, "content") {
fname = filepath.Join("content", fname)
}
require.NoError(t, create.NewContent(h.PathSpec, siteFactory, c.kind, c.path))
fname := filepath.Join("content", filepath.FromSlash(c.path))
content := readFileFromFs(t, fs.Source, fname)
for i, v := range c.expected {
for _, v := range c.expected {
found := strings.Contains(content, v)
if !found {
t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)
@ -79,17 +85,44 @@ func TestNewContent(t *testing.T) {
}
}
func initViper(v *viper.Viper) {
v.Set("metaDataFormat", "toml")
v.Set("archetypeDir", "archetypes")
v.Set("contentDir", "content")
v.Set("themesDir", "themes")
v.Set("layoutDir", "layouts")
v.Set("i18nDir", "i18n")
v.Set("theme", "sample")
v.Set("archetypeDir", "archetypes")
v.Set("resourceDir", "resources")
v.Set("publishDir", "public")
func TestNewContentFromDir(t *testing.T) {
assert := require.New(t)
cfg, fs := newTestCfg(assert)
assert.NoError(initFs(fs))
archetypeDir := filepath.Join("archetypes", "my-bundle")
assert.NoError(fs.Source.Mkdir(archetypeDir, 0755))
contentFile := `
File: %s
Site Lang: {{ .Site.Language.Lang }}
Name: {{ replace .Name "-" " " | title }}
i18n: {{ T "hugo" }}
`
assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))
assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755))
assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755))
assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))
assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755))
h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
assert.NoError(err)
assert.Equal(2, len(h.Sites))
assert.NoError(create.NewContent(h, "my-bundle", "post/my-post"))
assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo2.xml")), `hugo2: {{ printf "no template handling in here" }}`)
// Content files should get the correct site context.
// TODO(bep) archetype check i18n
assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Post`, `i18n: Hugo Rocks!`)
assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Site Lang: nn`, `Name: My Post`, `i18n: Hugo Rokkar!`)
assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: My Post`)
}
func initFs(fs *hugofs.Fs) error {
@ -132,6 +165,10 @@ title = "{{ .BaseFileName | upper }}"
path: filepath.Join("archetypes", "emptydate.md"),
content: "+++\ndate =\"\"\ntitle = \"Empty Date Arch title\"\ntest = \"test1\"\n+++\n",
},
{
path: filepath.Join("archetypes", "lang.md"),
content: `Site Lang: {{ .Site.Language.Lang }}|Name: {{ replace .Name "-" " " | title }}|i18n: {{ T "hugo" }}`,
},
// #3623x
{
path: filepath.Join("archetypes", "shortcodes.md"),
@ -166,6 +203,12 @@ Some text.
return nil
}
func assertContains(assert *require.Assertions, v interface{}, matches ...string) {
for _, m := range matches {
assert.Contains(v, m)
}
}
// TODO(bep) extract common testing package with this and some others
func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {
filename = filepath.FromSlash(filename)
@ -185,22 +228,33 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {
return string(b)
}
func newTestCfg() (*viper.Viper, *hugofs.Fs) {
func newTestCfg(assert *require.Assertions) (*viper.Viper, *hugofs.Fs) {
v := viper.New()
v.Set("contentDir", "content")
v.Set("dataDir", "data")
v.Set("i18nDir", "i18n")
v.Set("layoutDir", "layouts")
v.Set("archetypeDir", "archetypes")
v.Set("assetDir", "assets")
cfg := `
[languages]
[languages.en]
weight = 1
languageName = "English"
[languages.nn]
weight = 2
languageName = "Nynorsk"
contentDir = "content_nn"
fs := hugofs.NewMem(v)
`
v.SetFs(fs.Source)
mm := afero.NewMemMapFs()
initViper(v)
assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo]
other = "Hugo Rocks!"`), 0755))
assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "nn.toml"), []byte(`[hugo]
other = "Hugo Rokkar!"`), 0755))
return v, fs
assert.NoError(afero.WriteFile(mm, "config.toml", []byte(cfg), 0755))
v, _, err := hugolib.LoadConfig(hugolib.ConfigSourceDescriptor{Fs: mm, Filename: "config.toml"})
assert.NoError(err)
return v, hugofs.NewFrom(mm, v)
}

View File

@ -61,7 +61,7 @@ func (fi *fileInfo) isOwner() bool {
return fi.bundleTp > bundleNot
}
func isContentFile(filename string) bool {
func IsContentFile(filename string) bool {
return contentFileExtensionsSet[strings.TrimPrefix(helpers.Ext(filename), ".")]
}
@ -98,7 +98,7 @@ const (
// Returns the given file's name's bundle type and whether it is a content
// file or not.
func classifyBundledFile(name string) (bundleDirType, bool) {
if !isContentFile(name) {
if !IsContentFile(name) {
return bundleNot, false
}
if strings.HasPrefix(name, "_index.") {

View File

@ -57,6 +57,14 @@ func (h *HugoSites) IsMultihost() bool {
return h != nil && h.multihost
}
func (h *HugoSites) LanguageSet() map[string]bool {
set := make(map[string]bool)
for _, s := range h.Sites {
set[s.Language.Lang] = true
}
return set
}
func (h *HugoSites) NumLogErrors() int {
if h == nil {
return 0

View File

@ -76,7 +76,7 @@ func newCapturer(
isBundleHeader := func(filename string) bool {
base := filepath.Base(filename)
name := helpers.Filename(base)
return isContentFile(base) && (name == "index" || name == "_index")
return IsContentFile(base) && (name == "index" || name == "_index")
}
// Make sure that any bundle header files are processed before the others. This makes

View File

@ -795,7 +795,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) {
removed = true
}
}
if removed && isContentFile(ev.Name) {
if removed && IsContentFile(ev.Name) {
h.removePageByFilename(ev.Name)
}