Add emoji support
This uses the Emoji map from https://github.com/kyokomi/emoji -- but with a custom replacement implementation. The built-in are fine for most use cases, but in Hugo we do care about pure speed. The benchmarks below are skewed in Hugo's direction as the source and result is a byte slice, Kyokomi's implementation works best with strings. Curious: The easy-to-use `strings.Replacer` is also plenty fast. ``` BenchmarkEmojiKyokomiFprint-4 20000 86038 ns/op 33960 B/op 117 allocs/op BenchmarkEmojiKyokomiSprint-4 20000 83252 ns/op 38232 B/op 122 allocs/op BenchmarkEmojiStringsReplacer-4 100000 21092 ns/op 17248 B/op 25 allocs/op BenchmarkHugoEmoji-4 500000 5728 ns/op 624 B/op 13 allocs/op ``` Fixes #1891
This commit is contained in:
Bjørn Erik Pedersen
Cameron Moore
parent
5926c6c8d5
commit
cafb784799
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
// Copyright 2016 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
|
@ -301,6 +301,7 @@ func LoadDefaultSettings() {
|
|||
viper.SetDefault("SectionPagesMenu", "")
|
||||
viper.SetDefault("DisablePathToLower", false)
|
||||
viper.SetDefault("HasCJKLanguage", false)
|
||||
viper.SetDefault("EnableEmoji", false)
|
||||
}
|
||||
|
||||
// InitializeConfig initializes a config file with sensible default configuration flags.
|
||||
|
|
|
|||
|
|
@ -99,6 +99,9 @@ Following is a list of Hugo-defined variables that you can configure and their c
|
|||
disableRobotsTXT: false
|
||||
# edit new content with this editor, if provided
|
||||
editor: ""
|
||||
# Enable Emoji emoticons support for page content.
|
||||
# See www.emoji-cheat-sheet.com
|
||||
enableEmoji: false
|
||||
footnoteAnchorPrefix: ""
|
||||
footnoteReturnLinkContents: ""
|
||||
# google analytics tracking id
|
||||
|
|
|
|||
|
|
@ -413,6 +413,14 @@ These are formatted with the layout string.
|
|||
e.g. `{{ dateFormat "Monday, Jan 2, 2006" "2015-01-21" }}` → "Wednesday, Jan 21, 2015"
|
||||
|
||||
|
||||
### emojify
|
||||
|
||||
Runs the string through the Emoji emoticons processor. The result will be declared as "safe" so Go templates will not filter it.
|
||||
|
||||
See the [Emoji cheat sheet](http://www.emoji-cheat-sheet.com/) for available emoticons.
|
||||
|
||||
e.g. `{{ "I :heart: Hugo" | emojify }}`
|
||||
|
||||
### highlight
|
||||
Takes a string of code and a language, uses Pygments to return the syntax highlighted code in HTML.
|
||||
Used in the [highlight shortcode](/extras/highlighting/).
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
// Copyright 2016 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package helpers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/kyokomi/emoji"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
emojiInit sync.Once
|
||||
|
||||
emojis = make(map[string][]byte)
|
||||
|
||||
emojiDelim = []byte(":")
|
||||
emojiWordDelim = []byte(" ")
|
||||
emojiMaxSize int
|
||||
)
|
||||
|
||||
// Emojify "emojifies" the input source.
|
||||
// Note that the input byte slice will be modified if needed.
|
||||
// See http://www.emoji-cheat-sheet.com/
|
||||
func Emojify(source []byte) []byte {
|
||||
|
||||
emojiInit.Do(initEmoji)
|
||||
|
||||
start := 0
|
||||
k := bytes.Index(source[start:], emojiDelim)
|
||||
|
||||
for k != -1 {
|
||||
|
||||
j := start + k
|
||||
|
||||
upper := j + emojiMaxSize
|
||||
|
||||
if upper > len(source) {
|
||||
upper = len(source)
|
||||
}
|
||||
|
||||
endEmoji := bytes.Index(source[j+1:upper], emojiDelim)
|
||||
|
||||
if endEmoji < 0 {
|
||||
break
|
||||
}
|
||||
|
||||
nextWordDelim := bytes.Index(source[j:upper], emojiWordDelim)
|
||||
|
||||
if endEmoji == 0 || (nextWordDelim != -1 && nextWordDelim < endEmoji) {
|
||||
start += endEmoji + 1
|
||||
} else {
|
||||
endKey := endEmoji + j + 2
|
||||
emojiKey := source[j:endKey]
|
||||
|
||||
if emoji, ok := emojis[string(emojiKey)]; ok {
|
||||
source = append(source[:j], append(emoji, source[endKey:]...)...)
|
||||
}
|
||||
|
||||
start += endEmoji
|
||||
}
|
||||
|
||||
if start >= len(source) {
|
||||
break
|
||||
}
|
||||
|
||||
k = bytes.Index(source[start:], emojiDelim)
|
||||
}
|
||||
|
||||
return source
|
||||
|
||||
}
|
||||
|
||||
func initEmoji() {
|
||||
emojiMap := emoji.CodeMap()
|
||||
|
||||
for k, v := range emojiMap {
|
||||
emojis[k] = []byte(v + emoji.ReplacePadding)
|
||||
|
||||
if len(k) > emojiMaxSize {
|
||||
emojiMaxSize = len(k)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
// Copyright 2016 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package helpers
|
||||
|
||||
import (
|
||||
"github.com/kyokomi/emoji"
|
||||
"github.com/spf13/hugo/bufferpool"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEmojiCustom(t *testing.T) {
|
||||
for i, this := range []struct {
|
||||
input string
|
||||
expect []byte
|
||||
}{
|
||||
{"A :smile: a day", []byte(emoji.Sprint("A :smile: a day"))},
|
||||
{"A few :smile:s a day", []byte(emoji.Sprint("A few :smile:s a day"))},
|
||||
{"A :smile: and a :beer: makes the day for sure.", []byte(emoji.Sprint("A :smile: and a :beer: makes the day for sure."))},
|
||||
{"A :smile: and: a :beer:", []byte(emoji.Sprint("A :smile: and: a :beer:"))},
|
||||
{"A :diamond_shape_with_a_dot_inside: and then some.", []byte(emoji.Sprint("A :diamond_shape_with_a_dot_inside: and then some."))},
|
||||
{":smile:", []byte(emoji.Sprint(":smile:"))},
|
||||
{":smi", []byte(":smi")},
|
||||
{"A :smile:", []byte(emoji.Sprint("A :smile:"))},
|
||||
{":beer:!", []byte(emoji.Sprint(":beer:!"))},
|
||||
{"::smile:", []byte(emoji.Sprint("::smile:"))},
|
||||
{":beer::", []byte(emoji.Sprint(":beer::"))},
|
||||
{" :beer: :", []byte(emoji.Sprint(" :beer: :"))},
|
||||
{":beer: and :smile: and another :beer:!", []byte(emoji.Sprint(":beer: and :smile: and another :beer:!"))},
|
||||
{" :beer: : ", []byte(emoji.Sprint(" :beer: : "))},
|
||||
{"No smilies for you!", []byte("No smilies for you!")},
|
||||
{" The motto: no smiles! ", []byte(" The motto: no smiles! ")},
|
||||
{":hugo_is_the_best_static_gen:", []byte(":hugo_is_the_best_static_gen:")},
|
||||
{"은행 :smile: 은행", []byte(emoji.Sprint("은행 :smile: 은행"))},
|
||||
} {
|
||||
result := Emojify([]byte(this.input))
|
||||
|
||||
if !reflect.DeepEqual(result, this.expect) {
|
||||
t.Errorf("[%d] got '%q' but expected %q", i, result, this.expect)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// The Emoji benchmarks below are heavily skewed in Hugo's direction:
|
||||
//
|
||||
// Hugo have a byte slice, wants a byte slice and doesn't mind if the original is modified.
|
||||
|
||||
func BenchmarkEmojiKyokomiFprint(b *testing.B) {
|
||||
|
||||
f := func(in []byte) []byte {
|
||||
buff := bufferpool.GetBuffer()
|
||||
defer bufferpool.PutBuffer(buff)
|
||||
emoji.Fprint(buff, string(in))
|
||||
|
||||
bc := make([]byte, buff.Len(), buff.Len())
|
||||
copy(bc, buff.Bytes())
|
||||
return bc
|
||||
}
|
||||
|
||||
doBenchmarkEmoji(b, f)
|
||||
}
|
||||
|
||||
func BenchmarkEmojiKyokomiSprint(b *testing.B) {
|
||||
|
||||
f := func(in []byte) []byte {
|
||||
return []byte(emoji.Sprint(string(in)))
|
||||
}
|
||||
|
||||
doBenchmarkEmoji(b, f)
|
||||
}
|
||||
|
||||
func BenchmarkHugoEmoji(b *testing.B) {
|
||||
doBenchmarkEmoji(b, Emojify)
|
||||
}
|
||||
|
||||
func doBenchmarkEmoji(b *testing.B, f func(in []byte) []byte) {
|
||||
|
||||
type input struct {
|
||||
in []byte
|
||||
expect []byte
|
||||
}
|
||||
|
||||
data := []struct {
|
||||
input string
|
||||
expect string
|
||||
}{
|
||||
{"A :smile: a day", emoji.Sprint("A :smile: a day")},
|
||||
{"A :smile: and a :beer: day keeps the doctor away", emoji.Sprint("A :smile: and a :beer: day keeps the doctor away")},
|
||||
{"A :smile: a day and 10 " + strings.Repeat(":beer: ", 10), emoji.Sprint("A :smile: a day and 10 " + strings.Repeat(":beer: ", 10))},
|
||||
{"No smiles today.", "No smiles today."},
|
||||
{"No smiles for you or " + strings.Repeat("you ", 1000), "No smiles for you or " + strings.Repeat("you ", 1000)},
|
||||
}
|
||||
|
||||
var in []input = make([]input, b.N*len(data))
|
||||
var cnt = 0
|
||||
for i := 0; i < b.N; i++ {
|
||||
for _, this := range data {
|
||||
in[cnt] = input{[]byte(this.input), []byte(this.expect)}
|
||||
cnt++
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
cnt = 0
|
||||
for i := 0; i < b.N; i++ {
|
||||
for j := range data {
|
||||
currIn := in[cnt]
|
||||
cnt++
|
||||
result := f(currIn.in)
|
||||
if len(result) != len(currIn.expect) {
|
||||
b.Fatalf("[%d] emoji std, got \n%q but expected \n%q", j, result, currIn.expect)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
// Copyright 2016 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"github.com/spf13/hugo/source"
|
||||
"github.com/spf13/hugo/tpl"
|
||||
jww "github.com/spf13/jwalterweatherman"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -114,6 +115,10 @@ func commonConvert(p *Page, t tpl.Template) HandledResult {
|
|||
|
||||
var err error
|
||||
|
||||
if viper.GetBool("EnableEmoji") {
|
||||
p.rawContent = helpers.Emojify(p.rawContent)
|
||||
}
|
||||
|
||||
renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
|
||||
|
||||
if len(p.contentShortCodes) > 0 {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2015 The Hugo Authors. All rights reserved.
|
||||
// Copyright 2016 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Portions Copyright The Go Authors.
|
||||
|
||||
|
|
@ -1156,6 +1156,19 @@ func jsonify(v interface{}) (template.HTML, error) {
|
|||
return "", err
|
||||
}
|
||||
return template.HTML(b), nil
|
||||
|
||||
}
|
||||
|
||||
// emojify "emojifies" the given string.
|
||||
//
|
||||
// See http://www.emoji-cheat-sheet.com/
|
||||
func emojify(in interface{}) (template.HTML, error) {
|
||||
str, err := cast.ToStringE(in)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return template.HTML(helpers.Emojify([]byte(str))), nil
|
||||
}
|
||||
|
||||
func refPage(page interface{}, ref, methodName string) template.HTML {
|
||||
|
|
@ -1715,6 +1728,7 @@ func init() {
|
|||
"dict": dictionary,
|
||||
"div": func(a, b interface{}) (interface{}, error) { return doArithmetic(a, b, '/') },
|
||||
"echoParam": returnWhenSet,
|
||||
"emojify": emojify,
|
||||
"eq": eq,
|
||||
"first": first,
|
||||
"ge": ge,
|
||||
|
|
|
|||
Loading…
Reference in New Issue