tpl/data: Misc header improvements, tests, allow multiple headers of same key
Closes #5617
This commit is contained in:
Bjørn Erik Pedersen
parent
150d75738b
commit
fcd63de3a5
|
|
@ -15,21 +15,52 @@ package types
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"reflect"
|
||||
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
// ToStringSlicePreserveString converts v to a string slice.
|
||||
// If v is a string, it will be wrapped in a string slice.
|
||||
// ToStringSlicePreserveString is the same as ToStringSlicePreserveStringE,
|
||||
// but it never fails.
|
||||
func ToStringSlicePreserveString(v interface{}) []string {
|
||||
vv, _ := ToStringSlicePreserveStringE(v)
|
||||
return vv
|
||||
}
|
||||
|
||||
// ToStringSlicePreserveStringE converts v to a string slice.
|
||||
// If v is a string, it will be wrapped in a string slice.
|
||||
func ToStringSlicePreserveStringE(v interface{}) ([]string, error) {
|
||||
if v == nil {
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
if sds, ok := v.(string); ok {
|
||||
return []string{sds}
|
||||
return []string{sds}, nil
|
||||
}
|
||||
return cast.ToStringSlice(v)
|
||||
result, err := cast.ToStringSliceE(v)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Probably []int or similar. Fall back to reflect.
|
||||
vv := reflect.ValueOf(v)
|
||||
|
||||
switch vv.Kind() {
|
||||
case reflect.Slice, reflect.Array:
|
||||
result = make([]string, vv.Len())
|
||||
for i := 0; i < vv.Len(); i++ {
|
||||
s, err := cast.ToStringE(vv.Index(i).Interface())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[i] = s
|
||||
}
|
||||
return result, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("failed to convert %T to a string slice", v)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TypeToString converts v to a string if it's a valid string type.
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@ func TestToStringSlicePreserveString(t *testing.T) {
|
|||
c := qt.New(t)
|
||||
|
||||
c.Assert(ToStringSlicePreserveString("Hugo"), qt.DeepEquals, []string{"Hugo"})
|
||||
c.Assert(ToStringSlicePreserveString(qt.Commentf("Hugo")), qt.DeepEquals, []string{"Hugo"})
|
||||
c.Assert(ToStringSlicePreserveString([]interface{}{"A", "B"}), qt.DeepEquals, []string{"A", "B"})
|
||||
c.Assert(ToStringSlicePreserveString([]int{1, 3}), qt.DeepEquals, []string{"1", "3"})
|
||||
c.Assert(ToStringSlicePreserveString(nil), qt.IsNil)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -114,19 +114,10 @@ You can use the following code to render the `Short Description` in your layout:
|
|||
|
||||
Note the use of the [`markdownify` template function][markdownify]. This will send the description through the Blackfriday Markdown rendering engine.
|
||||
|
||||
<!-- begin "Data-drive Content" page -->
|
||||
|
||||
## Data-Driven Content
|
||||
## Get Remote Data
|
||||
|
||||
In addition to the [data files](/extras/datafiles/) feature, Hugo also has a "data-driven content" feature, which lets you load any [JSON](https://www.json.org/) or [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) file from nearly any resource.
|
||||
|
||||
Data-driven content currently consists of two functions, `getJSON` and `getCSV`, which are available in all template files.
|
||||
|
||||
## Implementation details
|
||||
|
||||
### Call the Functions with a URL
|
||||
|
||||
In your template, call the functions like this:
|
||||
Use `getJSON` or `getCSV` to get remote data:
|
||||
|
||||
```
|
||||
{{ $dataJ := getJSON "url" }}
|
||||
|
|
@ -155,19 +146,18 @@ This will resolve internally to the following:
|
|||
{{ $gistJ := getJSON "https://api.github.com/users/GITHUB_USERNAME/gists" }}
|
||||
```
|
||||
|
||||
Finally, you can range over an array. This example will output the
|
||||
first 5 gists for a GitHub user:
|
||||
### Add HTTP headers
|
||||
|
||||
{{< new-in "0.84.0" >}} Both `getJSON` and `getCSV` takes an optional map as the last argument, e.g.:
|
||||
|
||||
```
|
||||
<ul>
|
||||
{{ $urlPre := "https://api.github.com" }}
|
||||
{{ $gistJ := getJSON $urlPre "/users/GITHUB_USERNAME/gists" }}
|
||||
{{ range first 5 $gistJ }}
|
||||
{{ if .public }}
|
||||
<li><a href="{{ .html_url }}" target="_blank">{{ .description }}</a></li>
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
</ul>
|
||||
{{ $data := getJSON "https://example.org/api" (dict "Authorization" "Bearer abcd") }}
|
||||
```
|
||||
|
||||
If you need multiple values for the same header key, use a slice:
|
||||
|
||||
```
|
||||
{{ $data := getJSON "https://example.org/api" (dict "X-List" (slice "a" "b" "c")) }}
|
||||
```
|
||||
|
||||
### Example for CSV files
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ import (
|
|||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/common/maps"
|
||||
|
||||
"github.com/gohugoio/hugo/common/types"
|
||||
|
||||
"github.com/gohugoio/hugo/common/constants"
|
||||
"github.com/gohugoio/hugo/common/loggers"
|
||||
|
||||
|
|
@ -59,14 +63,10 @@ type Namespace struct {
|
|||
// If you provide multiple parts for the URL they will be joined together to the final URL.
|
||||
// GetCSV returns nil or a slice slice to use in a short code.
|
||||
func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err error) {
|
||||
url := joinURL(args)
|
||||
url, headers := toURLAndHeaders(args)
|
||||
cache := ns.cacheGetCSV
|
||||
|
||||
unmarshal := func(b []byte) (bool, error) {
|
||||
if !bytes.Contains(b, []byte(sep)) {
|
||||
return false, _errors.Errorf("cannot find separator %s in CSV for %s", sep, url)
|
||||
}
|
||||
|
||||
if d, err = parseCSV(b, sep); err != nil {
|
||||
err = _errors.Wrapf(err, "failed to parse CSV file %s", url)
|
||||
|
||||
|
|
@ -82,17 +82,9 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err
|
|||
return nil, _errors.Wrapf(err, "failed to create request for getCSV for resource %s", url)
|
||||
}
|
||||
|
||||
req.Header.Add("Accept", "text/csv")
|
||||
req.Header.Add("Accept", "text/plain")
|
||||
|
||||
// Add custom user headers to the get request
|
||||
finalArg := args[len(args)-1]
|
||||
|
||||
if userHeaders, ok := finalArg.(map[string]interface{}); ok {
|
||||
for key, val := range userHeaders {
|
||||
req.Header.Add(key, val.(string))
|
||||
}
|
||||
}
|
||||
// Add custom user headers.
|
||||
addUserProvidedHeaders(headers, req)
|
||||
addDefaultHeaders(req, "text/csv", "text/plain")
|
||||
|
||||
err = ns.getResource(cache, unmarshal, req)
|
||||
if err != nil {
|
||||
|
|
@ -108,7 +100,7 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err
|
|||
// GetJSON returns nil or parsed JSON to use in a short code.
|
||||
func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
|
||||
var v interface{}
|
||||
url := joinURL(args)
|
||||
url, headers := toURLAndHeaders(args)
|
||||
cache := ns.cacheGetJSON
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
|
|
@ -124,17 +116,8 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
|
|||
return false, nil
|
||||
}
|
||||
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("User-Agent", "Hugo Static Site Generator")
|
||||
|
||||
// Add custom user headers to the get request
|
||||
finalArg := args[len(args)-1]
|
||||
|
||||
if userHeaders, ok := finalArg.(map[string]interface{}); ok {
|
||||
for key, val := range userHeaders {
|
||||
req.Header.Add(key, val.(string))
|
||||
}
|
||||
}
|
||||
addUserProvidedHeaders(headers, req)
|
||||
addDefaultHeaders(req, "application/json")
|
||||
|
||||
err = ns.getResource(cache, unmarshal, req)
|
||||
if err != nil {
|
||||
|
|
@ -145,8 +128,64 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
|
|||
return v, nil
|
||||
}
|
||||
|
||||
func joinURL(urlParts []interface{}) string {
|
||||
return strings.Join(cast.ToStringSlice(urlParts), "")
|
||||
func addDefaultHeaders(req *http.Request, accepts ...string) {
|
||||
for _, accept := range accepts {
|
||||
if !hasHeaderValue(req.Header, "Accept", accept) {
|
||||
req.Header.Add("Accept", accept)
|
||||
}
|
||||
}
|
||||
if !hasHeaderKey(req.Header, "User-Agent") {
|
||||
req.Header.Add("User-Agent", "Hugo Static Site Generator")
|
||||
}
|
||||
}
|
||||
|
||||
func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
|
||||
if headers == nil {
|
||||
return
|
||||
}
|
||||
for key, val := range headers {
|
||||
vals := types.ToStringSlicePreserveString(val)
|
||||
for _, s := range vals {
|
||||
req.Header.Add(key, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func hasHeaderValue(m http.Header, key, value string) bool {
|
||||
var s []string
|
||||
var ok bool
|
||||
|
||||
if s, ok = m[key]; !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, v := range s {
|
||||
if v == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasHeaderKey(m http.Header, key string) bool {
|
||||
_, ok := m[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func toURLAndHeaders(urlParts []interface{}) (string, map[string]interface{}) {
|
||||
if len(urlParts) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// The last argument may be a map.
|
||||
headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1])
|
||||
if err == nil {
|
||||
urlParts = urlParts[:len(urlParts)-1]
|
||||
} else {
|
||||
headers = nil
|
||||
}
|
||||
|
||||
return strings.Join(cast.ToStringSlice(urlParts), ""), headers
|
||||
}
|
||||
|
||||
// parseCSV parses bytes of CSV data into a slice slice string or an error
|
||||
|
|
|
|||
|
|
@ -14,12 +14,16 @@
|
|||
package data
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/common/maps"
|
||||
|
||||
qt "github.com/frankban/quicktest"
|
||||
)
|
||||
|
||||
|
|
@ -46,12 +50,6 @@ func TestGetCSV(t *testing.T) {
|
|||
"gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm,EXTRA\n",
|
||||
false,
|
||||
},
|
||||
{
|
||||
",",
|
||||
`http://error.no.sep/`,
|
||||
"gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
|
||||
false,
|
||||
},
|
||||
{
|
||||
",",
|
||||
`http://nofound/404`,
|
||||
|
|
@ -73,66 +71,54 @@ func TestGetCSV(t *testing.T) {
|
|||
false,
|
||||
},
|
||||
} {
|
||||
msg := qt.Commentf("Test %d", i)
|
||||
|
||||
ns := newTestNs()
|
||||
c.Run(test.url, func(c *qt.C) {
|
||||
msg := qt.Commentf("Test %d", i)
|
||||
|
||||
// Setup HTTP test server
|
||||
var srv *httptest.Server
|
||||
srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !haveHeader(r.Header, "Accept", "text/csv") && !haveHeader(r.Header, "Accept", "text/plain") {
|
||||
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||
ns := newTestNs()
|
||||
|
||||
// Setup HTTP test server
|
||||
var srv *httptest.Server
|
||||
srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !hasHeaderValue(r.Header, "Accept", "text/csv") && !hasHeaderValue(r.Header, "Accept", "text/plain") {
|
||||
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/404" {
|
||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("Content-type", "text/csv")
|
||||
|
||||
w.Write([]byte(test.content))
|
||||
})
|
||||
defer func() { srv.Close() }()
|
||||
|
||||
// Setup local test file for schema-less URLs
|
||||
if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
|
||||
f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
|
||||
c.Assert(err, qt.IsNil, msg)
|
||||
f.WriteString(test.content)
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Get on with it
|
||||
got, err := ns.GetCSV(test.sep, test.url)
|
||||
|
||||
if _, ok := test.expect.(bool); ok {
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
|
||||
c.Assert(got, qt.IsNil)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/404" {
|
||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("Content-type", "text/csv")
|
||||
|
||||
w.Write([]byte(test.content))
|
||||
})
|
||||
defer func() { srv.Close() }()
|
||||
|
||||
// Setup local test file for schema-less URLs
|
||||
if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
|
||||
f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
|
||||
c.Assert(err, qt.IsNil, msg)
|
||||
f.WriteString(test.content)
|
||||
f.Close()
|
||||
}
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
|
||||
c.Assert(got, qt.Not(qt.IsNil), msg)
|
||||
c.Assert(got, qt.DeepEquals, test.expect, msg)
|
||||
})
|
||||
|
||||
// Get on with it
|
||||
got, err := ns.GetCSV(test.sep, test.url)
|
||||
|
||||
if _, ok := test.expect.(bool); ok {
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
|
||||
// c.Assert(err, msg, qt.Not(qt.IsNil))
|
||||
c.Assert(got, qt.IsNil)
|
||||
continue
|
||||
}
|
||||
|
||||
c.Assert(err, qt.IsNil, msg)
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
|
||||
c.Assert(got, qt.Not(qt.IsNil), msg)
|
||||
c.Assert(got, qt.DeepEquals, test.expect, msg)
|
||||
|
||||
// Test user-defined headers as well
|
||||
gotHeader, _ := ns.GetCSV(test.sep, test.url, map[string]interface{}{"Accept-Charset": "utf-8", "Max-Forwards": "10"})
|
||||
|
||||
if _, ok := test.expect.(bool); ok {
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
|
||||
// c.Assert(err, msg, qt.Not(qt.IsNil))
|
||||
c.Assert(got, qt.IsNil)
|
||||
continue
|
||||
}
|
||||
|
||||
c.Assert(err, qt.IsNil, msg)
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
|
||||
c.Assert(gotHeader, qt.Not(qt.IsNil), msg)
|
||||
c.Assert(gotHeader, qt.DeepEquals, test.expect, msg)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -178,68 +164,153 @@ func TestGetJSON(t *testing.T) {
|
|||
},
|
||||
} {
|
||||
|
||||
msg := qt.Commentf("Test %d", i)
|
||||
ns := newTestNs()
|
||||
c.Run(test.url, func(c *qt.C) {
|
||||
|
||||
// Setup HTTP test server
|
||||
var srv *httptest.Server
|
||||
srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !haveHeader(r.Header, "Accept", "application/json") {
|
||||
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||
msg := qt.Commentf("Test %d", i)
|
||||
ns := newTestNs()
|
||||
|
||||
// Setup HTTP test server
|
||||
var srv *httptest.Server
|
||||
srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !hasHeaderValue(r.Header, "Accept", "application/json") {
|
||||
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/404" {
|
||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("Content-type", "application/json")
|
||||
|
||||
w.Write([]byte(test.content))
|
||||
})
|
||||
defer func() { srv.Close() }()
|
||||
|
||||
// Setup local test file for schema-less URLs
|
||||
if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
|
||||
f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
|
||||
c.Assert(err, qt.IsNil, msg)
|
||||
f.WriteString(test.content)
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Get on with it
|
||||
got, _ := ns.GetJSON(test.url)
|
||||
|
||||
if _, ok := test.expect.(bool); ok {
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Path == "/404" {
|
||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg)
|
||||
c.Assert(got, qt.Not(qt.IsNil), msg)
|
||||
c.Assert(got, qt.DeepEquals, test.expect)
|
||||
|
||||
w.Header().Add("Content-type", "application/json")
|
||||
|
||||
w.Write([]byte(test.content))
|
||||
})
|
||||
defer func() { srv.Close() }()
|
||||
|
||||
// Setup local test file for schema-less URLs
|
||||
if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
|
||||
f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
|
||||
c.Assert(err, qt.IsNil, msg)
|
||||
f.WriteString(test.content)
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Get on with it
|
||||
got, _ := ns.GetJSON(test.url)
|
||||
|
||||
if _, ok := test.expect.(bool); ok {
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
|
||||
// c.Assert(err, msg, qt.Not(qt.IsNil))
|
||||
continue
|
||||
}
|
||||
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg)
|
||||
c.Assert(got, qt.Not(qt.IsNil), msg)
|
||||
c.Assert(got, qt.DeepEquals, test.expect)
|
||||
|
||||
// Test user-defined headers as well
|
||||
gotHeader, _ := ns.GetJSON(test.url, map[string]interface{}{"Accept-Charset": "utf-8", "Max-Forwards": "10"})
|
||||
|
||||
if _, ok := test.expect.(bool); ok {
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1)
|
||||
// c.Assert(err, msg, qt.Not(qt.IsNil))
|
||||
continue
|
||||
}
|
||||
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg)
|
||||
c.Assert(gotHeader, qt.Not(qt.IsNil), msg)
|
||||
c.Assert(gotHeader, qt.DeepEquals, test.expect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinURL(t *testing.T) {
|
||||
func TestHeaders(t *testing.T) {
|
||||
t.Parallel()
|
||||
c := qt.New(t)
|
||||
c.Assert(joinURL([]interface{}{"https://foo?id=", 32}), qt.Equals, "https://foo?id=32")
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
headers interface{}
|
||||
assert func(c *qt.C, headers string)
|
||||
}{
|
||||
{
|
||||
`Misc header variants`,
|
||||
map[string]interface{}{
|
||||
"Accept-Charset": "utf-8",
|
||||
"Max-forwards": "10",
|
||||
"X-Int": 32,
|
||||
"X-Templ": template.HTML("a"),
|
||||
"X-Multiple": []string{"a", "b"},
|
||||
"X-MultipleInt": []int{3, 4},
|
||||
},
|
||||
func(c *qt.C, headers string) {
|
||||
c.Assert(headers, qt.Contains, "Accept-Charset: utf-8")
|
||||
c.Assert(headers, qt.Contains, "Max-Forwards: 10")
|
||||
c.Assert(headers, qt.Contains, "X-Int: 32")
|
||||
c.Assert(headers, qt.Contains, "X-Templ: a")
|
||||
c.Assert(headers, qt.Contains, "X-Multiple: a")
|
||||
c.Assert(headers, qt.Contains, "X-Multiple: b")
|
||||
c.Assert(headers, qt.Contains, "X-Multipleint: 3")
|
||||
c.Assert(headers, qt.Contains, "X-Multipleint: 4")
|
||||
c.Assert(headers, qt.Contains, "User-Agent: Hugo Static Site Generator")
|
||||
},
|
||||
},
|
||||
{
|
||||
`Params`,
|
||||
maps.Params{
|
||||
"Accept-Charset": "utf-8",
|
||||
},
|
||||
func(c *qt.C, headers string) {
|
||||
c.Assert(headers, qt.Contains, "Accept-Charset: utf-8")
|
||||
},
|
||||
},
|
||||
{
|
||||
`Override User-Agent`,
|
||||
map[string]interface{}{
|
||||
"User-Agent": "007",
|
||||
},
|
||||
func(c *qt.C, headers string) {
|
||||
c.Assert(headers, qt.Contains, "User-Agent: 007")
|
||||
},
|
||||
},
|
||||
} {
|
||||
|
||||
c.Run(test.name, func(c *qt.C) {
|
||||
|
||||
ns := newTestNs()
|
||||
|
||||
// Setup HTTP test server
|
||||
var srv *httptest.Server
|
||||
var headers bytes.Buffer
|
||||
srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
|
||||
c.Assert(r.URL.String(), qt.Equals, "http://gohugo.io/api?foo")
|
||||
w.Write([]byte("{}"))
|
||||
r.Header.Write(&headers)
|
||||
|
||||
})
|
||||
defer func() { srv.Close() }()
|
||||
|
||||
testFunc := func(fn func(args ...interface{}) error) {
|
||||
defer headers.Reset()
|
||||
err := fn("http://example.org/api", "?foo", test.headers)
|
||||
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0)
|
||||
test.assert(c, headers.String())
|
||||
}
|
||||
|
||||
testFunc(func(args ...interface{}) error {
|
||||
_, err := ns.GetJSON(args...)
|
||||
return err
|
||||
})
|
||||
testFunc(func(args ...interface{}) error {
|
||||
_, err := ns.GetCSV(",", args...)
|
||||
return err
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestToURLAndHeaders(t *testing.T) {
|
||||
t.Parallel()
|
||||
c := qt.New(t)
|
||||
url, headers := toURLAndHeaders([]interface{}{"https://foo?id=", 32})
|
||||
c.Assert(url, qt.Equals, "https://foo?id=32")
|
||||
c.Assert(headers, qt.IsNil)
|
||||
|
||||
url, headers = toURLAndHeaders([]interface{}{"https://foo?id=", 32, map[string]interface{}{"a": "b"}})
|
||||
c.Assert(url, qt.Equals, "https://foo?id=32")
|
||||
c.Assert(headers, qt.DeepEquals, map[string]interface{}{"a": "b"})
|
||||
}
|
||||
|
||||
func TestParseCSV(t *testing.T) {
|
||||
|
|
@ -276,19 +347,3 @@ func TestParseCSV(t *testing.T) {
|
|||
c.Assert(act, qt.Equals, test.exp, msg)
|
||||
}
|
||||
}
|
||||
|
||||
func haveHeader(m http.Header, key, needle string) bool {
|
||||
var s []string
|
||||
var ok bool
|
||||
|
||||
if s, ok = m[key]; !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, v := range s {
|
||||
if v == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
package data
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -37,7 +38,9 @@ var (
|
|||
// getRemote loads the content of a remote file. This method is thread safe.
|
||||
func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error {
|
||||
url := req.URL.String()
|
||||
id := helpers.MD5String(url)
|
||||
var headers bytes.Buffer
|
||||
req.Header.Write(&headers)
|
||||
id := helpers.MD5String(url + headers.String())
|
||||
var handled bool
|
||||
var retry bool
|
||||
|
||||
|
|
@ -94,10 +97,6 @@ func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (b
|
|||
// getLocal loads the content of a local file
|
||||
func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
|
||||
filename := filepath.Join(cfg.GetString("workingDir"), url)
|
||||
if e, err := helpers.Exists(filename, fs); !e {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return afero.ReadFile(fs, filename)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue