More filters

README.md

@@ -28,16 +28,24 @@ We are currently ~5x faster than Jekyll. Some obvious improvements would include
   - [ ] Assets
     - [ ] Coffeescript
     - [x] Sass/SCSS
-      - [ ] Sass caching
+      - [ ] Sass cache
 - [ ] Customization
   - [x] Templates
-    - [ ] Jekyll filters (partial)
+    - [ ] Jekyll filters
-    - [ ] Jekyll tags (partial)
+      - [ ] array filters: group_by group_by_exp sample pop shift
+      - [ ] string filters: cgi_escape uri_escape scssify smartify slugify normalize_whitespace
+      - [x] everything else
+    - [ ] Jekyll tags
+      - [x] `include`
+      - [ ] `include_relative`
+      - [x] `link`
+      - [ ] `post_url`
+      - [ ] `gist`
+      - [ ] `highlight`
     - [ ] `markdown=1`
   - [x] Includes
-    - [ ] `include_relative`
+      - [x] `include` parameters
-    - [x] parameters
+      - [ ] `include` variables (e.g. `{% include {{ expr }} %}`)
-    - [ ] variables `{% include {{ expr }} %}`
   - [x] Permalinks
   - [ ] Pagination
   - [ ] Themes
@@ -45,7 +53,7 @@ We are currently ~5x faster than Jekyll. Some obvious improvements would include
 - [x] Server
   - [x] Directory watch
   - [x] Live reload
-- [ ] Windows -- not tested
+- [ ] Windows
 
 Intentional differences from Jekyll:
 

liquid/filters.go

@@ -1,13 +1,16 @@
 package liquid
 
 import (
+	"bytes"
 	"encoding/json"
 	"encoding/xml"
 	"reflect"
+	"regexp"
 	"strings"
 	"time"
 
 	"github.com/osteele/liquid/expressions"
+	"github.com/osteele/liquid/generics"
 	"github.com/russross/blackfriday"
 )
 
@@ -24,10 +27,19 @@ func (e *Wrapper) addJekyllFilters() {
 		}
 		return out
 	})
+	// sort overrides the Liquid filter with one that takes parameters
+	e.engine.DefineFilter("sort", sortFilter)
 	e.engine.DefineFilter("where", whereFilter) // TODO test case
 	e.engine.DefineFilter("where_exp", whereExpFilter)
 	e.engine.DefineFilter("xml_escape", xml.Marshal)
 
+	e.engine.DefineFilter("push", func(array []interface{}, item interface{}) interface{} {
+		return append(array, generics.MustConvertItem(item, array))
+	})
+	e.engine.DefineFilter("unshift", func(array []interface{}, item interface{}) interface{} {
+		return append([]interface{}{generics.MustConvertItem(item, array)}, array...)
+	})
+
 	// dates
 	e.engine.DefineFilter("date_to_rfc822", func(date time.Time) string {
 		return date.Format(time.RFC822)
@@ -55,6 +67,36 @@ func (e *Wrapper) addJekyllFilters() {
 	})
 	e.engine.DefineFilter("jsonify", json.Marshal)
 	e.engine.DefineFilter("markdownify", blackfriday.MarkdownCommon)
+	// e.engine.DefineFilter("normalize_whitespace", func(s string) string {
+	// 	wsPattern := regexp.MustCompile(`(?s:[\s\n]+)`)
+	// 	return wsPattern.ReplaceAllString(s, " ")
+	// })
+	e.engine.DefineFilter("to_integer", func(n int) int { return n })
+	e.engine.DefineFilter("number_of_words", func(s string) int {
+		wordPattern := regexp.MustCompile(`\w+`) // TODO what's the Jekyll spec for a word?
+		m := wordPattern.FindAllStringIndex(s, -1)
+		if m == nil {
+			return 0
+		}
+		return len(m)
+	})
+
+	// string escapes
+	// e.engine.DefineFilter("uri_escape", func(s string) string {
+	// 	parts := strings.SplitN(s, "?", 2)
+	// 	if len(parts) > 0 {
+	// TODO PathEscape is the wrong function
+	// 		parts[len(parts)-1] = url.PathEscape(parts[len(parts)-1])
+	// 	}
+	// 	return strings.Join(parts, "?")
+	// })
+	e.engine.DefineFilter("xml_escape", func(s string) string {
+		buf := new(bytes.Buffer)
+		if err := xml.EscapeText(buf, []byte(s)); err != nil {
+			panic(err)
+		}
+		return buf.String()
+	})
 }
 
 func arrayToSentenceStringFilter(value []string, conjunction interface{}) string {
@@ -73,6 +115,23 @@ func arrayToSentenceStringFilter(value []string, conjunction interface{}) string
 	return strings.Join(ar, ", ")
 }
 
+func sortFilter(in []interface{}, key interface{}, nilFirst interface{}) []interface{} {
+	nf, ok := nilFirst.(bool)
+	if !ok {
+		nf = true
+	}
+	out := make([]interface{}, len(in))
+	for i, v := range in {
+		out[i] = v
+	}
+	if key == nil {
+		generics.Sort(out)
+	} else {
+		generics.SortByProperty(out, key.(string), nf)
+	}
+	return out
+}
+
 // func xmlEscapeFilter(value interface{}) interface{} {
 // 	data, err := xml.Marshal(value)
 // 	// TODO can't handle maps
@@ -105,19 +164,19 @@ func whereExpFilter(in []interface{}, name string, expr expressions.Closure) ([]
 	return out, nil
 }
 
-func whereFilter(in []interface{}, key string, value interface{}) []interface{} {
+func whereFilter(in []map[string]interface{}, key string, value interface{}) []interface{} {
 	rt := reflect.ValueOf(in)
 	switch rt.Kind() {
 	case reflect.Array, reflect.Slice:
 	default:
-		return in
+		return nil
 	}
 	out := []interface{}{}
 	for i := 0; i < rt.Len(); i++ {
 		item := rt.Index(i)
 		if item.Kind() == reflect.Map && item.Type().Key().Kind() == reflect.String {
 			attr := item.MapIndex(reflect.ValueOf(key))
-			if attr != reflect.Zero(attr.Type()) && (value == nil || attr.Interface() == value) {
+			if attr.IsValid() && (value == nil || attr.Interface() == value) {
 				out = append(out, item.Interface())
 			}
 		}

liquid/filters_test.go

@@ -11,38 +11,68 @@ import (
 
 var filterTests = []struct{ in, expected string }{
 	// dates
-	// TODO date_to_xmlschema use local timezone
+	// TODO date_to_xmlschema should use local timezone?
-	{`{{time | date_to_xmlschema}}`, "2008-11-07T13:07:54+00:00"},
+	{`{{ time | date_to_xmlschema }}`, "2008-11-07T13:07:54+00:00"},
-	{`{{time | date_to_rfc822 }}`, "07 Nov 08 13:07 UTC"},
+	{`{{ time | date_to_rfc822 }}`, "07 Nov 08 13:07 UTC"},
-	{`{{time | date_to_string }}`, "07 Nov 2008"},
+	{`{{ time | date_to_string }}`, "07 Nov 2008"},
-	{`{{time | date_to_long_string }}`, "07 November 2008"},
+	{`{{ time | date_to_long_string }}`, "07 November 2008"},
 
 	// arrays
-	// TODO sort where group_by group_by_exp sample push pop shift
+	// TODO group_by group_by_exp sample pop shift
-	// site.members | where:"graduation_year","2014"
+	// pop and shift are challenging because they require lvalues
-	{`{{ar | array_to_sentence_string }}`, "first, second, and third"},
+	{`{{ ar | array_to_sentence_string }}`, "first, second, and third"},
+
+	// TODO what is the default for nil first?
+	{`{{ animals | sort | join: ", " }}`, "Sally Snake, giraffe, octopus, zebra"},
+	{`{{ site.pages | sort: "weight" | map: "name" | join }}`, "b, a, d, c"},
+	{`{{ site.pages | sort: "weight", true | map: "name" | join }}`, "b, a, d, c"},
+	{`{{ site.pages | sort: "weight", false | map: "name" | join }}`, "a, d, c, b"},
+
+	{`{{ site.members | where: "graduation_year", "2014" | map: "name" | join }}`, "yes"},
+
+	{`{{ page.tags | push: 'Spokane' | join }}`, "Seattle, Tacoma, Spokane"},
+	// {`{{ page.tags | pop }}`, "Seattle"},
+	// {`{{ page.tags | shift }}`, "Tacoma"},
+	{`{{ page.tags | unshift: "Olympia" | join }}`, "Olympia, Seattle, Tacoma"},
 
 	// strings
-	// TODO xml_escape cgi_escape uri_escape number_of_words
+	// TODO cgi_escape uri_escape scssify smartify slugify normalize_whitespace
-	// TODO scssify smartify slugify normalize_whitespace to_integer
 	{`{{ "/assets/style.css" | relative_url }}`, "/my-baseurl/assets/style.css"},
 	{`{{ "/assets/style.css" | absolute_url }}`, "http://example.com/my-baseurl/assets/style.css"},
-	{`{{"Markdown with _emphasis_ and *bold*." | markdownify}}`, "<p>Markdown with <em>emphasis</em> and <em>bold</em>.</p>"},
+	{`{{ "Markdown with _emphasis_ and *bold*." | markdownify }}`, "<p>Markdown with <em>emphasis</em> and <em>bold</em>.</p>"},
-	{`{{obj | jsonify }}`, `{"a":[1,2,3,4]}`},
+	{`{{ obj | jsonify }}`, `{"a":[1,2,3,4]}`},
-	{`{{pages | map: "name" | join}}`, "a, b, c, d"},
+	{`{{ site.pages | map: "name" | join }}`, "a, b, c, d"},
-	{`{{pages | filter: "weight" | map: "name" | join}}`, "a, c, d"},
+	{`{{ site.pages | filter: "weight" | map: "name" | join }}`, "a, c, d"},
+	// {"{{ \"a \n b\" | normalize_whitespace }}", "a b"},
+	{`{{ "123" | to_integer | type }}`, "int"},
+	{`{{ false | to_integer }}`, "0"},
+	{`{{ true | to_integer }}`, "1"},
+	{`{{ "here are some words" | number_of_words}}`, "4"},
+	{`{{ "1 < 2 & 3" | xml_escape }}`, "1 &lt; 2 &amp; 3"},
+	// {`{{ "http://foo.com/?q=foo, \bar?" | uri_escape }}`, "http://foo.com/?q=foo,%20%5Cbar?"},
 }
 
 var filterTestScope = map[string]interface{}{
-	"ar": []string{"first", "second", "third"},
+	"animals": []string{"zebra", "octopus", "giraffe", "Sally Snake"},
+	"ar":      []string{"first", "second", "third"},
 	"obj": map[string]interface{}{
 		"a": []int{1, 2, 3, 4},
 	},
-	"pages": []map[string]interface{}{
+	"page": map[string]interface{}{
-		{"name": "a", "weight": 10},
+		"tags": []string{"Seattle", "Tacoma"},
-		{"name": "b"},
+	},
-		{"name": "c", "weight": 50},
+	"site": map[string]interface{}{
-		{"name": "d", "weight": 30},
+		"members": []map[string]interface{}{
+			{"name": "yes", "graduation_year": "2014"},
+			{"name": "no", "graduation_year": "2015"},
+			{"name": "no"},
+		},
+		"pages": []map[string]interface{}{
+			{"name": "a", "weight": 10},
+			{"name": "b"},
+			{"name": "c", "weight": 50},
+			{"name": "d", "weight": 30},
+		},
 	},
 	"time": timeMustParse("2008-11-07T13:07:54Z"),
 }