Add comments and update tests

engine.go

@@ -98,9 +98,12 @@ func (e *Engine) ParseAndRenderString(source string, b Bindings) (string, Source
 	return string(bs), nil
 }
 
-// SetDelims sets the delimiters for parsing the template. This sets the character characters that
-// are used for '{', '}' and '%'
-func (e *Engine) SetDelims(objectLeft, objectRight, tag byte) *Engine {
-	e.cfg.Delims = []byte{objectLeft, objectRight, tag}
+// Delims sets the action delimiters to the specified strings, to be used in subsequent calls to
+// ParseTemplate, ParseTemplateLocation, ParseAndRender, or ParseAndRenderString. An empty delimiter
+// stands for the corresponding default: { and } for the general delimiters and % for the tag
+// modifier. This results in objects being delimited with "{{" and "}}" and tags "{%" and "%}". The
+// return value is the engine, so calls can be chained.
+func (e *Engine) Delims(left, right, tag byte) *Engine {
+	e.cfg.Delims = []byte{left, right, tag}
 	return e
 }

parser/scanner.go

@@ -8,15 +8,18 @@ import (
 
 // Scan breaks a string into a sequence of Tokens.
 func Scan(data string, loc SourceLoc, delims []byte) (tokens []Token) {
-	// Configure the token matcher to respect the delimeters passed to it
+	// delims = {, }, % => delimiters = {{, }}, {%, %}
 	if len(delims) != 3 {
 		delims = []byte{'{', '}', '%'}
 	}
-	objectLeft := string(delims[0]) + string(delims[0])
-	objectRight := string(delims[1]) + string(delims[1])
-	tagLeft := string(delims[0]) + string(delims[2])
-	tagRight := string(delims[2]) + string(delims[1])
-	var tokenMatcher = regexp.MustCompile(fmt.Sprintf(`%v-?\s*(.+?)\s*-?%v|%v-?\s*(\w+)(?:\s+((?:[^%%]|%%[^}])+?))?\s*-?%v`, objectLeft, objectRight, tagLeft, tagRight))
+	delimiters := formFullDelimiters(delims)
+	tokenMatcher := regexp.MustCompile(
+		fmt.Sprintf(`%s-?\s*(.+?)\s*-?%s|%s-?\s*(\w+)(?:\s+((?:[^%%]|%%[^}])+?))?\s*-?%s`,
+			// QuoteMeta will escape any of these that are regex commands
+			regexp.QuoteMeta(delimiters[0]), regexp.QuoteMeta(delimiters[1]),
+			regexp.QuoteMeta(delimiters[2]), regexp.QuoteMeta(delimiters[3]),
+		),
+	)
 
 	// TODO error on unterminated {{ and {%
 	// TODO probably an error when a tag contains a {{ or {%, at least outside of a string
@@ -61,3 +64,16 @@ func Scan(data string, loc SourceLoc, delims []byte) (tokens []Token) {
 	}
 	return tokens
 }
+
+// formFullDelimiters converts the single character byte delimiters into the full string actual
+// delimiters.
+func formFullDelimiters(delims []byte) []string {
+	// Configure the token matcher to respect the delimiters passed to it. The default delims are '{',
+	// '}', '%' which turn into "{{" and "}}" for objects and "{%" and "%}" for tags
+	fullDelimiters := make([]string, 4, 4)
+	fullDelimiters[0] = string([]byte{delims[0], delims[0]})
+	fullDelimiters[1] = string([]byte{delims[1], delims[1]})
+	fullDelimiters[2] = string([]byte{delims[0], delims[2]})
+	fullDelimiters[3] = string([]byte{delims[2], delims[1]})
+	return fullDelimiters
+}

parser/scanner_test.go

@@ -98,3 +98,63 @@ func TestScan_ws(t *testing.T) {
 		})
 	}
 }
+
+var scannerCountTestsDelims = []struct {
+	in  string
+	len int
+}{
+	{`<* tag arg *>`, 1},
+	{`<* tag arg *><* tag *>`, 2},
+	{`<* tag arg *><* tag arg *><* tag *>`, 3},
+	// {`<* tag *><* tag *>`, 2}, // Currently failing
+	// {`<* tag arg *><* tag arg *><* tag *><* tag *>`, 4}, // Currently failing
+	{`<< expr >>`, 1},
+	{`<< expr arg >>`, 1},
+	{`<< expr >><< expr >>`, 2},
+	{`<< expr arg >><< expr arg >>`, 2},
+}
+
+func TestScan_delims(t *testing.T) {
+	scan := func(src string) []Token { return Scan(src, SourceLoc{}, []byte{'<', '>', '*'}) }
+	tokens := scan("12")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, TextTokenType, tokens[0].Type)
+	require.Equal(t, "12", tokens[0].Source)
+
+	tokens = scan("<<obj>>")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, ObjTokenType, tokens[0].Type)
+	require.Equal(t, "obj", tokens[0].Args)
+
+	tokens = scan("<< obj >>")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, ObjTokenType, tokens[0].Type)
+	require.Equal(t, "obj", tokens[0].Args)
+
+	tokens = scan("<*tag args*>")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, TagTokenType, tokens[0].Type)
+	require.Equal(t, "tag", tokens[0].Name)
+	require.Equal(t, "args", tokens[0].Args)
+
+	tokens = scan("<* tag args *>")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, TagTokenType, tokens[0].Type)
+	require.Equal(t, "tag", tokens[0].Name)
+	require.Equal(t, "args", tokens[0].Args)
+
+	tokens = scan("pre<* tag args *>mid<< object >>post")
+	require.Equal(t, `[TextTokenType{"pre"} TagTokenType{Tag:"tag", Args:"args"} TextTokenType{"mid"} ObjTokenType{"object"} TextTokenType{"post"}]`, fmt.Sprint(tokens))
+
+	for i, test := range scannerCountTestsDelims {
+		t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
+			tokens := scan(test.in)
+			require.Len(t, tokens, test.len)
+		})
+	}
+}