diff --git a/engine.go b/engine.go index 3858e24..e9feb19 100644 --- a/engine.go +++ b/engine.go @@ -98,9 +98,12 @@ func (e *Engine) ParseAndRenderString(source string, b Bindings) (string, Source return string(bs), nil } -// SetDelims sets the delimiters for parsing the template. This sets the character characters that -// are used for '{', '}' and '%' -func (e *Engine) SetDelims(objectLeft, objectRight, tag byte) *Engine { - e.cfg.Delims = []byte{objectLeft, objectRight, tag} +// Delims sets the action delimiters to the specified strings, to be used in subsequent calls to +// ParseTemplate, ParseTemplateLocation, ParseAndRender, or ParseAndRenderString. An empty delimiter +// stands for the corresponding default: { and } for the general delimiters and % for the tag +// modifier. This results in objects being delimited with "{{" and "}}" and tags "{%" and "%}". The +// return value is the engine, so calls can be chained. +func (e *Engine) Delims(left, right, tag byte) *Engine { + e.cfg.Delims = []byte{left, right, tag} return e } diff --git a/parser/scanner.go b/parser/scanner.go index fa59314..a2110d2 100644 --- a/parser/scanner.go +++ b/parser/scanner.go @@ -8,15 +8,18 @@ import ( // Scan breaks a string into a sequence of Tokens. func Scan(data string, loc SourceLoc, delims []byte) (tokens []Token) { - // Configure the token matcher to respect the delimeters passed to it + // delims = {, }, % => delimiters = {{, }}, {%, %} if len(delims) != 3 { delims = []byte{'{', '}', '%'} } - objectLeft := string(delims[0]) + string(delims[0]) - objectRight := string(delims[1]) + string(delims[1]) - tagLeft := string(delims[0]) + string(delims[2]) - tagRight := string(delims[2]) + string(delims[1]) - var tokenMatcher = regexp.MustCompile(fmt.Sprintf(`%v-?\s*(.+?)\s*-?%v|%v-?\s*(\w+)(?:\s+((?:[^%%]|%%[^}])+?))?\s*-?%v`, objectLeft, objectRight, tagLeft, tagRight)) + delimiters := formFullDelimiters(delims) + tokenMatcher := regexp.MustCompile( + fmt.Sprintf(`%s-?\s*(.+?)\s*-?%s|%s-?\s*(\w+)(?:\s+((?:[^%%]|%%[^}])+?))?\s*-?%s`, + // QuoteMeta will escape any of these that are regex commands + regexp.QuoteMeta(delimiters[0]), regexp.QuoteMeta(delimiters[1]), + regexp.QuoteMeta(delimiters[2]), regexp.QuoteMeta(delimiters[3]), + ), + ) // TODO error on unterminated {{ and {% // TODO probably an error when a tag contains a {{ or {%, at least outside of a string @@ -61,3 +64,16 @@ func Scan(data string, loc SourceLoc, delims []byte) (tokens []Token) { } return tokens } + +// formFullDelimiters converts the single character byte delimiters into the full string actual +// delimiters. +func formFullDelimiters(delims []byte) []string { + // Configure the token matcher to respect the delimiters passed to it. The default delims are '{', + // '}', '%' which turn into "{{" and "}}" for objects and "{%" and "%}" for tags + fullDelimiters := make([]string, 4, 4) + fullDelimiters[0] = string([]byte{delims[0], delims[0]}) + fullDelimiters[1] = string([]byte{delims[1], delims[1]}) + fullDelimiters[2] = string([]byte{delims[0], delims[2]}) + fullDelimiters[3] = string([]byte{delims[2], delims[1]}) + return fullDelimiters +} diff --git a/parser/scanner_test.go b/parser/scanner_test.go index 7561d6e..c3ec549 100644 --- a/parser/scanner_test.go +++ b/parser/scanner_test.go @@ -98,3 +98,63 @@ func TestScan_ws(t *testing.T) { }) } } + +var scannerCountTestsDelims = []struct { + in string + len int +}{ + {`<* tag arg *>`, 1}, + {`<* tag arg *><* tag *>`, 2}, + {`<* tag arg *><* tag arg *><* tag *>`, 3}, + // {`<* tag *><* tag *>`, 2}, // Currently failing + // {`<* tag arg *><* tag arg *><* tag *><* tag *>`, 4}, // Currently failing + {`<< expr >>`, 1}, + {`<< expr arg >>`, 1}, + {`<< expr >><< expr >>`, 2}, + {`<< expr arg >><< expr arg >>`, 2}, +} + +func TestScan_delims(t *testing.T) { + scan := func(src string) []Token { return Scan(src, SourceLoc{}, []byte{'<', '>', '*'}) } + tokens := scan("12") + require.NotNil(t, tokens) + require.Len(t, tokens, 1) + require.Equal(t, TextTokenType, tokens[0].Type) + require.Equal(t, "12", tokens[0].Source) + + tokens = scan("<>") + require.NotNil(t, tokens) + require.Len(t, tokens, 1) + require.Equal(t, ObjTokenType, tokens[0].Type) + require.Equal(t, "obj", tokens[0].Args) + + tokens = scan("<< obj >>") + require.NotNil(t, tokens) + require.Len(t, tokens, 1) + require.Equal(t, ObjTokenType, tokens[0].Type) + require.Equal(t, "obj", tokens[0].Args) + + tokens = scan("<*tag args*>") + require.NotNil(t, tokens) + require.Len(t, tokens, 1) + require.Equal(t, TagTokenType, tokens[0].Type) + require.Equal(t, "tag", tokens[0].Name) + require.Equal(t, "args", tokens[0].Args) + + tokens = scan("<* tag args *>") + require.NotNil(t, tokens) + require.Len(t, tokens, 1) + require.Equal(t, TagTokenType, tokens[0].Type) + require.Equal(t, "tag", tokens[0].Name) + require.Equal(t, "args", tokens[0].Args) + + tokens = scan("pre<* tag args *>mid<< object >>post") + require.Equal(t, `[TextTokenType{"pre"} TagTokenType{Tag:"tag", Args:"args"} TextTokenType{"mid"} ObjTokenType{"object"} TextTokenType{"post"}]`, fmt.Sprint(tokens)) + + for i, test := range scannerCountTestsDelims { + t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) { + tokens := scan(test.in) + require.Len(t, tokens, test.len) + }) + } +}