From 58395a8ab476f1b1f42e224446b78521ba2a8c6d Mon Sep 17 00:00:00 2001 From: Oliver Steele Date: Sun, 25 Jun 2017 11:23:20 -0400 Subject: [PATCH] Initial --- README.md | 41 ++++++++++++++++++++++++++ ast.go | 64 ++++++++++++++++++++++++++++++++++++++++ chunk_parser.go | 67 +++++++++++++++++++++++++++++++++++++++++ chunk_scanner.go | 72 +++++++++++++++++++++++++++++++++++++++++++++ chunktype_string.go | 16 ++++++++++ control_tags.go | 72 +++++++++++++++++++++++++++++++++++++++++++++ liquid_test.go | 49 ++++++++++++++++++++++++++++++ render.go | 34 +++++++++++++++++++++ 8 files changed, 415 insertions(+) create mode 100644 README.md create mode 100644 ast.go create mode 100644 chunk_parser.go create mode 100644 chunk_scanner.go create mode 100644 chunktype_string.go create mode 100644 control_tags.go create mode 100644 liquid_test.go create mode 100644 render.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5ea25c --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# Go Liquid Template Parser + +`goliquid` is a Go implementation of the [Shopify Liquid template language](https://shopify.github.io/liquid/tags/variable/), for use in [Gojekyll](https://github.com/osteele/gojekyll). + +## Status + +- [ ] Basics + - [ ] Constants + - [ ] Variables + - [ ] Operators + - [ ] Arrays + - [ ] Whitespace Control +- [ ] Tags + - [ ] Comment + - [ ] Control Flow + - [ ] Iteration + - [ ] for + - [ ] limit, offset, range, reversed + - [ ] break, continue + - [ ] loop variables + - [ ] tablerow + - [ ] cycle + - [ ] Raw + - [ ] Variable + - [ ] Assign + - [ ] Capture +- [ ] Filters + +## Install + +`go get -u github.com/osteele/goliquid` + +## Develop + +```bash +go get golang.org/x/tools/cmd/stringer +``` + +```bash +go generate +``` diff --git a/ast.go b/ast.go new file mode 100644 index 0000000..eac93ac --- /dev/null +++ b/ast.go @@ -0,0 +1,64 @@ +package main + +import ( + "io" + + yaml "gopkg.in/yaml.v2" +) + +type AST interface { + Render(io.Writer, Context) error + // String() string +} + +type ASTSeq struct { + Children []AST +} + +type ASTChunks struct { + chunks []Chunk +} + +type ASTText struct { + chunk Chunk +} + +type ASTObject struct { + chunk Chunk +} + +type ASTControlTag struct { + chunk Chunk + cd *ControlTagDefinition + body []AST + branches []*ASTControlTag +} + +func (n ASTSeq) String() string { + b, err := yaml.Marshal(n) + if err != nil { + panic(err) + } + return string(b) +} + +func (n ASTChunks) MarshalYAML() (interface{}, error) { + return map[string]interface{}{"leaf": n.chunks}, nil +} + +func (n ASTControlTag) MarshalYAML() (interface{}, error) { + return map[string]map[string]interface{}{ + n.cd.Name: { + "args": n.chunk.Args, + "body": n.body, + "branches": n.branches, + }}, nil +} + +func (n ASTText) MarshalYAML() (interface{}, error) { + return n.chunk.MarshalYAML() +} + +func (n ASTObject) MarshalYAML() (interface{}, error) { + return n.chunk.MarshalYAML() +} diff --git a/chunk_parser.go b/chunk_parser.go new file mode 100644 index 0000000..ee21a55 --- /dev/null +++ b/chunk_parser.go @@ -0,0 +1,67 @@ +package main + +import ( + "fmt" +) + +func Parse(chunks []Chunk) (AST, error) { + type frame struct { + cd *ControlTagDefinition + cn *ASTControlTag + ap *[]AST + } + var ( + root = &ASTSeq{} + ap = &root.Children // pointer to current node accumulation slice + ccd *ControlTagDefinition + ccn *ASTControlTag + stack []frame // stack of control structures + ) + for _, c := range chunks { + switch c.Type { + case ObjChunk: + *ap = append(*ap, &ASTObject{chunk: c}) + case TextChunk: + *ap = append(*ap, &ASTText{chunk: c}) + case TagChunk: + if cd, ok := FindControlDefinition(c.Tag); ok { + switch { + case cd.RequiresParent() && cd.Parent != ccd: + suffix := "" + if ccd != nil { + suffix = "; immediate parent is " + ccd.Name + } + return nil, fmt.Errorf("%s not inside %s%s", cd.Name, cd.Parent.Name, suffix) + case cd.IsStartTag(): + stack = append(stack, frame{cd: ccd, cn: ccn, ap: ap}) + ccd, ccn = cd, &ASTControlTag{chunk: c, cd: cd} + *ap = append(*ap, ccn) + ap = &ccn.body + case cd.IsBranchTag: + n := &ASTControlTag{chunk: c, cd: cd} + ccn.branches = append(ccn.branches, n) + ap = &n.body + case cd.IsEndTag: + f := stack[len(stack)-1] + ccd, ccn, ap, stack = f.cd, f.cn, f.ap, stack[:len(stack)-1] + } + } else if len(*ap) > 0 { + switch n := ((*ap)[len(*ap)-1]).(type) { + case *ASTChunks: + n.chunks = append(n.chunks, c) + default: + *ap = append(*ap, &ASTChunks{chunks: []Chunk{c}}) + } + } else { + *ap = append(*ap, &ASTChunks{chunks: []Chunk{c}}) + } + } + } + if ccd != nil { + return nil, fmt.Errorf("unterminated %s tag", ccd.Name) + } + if len(root.Children) == 1 { + return root.Children[0], nil + } + return root, nil +} diff --git a/chunk_scanner.go b/chunk_scanner.go new file mode 100644 index 0000000..8c53b72 --- /dev/null +++ b/chunk_scanner.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "regexp" +) + +type Chunk struct { + Type ChunkType + SourceInfo SourceInfo + Source, Tag, Args string +} + +type SourceInfo struct { + Pathname string + lineNo int +} + +type ChunkType int + +//go:generate stringer -type=ChunkType +const ( + TextChunk ChunkType = iota + TagChunk + ObjChunk +) + +var chunkMatcher = regexp.MustCompile(`{{\s*(.+?)\s*}}|{%\s*(\w+)(?:\s+(.+?))?\s*%}`) + +// MarshalYAML, for debugging +func (c Chunk) MarshalYAML() (interface{}, error) { + switch c.Type { + case TextChunk: + return map[string]interface{}{"text": c.Source}, nil + case TagChunk: + return map[string]interface{}{"tag": c.Tag, "args": c.Args}, nil + case ObjChunk: + return map[string]interface{}{"obj": c.Tag}, nil + default: + return nil, fmt.Errorf("unknown chunk tag type: %v", c.Type) + } +} + +func ScanChunks(data string, pathname string) []Chunk { + var ( + sourceInfo = SourceInfo{pathname, 0} + out = make([]Chunk, 0) + p, pe = 0, len(data) + matches = chunkMatcher.FindAllStringSubmatchIndex(data, -1) + ) + for _, m := range matches { + ts, te := m[0], m[1] + if p < ts { + out = append(out, Chunk{TextChunk, sourceInfo, data[p:ts], "", ""}) + } + switch data[ts+1] { + case '{': + out = append(out, Chunk{ObjChunk, sourceInfo, data[ts:te], data[m[2]:m[3]], ""}) + case '%': + var args string + if m[6] > 0 { + args = data[m[6]:m[7]] + } + out = append(out, Chunk{TagChunk, sourceInfo, data[ts:te], data[m[4]:m[5]], args}) + } + p = te + } + if p < pe { + out = append(out, Chunk{TextChunk, sourceInfo, data[p:], "", ""}) + } + return out +} diff --git a/chunktype_string.go b/chunktype_string.go new file mode 100644 index 0000000..741c30b --- /dev/null +++ b/chunktype_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=ChunkType"; DO NOT EDIT. + +package main + +import "fmt" + +const _ChunkType_name = "TextChunkTagChunkObjChunk" + +var _ChunkType_index = [...]uint8{0, 9, 17, 25} + +func (i ChunkType) String() string { + if i < 0 || i >= ChunkType(len(_ChunkType_index)-1) { + return fmt.Sprintf("ChunkType(%d)", i) + } + return _ChunkType_name[_ChunkType_index[i]:_ChunkType_index[i+1]] +} diff --git a/control_tags.go b/control_tags.go new file mode 100644 index 0000000..422860f --- /dev/null +++ b/control_tags.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "io" +) + +func init() { + loopTags := []string{"break", "continue", "cycle"} + DefineControlTag("comment").Action(unimplementedControlTag) + DefineControlTag("if").Branch("else").Branch("elseif").Action(unimplementedControlTag) + DefineControlTag("unless").Action(unimplementedControlTag) + DefineControlTag("case").Branch("when").Action(unimplementedControlTag) + DefineControlTag("for").Governs(loopTags).Action(unimplementedControlTag) + DefineControlTag("tablerow").Governs(loopTags).Action(unimplementedControlTag) + DefineControlTag("capture").Action(unimplementedControlTag) +} + +// ControlTagDefinitions is a map of tag names to control tag definitions. +var ControlTagDefinitions = map[string]*ControlTagDefinition{} + +// ControlTagAction runs the interpreter. +type ControlTagAction func(io.Writer, Context) error + +// ControlTagDefinition tells the parser how to parse control tags. +type ControlTagDefinition struct { + Name string + IsBranchTag bool + IsEndTag bool + Parent *ControlTagDefinition +} + +func (c *ControlTagDefinition) RequiresParent() bool { + return c.IsBranchTag || c.IsEndTag +} + +func (c *ControlTagDefinition) IsStartTag() bool { + return !c.IsBranchTag && !c.IsEndTag +} + +// DefineControlTag defines a control tag and its matching end tag. +func DefineControlTag(name string) *ControlTagDefinition { + ct := &ControlTagDefinition{Name: name} + addControlTagDefinition(ct) + addControlTagDefinition(&ControlTagDefinition{Name: "end" + name, IsEndTag: true, Parent: ct}) + return ct +} + +func FindControlDefinition(name string) (*ControlTagDefinition, bool) { + ct, found := ControlTagDefinitions[name] + return ct, found +} + +func addControlTagDefinition(ct *ControlTagDefinition) { + ControlTagDefinitions[ct.Name] = ct +} + +func (ct *ControlTagDefinition) Branch(name string) *ControlTagDefinition { + addControlTagDefinition(&ControlTagDefinition{Name: name, IsBranchTag: true, Parent: ct}) + return ct +} + +func (ct *ControlTagDefinition) Governs(_ []string) *ControlTagDefinition { + return ct +} + +func (ct *ControlTagDefinition) Action(_ ControlTagAction) { +} + +func unimplementedControlTag(io.Writer, Context) error { + return fmt.Errorf("unimplementedControlTag") +} diff --git a/liquid_test.go b/liquid_test.go new file mode 100644 index 0000000..f327172 --- /dev/null +++ b/liquid_test.go @@ -0,0 +1,49 @@ +//go:generate ragel -Z liquid.rl + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +var tests = []struct{ in, out string }{ + {"prepost", "pre:insertion:post"}, + {"pre:insertion:post", "pre:insertion:post"}, + {"post", ":insertion:post"}, +} + +func TestLiquid(t *testing.T) { + tokens := ScanChunks("pre{%if 1%}left{{x}}right{%endif%}post", "") + // fmt.Println("tokens =", tokens) + ast, err := Parse(tokens) + require.NoError(t, err) + fmt.Println("ast =", ast) + err = ast.Render(os.Stdout, nil) + require.NoError(t, err) + fmt.Println() + + require.True(t, true) + + return + for _, test := range chunkTests { + tokens := ScanChunks(test.in, "") + ast, err := Parse(tokens) + require.NoError(t, err) + actual := ast.Render(os.Stdout, nil) + require.Equal(t, test.expected, actual) + } +} + +type chunkTest struct { + in string + expected string +} + +var chunkTests = []chunkTest{ + chunkTest{"{{var}}", "value"}, + chunkTest{"{{x}}", "1"}, +} diff --git a/render.go b/render.go new file mode 100644 index 0000000..b2cac0c --- /dev/null +++ b/render.go @@ -0,0 +1,34 @@ +package main + +import "io" + +type Context interface{} + +func (n *ASTSeq) Render(w io.Writer, ctx Context) error { + for _, c := range n.Children { + if err := c.Render(w, ctx); err != nil { + return err + } + } + return nil +} + +func (n *ASTChunks) Render(w io.Writer, _ Context) error { + _, err := w.Write([]byte("{chunks}")) + return err +} + +func (n *ASTText) Render(w io.Writer, _ Context) error { + _, err := w.Write([]byte(n.chunk.Source)) + return err +} + +func (n *ASTControlTag) Render(w io.Writer, _ Context) error { + _, err := w.Write([]byte("{control}")) + return err +} + +func (n *ASTObject) Render(w io.Writer, _ Context) error { + _, err := w.Write([]byte("{object}")) + return err +}