From a55b2615a4a5d23822661f416417d71dcb58bd47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Fri, 1 Apr 2016 10:44:22 +0300
Subject: [PATCH] Run Smartypants as a separate pass over the AST

Separate Smartypants somewhat from the HTML renderer. Move its flags
from HtmlFlags to Extensions (probably should be moved to its own set of
flags, but not now). With that done, do a separate walk of the tree and
either run Smartypants processor if it's enabled, or simply escape text
nodes.
---
 block_test.go  |  4 +--
 html.go        | 94 +++++++++++++++++++-------------------------------
 inline_test.go | 20 +++++------
 markdown.go    | 52 ++++++++++++++++------------
 ref_test.go    |  2 +-
 smartypants.go | 28 ++++++++++++++-
 6 files changed, 104 insertions(+), 96 deletions(-)

diff --git a/block_test.go b/block_test.go
index 06d5b5d..2f1616c 100644
--- a/block_test.go
+++ b/block_test.go
@@ -23,13 +23,13 @@ func runMarkdownBlockWithRenderer(input string, extensions Extensions, renderer
 }
 
 func runMarkdownBlock(input string, extensions Extensions) string {
-	renderer := HtmlRenderer(UseXHTML, "", "")
+	renderer := HtmlRenderer(UseXHTML, extensions, "", "")
 	return runMarkdownBlockWithRenderer(input, extensions, renderer)
 }
 
 func runnerWithRendererParameters(parameters HtmlRendererParameters) func(string, Extensions) string {
 	return func(input string, extensions Extensions) string {
-		renderer := HtmlRendererWithParameters(UseXHTML, "", "", parameters)
+		renderer := HtmlRendererWithParameters(UseXHTML, extensions, "", "", parameters)
 		return runMarkdownBlockWithRenderer(input, extensions, renderer)
 	}
 }
diff --git a/html.go b/html.go
index d5c8123..9433072 100644
--- a/html.go
+++ b/html.go
@@ -29,25 +29,20 @@ type HtmlFlags int
 
 // HTML renderer configuration options.
 const (
-	HtmlFlagsNone           HtmlFlags = 0
-	SkipHTML                HtmlFlags = 1 << iota // Skip preformatted HTML blocks
-	SkipStyle                                     // Skip embedded <style> elements
-	SkipImages                                    // Skip embedded images
-	SkipLinks                                     // Skip all links
-	Safelink                                      // Only link to trusted protocols
-	NofollowLinks                                 // Only link with rel="nofollow"
-	NoreferrerLinks                               // Only link with rel="noreferrer"
-	HrefTargetBlank                               // Add a blank target
-	Toc                                           // Generate a table of contents
-	OmitContents                                  // Skip the main contents (for a standalone table of contents)
-	CompletePage                                  // Generate a complete HTML page
-	UseXHTML                                      // Generate XHTML output instead of HTML
-	UseSmartypants                                // Enable smart punctuation substitutions
-	SmartypantsFractions                          // Enable smart fractions (with UseSmartypants)
-	SmartypantsDashes                             // Enable smart dashes (with UseSmartypants)
-	SmartypantsLatexDashes                        // Enable LaTeX-style dashes (with UseSmartypants)
-	SmartypantsAngledQuotes                       // Enable angled double quotes (with UseSmartypants) for double quotes rendering
-	FootnoteReturnLinks                           // Generate a link at the end of a footnote to return to the source
+	HtmlFlagsNone       HtmlFlags = 0
+	SkipHTML            HtmlFlags = 1 << iota // Skip preformatted HTML blocks
+	SkipStyle                                 // Skip embedded <style> elements
+	SkipImages                                // Skip embedded images
+	SkipLinks                                 // Skip all links
+	Safelink                                  // Only link to trusted protocols
+	NofollowLinks                             // Only link with rel="nofollow"
+	NoreferrerLinks                           // Only link with rel="noreferrer"
+	HrefTargetBlank                           // Add a blank target
+	Toc                                       // Generate a table of contents
+	OmitContents                              // Skip the main contents (for a standalone table of contents)
+	CompletePage                              // Generate a complete HTML page
+	UseXHTML                                  // Generate XHTML output instead of HTML
+	FootnoteReturnLinks                       // Generate a link at the end of a footnote to return to the source
 
 	TagName               = "[A-Za-z][A-Za-z0-9-]*"
 	AttributeName         = "[a-zA-Z_:][a-zA-Z0-9:._-]*"
@@ -109,10 +104,11 @@ type HTML struct {
 	// Track header IDs to prevent ID collision in a single generation.
 	headerIDs map[string]int
 
-	smartypants   *SPRenderer
 	w             HtmlWriter
 	lastOutputLen int
 	disableTags   int
+
+	extensions Extensions // This gives Smartypants renderer access to flags
 }
 
 const (
@@ -127,8 +123,8 @@ const (
 // title is the title of the document, and css is a URL for the document's
 // stylesheet.
 // title and css are only used when HTML_COMPLETE_PAGE is selected.
-func HtmlRenderer(flags HtmlFlags, title string, css string) Renderer {
-	return HtmlRendererWithParameters(flags, title, css, HtmlRendererParameters{})
+func HtmlRenderer(flags HtmlFlags, extensions Extensions, title string, css string) Renderer {
+	return HtmlRendererWithParameters(flags, extensions, title, css, HtmlRendererParameters{})
 }
 
 type HtmlWriter struct {
@@ -157,7 +153,7 @@ func (r *HTML) Write(b []byte) (int, error) {
 	return r.w.Write(b)
 }
 
-func HtmlRendererWithParameters(flags HtmlFlags, title string,
+func HtmlRendererWithParameters(flags HtmlFlags, extensions Extensions, title string,
 	css string, renderParameters HtmlRendererParameters) Renderer {
 	// configure the rendering engine
 	closeTag := htmlClose
@@ -172,6 +168,7 @@ func HtmlRendererWithParameters(flags HtmlFlags, title string,
 	var writer HtmlWriter
 	return &HTML{
 		flags:      flags,
+		extensions: extensions,
 		closeTag:   closeTag,
 		title:      title,
 		css:        css,
@@ -183,8 +180,7 @@ func HtmlRendererWithParameters(flags HtmlFlags, title string,
 
 		headerIDs: make(map[string]int),
 
-		smartypants: NewSmartypantsRenderer(flags),
-		w:           writer,
+		w: writer,
 	}
 }
 
@@ -688,41 +684,15 @@ func (r *HTML) Entity(entity []byte) {
 }
 
 func (r *HTML) NormalText(text []byte) {
-	if r.flags&UseSmartypants != 0 {
+	if r.extensions&Smartypants != 0 {
 		r.Smartypants(text)
 	} else {
 		r.attrEscape(text)
 	}
 }
 
-func (r *HTML) Smartypants2(text []byte) []byte {
-	var buff bytes.Buffer
-	// first do normal entity escaping
-	text = attrEscape2(text)
-	mark := 0
-	for i := 0; i < len(text); i++ {
-		if action := r.smartypants.callbacks[text[i]]; action != nil {
-			if i > mark {
-				buff.Write(text[mark:i])
-			}
-			previousChar := byte(0)
-			if i > 0 {
-				previousChar = text[i-1]
-			}
-			var tmp bytes.Buffer
-			i += action(&tmp, previousChar, text[i:])
-			buff.Write(tmp.Bytes())
-			mark = i + 1
-		}
-	}
-	if mark < len(text) {
-		buff.Write(text[mark:])
-	}
-	return buff.Bytes()
-}
-
 func (r *HTML) Smartypants(text []byte) {
-	r.w.Write(r.Smartypants2(text))
+	r.w.Write(NewSmartypantsRenderer(r.extensions).Process(text))
 }
 
 func (r *HTML) DocumentHeader() {
@@ -1133,12 +1103,7 @@ func (r *HTML) RenderNode(w io.Writer, node *Node, entering bool) {
 	attrs := []string{}
 	switch node.Type {
 	case Text:
-		if r.flags&UseSmartypants != 0 && isSmartypantable(node) {
-			// TODO: don't do that in renderer, do that at parse time!
-			r.out(w, r.Smartypants2(node.Literal))
-		} else {
-			r.out(w, esc(node.Literal, false))
-		}
+		r.out(w, node.Literal)
 		break
 	case Softbreak:
 		r.out(w, []byte("\n"))
@@ -1431,6 +1396,17 @@ func (r *HTML) RenderNode(w io.Writer, node *Node, entering bool) {
 func (r *HTML) Render(ast *Node) []byte {
 	//println("render_Blackfriday")
 	//dump(ast)
+	// Run Smartypants if it's enabled or simply escape text if not
+	sr := NewSmartypantsRenderer(r.extensions)
+	ForEachNode(ast, func(node *Node, entering bool) {
+		if node.Type == Text {
+			if r.extensions&Smartypants != 0 {
+				node.Literal = sr.Process(node.Literal)
+			} else {
+				node.Literal = esc(node.Literal, false)
+			}
+		}
+	})
 	var buff bytes.Buffer
 	ForEachNode(ast, func(node *Node, entering bool) {
 		r.RenderNode(&buff, node, entering)
diff --git a/inline_test.go b/inline_test.go
index 2882e45..6bb7da3 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -26,7 +26,7 @@ func runMarkdownInline(input string, opts Options, htmlFlags HtmlFlags, params H
 
 	htmlFlags |= UseXHTML
 
-	renderer := HtmlRendererWithParameters(htmlFlags, "", "", params)
+	renderer := HtmlRendererWithParameters(htmlFlags, opts.Extensions, "", "", params)
 
 	return string(MarkdownOptions([]byte(input), renderer, opts))
 }
@@ -1084,7 +1084,7 @@ func TestInlineComments(t *testing.T) {
 		"blahblah\n<!--- foo -->\nrhubarb\n",
 		"<p>blahblah\n<!--- foo -->\nrhubarb</p>\n",
 	}
-	doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{})
+	doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{})
 }
 
 func TestSmartDoubleQuotes(t *testing.T) {
@@ -1096,7 +1096,7 @@ func TestSmartDoubleQuotes(t *testing.T) {
 		"two pair of \"some\" quoted \"text\".\n",
 		"<p>two pair of &ldquo;some&rdquo; quoted &ldquo;text&rdquo;.</p>\n"}
 
-	doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{})
+	doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{})
 }
 
 func TestSmartAngledDoubleQuotes(t *testing.T) {
@@ -1108,7 +1108,7 @@ func TestSmartAngledDoubleQuotes(t *testing.T) {
 		"two pair of \"some\" quoted \"text\".\n",
 		"<p>two pair of &laquo;some&raquo; quoted &laquo;text&raquo;.</p>\n"}
 
-	doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsAngledQuotes, HtmlRendererParameters{})
+	doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsAngledQuotes}, 0, HtmlRendererParameters{})
 }
 
 func TestSmartFractions(t *testing.T) {
@@ -1118,7 +1118,7 @@ func TestSmartFractions(t *testing.T) {
 		"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n",
 		"<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"}
 
-	doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{})
+	doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{})
 
 	tests = []string{
 		"1/2, 2/3, 81/100 and 1000000/1048576.\n",
@@ -1126,7 +1126,7 @@ func TestSmartFractions(t *testing.T) {
 		"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n",
 		"<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"}
 
-	doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsFractions, HtmlRendererParameters{})
+	doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsFractions}, 0, HtmlRendererParameters{})
 }
 
 func TestDisableSmartDashes(t *testing.T) {
@@ -1145,7 +1145,7 @@ func TestDisableSmartDashes(t *testing.T) {
 		"<p>foo &mdash; bar</p>\n",
 		"foo --- bar\n",
 		"<p>foo &mdash;&ndash; bar</p>\n",
-	}, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{})
+	}, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{})
 	doTestsInlineParam(t, []string{
 		"foo - bar\n",
 		"<p>foo - bar</p>\n",
@@ -1153,7 +1153,7 @@ func TestDisableSmartDashes(t *testing.T) {
 		"<p>foo &ndash; bar</p>\n",
 		"foo --- bar\n",
 		"<p>foo &mdash; bar</p>\n",
-	}, Options{}, UseSmartypants|SmartypantsLatexDashes|SmartypantsDashes,
+	}, Options{Extensions: Smartypants | SmartypantsLatexDashes | SmartypantsDashes}, 0,
 		HtmlRendererParameters{})
 	doTestsInlineParam(t, []string{
 		"foo - bar\n",
@@ -1162,7 +1162,7 @@ func TestDisableSmartDashes(t *testing.T) {
 		"<p>foo -- bar</p>\n",
 		"foo --- bar\n",
 		"<p>foo --- bar</p>\n",
-	}, Options{},
-		UseSmartypants|SmartypantsLatexDashes,
+	}, Options{Extensions: Smartypants | SmartypantsLatexDashes},
+		0,
 		HtmlRendererParameters{})
 }
diff --git a/markdown.go b/markdown.go
index af4dae0..d46bb2a 100644
--- a/markdown.go
+++ b/markdown.go
@@ -32,30 +32,35 @@ type Extensions int
 // These are the supported markdown parsing extensions.
 // OR these values together to select multiple extensions.
 const (
-	NoExtensions           Extensions = 0
-	NoIntraEmphasis        Extensions = 1 << iota // Ignore emphasis markers inside words
-	Tables                                        // Render tables
-	FencedCode                                    // Render fenced code blocks
-	Autolink                                      // Detect embedded URLs that are not explicitly marked
-	Strikethrough                                 // Strikethrough text using ~~test~~
-	LaxHTMLBlocks                                 // Loosen up HTML block parsing rules
-	SpaceHeaders                                  // Be strict about prefix header rules
-	HardLineBreak                                 // Translate newlines into line breaks
-	TabSizeEight                                  // Expand tabs to eight spaces instead of four
-	Footnotes                                     // Pandoc-style footnotes
-	NoEmptyLineBeforeBlock                        // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
-	HeaderIDs                                     // specify header IDs  with {#id}
-	Titleblock                                    // Titleblock ala pandoc
-	AutoHeaderIDs                                 // Create the header ID from the text
-	BackslashLineBreak                            // Translate trailing backslashes into line breaks
-	DefinitionLists                               // Render definition lists
+	NoExtensions            Extensions = 0
+	NoIntraEmphasis         Extensions = 1 << iota // Ignore emphasis markers inside words
+	Tables                                         // Render tables
+	FencedCode                                     // Render fenced code blocks
+	Autolink                                       // Detect embedded URLs that are not explicitly marked
+	Strikethrough                                  // Strikethrough text using ~~test~~
+	LaxHTMLBlocks                                  // Loosen up HTML block parsing rules
+	SpaceHeaders                                   // Be strict about prefix header rules
+	HardLineBreak                                  // Translate newlines into line breaks
+	TabSizeEight                                   // Expand tabs to eight spaces instead of four
+	Footnotes                                      // Pandoc-style footnotes
+	NoEmptyLineBeforeBlock                         // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
+	HeaderIDs                                      // specify header IDs  with {#id}
+	Titleblock                                     // Titleblock ala pandoc
+	AutoHeaderIDs                                  // Create the header ID from the text
+	BackslashLineBreak                             // Translate trailing backslashes into line breaks
+	DefinitionLists                                // Render definition lists
+	Smartypants                                    // Enable smart punctuation substitutions
+	SmartypantsFractions                           // Enable smart fractions (with Smartypants)
+	SmartypantsDashes                              // Enable smart dashes (with Smartypants)
+	SmartypantsLatexDashes                         // Enable LaTeX-style dashes (with Smartypants)
+	SmartypantsAngledQuotes                        // Enable angled double quotes (with Smartypants) for double quotes rendering
 
-	CommonHtmlFlags HtmlFlags = UseXHTML | UseSmartypants |
-		SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
+	CommonHtmlFlags HtmlFlags = UseXHTML
 
 	CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
 		Autolink | Strikethrough | SpaceHeaders | HeaderIDs |
-		BackslashLineBreak | DefinitionLists
+		BackslashLineBreak | DefinitionLists | Smartypants |
+		SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
 )
 
 var DefaultOptions = Options{
@@ -340,7 +345,7 @@ type Options struct {
 func MarkdownBasic(input []byte) []byte {
 	// set up the HTML renderer
 	htmlFlags := UseXHTML
-	renderer := HtmlRenderer(htmlFlags, "", "")
+	renderer := HtmlRenderer(htmlFlags, CommonExtensions, "", "")
 
 	// set up the parser
 	return MarkdownOptions(input, renderer, Options{Extensions: 0})
@@ -367,7 +372,7 @@ func MarkdownBasic(input []byte) []byte {
 // * Custom Header IDs
 func MarkdownCommon(input []byte) []byte {
 	// set up the HTML renderer
-	renderer := HtmlRenderer(CommonHtmlFlags, "", "")
+	renderer := HtmlRenderer(CommonHtmlFlags, CommonExtensions, "", "")
 	return MarkdownOptions(input, renderer, DefaultOptions)
 }
 
@@ -441,10 +446,11 @@ func Parse(input []byte, opts Options) *Node {
 
 	first := firstPass(p, input)
 	secondPass(p, first)
-	// walk the tree and finish up some of unfinished blocks:
+	// Walk the tree and finish up some of unfinished blocks
 	for p.tip != nil {
 		p.finalize(p.tip)
 	}
+	// Walk the tree again and process inline markdown in each block
 	ForEachNode(p.doc, func(node *Node, entering bool) {
 		if node.Type == Paragraph || node.Type == Header || node.Type == TableCell {
 			p.currBlock = node
diff --git a/ref_test.go b/ref_test.go
index 6d4a628..85f6903 100644
--- a/ref_test.go
+++ b/ref_test.go
@@ -20,7 +20,7 @@ import (
 )
 
 func runMarkdownReference(input string, flag Extensions) string {
-	renderer := HtmlRenderer(0, "", "")
+	renderer := HtmlRenderer(0, flag, "", "")
 	return string(Markdown([]byte(input), renderer, flag))
 }
 
diff --git a/smartypants.go b/smartypants.go
index 123814d..020fab0 100644
--- a/smartypants.go
+++ b/smartypants.go
@@ -366,7 +366,7 @@ func (smrt *SPRenderer) smartLeftAngle(out *bytes.Buffer, previousChar byte, tex
 
 type smartCallback func(out *bytes.Buffer, previousChar byte, text []byte) int
 
-func NewSmartypantsRenderer(flags HtmlFlags) *SPRenderer {
+func NewSmartypantsRenderer(flags Extensions) *SPRenderer {
 	var r SPRenderer
 	if flags&SmartypantsAngledQuotes == 0 {
 		r.callbacks['"'] = r.smartDoubleQuote
@@ -397,3 +397,29 @@ func NewSmartypantsRenderer(flags HtmlFlags) *SPRenderer {
 	r.callbacks['`'] = r.smartBacktick
 	return &r
 }
+
+func (sr *SPRenderer) Process(text []byte) []byte {
+	var buff bytes.Buffer
+	// first do normal entity escaping
+	text = attrEscape2(text)
+	mark := 0
+	for i := 0; i < len(text); i++ {
+		if action := sr.callbacks[text[i]]; action != nil {
+			if i > mark {
+				buff.Write(text[mark:i])
+			}
+			previousChar := byte(0)
+			if i > 0 {
+				previousChar = text[i-1]
+			}
+			var tmp bytes.Buffer
+			i += action(&tmp, previousChar, text[i:])
+			buff.Write(tmp.Bytes())
+			mark = i + 1
+		}
+	}
+	if mark < len(text) {
+		buff.Write(text[mark:])
+	}
+	return buff.Bytes()
+}