danog/blackfriday
1
0
Fork 0
mirror of https://github.com/danog/blackfriday.git synced 2024-11-26 20:14:43 +01:00
Code Issues Projects Releases Wiki Activity

Run Smartypants as a separate pass over the AST

Browse Source 
Separate Smartypants somewhat from the HTML renderer. Move its flags
from HtmlFlags to Extensions (probably should be moved to its own set of
flags, but not now). With that done, do a separate walk of the tree and
either run Smartypants processor if it's enabled, or simply escape text
nodes.
This commit is contained in:
Vytautas Šaltenis 2016-04-01 10:44:22 +03:00
parent 7869a127bd
commit a55b2615a4
6 changed files with 104 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
block_test.go
View File

@ -23,13 +23,13 @@ func runMarkdownBlockWithRenderer(input string, extensions Extensions, renderer
} }
func runMarkdownBlock(input string, extensions Extensions) string { func runMarkdownBlock(input string, extensions Extensions) string {
renderer := HtmlRenderer(UseXHTML, "", "") renderer := HtmlRenderer(UseXHTML, extensions, "", "")
return runMarkdownBlockWithRenderer(input, extensions, renderer) return runMarkdownBlockWithRenderer(input, extensions, renderer)
} }
func runnerWithRendererParameters(parameters HtmlRendererParameters) func(string, Extensions) string { func runnerWithRendererParameters(parameters HtmlRendererParameters) func(string, Extensions) string {
return func(input string, extensions Extensions) string { return func(input string, extensions Extensions) string {
renderer := HtmlRendererWithParameters(UseXHTML, "", "", parameters) renderer := HtmlRendererWithParameters(UseXHTML, extensions, "", "", parameters)
return runMarkdownBlockWithRenderer(input, extensions, renderer) return runMarkdownBlockWithRenderer(input, extensions, renderer)
} }
} }

94
html.go
View File

@ -29,25 +29,20 @@ type HtmlFlags int
// HTML renderer configuration options. // HTML renderer configuration options.
const ( const (
HtmlFlagsNone HtmlFlags = 0 HtmlFlagsNone HtmlFlags = 0
SkipHTML HtmlFlags = 1 << iota // Skip preformatted HTML blocks SkipHTML HtmlFlags = 1 << iota // Skip preformatted HTML blocks
SkipStyle // Skip embedded <style> elements SkipStyle // Skip embedded <style> elements
SkipImages // Skip embedded images SkipImages // Skip embedded images
SkipLinks // Skip all links SkipLinks // Skip all links
Safelink // Only link to trusted protocols Safelink // Only link to trusted protocols
NofollowLinks // Only link with rel="nofollow" NofollowLinks // Only link with rel="nofollow"
NoreferrerLinks // Only link with rel="noreferrer" NoreferrerLinks // Only link with rel="noreferrer"
HrefTargetBlank // Add a blank target HrefTargetBlank // Add a blank target
Toc // Generate a table of contents Toc // Generate a table of contents
OmitContents // Skip the main contents (for a standalone table of contents) OmitContents // Skip the main contents (for a standalone table of contents)
CompletePage // Generate a complete HTML page CompletePage // Generate a complete HTML page
UseXHTML // Generate XHTML output instead of HTML UseXHTML // Generate XHTML output instead of HTML
UseSmartypants // Enable smart punctuation substitutions FootnoteReturnLinks // Generate a link at the end of a footnote to return to the source
SmartypantsFractions // Enable smart fractions (with UseSmartypants)
SmartypantsDashes // Enable smart dashes (with UseSmartypants)
SmartypantsLatexDashes // Enable LaTeX-style dashes (with UseSmartypants)
SmartypantsAngledQuotes // Enable angled double quotes (with UseSmartypants) for double quotes rendering
FootnoteReturnLinks // Generate a link at the end of a footnote to return to the source
TagName = "[A-Za-z][A-Za-z0-9-]*" TagName = "[A-Za-z][A-Za-z0-9-]*"
AttributeName = "[a-zA-Z_:][a-zA-Z0-9:._-]*" AttributeName = "[a-zA-Z_:][a-zA-Z0-9:._-]*"
@ -109,10 +104,11 @@ type HTML struct {
// Track header IDs to prevent ID collision in a single generation. // Track header IDs to prevent ID collision in a single generation.
headerIDs map[string]int headerIDs map[string]int
smartypants *SPRenderer
w HtmlWriter w HtmlWriter
lastOutputLen int lastOutputLen int
disableTags int disableTags int
extensions Extensions // This gives Smartypants renderer access to flags
} }
const ( const (
@ -127,8 +123,8 @@ const (
// title is the title of the document, and css is a URL for the document's // title is the title of the document, and css is a URL for the document's
// stylesheet. // stylesheet.
// title and css are only used when HTML_COMPLETE_PAGE is selected. // title and css are only used when HTML_COMPLETE_PAGE is selected.
func HtmlRenderer(flags HtmlFlags, title string, css string) Renderer { func HtmlRenderer(flags HtmlFlags, extensions Extensions, title string, css string) Renderer {
return HtmlRendererWithParameters(flags, title, css, HtmlRendererParameters{}) return HtmlRendererWithParameters(flags, extensions, title, css, HtmlRendererParameters{})
} }
type HtmlWriter struct { type HtmlWriter struct {
@ -157,7 +153,7 @@ func (r *HTML) Write(b []byte) (int, error) {
return r.w.Write(b) return r.w.Write(b)
} }
func HtmlRendererWithParameters(flags HtmlFlags, title string, func HtmlRendererWithParameters(flags HtmlFlags, extensions Extensions, title string,
css string, renderParameters HtmlRendererParameters) Renderer { css string, renderParameters HtmlRendererParameters) Renderer {
// configure the rendering engine // configure the rendering engine
closeTag := htmlClose closeTag := htmlClose
@ -172,6 +168,7 @@ func HtmlRendererWithParameters(flags HtmlFlags, title string,
var writer HtmlWriter var writer HtmlWriter
return &HTML{ return &HTML{
flags: flags, flags: flags,
extensions: extensions,
closeTag: closeTag, closeTag: closeTag,
title: title, title: title,
css: css, css: css,
@ -183,8 +180,7 @@ func HtmlRendererWithParameters(flags HtmlFlags, title string,
headerIDs: make(map[string]int), headerIDs: make(map[string]int),
smartypants: NewSmartypantsRenderer(flags), w: writer,
w: writer,
} }
} }
@ -688,41 +684,15 @@ func (r *HTML) Entity(entity []byte) {
} }
func (r *HTML) NormalText(text []byte) { func (r *HTML) NormalText(text []byte) {
if r.flags&UseSmartypants != 0 { if r.extensions&Smartypants != 0 {
r.Smartypants(text) r.Smartypants(text)
} else { } else {
r.attrEscape(text) r.attrEscape(text)
} }
} }
func (r *HTML) Smartypants2(text []byte) []byte {
var buff bytes.Buffer
// first do normal entity escaping
text = attrEscape2(text)
mark := 0
for i := 0; i < len(text); i++ {
if action := r.smartypants.callbacks[text[i]]; action != nil {
if i > mark {
buff.Write(text[mark:i])
}
previousChar := byte(0)
if i > 0 {
previousChar = text[i-1]
}
var tmp bytes.Buffer
i += action(&tmp, previousChar, text[i:])
buff.Write(tmp.Bytes())
mark = i + 1
}
}
if mark < len(text) {
buff.Write(text[mark:])
}
return buff.Bytes()
}
func (r *HTML) Smartypants(text []byte) { func (r *HTML) Smartypants(text []byte) {
r.w.Write(r.Smartypants2(text)) r.w.Write(NewSmartypantsRenderer(r.extensions).Process(text))
} }
func (r *HTML) DocumentHeader() { func (r *HTML) DocumentHeader() {
@ -1133,12 +1103,7 @@ func (r *HTML) RenderNode(w io.Writer, node *Node, entering bool) {
attrs := []string{} attrs := []string{}
switch node.Type { switch node.Type {
case Text: case Text:
if r.flags&UseSmartypants != 0 && isSmartypantable(node) { r.out(w, node.Literal)
// TODO: don't do that in renderer, do that at parse time!
r.out(w, r.Smartypants2(node.Literal))
} else {
r.out(w, esc(node.Literal, false))
}
break break
case Softbreak: case Softbreak:
r.out(w, []byte("\n")) r.out(w, []byte("\n"))
@ -1431,6 +1396,17 @@ func (r *HTML) RenderNode(w io.Writer, node *Node, entering bool) {
func (r *HTML) Render(ast *Node) []byte { func (r *HTML) Render(ast *Node) []byte {
//println("render_Blackfriday") //println("render_Blackfriday")
//dump(ast) //dump(ast)
// Run Smartypants if it's enabled or simply escape text if not
sr := NewSmartypantsRenderer(r.extensions)
ForEachNode(ast, func(node *Node, entering bool) {
if node.Type == Text {
if r.extensions&Smartypants != 0 {
node.Literal = sr.Process(node.Literal)
} else {
node.Literal = esc(node.Literal, false)
}
}
})
var buff bytes.Buffer var buff bytes.Buffer
ForEachNode(ast, func(node *Node, entering bool) { ForEachNode(ast, func(node *Node, entering bool) {
r.RenderNode(&buff, node, entering) r.RenderNode(&buff, node, entering)

20
inline_test.go
View File

@ -26,7 +26,7 @@ func runMarkdownInline(input string, opts Options, htmlFlags HtmlFlags, params H
htmlFlags |= UseXHTML htmlFlags |= UseXHTML
renderer := HtmlRendererWithParameters(htmlFlags, "", "", params) renderer := HtmlRendererWithParameters(htmlFlags, opts.Extensions, "", "", params)
return string(MarkdownOptions([]byte(input), renderer, opts)) return string(MarkdownOptions([]byte(input), renderer, opts))
} }
@ -1084,7 +1084,7 @@ func TestInlineComments(t *testing.T) {
"blahblah\n<!--- foo -->\nrhubarb\n", "blahblah\n<!--- foo -->\nrhubarb\n",
"<p>blahblah\n<!--- foo -->\nrhubarb</p>\n", "<p>blahblah\n<!--- foo -->\nrhubarb</p>\n",
} }
doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{}) doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{})
} }
func TestSmartDoubleQuotes(t *testing.T) { func TestSmartDoubleQuotes(t *testing.T) {
@ -1096,7 +1096,7 @@ func TestSmartDoubleQuotes(t *testing.T) {
"two pair of \"some\" quoted \"text\".\n", "two pair of \"some\" quoted \"text\".\n",
"<p>two pair of &ldquo;some&rdquo; quoted &ldquo;text&rdquo;.</p>\n"} "<p>two pair of &ldquo;some&rdquo; quoted &ldquo;text&rdquo;.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{}) doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{})
} }
func TestSmartAngledDoubleQuotes(t *testing.T) { func TestSmartAngledDoubleQuotes(t *testing.T) {
@ -1108,7 +1108,7 @@ func TestSmartAngledDoubleQuotes(t *testing.T) {
"two pair of \"some\" quoted \"text\".\n", "two pair of \"some\" quoted \"text\".\n",
"<p>two pair of &laquo;some&raquo; quoted &laquo;text&raquo;.</p>\n"} "<p>two pair of &laquo;some&raquo; quoted &laquo;text&raquo;.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsAngledQuotes, HtmlRendererParameters{}) doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsAngledQuotes}, 0, HtmlRendererParameters{})
} }
func TestSmartFractions(t *testing.T) { func TestSmartFractions(t *testing.T) {
@ -1118,7 +1118,7 @@ func TestSmartFractions(t *testing.T) {
"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n", "1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n",
"<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"} "<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{}) doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{})
tests = []string{ tests = []string{
"1/2, 2/3, 81/100 and 1000000/1048576.\n", "1/2, 2/3, 81/100 and 1000000/1048576.\n",
@ -1126,7 +1126,7 @@ func TestSmartFractions(t *testing.T) {
"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n", "1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n",
"<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"} "<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsFractions, HtmlRendererParameters{}) doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsFractions}, 0, HtmlRendererParameters{})
} }
func TestDisableSmartDashes(t *testing.T) { func TestDisableSmartDashes(t *testing.T) {
@ -1145,7 +1145,7 @@ func TestDisableSmartDashes(t *testing.T) {
"<p>foo &mdash; bar</p>\n", "<p>foo &mdash; bar</p>\n",
"foo --- bar\n", "foo --- bar\n",
"<p>foo &mdash;&ndash; bar</p>\n", "<p>foo &mdash;&ndash; bar</p>\n",
}, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{}) }, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{})
doTestsInlineParam(t, []string{ doTestsInlineParam(t, []string{
"foo - bar\n", "foo - bar\n",
"<p>foo - bar</p>\n", "<p>foo - bar</p>\n",
@ -1153,7 +1153,7 @@ func TestDisableSmartDashes(t *testing.T) {
"<p>foo &ndash; bar</p>\n", "<p>foo &ndash; bar</p>\n",
"foo --- bar\n", "foo --- bar\n",
"<p>foo &mdash; bar</p>\n", "<p>foo &mdash; bar</p>\n",
}, Options{}, UseSmartypants|SmartypantsLatexDashes|SmartypantsDashes, }, Options{Extensions: Smartypants | SmartypantsLatexDashes | SmartypantsDashes}, 0,
HtmlRendererParameters{}) HtmlRendererParameters{})
doTestsInlineParam(t, []string{ doTestsInlineParam(t, []string{
"foo - bar\n", "foo - bar\n",
@ -1162,7 +1162,7 @@ func TestDisableSmartDashes(t *testing.T) {
"<p>foo -- bar</p>\n", "<p>foo -- bar</p>\n",
"foo --- bar\n", "foo --- bar\n",
"<p>foo --- bar</p>\n", "<p>foo --- bar</p>\n",
}, Options{}, }, Options{Extensions: Smartypants | SmartypantsLatexDashes},
UseSmartypants|SmartypantsLatexDashes, 0,
HtmlRendererParameters{}) HtmlRendererParameters{})
} }

52
markdown.go
View File

@ -32,30 +32,35 @@ type Extensions int
// These are the supported markdown parsing extensions. // These are the supported markdown parsing extensions.
// OR these values together to select multiple extensions. // OR these values together to select multiple extensions.
const ( const (
NoExtensions Extensions = 0 NoExtensions Extensions = 0
NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words
Tables // Render tables Tables // Render tables
FencedCode // Render fenced code blocks FencedCode // Render fenced code blocks
Autolink // Detect embedded URLs that are not explicitly marked Autolink // Detect embedded URLs that are not explicitly marked
Strikethrough // Strikethrough text using ~~test~~ Strikethrough // Strikethrough text using ~~test~~
LaxHTMLBlocks // Loosen up HTML block parsing rules LaxHTMLBlocks // Loosen up HTML block parsing rules
SpaceHeaders // Be strict about prefix header rules SpaceHeaders // Be strict about prefix header rules
HardLineBreak // Translate newlines into line breaks HardLineBreak // Translate newlines into line breaks
TabSizeEight // Expand tabs to eight spaces instead of four TabSizeEight // Expand tabs to eight spaces instead of four
Footnotes // Pandoc-style footnotes Footnotes // Pandoc-style footnotes
NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
HeaderIDs // specify header IDs with {#id} HeaderIDs // specify header IDs with {#id}
Titleblock // Titleblock ala pandoc Titleblock // Titleblock ala pandoc
AutoHeaderIDs // Create the header ID from the text AutoHeaderIDs // Create the header ID from the text
BackslashLineBreak // Translate trailing backslashes into line breaks BackslashLineBreak // Translate trailing backslashes into line breaks
DefinitionLists // Render definition lists DefinitionLists // Render definition lists
Smartypants // Enable smart punctuation substitutions
SmartypantsFractions // Enable smart fractions (with Smartypants)
SmartypantsDashes // Enable smart dashes (with Smartypants)
SmartypantsLatexDashes // Enable LaTeX-style dashes (with Smartypants)
SmartypantsAngledQuotes // Enable angled double quotes (with Smartypants) for double quotes rendering
CommonHtmlFlags HtmlFlags = UseXHTML | UseSmartypants | CommonHtmlFlags HtmlFlags = UseXHTML
SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode | CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
Autolink | Strikethrough | SpaceHeaders | HeaderIDs | Autolink | Strikethrough | SpaceHeaders | HeaderIDs |
BackslashLineBreak | DefinitionLists BackslashLineBreak | DefinitionLists | Smartypants |
SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
) )
var DefaultOptions = Options{ var DefaultOptions = Options{
@ -340,7 +345,7 @@ type Options struct {
func MarkdownBasic(input []byte) []byte { func MarkdownBasic(input []byte) []byte {
// set up the HTML renderer // set up the HTML renderer
htmlFlags := UseXHTML htmlFlags := UseXHTML
renderer := HtmlRenderer(htmlFlags, "", "") renderer := HtmlRenderer(htmlFlags, CommonExtensions, "", "")
// set up the parser // set up the parser
return MarkdownOptions(input, renderer, Options{Extensions: 0}) return MarkdownOptions(input, renderer, Options{Extensions: 0})
@ -367,7 +372,7 @@ func MarkdownBasic(input []byte) []byte {
// * Custom Header IDs // * Custom Header IDs
func MarkdownCommon(input []byte) []byte { func MarkdownCommon(input []byte) []byte {
// set up the HTML renderer // set up the HTML renderer
renderer := HtmlRenderer(CommonHtmlFlags, "", "") renderer := HtmlRenderer(CommonHtmlFlags, CommonExtensions, "", "")
return MarkdownOptions(input, renderer, DefaultOptions) return MarkdownOptions(input, renderer, DefaultOptions)
} }
@ -441,10 +446,11 @@ func Parse(input []byte, opts Options) *Node {
first := firstPass(p, input) first := firstPass(p, input)
secondPass(p, first) secondPass(p, first)
// walk the tree and finish up some of unfinished blocks: // Walk the tree and finish up some of unfinished blocks
for p.tip != nil { for p.tip != nil {
p.finalize(p.tip) p.finalize(p.tip)
} }
// Walk the tree again and process inline markdown in each block
ForEachNode(p.doc, func(node *Node, entering bool) { ForEachNode(p.doc, func(node *Node, entering bool) {
if node.Type == Paragraph || node.Type == Header || node.Type == TableCell { if node.Type == Paragraph || node.Type == Header || node.Type == TableCell {
p.currBlock = node p.currBlock = node

2
ref_test.go
View File

@ -20,7 +20,7 @@ import (
) )
func runMarkdownReference(input string, flag Extensions) string { func runMarkdownReference(input string, flag Extensions) string {
renderer := HtmlRenderer(0, "", "") renderer := HtmlRenderer(0, flag, "", "")
return string(Markdown([]byte(input), renderer, flag)) return string(Markdown([]byte(input), renderer, flag))
} }

28
smartypants.go
View File

@ -366,7 +366,7 @@ func (smrt *SPRenderer) smartLeftAngle(out *bytes.Buffer, previousChar byte, tex
type smartCallback func(out *bytes.Buffer, previousChar byte, text []byte) int type smartCallback func(out *bytes.Buffer, previousChar byte, text []byte) int
func NewSmartypantsRenderer(flags HtmlFlags) *SPRenderer { func NewSmartypantsRenderer(flags Extensions) *SPRenderer {
var r SPRenderer var r SPRenderer
if flags&SmartypantsAngledQuotes == 0 { if flags&SmartypantsAngledQuotes == 0 {
r.callbacks['"'] = r.smartDoubleQuote r.callbacks['"'] = r.smartDoubleQuote
@ -397,3 +397,29 @@ func NewSmartypantsRenderer(flags HtmlFlags) *SPRenderer {
r.callbacks['`'] = r.smartBacktick r.callbacks['`'] = r.smartBacktick
return &r return &r
} }
func (sr *SPRenderer) Process(text []byte) []byte {
var buff bytes.Buffer
// first do normal entity escaping
text = attrEscape2(text)
mark := 0
for i := 0; i < len(text); i++ {
if action := sr.callbacks[text[i]]; action != nil {
if i > mark {
buff.Write(text[mark:i])
}
previousChar := byte(0)
if i > 0 {
previousChar = text[i-1]
}
var tmp bytes.Buffer
i += action(&tmp, previousChar, text[i:])
buff.Write(tmp.Bytes())
mark = i + 1
}
}
if mark < len(text) {
buff.Write(text[mark:])
}
return buff.Bytes()
}