danog/blackfriday
1
0
Fork 0
mirror of https://github.com/danog/blackfriday.git synced 2024-11-26 20:14:43 +01:00
Code Issues Projects Releases Wiki Activity

Run Smartypants as a separate pass over the AST

Browse Source 
Separate Smartypants somewhat from the HTML renderer. Move its flags
from HtmlFlags to Extensions (probably should be moved to its own set of
flags, but not now). With that done, do a separate walk of the tree and
either run Smartypants processor if it's enabled, or simply escape text
nodes.
This commit is contained in:
Vytautas Šaltenis 2016-04-01 10:44:22 +03:00
parent 7869a127bd
commit a55b2615a4
6 changed files with 104 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
block_test.go
View File

@ -23,13 +23,13 @@ func runMarkdownBlockWithRenderer(input string, extensions Extensions, renderer
}
func runMarkdownBlock(input string, extensions Extensions) string {
renderer := HtmlRenderer(UseXHTML, "", "")
renderer := HtmlRenderer(UseXHTML, extensions, "", "")
return runMarkdownBlockWithRenderer(input, extensions, renderer)
}
func runnerWithRendererParameters(parameters HtmlRendererParameters) func(string, Extensions) string {
return func(input string, extensions Extensions) string {
renderer := HtmlRendererWithParameters(UseXHTML, "", "", parameters)
renderer := HtmlRendererWithParameters(UseXHTML, extensions, "", "", parameters)
return runMarkdownBlockWithRenderer(input, extensions, renderer)
}
}

94
html.go
View File

@ -29,25 +29,20 @@ type HtmlFlags int
// HTML renderer configuration options.
const (
HtmlFlagsNone HtmlFlags = 0
SkipHTML HtmlFlags = 1 << iota // Skip preformatted HTML blocks
SkipStyle // Skip embedded <style> elements
SkipImages // Skip embedded images
SkipLinks // Skip all links
Safelink // Only link to trusted protocols
NofollowLinks // Only link with rel="nofollow"
NoreferrerLinks // Only link with rel="noreferrer"
HrefTargetBlank // Add a blank target
Toc // Generate a table of contents
OmitContents // Skip the main contents (for a standalone table of contents)
CompletePage // Generate a complete HTML page
UseXHTML // Generate XHTML output instead of HTML
UseSmartypants // Enable smart punctuation substitutions
SmartypantsFractions // Enable smart fractions (with UseSmartypants)
SmartypantsDashes // Enable smart dashes (with UseSmartypants)
SmartypantsLatexDashes // Enable LaTeX-style dashes (with UseSmartypants)
SmartypantsAngledQuotes // Enable angled double quotes (with UseSmartypants) for double quotes rendering
FootnoteReturnLinks // Generate a link at the end of a footnote to return to the source
HtmlFlagsNone HtmlFlags = 0
SkipHTML HtmlFlags = 1 << iota // Skip preformatted HTML blocks
SkipStyle // Skip embedded <style> elements
SkipImages // Skip embedded images
SkipLinks // Skip all links
Safelink // Only link to trusted protocols
NofollowLinks // Only link with rel="nofollow"
NoreferrerLinks // Only link with rel="noreferrer"
HrefTargetBlank // Add a blank target
Toc // Generate a table of contents
OmitContents // Skip the main contents (for a standalone table of contents)
CompletePage // Generate a complete HTML page
UseXHTML // Generate XHTML output instead of HTML
FootnoteReturnLinks // Generate a link at the end of a footnote to return to the source
TagName = "[A-Za-z][A-Za-z0-9-]*"
AttributeName = "[a-zA-Z_:][a-zA-Z0-9:._-]*"
@ -109,10 +104,11 @@ type HTML struct {
// Track header IDs to prevent ID collision in a single generation.
headerIDs map[string]int
smartypants *SPRenderer
w HtmlWriter
lastOutputLen int
disableTags int
extensions Extensions // This gives Smartypants renderer access to flags
}
const (
@ -127,8 +123,8 @@ const (
// title is the title of the document, and css is a URL for the document's
// stylesheet.
// title and css are only used when HTML_COMPLETE_PAGE is selected.
func HtmlRenderer(flags HtmlFlags, title string, css string) Renderer {
return HtmlRendererWithParameters(flags, title, css, HtmlRendererParameters{})
func HtmlRenderer(flags HtmlFlags, extensions Extensions, title string, css string) Renderer {
return HtmlRendererWithParameters(flags, extensions, title, css, HtmlRendererParameters{})
}
type HtmlWriter struct {
@ -157,7 +153,7 @@ func (r *HTML) Write(b []byte) (int, error) {
return r.w.Write(b)
}
func HtmlRendererWithParameters(flags HtmlFlags, title string,
func HtmlRendererWithParameters(flags HtmlFlags, extensions Extensions, title string,
css string, renderParameters HtmlRendererParameters) Renderer {
// configure the rendering engine
closeTag := htmlClose
@ -172,6 +168,7 @@ func HtmlRendererWithParameters(flags HtmlFlags, title string,
var writer HtmlWriter
return &HTML{
flags: flags,
extensions: extensions,
closeTag: closeTag,
title: title,
css: css,
@ -183,8 +180,7 @@ func HtmlRendererWithParameters(flags HtmlFlags, title string,
headerIDs: make(map[string]int),
smartypants: NewSmartypantsRenderer(flags),
w: writer,
w: writer,
}
}
@ -688,41 +684,15 @@ func (r *HTML) Entity(entity []byte) {
}
func (r *HTML) NormalText(text []byte) {
if r.flags&UseSmartypants != 0 {
if r.extensions&Smartypants != 0 {
r.Smartypants(text)
} else {
r.attrEscape(text)
}
}
func (r *HTML) Smartypants2(text []byte) []byte {
var buff bytes.Buffer
// first do normal entity escaping
text = attrEscape2(text)
mark := 0
for i := 0; i < len(text); i++ {
if action := r.smartypants.callbacks[text[i]]; action != nil {
if i > mark {
buff.Write(text[mark:i])
}
previousChar := byte(0)
if i > 0 {
previousChar = text[i-1]
}
var tmp bytes.Buffer
i += action(&tmp, previousChar, text[i:])
buff.Write(tmp.Bytes())
mark = i + 1
}
}
if mark < len(text) {
buff.Write(text[mark:])
}
return buff.Bytes()
}
func (r *HTML) Smartypants(text []byte) {
r.w.Write(r.Smartypants2(text))
r.w.Write(NewSmartypantsRenderer(r.extensions).Process(text))
}
func (r *HTML) DocumentHeader() {
@ -1133,12 +1103,7 @@ func (r *HTML) RenderNode(w io.Writer, node *Node, entering bool) {
attrs := []string{}
switch node.Type {
case Text:
if r.flags&UseSmartypants != 0 && isSmartypantable(node) {
// TODO: don't do that in renderer, do that at parse time!
r.out(w, r.Smartypants2(node.Literal))
} else {
r.out(w, esc(node.Literal, false))
}
r.out(w, node.Literal)
break
case Softbreak:
r.out(w, []byte("\n"))
@ -1431,6 +1396,17 @@ func (r *HTML) RenderNode(w io.Writer, node *Node, entering bool) {
func (r *HTML) Render(ast *Node) []byte {
//println("render_Blackfriday")
//dump(ast)
// Run Smartypants if it's enabled or simply escape text if not
sr := NewSmartypantsRenderer(r.extensions)
ForEachNode(ast, func(node *Node, entering bool) {
if node.Type == Text {
if r.extensions&Smartypants != 0 {
node.Literal = sr.Process(node.Literal)
} else {
node.Literal = esc(node.Literal, false)
}
}
})
var buff bytes.Buffer
ForEachNode(ast, func(node *Node, entering bool) {
r.RenderNode(&buff, node, entering)

20
inline_test.go
View File

@ -26,7 +26,7 @@ func runMarkdownInline(input string, opts Options, htmlFlags HtmlFlags, params H
htmlFlags |= UseXHTML
renderer := HtmlRendererWithParameters(htmlFlags, "", "", params)
renderer := HtmlRendererWithParameters(htmlFlags, opts.Extensions, "", "", params)
return string(MarkdownOptions([]byte(input), renderer, opts))
}
@ -1084,7 +1084,7 @@ func TestInlineComments(t *testing.T) {
"blahblah\n<!--- foo -->\nrhubarb\n",
"<p>blahblah\n<!--- foo -->\nrhubarb</p>\n",
}
doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{})
doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{})
}
func TestSmartDoubleQuotes(t *testing.T) {
@ -1096,7 +1096,7 @@ func TestSmartDoubleQuotes(t *testing.T) {
"two pair of \"some\" quoted \"text\".\n",
"<p>two pair of &ldquo;some&rdquo; quoted &ldquo;text&rdquo;.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{})
doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{})
}
func TestSmartAngledDoubleQuotes(t *testing.T) {
@ -1108,7 +1108,7 @@ func TestSmartAngledDoubleQuotes(t *testing.T) {
"two pair of \"some\" quoted \"text\".\n",
"<p>two pair of &laquo;some&raquo; quoted &laquo;text&raquo;.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsAngledQuotes, HtmlRendererParameters{})
doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsAngledQuotes}, 0, HtmlRendererParameters{})
}
func TestSmartFractions(t *testing.T) {
@ -1118,7 +1118,7 @@ func TestSmartFractions(t *testing.T) {
"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n",
"<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{})
doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{})
tests = []string{
"1/2, 2/3, 81/100 and 1000000/1048576.\n",
@ -1126,7 +1126,7 @@ func TestSmartFractions(t *testing.T) {
"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n",
"<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"}
doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsFractions, HtmlRendererParameters{})
doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsFractions}, 0, HtmlRendererParameters{})
}
func TestDisableSmartDashes(t *testing.T) {
@ -1145,7 +1145,7 @@ func TestDisableSmartDashes(t *testing.T) {
"<p>foo &mdash; bar</p>\n",
"foo --- bar\n",
"<p>foo &mdash;&ndash; bar</p>\n",
}, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{})
}, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{})
doTestsInlineParam(t, []string{
"foo - bar\n",
"<p>foo - bar</p>\n",
@ -1153,7 +1153,7 @@ func TestDisableSmartDashes(t *testing.T) {
"<p>foo &ndash; bar</p>\n",
"foo --- bar\n",
"<p>foo &mdash; bar</p>\n",
}, Options{}, UseSmartypants|SmartypantsLatexDashes|SmartypantsDashes,
}, Options{Extensions: Smartypants | SmartypantsLatexDashes | SmartypantsDashes}, 0,
HtmlRendererParameters{})
doTestsInlineParam(t, []string{
"foo - bar\n",
@ -1162,7 +1162,7 @@ func TestDisableSmartDashes(t *testing.T) {
"<p>foo -- bar</p>\n",
"foo --- bar\n",
"<p>foo --- bar</p>\n",
}, Options{},
UseSmartypants|SmartypantsLatexDashes,
}, Options{Extensions: Smartypants | SmartypantsLatexDashes},
0,
HtmlRendererParameters{})
}

52
markdown.go
View File

@ -32,30 +32,35 @@ type Extensions int
// These are the supported markdown parsing extensions.
// OR these values together to select multiple extensions.
const (
NoExtensions Extensions = 0
NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words
Tables // Render tables
FencedCode // Render fenced code blocks
Autolink // Detect embedded URLs that are not explicitly marked
Strikethrough // Strikethrough text using ~~test~~
LaxHTMLBlocks // Loosen up HTML block parsing rules
SpaceHeaders // Be strict about prefix header rules
HardLineBreak // Translate newlines into line breaks
TabSizeEight // Expand tabs to eight spaces instead of four
Footnotes // Pandoc-style footnotes
NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
HeaderIDs // specify header IDs with {#id}
Titleblock // Titleblock ala pandoc
AutoHeaderIDs // Create the header ID from the text
BackslashLineBreak // Translate trailing backslashes into line breaks
DefinitionLists // Render definition lists
NoExtensions Extensions = 0
NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words
Tables // Render tables
FencedCode // Render fenced code blocks
Autolink // Detect embedded URLs that are not explicitly marked
Strikethrough // Strikethrough text using ~~test~~
LaxHTMLBlocks // Loosen up HTML block parsing rules
SpaceHeaders // Be strict about prefix header rules
HardLineBreak // Translate newlines into line breaks
TabSizeEight // Expand tabs to eight spaces instead of four
Footnotes // Pandoc-style footnotes
NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
HeaderIDs // specify header IDs with {#id}
Titleblock // Titleblock ala pandoc
AutoHeaderIDs // Create the header ID from the text
BackslashLineBreak // Translate trailing backslashes into line breaks
DefinitionLists // Render definition lists
Smartypants // Enable smart punctuation substitutions
SmartypantsFractions // Enable smart fractions (with Smartypants)
SmartypantsDashes // Enable smart dashes (with Smartypants)
SmartypantsLatexDashes // Enable LaTeX-style dashes (with Smartypants)
SmartypantsAngledQuotes // Enable angled double quotes (with Smartypants) for double quotes rendering
CommonHtmlFlags HtmlFlags = UseXHTML | UseSmartypants |
SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
CommonHtmlFlags HtmlFlags = UseXHTML
CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
Autolink | Strikethrough | SpaceHeaders | HeaderIDs |
BackslashLineBreak | DefinitionLists
BackslashLineBreak | DefinitionLists | Smartypants |
SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
)
var DefaultOptions = Options{
@ -340,7 +345,7 @@ type Options struct {
func MarkdownBasic(input []byte) []byte {
// set up the HTML renderer
htmlFlags := UseXHTML
renderer := HtmlRenderer(htmlFlags, "", "")
renderer := HtmlRenderer(htmlFlags, CommonExtensions, "", "")
// set up the parser
return MarkdownOptions(input, renderer, Options{Extensions: 0})
@ -367,7 +372,7 @@ func MarkdownBasic(input []byte) []byte {
// * Custom Header IDs
func MarkdownCommon(input []byte) []byte {
// set up the HTML renderer
renderer := HtmlRenderer(CommonHtmlFlags, "", "")
renderer := HtmlRenderer(CommonHtmlFlags, CommonExtensions, "", "")
return MarkdownOptions(input, renderer, DefaultOptions)
}
@ -441,10 +446,11 @@ func Parse(input []byte, opts Options) *Node {
first := firstPass(p, input)
secondPass(p, first)
// walk the tree and finish up some of unfinished blocks:
// Walk the tree and finish up some of unfinished blocks
for p.tip != nil {
p.finalize(p.tip)
}
// Walk the tree again and process inline markdown in each block
ForEachNode(p.doc, func(node *Node, entering bool) {
if node.Type == Paragraph || node.Type == Header || node.Type == TableCell {
p.currBlock = node

2
ref_test.go
View File

@ -20,7 +20,7 @@ import (
)
func runMarkdownReference(input string, flag Extensions) string {
renderer := HtmlRenderer(0, "", "")
renderer := HtmlRenderer(0, flag, "", "")
return string(Markdown([]byte(input), renderer, flag))
}

28
smartypants.go
View File

@ -366,7 +366,7 @@ func (smrt *SPRenderer) smartLeftAngle(out *bytes.Buffer, previousChar byte, tex
type smartCallback func(out *bytes.Buffer, previousChar byte, text []byte) int
func NewSmartypantsRenderer(flags HtmlFlags) *SPRenderer {
func NewSmartypantsRenderer(flags Extensions) *SPRenderer {
var r SPRenderer
if flags&SmartypantsAngledQuotes == 0 {
r.callbacks['"'] = r.smartDoubleQuote
@ -397,3 +397,29 @@ func NewSmartypantsRenderer(flags HtmlFlags) *SPRenderer {
r.callbacks['`'] = r.smartBacktick
return &r
}
func (sr *SPRenderer) Process(text []byte) []byte {
var buff bytes.Buffer
// first do normal entity escaping
text = attrEscape2(text)
mark := 0
for i := 0; i < len(text); i++ {
if action := sr.callbacks[text[i]]; action != nil {
if i > mark {
buff.Write(text[mark:i])
}
previousChar := byte(0)
if i > 0 {
previousChar = text[i-1]
}
var tmp bytes.Buffer
i += action(&tmp, previousChar, text[i:])
buff.Write(tmp.Bytes())
mark = i + 1
}
}
if mark < len(text) {
buff.Write(text[mark:])
}
return buff.Bytes()
}