Add robots.txt to sitemap

README.md

@@ -7,11 +7,11 @@
 Gojekyll is a clone of the [Jekyll](https://jekyllrb.com) static site generator, written in the [Go](https://golang.org) programming language. It provides `build` and `serve` commands, with directory watch and live reload for the latter.
 
 |                 | Gojekyll                             | Jekyll | Hugo         |
-|------------------------|--------------------------------|--------|--------------|
+|----------------------|--------------------------------------|--------|--------------|
-| Fast                   | [✓ 20x](./docs/benchmarks.md)  |        | ✓            |
+| Fast                 | ✓ [~20×Jekyll](./docs/benchmarks.md) |        | ✓            |
 | Stable               |                                      | ✓      | ✓            |
 | Template language    | Liquid                               | Liquid | Go templates |
-| Compatible with Jekyll | [partly](#current-limitations) | ✓      |              |
+| Jekyll compatibility | [partial](#current-limitations)      | ✓      |              |
 | Plugins              | [some](./docs/plugins.md)            | yes    | ?            |
 | Windows              |                                      | ✓      | ✓            |
 | Implementation       | Go                                   | Ruby   | Go           |

docs/plugins.md

@@ -21,7 +21,7 @@ The functionality of some plugins is built into the core program:
 | jekyll-relative-links        | GitHub Pages  |                     |                                                                         |
 | jekyll-sass-converter        | core          | ✓ (always enabled)  |                                                                         |
 | jekyll-seo_tag               | GitHub Pages  | ✓                   | SEO and JSON LD are not fully populated; site? and other tag parameters |
-| jekyll-sitemap               | GitHub Pages  |                     |                                                                         |
+| jekyll-sitemap               | GitHub Pages  | ✓                   | file modified dates⁴                                                    |
 | jemoji                       | GitHub Pages  | ✓                   | image tag fallback                                                      |
 
 ¹ (1) The code and internal APIs are too immature for this; and (2) The [natural way](https://golang.org/pkg/plugin/) of implementing this only works on Linux.
@@ -30,3 +30,5 @@ The functionality of some plugins is built into the core program:
 
 ³ “Core” plugins are referenced in the main [Jekyll documentation](https://jekyllrb.com/docs/home/).
 The Jekyll documentation [Official Plugins](https://jekyllrb.com/docs/plugins/#available-plugins) / #Official tag of [Awesome Jekyll Plugins](https://github.com/planetjekyll/awesome-jekyll-plugins) look dated; I didn't use those.
+
+⁴ These don't seem that useful with source control and CI. (Post dates are included.)

filters/filters.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/osteele/gojekyll/config"
+	"github.com/osteele/gojekyll/utils"
 	"github.com/osteele/liquid"
 	"github.com/osteele/liquid/evaluator"
 	"github.com/osteele/liquid/expression"
@@ -79,7 +80,7 @@ func AddJekyllFilters(e *liquid.Engine, c *config.Config) {
 
 	// strings
 	e.RegisterFilter("absolute_url", func(s string) string {
-		return c.AbsoluteURL + c.BaseURL + s
+		return utils.URLJoin(c.AbsoluteURL, c.BaseURL, s)
 	})
 	e.RegisterFilter("relative_url", func(s string) string {
 		return c.BaseURL + s

plugins/sitemap.go

@@ -1,9 +1,7 @@
 package plugins
 
 import (
-	"io"
+	"github.com/osteele/gojekyll/pages"
-
-	"github.com/osteele/liquid"
 )
 
 type sitemapPlugin struct{ plugin }
@@ -18,50 +16,17 @@ func init() {
 // }
 
 func (p *sitemapPlugin) PostRead(s Site) error {
-	tpl, err := s.TemplateEngine().ParseTemplate([]byte(sitemapTemplateSource))
+	s.AddDocument(newTemplateDoc(s, "sitemap.xml", sitemapTemplateSource), true)
-	if err != nil {
+	s.AddDocument(newTemplateDoc(s, "robots.txt", `Sitemap: {{ "sitemap.xml" | absolute_url }}`), true)
-		panic(err)
-	}
-	d := sitemapDoc{s, tpl}
-	s.AddDocument(&d, true)
 	return nil
 }
 
-// func (p *sitemapPlugin) feedMetaTag(ctx render.Context) (string, error) {
+func newTemplateDoc(s Site, path, src string) pages.Document {
-// 	cfg := p.site.Config()
+	tpl, err := s.TemplateEngine().ParseTemplate([]byte(src))
-// 	name, _ := cfg.Variables["name"].(string)
-// 	tag := fmt.Sprintf(`<link type="application/atom+xml" rel="alternate" href="%s/feed.xml" title="%s">`,
-// 		html.EscapeString(cfg.AbsoluteURL), html.EscapeString(name))
-// 	return tag, nil
-// }
-
-type sitemapDoc struct {
-	site Site
-	tpl  *liquid.Template
-	// plugin *sitemapPlugin
-	// path   string
-}
-
-func (d *sitemapDoc) Permalink() string    { return "/sitemap.xml" }
-func (d *sitemapDoc) SourcePath() string   { return "" }
-func (d *sitemapDoc) OutputExt() string    { return ".xml" }
-func (d *sitemapDoc) Published() bool      { return true }
-func (d *sitemapDoc) Static() bool         { return false } // FIXME means different things to different callers
-func (d *sitemapDoc) Categories() []string { return []string{} }
-func (d *sitemapDoc) Tags() []string       { return []string{} }
-
-func (d *sitemapDoc) Content() []byte {
-	bindings := map[string]interface{}{"site": d.site}
-	b, err := d.tpl.Render(bindings)
 	if err != nil {
 		panic(err)
 	}
-	return b
+	return &templateDoc{s, path, tpl}
-}
-
-func (d *sitemapDoc) Write(w io.Writer) error {
-	_, err := w.Write(d.Content())
-	return err
 }
 
 // Taken verbatim from https://github.com/jekyll/jekyll-sitemap-plugin/

plugins/template_doc.go

@@ -0,0 +1,36 @@
+package plugins
+
+import (
+	"io"
+	"path"
+
+	"github.com/osteele/liquid"
+)
+
+type templateDoc struct {
+	site Site
+	path string
+	tpl  *liquid.Template
+}
+
+func (d *templateDoc) Permalink() string    { return "/" + d.path }
+func (d *templateDoc) SourcePath() string   { return "" }
+func (d *templateDoc) OutputExt() string    { return path.Ext(d.path) }
+func (d *templateDoc) Published() bool      { return true }
+func (d *templateDoc) Static() bool         { return false } // FIXME means different things to different callers
+func (d *templateDoc) Categories() []string { return []string{} }
+func (d *templateDoc) Tags() []string       { return []string{} }
+
+func (d *templateDoc) Content() []byte {
+	bindings := map[string]interface{}{"site": d.site}
+	b, err := d.tpl.Render(bindings)
+	if err != nil {
+		panic(err)
+	}
+	return b
+}
+
+func (d *templateDoc) Write(w io.Writer) error {
+	_, err := w.Write(d.Content())
+	return err
+}

utils/urls.go

@@ -0,0 +1,21 @@
+package utils
+
+import "strings"
+
+// URLJoin interpolates paths with "/", skipping empty paths and avoiding "//".
+func URLJoin(paths ...string) string {
+	url := ""
+loop:
+	for _, p := range paths {
+		switch {
+		case p == "":
+			continue loop
+		case url != "" && !strings.HasSuffix(url, "/") && !strings.HasPrefix(p, "/"):
+			url += "/"
+		case strings.HasSuffix(url, "/") && strings.HasPrefix(p, "/"):
+			p = p[1:]
+		}
+		url += p
+	}
+	return url
+}