From b14c8c5b448c3c4f473f7a49fcab30b0b2f3beba Mon Sep 17 00:00:00 2001 From: Oliver Steele Date: Sat, 10 Jun 2017 15:38:09 -0400 Subject: [PATCH] Initial There was actually a day of work with some ~10 commits preceding this, with history lost due to a bug and a backup system failure. --- .gitignore | 1 + README.md | 20 ++++++++++ build.go | 62 ++++++++++++++++++++++++++++++ helpers.go | 67 ++++++++++++++++++++++++++++++++ link_tag.go | 60 +++++++++++++++++++++++++++++ main.go | 85 +++++++++++++++++++++++++++++++++++++++++ page.go | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 43 +++++++++++++++++++++ site.go | 98 +++++++++++++++++++++++++++++++++++++++++++++++ test/about.md | 3 ++ test/index.md | 15 ++++++++ 11 files changed, 558 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 build.go create mode 100644 helpers.go create mode 100644 link_tag.go create mode 100644 main.go create mode 100644 page.go create mode 100755 server.go create mode 100644 site.go create mode 100644 test/about.md create mode 100644 test/index.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ca35be0 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +_site diff --git a/README.md b/README.md new file mode 100644 index 0000000..0a918a2 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# Go Jekyll + +When I grow up, I want to be a golang implementation of Jekyll. + +## Status + +I'm writing this to learn my way around Go. It's not good for anytihng yet, and it may never come to anything. + +## Install + +```bash +go get +``` + +## Run + +```bash +go run *.go --source test build +go run *.go --source test serve +``` diff --git a/build.go b/build.go new file mode 100644 index 0000000..599159d --- /dev/null +++ b/build.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +func cleanDirectory() error { + removeFiles := func(path string, info os.FileInfo, err error) error { + stat, err := os.Stat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + if stat.IsDir() { + return nil + } + // TODO check for inclusion in KeepFiles + err = os.Remove(path) + return err + } + err := filepath.Walk(siteConfig.DestinationDir, removeFiles) + if err == nil { + err = removeEmptyDirectories(siteConfig.DestinationDir) + } + return err +} + +func build() error { + err := cleanDirectory() + if err != nil { + return err + } + for _, page := range siteMap { + if !page.Static { + page, err = readFile(page.Path, true) + } + if err != nil { + return err + } + path := page.Permalink + // TODO only do this for MIME pages + if !page.Static && !strings.HasSuffix(path, ".html") { + path += "/index.html" + } + destPath := filepath.Join(siteConfig.DestinationDir, path) + os.MkdirAll(filepath.Dir(destPath), 0777) + if page.Static { + os.Link(filepath.Join(siteConfig.SourceDir, page.Path), destPath) + } else { + fmt.Println("render", filepath.Join(siteConfig.SourceDir, page.Path), "->", destPath) + ioutil.WriteFile(destPath, page.Body, 0644) + } + } + return nil +} diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..b31ad1d --- /dev/null +++ b/helpers.go @@ -0,0 +1,67 @@ +package main + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// alternative to http://left-pad.io +func leftPad(s string, n int) string { + ws := make([]byte, n) + for i := range ws { + ws[i] = ' ' + } + return string(ws) + s +} + +func postfixWalk(path string, walkFn filepath.WalkFunc) error { + files, err := ioutil.ReadDir(path) + if err != nil { + return err + } + + for _, stat := range files { + if stat.IsDir() { + postfixWalk(filepath.Join(path, stat.Name()), walkFn) + } + } + + info, err := os.Stat(path) + err = walkFn(path, info, err) + if err != nil { + return err + } + return nil +} + +func removeEmptyDirectories(path string) error { + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + stat, err := os.Stat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return nil + } + if stat.IsDir() { + err = os.Remove(path) + // TODO swallow the error if it's because the directory isn't + // empty. This can happen if there's an entry in _config.keepfiles + } + return err + } + return postfixWalk(path, walkFn) +} + +func stringArrayToMap(strings []string) map[string]bool { + stringMap := map[string]bool{} + for _, s := range strings { + stringMap[s] = true + } + return stringMap +} diff --git a/link_tag.go b/link_tag.go new file mode 100644 index 0000000..068dde7 --- /dev/null +++ b/link_tag.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + "io" + "strings" + + "github.com/acstech/liquid/core" +) + +// LinkFactory creates a link tag +func LinkFactory(p *core.Parser, config *core.Configuration) (core.Tag, error) { + start := p.Position + p.SkipPastTag() + end := p.Position - 2 + path := strings.Trim(string(p.Data[start:end]), " ") + + permalink, ok := getFilePermalink(path) + if !ok { + return nil, p.Error(fmt.Sprintf("%s not found", path)) + } + + return &Link{path: permalink}, nil +} + +// Link tag data, for passing information from the factory to Execute +type Link struct { + path string +} + +// AddCode is equired by the Liquid tag interface +func (l *Link) AddCode(code core.Code) { + panic("AddCode should not have been called on a Link") +} + +// AddSibling is required by the Liquid tag interface +func (l *Link) AddSibling(tag core.Tag) error { + panic("AddSibling should not have been called on a Link") +} + +// LastSibling is required by the Liquid tag interface +func (l *Link) LastSibling() core.Tag { + return nil +} + +// Execute is required by the Liquid tag interface +func (l *Link) Execute(writer io.Writer, data map[string]interface{}) core.ExecuteState { + writer.Write([]byte(l.path)) + return core.Normal +} + +// Name is required by the Liquid tag interface +func (l *Link) Name() string { + return "link" +} + +// Type is required by the Liquid tag interface +func (l *Link) Type() core.TagType { + return core.StandaloneTag +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..1e0d638 --- /dev/null +++ b/main.go @@ -0,0 +1,85 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/acstech/liquid" +) + +// This is the longest label. Pull it out here so we can both use it, and measure it for alignment. +const configurationFileLabel = "Configuration file:" + +func printSetting(label string, value string) { + fmt.Printf("%s %s\n", + leftPad(label, len(configurationFileLabel)-len(label)), value) +} + +func printPathSetting(label string, path string) { + path, err := filepath.Abs(path) + if err != nil { + panic("Couldn't convert to absolute path") + } + printSetting(label, path) +} + +func main() { + liquid.Tags["link"] = LinkFactory + + flag.StringVar(&siteConfig.DestinationDir, "destination", siteConfig.DestinationDir, "Destination directory") + flag.StringVar(&siteConfig.SourceDir, "source", siteConfig.SourceDir, "Source directory") + flag.Parse() + + configPath := filepath.Join(siteConfig.SourceDir, "_siteConfig.yml") + // TODO error if file is e.g. unreadable + if _, err := os.Stat(configPath); err == nil { + err := siteConfig.readFromDirectory(siteConfig.SourceDir) + if err != nil { + fmt.Println(err) + return + } + printPathSetting(configurationFileLabel, configPath) + } else { + printSetting(configurationFileLabel, "none") + } + printPathSetting("Source:", siteConfig.SourceDir) + printPathSetting("Destination:", siteConfig.DestinationDir) + + fileMap, err := buildFileMap() + if err != nil { + fmt.Println(err) + return + } + siteMap = fileMap + + switch flag.Arg(0) { + case "s", "serve", "server": + err = server() + case "b", "build": + printSetting("Generating...", "") + start := time.Now() + err = build() + elapsed := time.Since(start) + printSetting("", fmt.Sprintf("done in %.2fs.", elapsed.Seconds())) + case "routes": + fmt.Printf("\nRoutes:\n") + for urlPath, p := range siteMap { + fmt.Printf(" %s -> %s\n", urlPath, p.Path) + } + case "build1": + page, err2 := readFile("index.md", true) + if err2 != nil { + err = err2 + break + } + fmt.Println(string(page.Body)) + default: + fmt.Println("A subcommand is required.") + } + if err != nil { + fmt.Println(err) + } +} diff --git a/page.go b/page.go new file mode 100644 index 0000000..f727fc4 --- /dev/null +++ b/page.go @@ -0,0 +1,104 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "path/filepath" + "regexp" + + "github.com/acstech/liquid" + "github.com/russross/blackfriday" + yaml "gopkg.in/yaml.v2" +) + +var frontmatterRe = regexp.MustCompile(`(?s)^---\n(.+?)\n---\n`) + +// A Page represents an HTML page. +type Page struct { + Path string + Permalink string + Static bool + Expanded bool + Body []byte +} + +func (p Page) String() string { + return fmt.Sprintf("Page{Path=%v, Permalink=%v}", p.Path, p.Permalink) +} + +func readFile(path string, expand bool) (*Page, error) { + // TODO don't read, parse binary files + + source, err := ioutil.ReadFile(filepath.Join(siteConfig.SourceDir, path)) + if err != nil { + return nil, err + } + + static := true + data := map[string]interface{}{} + body := source + + fmMatchIndex := frontmatterRe.FindSubmatchIndex(source) + if fmMatchIndex != nil { + static = false + body = source[fmMatchIndex[1]:] + fmBytes := source[fmMatchIndex[2]:fmMatchIndex[3]] + var fmMap interface{} + err = yaml.Unmarshal(fmBytes, &fmMap) + if err != nil { + return nil, err + } + fmStringMap, ok := fmMap.(map[interface{}]interface{}) + if !ok { + return nil, errors.New("YAML frontmatter is not a map") + } + for k, v := range fmStringMap { + stringer, ok := k.(fmt.Stringer) + if ok { + data[stringer.String()] = v + } else { + data[fmt.Sprintf("%v", k)] = v + } + } + } + + ext := filepath.Ext(path) + + var title string + if val, ok := data["permalink"]; ok { + title = fmt.Sprintf("%v", val) + } else { + title = filepath.Base(path) + title = title[:len(title)-len(ext)] + } + + // TODO use site, collection default; expand components + permalink := "/" + path[:len(path)-len(ext)] + if val, ok := data["permalink"]; ok { + permalink = val.(string) // TODO what if it's not a string? + } + + if expand && ext == ".md" { + template, err := liquid.Parse(body, nil) + if err != nil { + return nil, err + } + writer := new(bytes.Buffer) + template.Render(writer, data) + body = blackfriday.MarkdownBasic(writer.Bytes()) + } + + if !expand { + body = []byte{} + } + + return &Page{ + Path: path, + Permalink: permalink, + Expanded: expand, + Static: static, + Body: body, + }, nil +} diff --git a/server.go b/server.go new file mode 100755 index 0000000..2c00684 --- /dev/null +++ b/server.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "net/http" +) + +func server() error { + address := "localhost:4000" + printSetting("Server address:", "http://"+address+"/") + printSetting("Server running...", "press ctrl-c to stop.") + http.HandleFunc("/", handler) + err := http.ListenAndServe(address, nil) + if err != nil { + // TODO pick another port + return err + } + return nil +} + +func handler(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + + // w.Header().Set("Content-Type", "text/plain; charset=utf-8") // normal header + + p, found := siteMap[path] + if !found { + w.WriteHeader(http.StatusNotFound) + p, found = siteMap["404.html"] + } + if !found { + fmt.Fprintf(w, "404 page not found: %s", path) + return + } + + p, err := readFile(p.Path, true) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + fmt.Printf("Error expanding %s\n%s", p.Path, err) + fmt.Fprintf(w, "Error expanding %s\n%s", p.Path, err) + } + w.Write(p.Body) +} diff --git a/site.go b/site.go new file mode 100644 index 0000000..7201e86 --- /dev/null +++ b/site.go @@ -0,0 +1,98 @@ +package main + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + yaml "gopkg.in/yaml.v2" +) + +// SiteConfig is the Jekyll site configuration, typically read from _config.yml. +type SiteConfig struct { + Permalink string + SourceDir string + DestinationDir string + // Safe bool + Exclude []string + Include []string + // KeepFiles []string + // TimeZone string + // Encoding string + Collections map[string]interface{} +} + +// Initialize with defaults. +var siteConfig = SiteConfig{ + SourceDir: "./", + DestinationDir: "./_site", + Permalink: "/:categories/:year/:month/:day/:title.html", +} + +// A map from URL path -> *Page +var siteMap map[string]*Page + +func (config *SiteConfig) readFromDirectory(path string) error { + configBytes, err := ioutil.ReadFile(path) + if err == nil { + err = yaml.Unmarshal(configBytes, &config) + } else if os.IsNotExist(err) { + err = nil + } + return err +} + +func buildFileMap() (map[string]*Page, error) { + basePath := siteConfig.SourceDir + fileMap := map[string]*Page{} + exclusionMap := stringArrayToMap(siteConfig.Exclude) + + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == siteConfig.SourceDir { + return nil + } + // TODO replace by info.IsDir + stat, err := os.Stat(path) + if err != nil { + return err + } + + relPath, err := filepath.Rel(basePath, path) + if err != nil { + return err + } + base := filepath.Base(relPath) + // TODO exclude based on glob, not exact match + _, exclude := exclusionMap[relPath] + exclude = exclude || strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") + if exclude { + if stat.IsDir() { + return filepath.SkipDir + } + return nil + } + if !stat.IsDir() { + page, err := readFile(relPath, false) + if err != nil { + return err + } + fileMap[page.Permalink] = page + } + return nil + } + err := filepath.Walk(basePath, walkFn) + return fileMap, err +} + +func getFilePermalink(path string) (string, bool) { + for _, v := range siteMap { + if v.Path == path { + return v.Permalink, true + } + } + return "", false +} diff --git a/test/about.md b/test/about.md new file mode 100644 index 0000000..e7c86d8 --- /dev/null +++ b/test/about.md @@ -0,0 +1,3 @@ +# About + +A page without frontmatter. diff --git a/test/index.md b/test/index.md new file mode 100644 index 0000000..9384c6f --- /dev/null +++ b/test/index.md @@ -0,0 +1,15 @@ +--- +permalink: / +variable: variable substitution +--- + +# Site Title + +Here is a test with {{ variable }} and [a link]({% link about.md %}). + +## Subsection + +* A +* list +* of +* items