Add native chroma highlighting

README.md

@@ -59,9 +59,7 @@ gojekyll help build
 
 1. Ubuntu (64-bit) and macOS binaries are available from the [releases
    page](https://github.com/osteele/gojekyll/releases).
-2. [Optional] **Highlight**. To use the `{% highlight %}` tag, you need
-   [Pygments](http://pygments.org): `pip install Pygments`.
-3. [Optional] **Themes**. To use a theme, you need to install Ruby and
+2. [Optional] **Themes**. To use a theme, you need to install Ruby and
    [bundler](http://bundler.io/). Create a `Gemfile` that lists the theme., and
    run `bundle install`. The [Jekyll theme
    instructions](https://jekyllrb.com/docs/themes/) provide more detail, and

docs/benchmarks.md

@@ -1,6 +1,6 @@
 # Benchmarks
 
-`[go]jekyll build` on a late-2015 MacBook Pro, running current versions of everything as of 2017-07-09.
+`[go]jekyll build` on an Intel Xeon E5620 @ 2.40GHz, running current versions of everything as of 2022-01-29.
 
 Disable the cache by setting the environment variable `GOJEKYLL_DISABLE_CACHE=1`.
 Disable threading by setting `GOMAXPROCS=1`.
@@ -13,22 +13,19 @@ This site contains only one SASS file.
 It contains a few instances of `{\% highlight \%}`.
 Each of these results in a call to Pygment. This dominates the un-cached times.
 
-| Executable | Options                     | Time          |
-|------------|-----------------------------|---------------|
-| jekyll     |                             | 18.53s        |
-| gojekyll   | single-threaded; cold cache | 3.14s ± 0.23s |
-| gojekyll   | single-threaded; warm cache | 2.19s ± 0.03s |
-| gojekyll   | multi-threaded; cold cache  | 1.19s ± 0.03s |
-| gojekyll   | multi-threaded; warm cache  | 0.63s ± 0.03s |
-
-## Software Design web site
-
-This site makes heavy use of SASS.
-
-| Executable | Options                     | Time          |
-|------------|-----------------------------|---------------|
-| jekyll     |                             | 8.07s         |
-| gojekyll   | single-threaded; cold cache | 1.46s ± 0.21s |
-| gojekyll   | single-threaded; warm cache | 0.60s ± 0.23s |
-| gojekyll   | multi-threaded; cold cache  | 1.23s ± 0.10s |
-| gojekyll   | multi-threaded; warm cache  | 0.35s ± 0.04s |
+| Executable | Options         | Time   |
+|------------|-----------------|--------|
+| jekyll     |                 | 9.086s |
+| gojekyll   | single-threaded | 5.35s  |
+| gojekyll   | multi-threaded  | 2.50s  |
+
+
+## MadelineProto Docs
+
+This site contains 1873 markdown files, and runs a modified version of the complex [Just The Docs theme](https://pmarsceill.github.io/just-the-docs/), with many SASS files, sitemap, search index generation.
+
+| Executable | Options         | Time             |
+|------------|-----------------|------------------|
+| jekyll     |                 | Timeout @ 1 hour |
+| gojekyll   | single-threaded | 750.61s          |
+| gojekyll   | multi-threaded  | 142.16s          |

filters/filters.go

@@ -12,12 +12,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/danog/blackfriday/v2"
 	"github.com/osteele/gojekyll/config"
 	"github.com/osteele/gojekyll/utils"
 	"github.com/osteele/liquid"
 	"github.com/osteele/liquid/evaluator"
 	"github.com/osteele/liquid/expressions"
-	"github.com/russross/blackfriday/v2"
 	libsass "github.com/wellington/go-libsass"
 )
 

go.mod

@@ -3,9 +3,10 @@ module github.com/osteele/gojekyll
 go 1.16
 
 require (
+	github.com/alecthomas/chroma v0.9.4
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/danog/blackfriday/v2 v2.1.6
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-github v17.0.0+incompatible
@@ -22,16 +23,14 @@ require (
 	github.com/osteele/tuesday v1.0.3 // indirect
 	github.com/pkg/browser v0.0.0-20210621091255-c198bc921a84
 	github.com/radovskyb/watcher v1.0.7
-	github.com/russross/blackfriday/v2 v2.1.0
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/tdewolff/minify v2.3.6+incompatible
 	github.com/tdewolff/parse v2.3.4+incompatible // indirect
 	github.com/tdewolff/test v1.0.6 // indirect
 	github.com/wellington/go-libsass v0.9.2
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
 	golang.org/x/oauth2 v0.0.0-20210622215436-a8dc77f794b6
-	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6

go.sum

@@ -33,6 +33,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/alecthomas/chroma v0.9.4 h1:YL7sOAE3p8HS96T9km7RgvmsZIctqbK1qJ0b7hzed44=
+github.com/alecthomas/chroma v0.9.4/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4=
@@ -43,9 +45,13 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/danog/blackfriday/v2 v2.1.6 h1:X8eMk7/+AgFxnkINl3k8P2M7k1LrPSrt/GajW7duyMM=
+github.com/danog/blackfriday/v2 v2.1.6/go.mod h1:LwBcWUEJB70NEUayhCkcsNRhiIG48Umt9idrfVfygbQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
+github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -154,8 +160,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
 github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
@@ -241,8 +245,8 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -287,9 +291,8 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210319071255-635bc2c9138d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

renderers/markdown.go

@@ -5,8 +5,8 @@ import (
 	"io"
 	"regexp"
 
+	"github.com/danog/blackfriday/v2"
 	"github.com/osteele/gojekyll/utils"
-	"github.com/russross/blackfriday/v2"
 	"golang.org/x/net/html"
 )
 
@@ -27,6 +27,7 @@ const blackfridayExtensions = 0 |
 	blackfriday.HeadingIDs |
 	blackfriday.BackslashLineBreak |
 	blackfriday.DefinitionLists |
+	blackfriday.NoEmptyLineBeforeBlock |
 	// added relative to commonExtensions
 	blackfriday.AutoHeadingIDs
 

tags/highlight.go

@@ -3,20 +3,15 @@ package tags
 import (
 	"bytes"
 	"fmt"
-	"html"
-	"os"
-	"os/exec"
 	"regexp"
-	"strings"
 
-	"github.com/osteele/gojekyll/cache"
+	"github.com/alecthomas/chroma"
+	"github.com/alecthomas/chroma/formatters/html"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
 	"github.com/osteele/liquid/render"
 )
 
-const pygmentizeCmd = "pygmentize"
-
-// warn once per execution, even on watch/rebuilds
-var warnedMissingPygmentize = false
 var highlightArgsRE = regexp.MustCompile(`^\s*(\S+)(\s+linenos)?\s*$`)
 
 func highlightTag(rc render.Context) (string, error) {
@@ -28,55 +23,43 @@ func highlightTag(rc render.Context) (string, error) {
 	if args == nil {
 		return "", fmt.Errorf("syntax error")
 	}
-	cmdArgs := []string{"-f", "html"}
-	cmdArgs = append(cmdArgs, "-l"+args[1])
-	if args[2] != "" {
-		cmdArgs = append(cmdArgs, "-O", "linenos=1")
-	}
-	s, err := rc.InnerString()
+	source, err := rc.InnerString()
 	if err != nil {
 		return "", err
 	}
-	r, err := cache.WithFile(fmt.Sprintf("pygments %s", args), s, func() (string, error) {
-		buf := new(bytes.Buffer)
-		cmd := exec.Command(pygmentizeCmd, cmdArgs...) // nolint: gas
-		cmd.Stdin = strings.NewReader(s)
-		cmd.Stdout = buf
-		cmd.Stderr = os.Stderr
-		if e := cmd.Run(); e != nil {
-			return "", e
-		}
-		return buf.String(), nil
-	})
-	if e, ok := err.(*exec.Error); ok {
-		// This only works in go < 1.16:
-		if e.Err == exec.ErrNotFound {
-			r, err = maybeWarnMissingPygmentize(s, err)
-		}
-		// This is language-dependent, but works in go 1.16 too
-		if strings.Contains(e.Err.Error(), "executable file not found") && e.Name == pygmentizeCmd {
-			r, err = maybeWarnMissingPygmentize(s, err)
-		}
+
+	// Determine lexer.
+	l := lexers.Get(args[1])
+	if l == nil {
+		l = lexers.Analyse(source)
 	}
-	// TODO: replace the test above by the following once support for go < 1.16
-	// is dropped if pathErr, ok := err.(*fs.PathError); ok {
-	//  if filepath.Base(pathErr.Path) == pygmentizeCmd {
-	//      r, err = maybeWarnMissingPygmentize(s, err)
-	//  }
-	// }
-	return r, err
-}
+	if l == nil {
+		l = lexers.Fallback
+	}
+	l = chroma.Coalesce(l)
+
+	lineNum := args[2] != ""
+
+	// Determine formatter.
+	f := html.New(
+		html.WithClasses(true),
+		html.WithLineNumbers(lineNum),
+		html.LineNumbersInTable(true),
+	)
 
-func maybeWarnMissingPygmentize(s string, err error) (string, error) {
-	r := `<code>` + html.EscapeString(s) + `</code>`
-	if warnedMissingPygmentize {
-		return r, nil
+	// Determine style.
+	s := styles.Get("")
+	if s == nil {
+		s = styles.Fallback
+	}
+
+	it, err := l.Tokenise(nil, source)
+	if err != nil {
+		return "", err
+	}
+	buf := new(bytes.Buffer)
+	if err = f.Format(buf, s, it); err != nil {
+		return "", err
 	}
-	warnedMissingPygmentize = true
-	_, err = fmt.Fprintf(os.Stderr,
-		"Error: %s\n"+
-			"Run `pip install Pygments` to install %s.\n"+
-			"The {%% highlight %%} tag will use <code>…</code> instead.\n",
-		err, pygmentizeCmd)
-	return r, err
+	return buf.String(), nil
 }

tags/highlight_test.go

@@ -16,12 +16,12 @@ var highlightTagTests = []struct{ in, out string }{
 	def foo
 	  puts 'foo'
 	end
-	{% endhighlight %}`, "highlight"},
+	{% endhighlight %}`, "chroma"},
 	{`{% highlight ruby linenos %}
 	def foo
 	  puts 'foo'
 	end
-	{% endhighlight %}`, "highlighttable"},
+	{% endhighlight %}`, "lntable"},
 }
 
 func TestHighlightTag(t *testing.T) {

tags/tags.go

@@ -24,9 +24,9 @@ func AddJekyllTags(e *liquid.Engine, c *config.Config, includeDirs []string, lh
 
 // tagContext provides the context to a tag renderer.
 type tagContext struct {
-	cfg *config.Config
+	cfg         *config.Config
 	includeDirs []string
-	lh  LinkTagHandler
+	lh          LinkTagHandler
 }
 
 // CreateUnimplementedTag creates a tag definition that prints a warning the first