From 2bb65c05594339f115c001f6163e94f2fb7d9a0c Mon Sep 17 00:00:00 2001 From: tengu-alt Date: Tue, 3 Sep 2024 16:45:54 +0300 Subject: [PATCH] Upgrade for the native protocols spec page --- cqlprotodoc/README.md | 12 + cqlprotodoc/go.mod | 5 + cqlprotodoc/go.sum | 2 + cqlprotodoc/main.go | 177 ++++++++++++++ cqlprotodoc/spec/spec.go | 226 ++++++++++++++++++ cqlprotodoc/template-notice.txt | 5 + cqlprotodoc/template.gohtml | 61 +++++ site-content/Dockerfile | 3 + site-content/docker-entrypoint.sh | 3 + ...process-native-protocol-specs-in-docker.sh | 35 +++ .../modules/ROOT/pages/native_protocol.adoc | 84 ++++++- site-ui/Dockerfile | 4 +- 12 files changed, 603 insertions(+), 14 deletions(-) create mode 100644 cqlprotodoc/README.md create mode 100644 cqlprotodoc/go.mod create mode 100644 cqlprotodoc/go.sum create mode 100644 cqlprotodoc/main.go create mode 100644 cqlprotodoc/spec/spec.go create mode 100644 cqlprotodoc/template-notice.txt create mode 100644 cqlprotodoc/template.gohtml create mode 100755 site-content/process-native-protocol-specs-in-docker.sh diff --git a/cqlprotodoc/README.md b/cqlprotodoc/README.md new file mode 100644 index 000000000..0f72be674 --- /dev/null +++ b/cqlprotodoc/README.md @@ -0,0 +1,12 @@ +Comes from https://github.com/martin-sucha/cqlprotodoc + +cqlprotodoc converts +[CQL protocol specification files](https://github.com/apache/cassandra/tree/trunk/doc) +to HTML for easier browsing. + +Usage: + +1. Update template-notice.txt based on your Cassandra distribution. +2. `cqlprotodoc ` + +The generated files are published at https://martin-sucha.github.io/cqlprotodoc/ diff --git a/cqlprotodoc/go.mod b/cqlprotodoc/go.mod new file mode 100644 index 000000000..cc578b1df --- /dev/null +++ b/cqlprotodoc/go.mod @@ -0,0 +1,5 @@ +module cqlprotodoc + +go 1.17 + +require github.com/mvdan/xurls v1.1.0 diff --git a/cqlprotodoc/go.sum b/cqlprotodoc/go.sum new file mode 100644 index 000000000..9fcdd8c17 --- /dev/null +++ b/cqlprotodoc/go.sum @@ -0,0 +1,2 @@ +github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww= +github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= diff --git a/cqlprotodoc/main.go b/cqlprotodoc/main.go new file mode 100644 index 000000000..aa3a256a8 --- /dev/null +++ b/cqlprotodoc/main.go @@ -0,0 +1,177 @@ +package main + +import ( + "cqlprotodoc/spec" + "embed" + "fmt" + "html/template" + "os" + "path/filepath" + "strings" +) + +//go:embed template.gohtml +var templateFS embed.FS + +//go:embed template-notice.txt +var notice []byte + +type TOCNode struct { + spec.TOCEntry + Exists bool + Children []*TOCNode +} + +func buildTOCTree(entries []spec.TOCEntry, sectionNumbers map[string]struct{}) []*TOCNode { + var root TOCNode + stack := []*TOCNode{&root} + for _, e := range entries { + level := strings.Count(e.Number, ".") + 1 + if len(stack) > level { + stack = stack[:level] + } + parent := stack[len(stack)-1] + _, exists := sectionNumbers[e.Number] + node := &TOCNode{ + TOCEntry: e, + Children: nil, + Exists: exists, + } + parent.Children = append(parent.Children, node) + stack = append(stack, node) + } + return root.Children +} + +type templateData struct { + spec.Document + LicenseHTML template.HTML + TOCTree []*TOCNode + Sections []Section +} + +type Section struct { + spec.Section + Level int + BodyHTML template.HTML +} + +func link(sb *strings.Builder, href, text string) { + sb.WriteString(``) + sb.WriteString(template.HTMLEscapeString(text)) + sb.WriteString(``) +} + +func formatBody(text []spec.Text, sectionNumbers map[string]struct{}) template.HTML { + var sb strings.Builder + for _, t := range text { + switch { + case t.SectionRef != "": + if _, ok := sectionNumbers[t.SectionRef]; ok { + link(&sb, "#s"+t.SectionRef, t.Text) + } else { + sb.WriteString(template.HTMLEscapeString(t.Text)) + } + case t.Href != "": + link(&sb, t.Href, t.Text) + default: + sb.WriteString(template.HTMLEscapeString(t.Text)) + } + } + return template.HTML(sb.String()) +} + +func buildSections(in []spec.Section, sectionNumbers map[string]struct{}) []Section { + ret := make([]Section, len(in)) + for i, s := range in { + ret[i].Section = in[i] + ret[i].Level = strings.Count(s.Number, ".") + 2 + ret[i].BodyHTML = formatBody(in[i].Body, sectionNumbers) + } + return ret +} + +func checkSectionLinks(d spec.Document, sectionNumbers, tocNumbers map[string]struct{}) { + + for _, t := range d.TOC { + if _, ok := sectionNumbers[t.Number]; !ok { + fmt.Fprintf(os.Stderr, "section %q exists in TOC, but not in sections\n", t.Number) + } + } + for _, s := range d.Sections { + if _, ok := tocNumbers[s.Number]; !ok { + fmt.Fprintf(os.Stderr, "section %q exists in sections, but not in TOC\n", s.Number) + } + for _, tt := range s.Body { + if tt.SectionRef != "" { + if _, ok := sectionNumbers[tt.SectionRef]; !ok { + fmt.Fprintf(os.Stderr, "non-existing section %q is referenced from section %q\n", + tt.SectionRef, s.Number) + } + } + } + } +} + +var tmpl = template.Must(template.ParseFS(templateFS, "template.gohtml")) + +func processDocument(inputPath, outputPath string) (outErr error) { + data, err := os.ReadFile(inputPath) + if err != nil { + return err + } + doc, err := spec.Parse(string(data)) + if err != nil { + return err + } + sectionNumbers := make(map[string]struct{}) + for _, s := range doc.Sections { + sectionNumbers[s.Number] = struct{}{} + } + tocNumbers := make(map[string]struct{}) + for _, t := range doc.TOC { + tocNumbers[t.Number] = struct{}{} + } + checkSectionLinks(doc, sectionNumbers, tocNumbers) + f, err := os.Create(outputPath) + if err != nil { + return err + } + defer func() { + closeErr := f.Close() + if closeErr != nil && outErr == nil { + outErr = closeErr + } + }() + return tmpl.Execute(f, templateData{ + Document: doc, + LicenseHTML: formatBody(doc.License, sectionNumbers), + TOCTree: buildTOCTree(doc.TOC, sectionNumbers), + Sections: buildSections(doc.Sections, sectionNumbers), + }) +} + +func main() { + if len(os.Args) != 3 { + fmt.Fprintln(os.Stderr, "Usage: ") + return + } + inputDir := os.Args[1] + outputDir := os.Args[2] + for _, name := range []string{"v5", "v4", "v3"} { + fmt.Fprintln(os.Stderr, name) + err := processDocument(filepath.Join(inputDir, fmt.Sprintf("native_protocol_%s.spec", name)), + filepath.Join(outputDir, fmt.Sprintf("native_protocol_%s.html", name))) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", name, err) + return + } + } + err := os.WriteFile(filepath.Join(outputDir, "NOTICE"), notice, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "NOTICE: %v\n", err) + return + } +} diff --git a/cqlprotodoc/spec/spec.go b/cqlprotodoc/spec/spec.go new file mode 100644 index 000000000..67ad40b84 --- /dev/null +++ b/cqlprotodoc/spec/spec.go @@ -0,0 +1,226 @@ +// Package spec implements parser for Cassandra protocol specification. +package spec + +import ( + "fmt" + "github.com/mvdan/xurls" + "regexp" + "strings" +) + +type Document struct { + License []Text + Title string + TOC []TOCEntry + Sections []Section +} + +type TOCEntry struct { + Number string + Title string +} + +type Section struct { + Number string + Title string + Body []Text +} + +func (s Section) Empty() bool { + return s.Number == "" && s.Title == "" && len(s.Body) == 0 +} + +// Text token in section body. +type Text struct { + // Text that is displayed. + Text string + // SectionRef is the number of section this text links to. + SectionRef string + // Href is URL this text links to. + Href string +} + +var commentRegexp = regexp.MustCompile("^# ?(.*)$") +var emptyRegexp = regexp.MustCompile(`^\s*$`) +var titleRegexp = regexp.MustCompile(`^\s+(.*)\s*$`) +var headingRegexp = regexp.MustCompile(`^(\s*)(\d+(?:\.\d+)*)\.? (.*)$`) + +const ( + mhSpaces = 1 + mhNumber = 2 + mhTitle = 3 +) + +func Parse(data string) (Document, error) { + lines := strings.Split(data, "\n") + var license strings.Builder + var doc Document + l := 0 + // license + for l < len(lines) { + m := commentRegexp.FindStringSubmatch(lines[l]) + if len(m) != 2 { + break + } + license.WriteString(m[1]) + license.WriteString("\n") + l++ + } + doc.License = parseBody(strings.Trim(license.String(), "\n ")) + // empty lines + for l < len(lines) && emptyRegexp.MatchString(lines[l]) { + l++ + } + // title + if l >= len(lines) { + return Document{}, fmt.Errorf("missing title") + } + m := titleRegexp.FindStringSubmatch(lines[l]) + if len(m) != 2 { + return Document{}, fmt.Errorf("line %d: title expected on line", l) + } + doc.Title = m[1] + l++ + // empty lines + for l < len(lines) && emptyRegexp.MatchString(lines[l]) { + l++ + } + // table of contents header + if lines[l] != "Table of Contents" { + return Document{}, fmt.Errorf("line %d: expected table of contents", l) + } + l++ + // empty lines + for l < len(lines) && emptyRegexp.MatchString(lines[l]) { + l++ + } + // toc entries + for l < len(lines) { + if emptyRegexp.MatchString(lines[l]) { + // end of toc + break + } + mh := headingRegexp.FindStringSubmatch(lines[l]) + if len(mh) != 4 { + return Document{}, fmt.Errorf("line %d: expected toc entry", l) + } + doc.TOC = append(doc.TOC, TOCEntry{ + Number: mh[mhNumber], + Title: mh[mhTitle], + }) + l++ + } + // empty lines + for l < len(lines) && emptyRegexp.MatchString(lines[l]) { + l++ + } + // content + tocIdx := 0 + var section Section + var body []string + + for l < len(lines) { + var sectionStart bool + var newSection Section + sectionStart, tocIdx, newSection = checkSectionStart(doc.TOC, tocIdx, lines[l]) + if sectionStart { + section.Body = parseBody(strings.Join(body, "\n")) + doc.Sections = append(doc.Sections, section) + section = newSection + body = nil + l++ + // Eat empty lines + for l < len(lines) && emptyRegexp.MatchString(lines[l]) { + l++ + } + continue + } + body = append(body, lines[l]) + l++ + } + + if len(body) > 0 || !section.Empty() { + section.Body = parseBody(strings.Join(body, "\n")) + doc.Sections = append(doc.Sections, section) + } + + return doc, nil +} + +// checkSectionStart checks if the line starts a new section and returns a new tocIdx. +func checkSectionStart(toc []TOCEntry, tocIdx int, line string) (bool, int, Section) { + mh := headingRegexp.FindStringSubmatch(line) + if len(mh) != 4 || tocIdx >= len(toc) { + return false, tocIdx, Section{} + } + + if mh[mhSpaces] == "" { + if mh[mhNumber] == toc[tocIdx].Number { + tocIdx++ + } + return true, tocIdx, Section{ + Number: mh[mhNumber], + Title: mh[mhTitle], + } + } + + t := strings.ToLower(mh[3]) + for i := tocIdx; i < len(toc); i++ { + t2 := strings.ToLower(toc[i].Title) + if mh[mhNumber] == toc[i].Number && (strings.Contains(t, t2) || strings.Contains(t2, t)) { + return true, i + 1, Section{ + Number: mh[mhNumber], + Title: mh[mhTitle], + } + } + } + + return false, tocIdx, Section{} +} + +var linkifyRegexp *regexp.Regexp +var sectionSubexpIdx int +var sectionsSubexpIdx int + +func init() { + s := xurls.Strict.String() + r := `(?:)|[Ss]ection (\d+(?:\.\d+)*)|[Ss]ections (\d+(?:\.\d+)*(?:(?:, (?:and )?| and )\d+(?:\.\d+)*)*)` + linkifyRegexp = regexp.MustCompile(strings.ReplaceAll(r, "", s)) + sectionSubexpIdx = xurls.Strict.NumSubexp()*2 + 2 + sectionsSubexpIdx = (xurls.Strict.NumSubexp()+1)*2 + 2 +} + +var sectionsSplitRegexp = regexp.MustCompile("(?:, (?:and )?| and )") + +func parseBody(s string) []Text { + var body []Text + lastIdx := 0 + for _, m := range linkifyRegexp.FindAllStringSubmatchIndex(s, -1) { + body = append(body, Text{Text: s[lastIdx:m[0]]}) + + switch { + case m[sectionSubexpIdx] != -1: + sectionNo := s[m[sectionSubexpIdx]:m[sectionSubexpIdx+1]] + body = append(body, Text{Text: s[m[0]:m[1]], SectionRef: sectionNo}) + case m[sectionsSubexpIdx] != -1: + body = append(body, Text{Text: s[m[0]:m[sectionsSubexpIdx]]}) + sections := s[m[sectionsSubexpIdx]:m[sectionsSubexpIdx+1]] + lastIdx2 := 0 + for _, m2 := range sectionsSplitRegexp.FindAllStringIndex(sections, -1) { + sectionNo := sections[lastIdx2:m2[0]] + body = append(body, Text{Text: sectionNo, SectionRef: sectionNo}) + // separator + body = append(body, Text{Text: sections[m2[0]:m2[1]]}) + lastIdx2 = m2[1] + } + sectionNo := sections[lastIdx2:] + body = append(body, Text{Text: sectionNo, SectionRef: sectionNo}) + default: + href := s[m[0]:m[1]] + body = append(body, Text{Text: href, Href: href}) + } + lastIdx = m[1] + } + body = append(body, Text{Text: s[lastIdx:]}) + return body +} diff --git a/cqlprotodoc/template-notice.txt b/cqlprotodoc/template-notice.txt new file mode 100644 index 000000000..5bf4b363a --- /dev/null +++ b/cqlprotodoc/template-notice.txt @@ -0,0 +1,5 @@ +Apache Cassandra +Copyright 2009-2022 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/cqlprotodoc/template.gohtml b/cqlprotodoc/template.gohtml new file mode 100644 index 000000000..12ed19f9d --- /dev/null +++ b/cqlprotodoc/template.gohtml @@ -0,0 +1,61 @@ +{{- /*gotype: cqlprotodoc.templateData*/ -}} + + + + {{.Title}} + + + + + 
{{.LicenseHTML}}
+

{{.Title}}

+

Table of Contents

+ + {{ range .Sections }} + {{.Number}} {{.Title}} + 
{{.BodyHTML}}
+ {{ end }} + + +{{ define "tocNodes" }} + {{- /*gotype: cqlprotodoc.TOCNode*/ -}} + {{ range . }} +
  • {{.Number}} + {{if .Exists}}{{.Title}}{{else}}{{.Title}}{{end}} + {{ with .Children }} +
      + {{ template "tocNodes" . }} +
    + {{ end}} +
    • + {{ end }} +{{ end }} \ No newline at end of file diff --git a/site-content/Dockerfile b/site-content/Dockerfile index 1fbf4d769..15797d3a4 100644 --- a/site-content/Dockerfile +++ b/site-content/Dockerfile @@ -123,6 +123,9 @@ EXPOSE 5151/tcp USER ${BUILD_USER} WORKDIR ${BUILD_DIR} COPY docker-entrypoint.sh /usr/local/bin/ + +COPY process-native-protocol-specs-in-docker.sh /usr/local/bin/ + ENTRYPOINT ["docker-entrypoint.sh"] # Possible commands are listed below. The entrypoint will accept any combination of these commands. diff --git a/site-content/docker-entrypoint.sh b/site-content/docker-entrypoint.sh index 889bbddc1..1f30d9824 100755 --- a/site-content/docker-entrypoint.sh +++ b/site-content/docker-entrypoint.sh @@ -206,6 +206,9 @@ render_site_content_to_html() { prepare_site_html_for_publication() { + log_message "INFO" "Processing native protocols spec page" + sudo /usr/local/bin/process-native-protocol-specs-in-docker.sh + pushd "${CASSANDRA_WEBSITE_DIR}" > /dev/null # copy everything to content/ directory diff --git a/site-content/process-native-protocol-specs-in-docker.sh b/site-content/process-native-protocol-specs-in-docker.sh new file mode 100755 index 000000000..4600f323b --- /dev/null +++ b/site-content/process-native-protocol-specs-in-docker.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +set -e + +# Variables +GO_VERSION="1.20.6" +GO_TAR="go${GO_VERSION}.linux-amd64.tar.gz" +PARSER_DIR="/home/build/cassandra-website/cqlprotodoc" +WEBSITE_DIR="/home/build/cassandra-website" + +# Step 0: Download and install Go +echo "Downloading Go $GO_VERSION..." +wget https://golang.org/dl/$GO_TAR + +echo "Installing Go..." +tar -C /usr/local -xzf $GO_TAR + +# Set Go environment variables +export PATH=$PATH:/usr/local/go/bin +export GOPATH=$HOME/go + +echo "Building the cqlprotodoc..." +cd "$PARSER_DIR" +go build -o cqlprotodoc + +# Step 3: Process the spec files using the parser +echo "Processing the .spec files..." +"$PARSER_DIR"/cqlprotodoc "$WEBSITE_DIR/site-content/source/modules/ROOT/examples/TEXT" "$WEBSITE_DIR/site-content/build/html/_" + +# Step 4: Cleanup - Remove the Cassandra and parser directories +echo "Cleaning up..." +rm -rf "$PARSER_DIR/cqlprotodoc" /usr/local/go $GO_TAR + +echo "Script completed successfully." diff --git a/site-content/source/modules/ROOT/pages/native_protocol.adoc b/site-content/source/modules/ROOT/pages/native_protocol.adoc index dbeb4065a..ca218ef2c 100644 --- a/site-content/source/modules/ROOT/pages/native_protocol.adoc +++ b/site-content/source/modules/ROOT/pages/native_protocol.adoc @@ -3,21 +3,81 @@ == Native Protocol Version 3 -[source, plaintext] ----- -include::example$TEXT/native_protocol_v3.spec[Version 3] ----- +[source, js] +++++ +
    + +++++ == Native Protocol Version 4 -[source, plaintext] ----- -include::example$TEXT/native_protocol_v4.spec[Version 4] ----- +[source, js] +++++ +
    + +++++ == Native Protocol Version 5 -[source, plaintext] ----- -include::example$TEXT/native_protocol_v5.spec[Version 5] ----- +[source, js] +++++ +
    + +++++ + +[source, js] +++++ + +++++ \ No newline at end of file diff --git a/site-ui/Dockerfile b/site-ui/Dockerfile index b7ea6246e..366d7f024 100644 --- a/site-ui/Dockerfile +++ b/site-ui/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:22.04 # # Set up non-root user, 'build', with default uid:gid # This allows passing --build-arg to use localhost username, and uid:gid: @@ -15,7 +15,7 @@ FROM ubuntu:18.04 ARG BUILD_USER_ARG="build" ARG UID_ARG=1000 ARG GID_ARG=1000 -ARG NODE_VERSION_ARG="v12.16.2" +ARG NODE_VERSION_ARG="v20.16.0" ENV BUILD_USER=${BUILD_USER_ARG}