content,internal: create styleguide page

The content of the styleguide page is generated from the markdown files contained in the component directories of content/static. We use goldmark to parse the files into html sections and generate an outline for the sidenav. Goldmark is extended to render html code blocks as both html and code snippets. Change-Id: I023edb77114001439be58e00d5a48198181f165a Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/320654 Trust: Jamal Carvalho <jamal@golang.org> Run-TryBot: Jamal Carvalho <jamal@golang.org> Reviewed-by: Julie Qiu <julie@golang.org>
2021-05-17 21:30:02 -04:00 · 2021-05-17 21:30:02 -04:00 · 737dbf6d9a
--- a/all.bash
+++ b/all.bash
@ -166,7 +166,7 @@ check_templates() {
 }


-script_hash_glob='content/static/html/**/*.tmpl'
+script_hash_glob='content/static/**/*.tmpl'

 # check_script_hashes checks that our CSP hashes match the ones
 # for our HTML scripts.
--- a/content/static/base/base.tmpl
+++ b/content/static/base/base.tmpl
@ -8,7 +8,9 @@
 <html lang="en">
  <head>
    <!-- This will capture unhandled errors during page load for reporting later. -->
-    <script>window.addEventListener('error', window.__err=function f(e){f.p=f.p||[];f.p.push(e)});</script>
+    <script>
+      window.addEventListener('error', window.__err=function f(e){f.p=f.p||[];f.p.push(e)});
+    </script>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
@ -23,13 +25,15 @@
    {{end}}
  </head>
  <body>
+    <!-- loadScript appends JS sources to the document head. It loads scripts as asynchronous
+        modules eliminating parser-blocking JavaScript. -->
    <script>
-      function loadScript(src, props = {}) {
+      function loadScript(src) {
        let s = document.createElement('script');
        s.src = src;
-        for (const [k, v] of Object.entries(props)) {
-          s[k] = v
-        }
+        s.type = 'module';
+        s.async = true;
+        s.defer = true
        document.head.appendChild(s);
      }
    </script>
@ -37,7 +41,7 @@
    {{template "main" .}}
    {{template "footer" .}}
    {{if .GoogleTagManagerID}}
-      <script async>
+      <script>
        // this will throw if the querySelector can’t find the element
        const gtmId = document.querySelector('.js-gtmID').dataset.gtmid;
        if (!gtmId) {
--- a/content/static/footer/footer.partial.tmpl
+++ b/content/static/footer/footer.partial.tmpl
--- a/content/static/header/header.partial.tmpl
+++ b/content/static/header/header.partial.tmpl
--- a/content/static/main-layout/main.css
+++ b/content/static/main-layout/main.css
@ -201,6 +201,7 @@
  :root[data-layout='compact'] .go-Main {
    grid-template-areas:
      'banner  banner'
+      'header  .'
      'header  nav'
      'aside   aside'
      'article article'
@ -217,7 +218,9 @@
    box-shadow: none;
  }
  :root[data-layout='compact'] .go-Main-nav--sticky {
+    height: var(--js-sticky-header-height);
    position: sticky;
+    top: 0;
  }
  :root[data-layout='compact'] .go-Main-nav--fixed {
    box-shadow: none;
@ -227,8 +230,6 @@
  }
  :root[data-layout='compact'] .go-Main-navMobile {
    display: flex;
-    margin-bottom: var(--gap);
-    margin-top: auto;
  }
 }

--- a/content/static/outline/outline.md
+++ b/content/static/outline/outline.md
@ -2,7 +2,7 @@

 ---

-### Tree
+### Tree {#outline-tree}

 ```html
 <ul class="go-Tree js-tree" role="tree">
@ -49,6 +49,8 @@
 </ul>
 ```

-### Dropdown
+### Select {#outline-select}

+```html
 <div class="js-select"></div>
+```
--- a/content/static/outline/tree.partial.tmpl
+++ b/content/static/outline/tree.partial.tmpl
--- a/content/static/styleguide/styleguide.css
+++ b/content/static/styleguide/styleguide.css
@ -41,6 +41,12 @@
 .StyleGuide .ColorIntent {
  grid-template-columns: repeat(auto-fit, 5rem [col-start] minmax(12rem, auto) [col-end]);
 }
+.StyleGuide .Outline {
+  align-items: flex-start;
+}
+.StyleGuide .Outline > span {
+  margin-top: 0.5rem;
+}
@media (min-width: 50rem) {
  .StyleGuide .Icon {
    grid-template-columns: 10rem 8rem auto;
@ -53,6 +59,7 @@
  .StyleGuide .Breadcrumb,
  .StyleGuide .Chip,
  .StyleGuide .Tooltip,
+  .StyleGuide .Outline,
  .StyleGuide .Clipboard {
    grid-template-columns: 20rem auto;
  }
@ -69,6 +76,7 @@
  .StyleGuide .Breadcrumb,
  .StyleGuide .Chip,
  .StyleGuide .Tooltip,
+  .StyleGuide .Outline,
  .StyleGuide .Clipboard {
    grid-template-columns: auto 50%;
  }
--- a/content/static/styleguide/styleguide.js
+++ b/content/static/styleguide/styleguide.js
--- a/content/static/styleguide/styleguide.js.map
+++ b/content/static/styleguide/styleguide.js.map
--- a/content/static/styleguide/styleguide.tmpl
+++ b/content/static/styleguide/styleguide.tmpl
@ -4,7 +4,7 @@
  license that can be found in the LICENSE file.
 -->

-{{define "title"}}<title>Stuide Guide</title>{{end}}
+{{define "title"}}<title>Style Guide · pkg.go.dev</title>{{end}}

 {{define "main-styles"}}
  <link href="/static/styleguide/styleguide.css" rel="stylesheet">
@ -33,7 +33,7 @@
    </ol>
  </nav>
  <div class="go-Main-headerContent">
-    <a class="go-Main-headerLogo" href="https://go.dev/" tabindex="-1" data-gtmc="header link"
+    <a class="go-Main-headerLogo" href="https://go.dev/" aria-hidden="true" tabindex="-1" data-gtmc="header link"
        aria-label="Link to Go Homepage">
      <img height="78" width="207" src="/static/logo/go-blue.svg" alt="Go">
    </a>
--- a/internal/frontend/server.go
+++ b/internal/frontend/server.go
@ -75,12 +75,13 @@ type ServerConfig struct {
 // NewServer creates a new Server for the given database and template directory.
 func NewServer(scfg ServerConfig) (_ *Server, err error) {
 	defer derrors.Wrap(&err, "NewServer(...)")
-	templateDir := template.TrustedSourceJoin(scfg.StaticPath, template.TrustedSourceFromConstant("html"))
+	templateDir := template.TrustedSourceJoin(scfg.StaticPath)
 	ts, err := parsePageTemplates(templateDir)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing templates: %v", err)
 	}
-	docTemplateDir := template.TrustedSourceJoin(templateDir, template.TrustedSourceFromConstant("doc"))
+	docTemplateDir := template.TrustedSourceJoin(templateDir, template.TrustedSourceFromConstant("html"),
+		template.TrustedSourceFromConstant("doc"))
 	dochtml.LoadTemplates(docTemplateDir)
 	s := &Server{
 		getDataSource:        scfg.DataSourceGetter,
@ -145,6 +146,7 @@ func (s *Server) Install(handle func(string, http.Handler), redisClient *redis.C
 	handle("/license-policy", s.licensePolicyHandler())
 	handle("/about", http.RedirectHandler("https://go.dev/about", http.StatusFound))
 	handle("/badge/", http.HandlerFunc(s.badgeHandler))
+	handle("/styleguide", http.HandlerFunc(s.errorHandler(s.serveStyleGuide)))
 	handle("/C", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Package "C" is a special case: redirect to /cmd/cgo.
 		// (This is what golang.org/C does.)
@ -553,24 +555,46 @@ func parsePageTemplates(base template.TrustedSource) (map[string]*template.Templ

 	templates := make(map[string]*template.Template)
 	for _, set := range htmlSets {
-		t, err := template.New("base.tmpl").Funcs(templateFuncs).ParseFilesFromTrustedSources(join(base, tsc("base.tmpl")))
+		t, err := template.New("base.tmpl").Funcs(templateFuncs).ParseFilesFromTrustedSources(join(base, tsc("html"), tsc("base.tmpl")))
 		if err != nil {
 			return nil, fmt.Errorf("ParseFiles: %v", err)
 		}
-		helperGlob := join(base, tsc("helpers"), tsc("*.tmpl"))
+		helperGlob := join(base, tsc("html"), tsc("helpers"), tsc("*.tmpl"))
 		if _, err := t.ParseGlobFromTrustedSource(helperGlob); err != nil {
 			return nil, fmt.Errorf("ParseGlob(%q): %v", helperGlob, err)
 		}

 		var files []template.TrustedSource
 		for _, f := range set {
-			files = append(files, join(base, tsc("pages"), f))
+			files = append(files, join(base, tsc("html"), tsc("pages"), f))
 		}
 		if _, err := t.ParseFilesFromTrustedSources(files...); err != nil {
 			return nil, fmt.Errorf("ParseFilesFromTrustedSources(%v): %v", files, err)
 		}
 		templates[set[0].String()] = t
 	}
+
+	styleGuideSets := [][]template.TrustedSource{
+		{tsc("styleguide"), tsc("main-layout")},
+	}
+	for _, set := range styleGuideSets {
+		t, err := template.New("base.tmpl").Funcs(templateFuncs).ParseFilesFromTrustedSources(join(base, tsc("base/base.tmpl")))
+		if err != nil {
+			return nil, fmt.Errorf("ParseFilesFromTrustedSources: %v", err)
+		}
+		helperGlob := join(base, tsc("**/*.partial.tmpl"))
+		if _, err := t.ParseGlobFromTrustedSource(helperGlob); err != nil {
+			return nil, fmt.Errorf("ParseGlobFromTrustedSource(%q): %v", helperGlob, err)
+		}
+		var files []template.TrustedSource
+		for _, f := range set {
+			if _, err := t.ParseGlobFromTrustedSource(join(base, f, tsc("*.tmpl"))); err != nil {
+				return nil, fmt.Errorf("ParseGlobFromTrustedSource(%v): %v", files, err)
+			}
+		}
+		templates[set[0].String()] = t
+	}
+
 	return templates, nil
 }

@ -581,7 +605,7 @@ func (s *Server) staticHandler() http.Handler {
 	// and rebuild them on file changes.
 	if s.devMode {
 		ctx := context.Background()
-		_, err := static.Build(static.Config{StaticPath: staticPath, Watch: true, Write: true})
+		_, err := static.Build(static.Config{StaticPath: staticPath + "/js", Watch: true, Write: true})
 		if err != nil {
 			log.Error(ctx, err)
 		}
--- a/internal/frontend/server_test.go
+++ b/internal/frontend/server_test.go
@ -1541,8 +1541,7 @@ func newTestServer(t *testing.T, proxyModules []*proxy.Module, redisClient *redi
 func TestCheckTemplates(t *testing.T) {
 	// Perform additional checks on parsed templates.
 	staticPath := template.TrustedSourceFromConstant("../../content/static")
-	templateDir := template.TrustedSourceJoin(staticPath, template.TrustedSourceFromConstant("html"))
-	templates, err := parsePageTemplates(templateDir)
+	templates, err := parsePageTemplates(staticPath)
 	if err != nil {
 		t.Fatal(err)
 	}
--- a/internal/frontend/styleguide.go
+++ b/internal/frontend/styleguide.go
@ -0,0 +1,219 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package frontend
+
+import (
+	"bytes"
+	"context"
+	"html"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/safehtml"
+	"github.com/google/safehtml/uncheckedconversions"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer"
+	ghtml "github.com/yuin/goldmark/renderer/html"
+	"github.com/yuin/goldmark/util"
+	"golang.org/x/pkgsite/internal"
+	"golang.org/x/pkgsite/internal/derrors"
+)
+
+// serveStyleGuide serves the styleguide page, the content of which is
+// generated from the markdown files in content/static.
+func (s *Server) serveStyleGuide(w http.ResponseWriter, r *http.Request, ds internal.DataSource) error {
+	ctx := r.Context()
+	page, err := styleGuide(ctx, s.staticPath.String())
+	page.basePage = s.newBasePage(r, "")
+	if err != nil {
+		return err
+	}
+	s.servePage(ctx, w, "styleguide", page)
+	return nil
+}
+
+type styleGuidePage struct {
+	basePage
+	Sections []*StyleSection
+	Outline  []*Heading
+}
+
+// styleGuide collects the paths to the markdown files in content/static,
+// renders them into sections for the styleguide, and merges the document
+// outlines into a single page outline.
+func styleGuide(ctx context.Context, staticPath string) (_ *styleGuidePage, err error) {
+	defer derrors.WrapStack(&err, "styleGuide(%q)", staticPath)
+	files, err := markdownFiles(staticPath)
+	if err != nil {
+		return nil, err
+	}
+	var sections []*StyleSection
+	for _, f := range files {
+		doc, err := styleSection(ctx, f)
+		if err != nil {
+			return nil, err
+		}
+		sections = append(sections, doc)
+	}
+	var outline []*Heading
+	for _, s := range sections {
+		outline = append(outline, s.Outline...)
+	}
+	return &styleGuidePage{
+		Sections: sections,
+		Outline:  outline,
+	}, nil
+}
+
+// StyleSection represents a section on the styleguide page.
+type StyleSection struct {
+	// ID is the ID for the header element of the section.
+	ID string
+
+	// Title is the title of the section, taken from the name
+	// of the markdown file.
+	Title string
+
+	// Content is the HTML rendered from the parsed markdown file.
+	Content safehtml.HTML
+
+	// Outline is a collection of headings used in the navigation.
+	Outline []*Heading
+}
+
+// styleSection uses goldmark to parse a markdown file and render
+// a section of the styleguide.
+func styleSection(ctx context.Context, filename string) (_ *StyleSection, err error) {
+	defer derrors.WrapStack(&err, "styleSection(%q)", filename)
+	var buf bytes.Buffer
+	source, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	// We set priority values so that we always use our custom transformer
+	// instead of the default ones. The default values are in:
+	// https://github.com/yuin/goldmark/blob/7b90f04af43131db79ec320be0bd4744079b346f/parser/parser.go#L567
+	const (
+		astTransformerPriority = 10000
+		nodeRenderersPriority  = 100
+	)
+	et := &extractTOC{ctx: ctx}
+	md := goldmark.New(
+		goldmark.WithExtensions(extension.GFM),
+		goldmark.WithParserOptions(
+			parser.WithAutoHeadingID(),
+			parser.WithAttribute(),
+			parser.WithASTTransformers(
+				util.Prioritized(et, astTransformerPriority),
+			),
+		),
+		goldmark.WithRendererOptions(
+			renderer.WithNodeRenderers(
+				util.Prioritized(&guideRenderer{}, nodeRenderersPriority),
+			),
+			ghtml.WithUnsafe(),
+			ghtml.WithXHTML(),
+		),
+	)
+
+	if err := md.Convert(source, &buf); err != nil {
+		return nil, err
+	}
+
+	id := strings.TrimSuffix(filepath.Base(filename), ".md")
+	return &StyleSection{
+		ID:      id,
+		Title:   camelCase(id),
+		Content: uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(buf.String()),
+		Outline: et.Headings,
+	}, nil
+}
+
+// guideRenderer is a renderer.NodeRenderer implementation that renders
+// styleguide sections.
+type guideRenderer struct {
+	ghtml.Config
+}
+
+func (r *guideRenderer) writeLines(w util.BufWriter, source []byte, n ast.Node) {
+	l := n.Lines().Len()
+	for i := 0; i < l; i++ {
+		line := n.Lines().At(i)
+		w.Write(line.Value(source))
+	}
+}
+
+func (r *guideRenderer) writeEscapedLines(w util.BufWriter, source []byte, n ast.Node) {
+	l := n.Lines().Len()
+	for i := 0; i < l; i++ {
+		line := n.Lines().At(i)
+		w.Write([]byte(html.EscapeString(string(line.Value(source)))))
+	}
+}
+
+// renderFencedCodeBlock writes html code snippets twice, once as actual
+// html for the page and again as a code snippet.
+func (r *guideRenderer) renderFencedCodeBlock(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	n := node.(*ast.FencedCodeBlock)
+	w.WriteString("<span>\n")
+	r.writeLines(w, source, n)
+	w.WriteString("</span>\n")
+	w.WriteString("<pre class=\"StringifyElement-markup js-clipboard\">\n")
+	r.writeEscapedLines(w, source, n)
+	w.WriteString("</pre>\n")
+	return ast.WalkContinue, nil
+}
+
+// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs.
+func (r *guideRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
+	reg.Register(ast.KindFencedCodeBlock, r.renderFencedCodeBlock)
+}
+
+// markdownFiles walks the content/static directory and collects
+// the paths to markdown files.
+func markdownFiles(dir string) ([]string, error) {
+	var matches []string
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if matched, err := filepath.Match("*.md", filepath.Base(path)); err != nil {
+			return err
+		} else if matched {
+			matches = append(matches, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return matches, nil
+}
+
+// camelCase turns a snake cased strink into a camel case string.
+// For example, hello-world becomes HelloWorld. This function is
+// used to ensure proper casing in the classnames of the style
+// sections.
+func camelCase(s string) string {
+	p := strings.Split(s, "-")
+	var o []string
+	for _, v := range p {
+		o = append(o, strings.Title(v))
+	}
+	return strings.Join(o, "")
+}
--- a/internal/middleware/secureheaders.go
+++ b/internal/middleware/secureheaders.go
@ -11,6 +11,10 @@ import (
 )

 var scriptHashes = []string{
+	// From content/static/base/base.tmpl
+	"'sha256-CoGrkqEM1Kjjf5b1bpcnDLl8ZZLAsVX+BoAzZ5+AOmc='",
+	"'sha256-3YbNePu1zD/B/1vcR3xg4CvNdno3XxbHPOPB+s4Sc0U='",
+	"'sha256-karKh1IrXOF1g+uoSxK+k9BuciCwYY/ytGuQVUiRzcM='",
 	// From content/static/html/base.tmpl
 	"'sha256-CgM7SjnSbDyuIteS+D1CQuSnzyKwL0qtXLU6ZW2hB+g='",
 	"'sha256-dwce5DnVX7uk6fdvvNxQyLTH/cJrTMDK6zzrdKwdwcg='",
@ -29,6 +33,8 @@ var scriptHashes = []string{
 	"'sha256-hb8VdkRSeBmkNlbshYmBnkYWC/BYHCPiz5s7liRcZNM='",
 	// From content/static/html/pages/unit_versions.tmpl
 	"'sha256-KBdPSv2Ajjw3jsa29qBhRW49nNx3jXxOLZIWX545FCA='",
+	// From content/static/styleguide/styleguide.tmpl
+	"'sha256-Z9STHpM3Fz5XojcH5dbUK50Igi6qInBbVVaqNpjL/HY='",
 }

 // SecureHeaders adds a content-security-policy and other security-related