content/static: use embed and io/fs

This lets us delete the generated static.go.

For golang/go#41102

Change-Id: Ie09f34a83f114592eec4ba2dd9263285169374ae
Reviewed-on: https://go-review.googlesource.com/c/website/+/291690
Trust: Russ Cox <rsc@golang.org>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
This commit is contained in:
Russ Cox 2021-02-12 14:29:00 -05:00
Родитель 546f845b33
Коммит aefff79d07
16 изменённых файлов: 36 добавлений и 559 удалений

Просмотреть файл

@ -4,7 +4,7 @@
.PHONY: usage
GO_REF ?= release-branch.go1.15
GO_REF ?= release-branch.go1.16
WEBSITE_HEAD := $(shell git rev-parse HEAD)
WEBSITE_CLEAN := $(shell (git status --porcelain | grep -q .) && echo dirty || echo clean)
ifeq ($(WEBSITE_CLEAN),clean)

Просмотреть файл

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.16
// +build golangorg
package main
@ -25,7 +26,6 @@ import (
"golang.org/x/tools/godoc"
"golang.org/x/tools/godoc/vfs"
"golang.org/x/tools/godoc/vfs/gatefs"
"golang.org/x/tools/godoc/vfs/mapfs"
"golang.org/x/tools/godoc/vfs/zipfs"
"golang.org/x/website/content/static"
"golang.org/x/website/internal/dl"
@ -79,8 +79,8 @@ func main() {
// go repository. This lets us update some documentation outside the
// Go release cycle. This includes root.html, which redirects to "/".
// See golang.org/issue/29206.
fs.Bind("/doc", mapfs.New(static.Files), "/doc", vfs.BindBefore)
fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
fs.Bind("/doc", vfs.FromFS(static.FS), "/doc", vfs.BindBefore)
fs.Bind("/lib/godoc", vfs.FromFS(static.FS), "/", vfs.BindReplace)
webroot := getFullPath("/src/golang.org/x/website")
fs.Bind("/favicon.ico", gatefs.New(vfs.OS(webroot), fsGate), "/favicon.ico", vfs.BindBefore)

13
cmd/golangorg/go115.go Normal file
Просмотреть файл

@ -0,0 +1,13 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !go1.16
package main
import "log"
func main() {
log.Fatalf("golangorg requires Go 1.16 or later")
}

Просмотреть файл

@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.16
package main_test
import (

Просмотреть файл

@ -18,6 +18,7 @@
// Some pages are being transitioned from $GOROOT to content/static/doc.
// See golang.org/issue/29206 and golang.org/issue/33637.
// +build go1.16
// +build !golangorg
package main
@ -39,7 +40,6 @@ import (
"golang.org/x/tools/godoc"
"golang.org/x/tools/godoc/vfs"
"golang.org/x/tools/godoc/vfs/gatefs"
"golang.org/x/tools/godoc/vfs/mapfs"
"golang.org/x/tools/godoc/vfs/zipfs"
"golang.org/x/website/content/static"
)
@ -172,8 +172,8 @@ func main() {
fs.Bind("/doc", vfs.OS(*templateDir), "/doc", vfs.BindBefore)
fs.Bind("/lib/godoc", vfs.OS(*templateDir), "/", vfs.BindBefore)
} else {
fs.Bind("/doc", mapfs.New(static.Files), "/doc", vfs.BindBefore)
fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
fs.Bind("/doc", vfs.FromFS(static.FS), "/doc", vfs.BindBefore)
fs.Bind("/lib/godoc", vfs.FromFS(static.FS), "/", vfs.BindReplace)
}
// Bind $GOPATH trees into Go root.

Просмотреть файл

@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.16
package main
import (
@ -12,7 +14,6 @@ import (
"golang.org/x/tools/godoc"
"golang.org/x/tools/godoc/vfs"
"golang.org/x/tools/godoc/vfs/mapfs"
"golang.org/x/website/content/static"
)
@ -26,8 +27,8 @@ func TestReleaseHistory(t *testing.T) {
origFS, origPres := fs, pres
defer func() { fs, pres = origFS, origPres }()
fs = vfs.NameSpace{}
fs.Bind("/doc", mapfs.New(static.Files), "/doc", vfs.BindBefore)
fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
fs.Bind("/doc", vfs.FromFS(static.FS), "/doc", vfs.BindBefore)
fs.Bind("/lib/godoc", vfs.FromFS(static.FS), "/", vfs.BindReplace)
pres = godoc.NewPresentation(godoc.NewCorpus(fs))
readTemplates(pres)
mux := registerHandlers(pres)

Просмотреть файл

@ -6,46 +6,3 @@ binary when serving the [golang.org](https://golang.org) website.
The details of the directory to path mapping are documented at the top of
[`cmd/golangorg/main.go`](https://go.googlesource.com/website/+/refs/heads/master/cmd/golangorg/main.go).
TODO(dmitshur): The process below can be simplified.
See [golang.org/issue/29206#issuecomment-536099768](https://golang.org/issue/29206#issuecomment-536099768).
## Development mode
In production, CSS/JS/template assets need to be compiled into the `golangorg`
binary. It can be tedious to recompile assets every time, but you can pass a
flag to load CSS/JS/templates from disk every time a page loads:
```
golangorg -templates=$GOPATH/src/golang.org/x/website/content/static -http=:6060
```
## Recompiling static assets
Files such as `static/style.css`, `static/doc/copyright.html` and so on are not
present in the final binary. They are embedded into `static/static.go` by running
`go generate`. To compile a change and test it in your browser:
1) Make changes to an existing file such as `static/style.css`.
2) If a new file is being added to the `static` directory, add it to the `files`
slice in `static/internal/gen/gen.go`.
3) Run `go generate golang.org/x/website/content/static` so `static/static.go` is
up to date.
4) Run `go run golang.org/x/website/cmd/golangorg -http=:6060` and view your changes
in the browser at http://localhost:6060. You may need to disable your browser's cache
to avoid reloading a stale file.
A test exists to catch a possible mistake of forgetting to regenerate static assets:
```
website $ go test ./...
--- FAIL: TestStaticIsUpToDate (0.06s)
gen_test.go:27: static.go is stale. Run:
$ go generate golang.org/x/website/content/static
$ git diff
to see the differences.
FAIL
FAIL golang.org/x/website/content/static 0.650s
```

Просмотреть файл

@ -2,9 +2,13 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package static exports a map of static file content that supports the godoc
// user interface. The map should be used with the mapfs package, see
// golang.org/x/tools/godoc/vfs/mapfs.
package static // import "golang.org/x/website/content/static"
// +build go1.16
//go:generate go run makestatic.go
// Package static exports the static content as an embed.FS.
package static
import "embed"
// FS is the static content as a file system.
//go:embed *
var FS embed.FS

Просмотреть файл

@ -1,176 +0,0 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package gen is used by content/static/makestatic.go
// to generate content/static/static.go.
//
// This is a separate package so that it can be tested without
// build constraints. cmd/golangorg and other binaries should not
// depend on it.
package gen
import (
"bytes"
"fmt"
"go/format"
"io/ioutil"
"strings"
"unicode"
"golang.org/x/website/internal/markdown"
)
var files = []string{
"analysis/call-eg.png",
"analysis/call3.png",
"analysis/callers1.png",
"analysis/callers2.png",
"analysis/chan1.png",
"analysis/chan2a.png",
"analysis/chan2b.png",
"analysis/error1.png",
"analysis/help.html",
"analysis/ident-def.png",
"analysis/ident-field.png",
"analysis/ident-func.png",
"analysis/ipcg-func.png",
"analysis/ipcg-pkg.png",
"analysis/typeinfo-pkg.png",
"analysis/typeinfo-src.png",
"codewalk.html",
"codewalkdir.html",
"dirlist.html",
"doc/code.html",
"doc/conduct.html",
"doc/contrib.html",
"doc/copyright.html",
"doc/devel/pre_go1.html",
"doc/devel/release.html",
"doc/devel/weekly.html",
"doc/docs.html",
"doc/download.js",
"doc/gopath_code.html",
"doc/hats.js",
"doc/install.html",
"doc/install-source.html",
"doc/manage-install.html",
"doc/modules/images/multiple-modules.png",
"doc/modules/images/single-module.png",
"doc/modules/images/source-hierarchy.png",
"doc/modules/images/v2-branch-module.png",
"doc/modules/images/v2-module.png",
"doc/modules/images/version-number.png",
"doc/mvs/buildlist.svg",
"doc/mvs/downgrade.svg",
"doc/mvs/exclude.svg",
"doc/mvs/get-downgrade.svg",
"doc/mvs/get-upgrade.svg",
"doc/mvs/replace.svg",
"doc/mvs/upgrade.svg",
"doc/root.html",
"doc/security.html",
"doc/tutorial/add-a-test.html",
"doc/tutorial/call-module-code.html",
"doc/tutorial/compile-install.html",
"doc/tutorial/create-module.html",
"doc/tutorial/getting-started.html",
"doc/tutorial/greetings-multiple-people.html",
"doc/tutorial/handle-errors.html",
"doc/tutorial/images/function-syntax.png",
"doc/tutorial/index.html",
"doc/tutorial/random-greeting.html",
"error.html",
"example.html",
"godoc.html",
"godocs.js",
"images/cloud-download.svg",
"images/footer-gopher.jpg",
"images/go-logo-blue.svg",
"images/home-gopher.png",
"images/minus.gif",
"images/play-link.svg",
"images/plus.gif",
"jquery.js",
"opensearch.xml",
"package.html",
"packageroot.html",
"play.js",
"playground.js",
"search.html",
"searchcode.html",
"searchdoc.html",
"searchtxt.html",
"style.css",
}
var markdownFiles = []string{
"doc/mod.md",
"doc/modules/developing.md",
"doc/modules/gomod-ref.md",
"doc/modules/major-version.md",
"doc/modules/managing-dependencies.md",
"doc/modules/managing-source.md",
"doc/modules/publishing.md",
"doc/modules/release-workflow.md",
"doc/modules/version-numbers.md",
}
// Generate reads a set of files and returns a file buffer that declares
// a map of string constants containing contents of the input files.
func Generate() ([]byte, error) {
buf := new(bytes.Buffer)
fmt.Fprintf(buf, "%v\n\n%v\n\npackage static\n\n", license, warning)
fmt.Fprintf(buf, "var Files = map[string]string{\n")
for _, fn := range files {
b, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
}
fmt.Fprintf(buf, "\t%q: ", fn)
appendQuote(buf, b)
fmt.Fprintf(buf, ",\n\n")
}
for _, fn := range markdownFiles {
src, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
}
gen, err := markdown.Render(src)
if err != nil {
return nil, fmt.Errorf("%s: %v", fn, err)
}
htmlName := strings.TrimSuffix(fn, ".md") + ".html"
fmt.Fprintf(buf, "\t%q: ", htmlName)
appendQuote(buf, gen)
fmt.Fprintf(buf, ",\n\n")
}
fmt.Fprintln(buf, "}")
return format.Source(buf.Bytes())
}
// appendQuote is like strconv.AppendQuote, but we avoid the latter
// because it changes when Unicode evolves, breaking gen_test.go.
func appendQuote(out *bytes.Buffer, data []byte) {
out.WriteByte('"')
for _, b := range data {
if b == '\\' || b == '"' {
out.WriteByte('\\')
out.WriteByte(b)
} else if b <= unicode.MaxASCII && unicode.IsPrint(rune(b)) && !unicode.IsSpace(rune(b)) {
out.WriteByte(b)
} else {
fmt.Fprintf(out, "\\x%02x", b)
}
}
out.WriteByte('"')
}
const warning = `// Code generated by "makestatic"; DO NOT EDIT.`
const license = `// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.`

Просмотреть файл

@ -1,28 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gen
import (
"bytes"
"strconv"
"testing"
"unicode"
)
// TestAppendQuote ensures that AppendQuote produces a valid literal.
func TestAppendQuote(t *testing.T) {
var in, out bytes.Buffer
for r := rune(0); r < unicode.MaxRune; r++ {
in.WriteRune(r)
}
appendQuote(&out, in.Bytes())
in2, err := strconv.Unquote(out.String())
if err != nil {
t.Fatalf("AppendQuote produced invalid string literal: %v", err)
}
if got, want := in2, in.String(); got != want {
t.Fatal("AppendQuote modified string") // no point printing got/want: huge
}
}

Просмотреть файл

@ -1,36 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
// Command makestatic writes the generated file buffer to "static.go".
// It is intended to be invoked via "go generate" (directive in "doc.go").
package main
import (
"fmt"
"io/ioutil"
"os"
"golang.org/x/website/content/static/internal/gen"
)
func main() {
if err := makestatic(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func makestatic() error {
buf, err := gen.Generate()
if err != nil {
return fmt.Errorf("error while generating static.go: %v\n", err)
}
err = ioutil.WriteFile("static.go", buf, 0666)
if err != nil {
return fmt.Errorf("error while writing static.go: %v\n", err)
}
return nil
}

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Просмотреть файл

@ -1,33 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package static
import (
"bytes"
"io/ioutil"
"testing"
"golang.org/x/website/content/static/internal/gen"
)
func TestStaticIsUpToDate(t *testing.T) {
oldBuf, err := ioutil.ReadFile("static.go")
if err != nil {
t.Errorf("error while reading static.go: %v\n", err)
}
newBuf, err := gen.Generate()
if err != nil {
t.Errorf("error while generating static.go: %v\n", err)
}
if bytes.Compare(oldBuf, newBuf) != 0 {
t.Error(`static.go is stale. Run:
$ go generate golang.org/x/website/content/static
$ git diff
to see the differences.`)
}
}

2
go.mod
Просмотреть файл

@ -1,6 +1,6 @@
module golang.org/x/website
go 1.11
go 1.16
require (
cloud.google.com/go v0.58.0 // indirect

2
go.sum
Просмотреть файл

@ -32,7 +32,6 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0 h1:86K1Gel7BQ9/WmNWn7dTKMvTLFzwtBe5FNqYbi9X35g=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@ -98,7 +97,6 @@ github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=

Просмотреть файл

@ -1,38 +0,0 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package markdown provides a wrapper for rendering Markdown. It is intended
// to be used on the golang.org website.
//
// This package is not intended for general use, and its API is not guaranteed
// to be stable.
package markdown
import (
"bytes"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer/html"
)
// Render converts a limited and opinionated flavor of Markdown (compliant with
// CommonMark 0.29) to HTML for the purposes of golang.org websites. This should
// not be adjusted except for the needs of *.golang.org.
//
// The Markdown source may contain raw HTML and Go templates. Sanitization of
// untrusted content is not performed: the caller is responsible for ensuring
// that only trusted content is provided.
func Render(src []byte) ([]byte, error) {
// parser.WithHeadingAttribute allows custom ids on headings.
// html.WithUnsafe allows use of raw HTML, which we need for tables.
md := goldmark.New(
goldmark.WithParserOptions(parser.WithHeadingAttribute()),
goldmark.WithRendererOptions(html.WithUnsafe()))
var buf bytes.Buffer
if err := md.Convert(src, &buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}