integrate cli yamldocs code (with history)

Merge remote-tracking branch 'cli_yamldocs/master' into HEAD

    # install filter-repo (https://github.com/newren/git-filter-repo/blob/main/INSTALL.md)
    brew install git-filter-repo

    # create a temporary clone of docker/cli
    git clone https://github.com/docker/cli.git ~/Projects/cli_yamldocs
    cd cli_yamldocs

    # remove all code, except for docs/yaml, and rename to docs/yamlgen
    git filter-repo  --path docs/yaml --path-rename docs/yaml:docs/yamlgen

    # exclude the Dockerfile
    git filter-repo --path-glob 'docs/yamlgen/Dockerfile' --invert-paths

    # go to the target github.com/docker/docgen repository
    cd ~/projects/docgen

    # add the temporary repository as an upstream and make sure it's up-to-date
    git remote add cli_yamldocs ~/projects/cli_yamldocs
    git fetch cli_yamldocs

    # create a branch to work with
    git checkout -b import_history

    # do an interactive rebase
    git rebase -i -S aaa47f8fbd

    # set breakpoint before first import of yamldocs
    break
    pick 1e7a862 Add YAML-docs generator
    pick b3375b7 Move sources and add go mod
    pick 7b04bac README
    pick 2bf814d Example
    pick b5743a3 Container based developer flow
    pick e3997d7 Rename
    pick cd391b6 Dependabot
    pick 08d8459 GitHub workflow
    pick 75d4c97 Fix linter and add tests
    pick aaff066 Rename package and move `LoadLongDescription` func
    pick 05a4448 docs: add md generation
    pick e7a1fd1 Refactor markdown generator
    pick 10c276e Add `GenTree` func
    pick 8efbe43 Cleanup
    pick 5ca2026 Fix godev
    pick 4ed3e61 Add CodeQL workflow
    pick d739cb1 Basic test
    pick b75cd36 docs: add external docs links support
    pick e24b031 yamldocs: various improvements
    pick 80ca920 remove codeql for now
    pick 85eca43 yaml: fix tests
    pick 9b28897 lint
    pick f85801e add code of conduct and contributing notes
    pick e1298a2 readme for example
    pick 450385b Update README
    pick 0435ec1 Call pkg.go.dev when a new tag is pushed (force refresh)

    # merge the upstream code
    git merge --allow-unrelated-histories --signoff -S cli_yamldocs/master

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Sebastiaan van Stijn 2021-08-03 21:44:54 +02:00
Родитель aaa47f8fbd 77f5372c6c
Коммит 28aa8d80a0
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 76698F39D527CE8C
4 изменённых файлов: 669 добавлений и 0 удалений

117
docs/yamlgen/generate.go Normal file
Просмотреть файл

@ -0,0 +1,117 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/command/commands"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
const descriptionSourcePath = "docs/reference/commandline/"
func generateCliYaml(opts *options) error {
dockerCLI, err := command.NewDockerCli()
if err != nil {
return err
}
cmd := &cobra.Command{
Use: "docker [OPTIONS] COMMAND [ARG...]",
Short: "The base command for the Docker CLI.",
}
commands.AddCommands(cmd, dockerCLI)
disableFlagsInUseLine(cmd)
source := filepath.Join(opts.source, descriptionSourcePath)
fmt.Println("Markdown source:", source)
if err := loadLongDescription(cmd, source); err != nil {
return err
}
if err := os.MkdirAll(opts.target, 0755); err != nil {
return err
}
cmd.DisableAutoGenTag = true
return GenYamlTree(cmd, opts.target)
}
func disableFlagsInUseLine(cmd *cobra.Command) {
visitAll(cmd, func(ccmd *cobra.Command) {
// do not add a `[flags]` to the end of the usage line.
ccmd.DisableFlagsInUseLine = true
})
}
// visitAll will traverse all commands from the root.
// This is different from the VisitAll of cobra.Command where only parents
// are checked.
func visitAll(root *cobra.Command, fn func(*cobra.Command)) {
for _, cmd := range root.Commands() {
visitAll(cmd, fn)
}
fn(root)
}
func loadLongDescription(parentCmd *cobra.Command, path string) error {
for _, cmd := range parentCmd.Commands() {
if cmd.HasSubCommands() {
if err := loadLongDescription(cmd, path); err != nil {
return err
}
}
name := cmd.CommandPath()
log.Println("INFO: Generating docs for", name)
if i := strings.Index(name, " "); i >= 0 {
// remove root command / binary name
name = name[i+1:]
}
if name == "" {
continue
}
mdFile := strings.ReplaceAll(name, " ", "_") + ".md"
fullPath := filepath.Join(path, mdFile)
content, err := ioutil.ReadFile(fullPath)
if os.IsNotExist(err) {
log.Printf("WARN: %s does not exist, skipping\n", mdFile)
continue
}
if err != nil {
return err
}
applyDescriptionAndExamples(cmd, string(content))
}
return nil
}
type options struct {
source string
target string
}
func parseArgs() (*options, error) {
opts := &options{}
cwd, _ := os.Getwd()
flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
flags.StringVar(&opts.source, "root", cwd, "Path to project root")
flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files")
err := flags.Parse(os.Args[1:])
return opts, err
}
func main() {
opts, err := parseArgs()
if err != nil {
log.Println(err)
}
fmt.Println("Project root: ", opts.source)
fmt.Println("YAML output dir:", opts.target)
if err := generateCliYaml(opts); err != nil {
log.Println("Failed to generate yaml files:", err)
}
}

73
docs/yamlgen/markdown.go Normal file
Просмотреть файл

@ -0,0 +1,73 @@
package main
import (
"regexp"
"strings"
"unicode"
)
var (
// mdHeading matches MarkDown H1..h6 headings. Note that this regex may produce
// false positives for (e.g.) comments in code-blocks (# this is a comment),
// so should not be used as a generic regex for other purposes.
mdHeading = regexp.MustCompile(`^([#]{1,6})\s(.*)$`)
// htmlAnchor matches inline HTML anchors. This is intended to only match anchors
// for our use-case; DO NOT consider using this as a generic regex, or at least
// not before reading https://stackoverflow.com/a/1732454/1811501.
htmlAnchor = regexp.MustCompile(`<a\s+(?:name|id)="?([^"]+)"?\s*></a>\s*`)
)
// getSections returns all H2 sections by title (lowercase)
func getSections(mdString string) map[string]string {
parsedContent := strings.Split("\n"+mdString, "\n## ")
sections := make(map[string]string, len(parsedContent))
for _, s := range parsedContent {
if strings.HasPrefix(s, "#") {
// not a H2 Section
continue
}
parts := strings.SplitN(s, "\n", 2)
if len(parts) == 2 {
sections[strings.ToLower(parts[0])] = parts[1]
}
}
return sections
}
// cleanupMarkDown cleans up the MarkDown passed in mdString for inclusion in
// YAML. It removes trailing whitespace and substitutes tabs for four spaces
// to prevent YAML switching to use "compact" form; ("line1 \nline\t2\n")
// which, although equivalent, is hard to read.
func cleanupMarkDown(mdString string) (md string, anchors []string) {
// remove leading/trailing whitespace, and replace tabs in the whole content
mdString = strings.TrimSpace(mdString)
mdString = strings.ReplaceAll(mdString, "\t", " ")
mdString = strings.ReplaceAll(mdString, "https://docs.docker.com", "")
var id string
// replace trailing whitespace per line, and handle custom anchors
lines := strings.Split(mdString, "\n")
for i := 0; i < len(lines); i++ {
lines[i] = strings.TrimRightFunc(lines[i], unicode.IsSpace)
lines[i], id = convertHTMLAnchor(lines[i])
if id != "" {
anchors = append(anchors, id)
}
}
return strings.Join(lines, "\n"), anchors
}
// convertHTMLAnchor converts inline anchor-tags in headings (<a name=myanchor></a>)
// to an extended-markdown property ({#myanchor}). Extended Markdown properties
// are not supported in GitHub Flavored Markdown, but are supported by Jekyll,
// and lead to cleaner HTML in our docs, and prevents duplicate anchors.
// It returns the converted MarkDown heading and the custom ID (if present)
func convertHTMLAnchor(mdLine string) (md string, customID string) {
if m := mdHeading.FindStringSubmatch(mdLine); len(m) > 0 {
if a := htmlAnchor.FindStringSubmatch(m[2]); len(a) > 0 {
customID = a[1]
mdLine = m[1] + " " + htmlAnchor.ReplaceAllString(m[2], "") + " {#" + customID + "}"
}
}
return mdLine, customID
}

Просмотреть файл

@ -0,0 +1,132 @@
package main
import "testing"
func TestCleanupMarkDown(t *testing.T) {
tests := []struct {
doc, in, expected string
}{
{
doc: "whitespace around sections",
in: `
## Section start
Some lines.
And more lines.
`,
expected: `## Section start
Some lines.
And more lines.`,
},
{
doc: "lines with inline tabs",
in: `## Some Heading
A line with tabs in it.
Tabs should be replaced by spaces`,
expected: `## Some Heading
A line with tabs in it.
Tabs should be replaced by spaces`,
},
{
doc: "lines with trailing spaces",
in: `## Some Heading with spaces
This is a line.
This is an indented line
### Some other heading
Last line.`,
expected: `## Some Heading with spaces
This is a line.
This is an indented line
### Some other heading
Last line.`,
},
{
doc: "lines with trailing tabs",
in: `## Some Heading with tabs
This is a line.
This is an indented line
### Some other heading
Last line.`,
expected: `## Some Heading with tabs
This is a line.
This is an indented line
### Some other heading
Last line.`,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.doc, func(t *testing.T) {
out, _ := cleanupMarkDown(tc.in)
if out != tc.expected {
t.Fatalf("\nexpected:\n%q\nactual:\n%q\n", tc.expected, out)
}
})
}
}
func TestConvertHTMLAnchor(t *testing.T) {
tests := []struct {
in, id, expected string
}{
{
in: `# <a name=heading1></a> Heading 1`,
id: "heading1",
expected: `# Heading 1 {#heading1}`,
},
{
in: `## Heading 2<a name=heading2></a> `,
id: "heading2",
expected: `## Heading 2 {#heading2}`,
},
{
in: `### <a id=heading3></a>Heading 3`,
id: "heading3",
expected: `### Heading 3 {#heading3}`,
},
{
in: `#### <a id="heading4"></a> Heading 4`,
id: "heading4",
expected: `#### Heading 4 {#heading4}`,
},
{
in: `##### <a id="heading5" ></a> Heading 5`,
id: "heading5",
expected: `##### Heading 5 {#heading5}`,
},
{
in: `###### <a id=hello href=foo>hello!</a>Heading 6`,
id: "",
expected: `###### <a id=hello href=foo>hello!</a>Heading 6`,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.in, func(t *testing.T) {
out, id := convertHTMLAnchor(tc.in)
if id != tc.id {
t.Fatalf("expected: %s, actual: %s\n", tc.id, id)
}
if out != tc.expected {
t.Fatalf("\nexpected: %s\nactual: %s\n", tc.expected, out)
}
})
}
}

347
docs/yamlgen/yaml.go Normal file
Просмотреть файл

@ -0,0 +1,347 @@
package main
import (
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
yaml "gopkg.in/yaml.v2"
)
type cmdOption struct {
Option string
Shorthand string `yaml:",omitempty"`
ValueType string `yaml:"value_type,omitempty"`
DefaultValue string `yaml:"default_value,omitempty"`
Description string `yaml:",omitempty"`
DetailsURL string `yaml:"details_url,omitempty"` // DetailsURL contains an anchor-id or link for more information on this flag
Deprecated bool
MinAPIVersion string `yaml:"min_api_version,omitempty"`
Experimental bool
ExperimentalCLI bool
Kubernetes bool
Swarm bool
OSType string `yaml:"os_type,omitempty"`
}
type cmdDoc struct {
Name string `yaml:"command"`
SeeAlso []string `yaml:"parent,omitempty"`
Version string `yaml:"engine_version,omitempty"`
Aliases string `yaml:",omitempty"`
Short string `yaml:",omitempty"`
Long string `yaml:",omitempty"`
Usage string `yaml:",omitempty"`
Pname string `yaml:",omitempty"`
Plink string `yaml:",omitempty"`
Cname []string `yaml:",omitempty"`
Clink []string `yaml:",omitempty"`
Options []cmdOption `yaml:",omitempty"`
InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"`
Example string `yaml:"examples,omitempty"`
Deprecated bool
MinAPIVersion string `yaml:"min_api_version,omitempty"`
Experimental bool
ExperimentalCLI bool
Kubernetes bool
Swarm bool
OSType string `yaml:"os_type,omitempty"`
}
// GenYamlTree creates yaml structured ref files
func GenYamlTree(cmd *cobra.Command, dir string) error {
emptyStr := func(s string) string { return "" }
return GenYamlTreeCustom(cmd, dir, emptyStr)
}
// GenYamlTreeCustom creates yaml structured ref files
func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string) string) error {
for _, c := range cmd.Commands() {
if !c.Runnable() && !c.HasAvailableSubCommands() {
// skip non-runnable commands without subcommands
// but *do* generate YAML for hidden and deprecated commands
// the YAML will have those included as metadata, so that the
// documentation repository can decide whether or not to present them
continue
}
if err := GenYamlTreeCustom(c, dir, filePrepender); err != nil {
return err
}
}
// TODO: conditionally skip the root command (for plugins)
//
// The "root" command used in the generator is just a "stub", and only has a
// list of subcommands, but not (e.g.) global options/flags. We should fix
// that, so that the YAML file for the docker "root" command contains the
// global flags.
//
// If we're using this code to generate YAML docs for a plugin, the root-
// command is even less useful; in that case, the root command represents
// the "docker" command, and is a "dummy" with no flags, and only a single
// subcommand (the plugin's top command). For plugins, we should skip the
// root command altogether, to prevent generating a useless YAML file.
// if !cmd.HasParent() {
// return nil
// }
basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml"
filename := filepath.Join(dir, basename)
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
if _, err := io.WriteString(f, filePrepender(filename)); err != nil {
return err
}
return GenYamlCustom(cmd, f)
}
// GenYamlCustom creates custom yaml output
// nolint: gocyclo
func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
const (
// shortMaxWidth is the maximum width for the "Short" description before
// we force YAML to use multi-line syntax. The goal is to make the total
// width fit within 80 characters. This value is based on 80 characters
// minus the with of the field, colon, and whitespace ('short: ').
shortMaxWidth = 73
// longMaxWidth is the maximum width for the "Short" description before
// we force YAML to use multi-line syntax. The goal is to make the total
// width fit within 80 characters. This value is based on 80 characters
// minus the with of the field, colon, and whitespace ('long: ').
longMaxWidth = 74
)
cliDoc := cmdDoc{
Name: cmd.CommandPath(),
Aliases: strings.Join(cmd.Aliases, ", "),
Short: forceMultiLine(cmd.Short, shortMaxWidth),
Long: forceMultiLine(cmd.Long, longMaxWidth),
Example: cmd.Example,
Deprecated: len(cmd.Deprecated) > 0,
}
if len(cliDoc.Long) == 0 {
cliDoc.Long = cliDoc.Short
}
if cmd.Runnable() {
cliDoc.Usage = cmd.UseLine()
}
// Check recursively so that, e.g., `docker stack ls` returns the same output as `docker stack`
for curr := cmd; curr != nil; curr = curr.Parent() {
if v, ok := curr.Annotations["version"]; ok && cliDoc.MinAPIVersion == "" {
cliDoc.MinAPIVersion = v
}
if _, ok := curr.Annotations["experimental"]; ok && !cliDoc.Experimental {
cliDoc.Experimental = true
}
if _, ok := curr.Annotations["experimentalCLI"]; ok && !cliDoc.ExperimentalCLI {
cliDoc.ExperimentalCLI = true
}
if _, ok := curr.Annotations["kubernetes"]; ok && !cliDoc.Kubernetes {
cliDoc.Kubernetes = true
}
if _, ok := curr.Annotations["swarm"]; ok && !cliDoc.Swarm {
cliDoc.Swarm = true
}
if o, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" {
cliDoc.OSType = o
}
}
anchors := make(map[string]struct{})
if a, ok := cmd.Annotations["anchors"]; ok && a != "" {
for _, anchor := range strings.Split(a, ",") {
anchors[anchor] = struct{}{}
}
}
flags := cmd.NonInheritedFlags()
if flags.HasFlags() {
cliDoc.Options = genFlagResult(flags, anchors)
}
flags = cmd.InheritedFlags()
if flags.HasFlags() {
cliDoc.InheritedOptions = genFlagResult(flags, anchors)
}
if hasSeeAlso(cmd) {
if cmd.HasParent() {
parent := cmd.Parent()
cliDoc.Pname = parent.CommandPath()
cliDoc.Plink = strings.Replace(cliDoc.Pname, " ", "_", -1) + ".yaml"
cmd.VisitParents(func(c *cobra.Command) {
if c.DisableAutoGenTag {
cmd.DisableAutoGenTag = c.DisableAutoGenTag
}
})
}
children := cmd.Commands()
sort.Sort(byName(children))
for _, child := range children {
if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() {
continue
}
cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name())
cliDoc.Clink = append(cliDoc.Clink, strings.Replace(cliDoc.Name+"_"+child.Name(), " ", "_", -1)+".yaml")
}
}
final, err := yaml.Marshal(&cliDoc)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if _, err := fmt.Fprintln(w, string(final)); err != nil {
return err
}
return nil
}
func genFlagResult(flags *pflag.FlagSet, anchors map[string]struct{}) []cmdOption {
var (
result []cmdOption
opt cmdOption
)
const (
// shortMaxWidth is the maximum width for the "Short" description before
// we force YAML to use multi-line syntax. The goal is to make the total
// width fit within 80 characters. This value is based on 80 characters
// minus the with of the field, colon, and whitespace (' default_value: ').
defaultValueMaxWidth = 64
// longMaxWidth is the maximum width for the "Short" description before
// we force YAML to use multi-line syntax. The goal is to make the total
// width fit within 80 characters. This value is based on 80 characters
// minus the with of the field, colon, and whitespace (' description: ').
descriptionMaxWidth = 66
)
flags.VisitAll(func(flag *pflag.Flag) {
opt = cmdOption{
Option: flag.Name,
ValueType: flag.Value.Type(),
DefaultValue: forceMultiLine(flag.DefValue, defaultValueMaxWidth),
Description: forceMultiLine(flag.Usage, descriptionMaxWidth),
Deprecated: len(flag.Deprecated) > 0,
}
if v, ok := flag.Annotations["docs.external.url"]; ok && len(v) > 0 {
opt.DetailsURL = strings.TrimPrefix(v[0], "https://docs.docker.com")
} else if _, ok = anchors[flag.Name]; ok {
opt.DetailsURL = "#" + flag.Name
}
// Todo, when we mark a shorthand is deprecated, but specify an empty message.
// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
opt.Shorthand = flag.Shorthand
}
if _, ok := flag.Annotations["experimental"]; ok {
opt.Experimental = true
}
if _, ok := flag.Annotations["deprecated"]; ok {
opt.Deprecated = true
}
if v, ok := flag.Annotations["version"]; ok {
opt.MinAPIVersion = v[0]
}
if _, ok := flag.Annotations["experimentalCLI"]; ok {
opt.ExperimentalCLI = true
}
if _, ok := flag.Annotations["kubernetes"]; ok {
opt.Kubernetes = true
}
if _, ok := flag.Annotations["swarm"]; ok {
opt.Swarm = true
}
// Note that the annotation can have multiple ostypes set, however, multiple
// values are currently not used (and unlikely will).
//
// To simplify usage of the os_type property in the YAML, and for consistency
// with the same property for commands, we're only using the first ostype that's set.
if ostypes, ok := flag.Annotations["ostype"]; ok && len(opt.OSType) == 0 && len(ostypes) > 0 {
opt.OSType = ostypes[0]
}
result = append(result, opt)
})
return result
}
// forceMultiLine appends a newline (\n) to strings that are longer than max
// to force the yaml lib to use block notation (https://yaml.org/spec/1.2/spec.html#Block)
// instead of a single-line string with newlines and tabs encoded("string\nline1\nline2").
//
// This makes the generated YAML more readable, and easier to review changes.
// max can be used to customize the width to keep the whole line < 80 chars.
func forceMultiLine(s string, max int) string {
s = strings.TrimSpace(s)
if len(s) > max && !strings.Contains(s, "\n") {
s = s + "\n"
}
return s
}
// Small duplication for cobra utils
func hasSeeAlso(cmd *cobra.Command) bool {
if cmd.HasParent() {
return true
}
for _, c := range cmd.Commands() {
if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
continue
}
return true
}
return false
}
// applyDescriptionAndExamples fills in cmd.Long and cmd.Example with the
// "Description" and "Examples" H2 sections in mdString (if present).
func applyDescriptionAndExamples(cmd *cobra.Command, mdString string) {
sections := getSections(mdString)
var (
anchors []string
md string
)
if sections["description"] != "" {
md, anchors = cleanupMarkDown(sections["description"])
cmd.Long = md
anchors = append(anchors, md)
}
if sections["examples"] != "" {
md, anchors = cleanupMarkDown(sections["examples"])
cmd.Example = md
anchors = append(anchors, md)
}
if len(anchors) > 0 {
if cmd.Annotations == nil {
cmd.Annotations = make(map[string]string)
}
cmd.Annotations["anchors"] = strings.Join(anchors, ",")
}
}
type byName []*cobra.Command
func (s byName) Len() int { return len(s) }
func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() }