cmd/relui: add build release workflow

Adds the core of the build release workflow to relui. We build artifacts and run tests. Adds the necessary dependencies, currently just the coordinator client, to relui's main function. The filesystem stuff will be replaced in the next CL. For golang/go#51797. Change-Id: I4eea478d026b08540ac3bb48a684a545d862de6c Reviewed-on: https://go-review.googlesource.com/c/build/+/398694 Run-TryBot: Heschi Kreinick <heschi@google.com> Reviewed-by: Dmitri Shuralyov <dmitshur@google.com> Reviewed-by: Alex Rakoczy <alex@golang.org> Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org> TryBot-Result: Gopher Robot <gobot@golang.org>
2022-04-11 15:46:23 -04:00 · 2022-04-11 15:46:23 -04:00 · 4c4b95a29d
--- a/cmd/relui/deployment-prod.yaml
+++ b/cmd/relui/deployment-prod.yaml
@ -30,6 +30,7 @@ spec:
            - "--site-header-css=Site-header--production"
            - "--gerrit-api-secret=secret:symbolic-datum-552/gobot-password"
            - "--twitter-api-secret=secret:symbolic-datum-552/twitter-api-secret"
+            - "--builder-master-key=secret:symbolic-datum-552/builder-master-key"
          ports:
            - containerPort: 444
          env:
--- a/cmd/relui/main.go
+++ b/cmd/relui/main.go
@ -7,11 +7,19 @@ package main

 import (
 	"context"
+	"crypto/hmac"
+	"crypto/md5"
 	"flag"
+	"fmt"
+	"io"
 	"log"
 	"net/url"
+	"os"
+	"path/filepath"

 	"github.com/jackc/pgx/v4/pgxpool"
+	"golang.org/x/build"
+	"golang.org/x/build/buildlet"
 	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal/https"
 	"golang.org/x/build/internal/relui"
@ -36,6 +44,7 @@ func main() {
 	gerritAPIFlag := secret.Flag("gerrit-api-secret", "Gerrit API secret to use for workflows that interact with Gerrit.")
 	var twitterAPI secret.TwitterCredentials
 	secret.JSONVarFlag(&twitterAPI, "twitter-api-secret", "Twitter API secret to use for workflows involving tweeting.")
+	masterKey := secret.Flag("builder-master-key", "Builder master key")
 	https.RegisterFlags(flag.CommandLine)
 	flag.Parse()

@ -75,6 +84,18 @@ func main() {
 	dh := relui.NewDefinitionHolder()
 	relui.RegisterMailDLCLDefinition(dh, extCfg)
 	relui.RegisterTweetDefinitions(dh, extCfg)
+	coordinator := &buildlet.CoordinatorClient{
+		Auth: buildlet.UserPass{
+			Username: "user-relui",
+			Password: key(*masterKey, "user-relui"),
+		},
+		Instance: build.ProdCoordinator,
+	}
+	log.Printf("Coordinator client: %#v", coordinator)
+	if _, err := coordinator.RemoteBuildlets(); err != nil {
+		log.Printf("Broken coordinator client: %v", err)
+	}
+	relui.RegisterBuildReleaseWorkflows(dh, &osFiles{"/tmp"}, coordinator.CreateBuildlet)
 	db, err := pgxpool.Connect(ctx, *pgConnect)
 	if err != nil {
 		log.Fatal(err)
@ -98,3 +119,21 @@ func main() {
 	}
 	log.Fatalln(https.ListenAndServe(ctx, s))
 }
+
+func key(masterKey, principal string) string {
+	h := hmac.New(md5.New, []byte(masterKey))
+	io.WriteString(h, principal)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+type osFiles struct {
+	basePath string
+}
+
+func (f *osFiles) Create(path string) (io.WriteCloser, error) {
+	return os.Create(filepath.Join(f.basePath, path))
+}
+
+func (f *osFiles) Open(path string) (io.ReadCloser, error) {
+	return os.Open(filepath.Join(f.basePath, path))
+}
--- a/internal/relui/listener.go
+++ b/internal/relui/listener.go
@ -32,7 +32,7 @@ type PGListener struct {
 // workflow. The workflow.TaskState is persisted as a db.Task,
 // creating or updating a row as necessary.
 func (l *PGListener) TaskStateChanged(workflowID uuid.UUID, taskName string, state *workflow.TaskState) error {
-	log.Printf("TaskStateChanged(%q, %q, %v)", workflowID, taskName, state)
+	log.Printf("TaskStateChanged(%q, %q, %#v)", workflowID, taskName, state)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	result, err := json.Marshal(state.Result)
@ -54,7 +54,7 @@ func (l *PGListener) TaskStateChanged(workflowID uuid.UUID, taskName string, sta
 		return err
 	})
 	if err != nil {
-		log.Printf("TaskStateChanged(%q, %q, %v) = %v", workflowID, taskName, state, err)
+		log.Printf("TaskStateChanged(%q, %q, %#v) = %v", workflowID, taskName, state, err)
 	}
 	return err
 }
--- a/internal/relui/workflows.go
+++ b/internal/relui/workflows.go
@ -5,8 +5,14 @@
 package relui

 import (
+	"fmt"
+	"io"
+	"strings"
 	"sync"

+	"golang.org/x/build/buildlet"
+	"golang.org/x/build/dashboard"
+	"golang.org/x/build/internal/releasetargets"
 	"golang.org/x/build/internal/task"
 	"golang.org/x/build/internal/workflow"
 )
@ -169,3 +175,191 @@ func echo(ctx *workflow.TaskContext, arg string) (string, error) {
 	ctx.Printf("echo(%v, %q)", ctx, arg)
 	return arg, nil
 }
+
+func RegisterBuildReleaseWorkflows(h *DefinitionHolder, files WorkingFiles, createBuildlet func(string) (buildlet.Client, error)) {
+	go117, err := newBuildReleaseWorkflow("go1.17", files, createBuildlet)
+	if err != nil {
+		panic(err)
+	}
+	h.RegisterDefinition("Release Go 1.17", go117)
+	go118, err := newBuildReleaseWorkflow("go1.18", files, createBuildlet)
+	if err != nil {
+		panic(err)
+	}
+	h.RegisterDefinition("Release Go 1.18", go118)
+
+}
+
+func newBuildReleaseWorkflow(majorVersion string, files WorkingFiles, createBuildlet func(string) (buildlet.Client, error)) (*workflow.Definition, error) {
+	wd := workflow.New()
+	targets, ok := releasetargets.TargetsForVersion(majorVersion)
+	if !ok {
+		return nil, fmt.Errorf("malformed/unknown version %q", majorVersion)
+	}
+	version := wd.Parameter(workflow.Parameter{Name: "Version", Example: "go1.10.1"})
+	revision := wd.Parameter(workflow.Parameter{Name: "Revision", Example: "release-branch.go1.10"})
+	skipTests := wd.Parameter(workflow.Parameter{Name: "Targets to skip testing (space-separated target names or 'all') (optional)"})
+	tasks := buildReleaseTasks{files, createBuildlet}
+
+	source := wd.Task("Build source archive", tasks.buildSource, revision, version)
+	// Artifact file paths.
+	var artifacts []workflow.Value
+	// Empty values that represent the dependency on tests passing.
+	var testResults []workflow.Value
+	for _, target := range targets {
+		targetVal := wd.Constant(target)
+		taskName := func(step string) string { return target.Name + ": " + step }
+
+		// Build release artifacts for the platform.
+		bin := wd.Task(taskName("Build binary archive"), tasks.buildBinary, targetVal, source)
+		if target.GOOS == "windows" {
+			zip := wd.Task(taskName("Convert to .zip"), tasks.convertToZip, targetVal, bin)
+			msi := wd.Task(taskName("Build MSI"), tasks.buildMSI, targetVal, bin)
+			artifacts = append(artifacts, msi, zip)
+		} else {
+			artifacts = append(artifacts, bin)
+		}
+
+		if target.BuildOnly {
+			continue
+		}
+		short := wd.Task(taskName("Run short tests"), tasks.runTests, targetVal, wd.Constant(target.Builder), skipTests, bin)
+		testResults = append(testResults, short)
+		if target.LongTestBuilder != "" {
+			long := wd.Task(taskName("Run long tests"), tasks.runTests, targetVal, wd.Constant(target.LongTestBuilder), skipTests, bin)
+			testResults = append(testResults, long)
+		}
+	}
+	// Eventually we need to upload artifacts and perhaps summarize test results.
+	// For now, just mush them all together.
+	results := wd.Task("Combine results", combineResults, wd.Slice(artifacts), wd.Slice(testResults))
+	wd.Output("Build results", results)
+	return wd, nil
+}
+
+// buildReleaseTasks serves as an adapter to the various build tasks in the task package.
+type buildReleaseTasks struct {
+	fs             WorkingFiles
+	createBuildlet func(string) (buildlet.Client, error)
+}
+
+func (b *buildReleaseTasks) buildSource(ctx *workflow.TaskContext, revision, version string) (string, error) {
+	return b.runBuildStep(ctx, nil, "", "", "source.tar.gz", func(_ *task.BuildletStep, _ io.Reader, w io.Writer) error {
+		return task.WriteSourceArchive(ctx, revision, version, w)
+	})
+}
+
+func (b *buildReleaseTasks) buildBinary(ctx *workflow.TaskContext, target *releasetargets.Target, source string) (string, error) {
+	return b.runBuildStep(ctx, target, target.Builder, source, target.Name+"-binary.tar.gz", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
+		return bs.BuildBinary(ctx, r, w)
+	})
+}
+
+func (b *buildReleaseTasks) buildMSI(ctx *workflow.TaskContext, target *releasetargets.Target, binary string) (string, error) {
+	return b.runBuildStep(ctx, target, target.Builder, binary, target.Name+".msi", func(bs *task.BuildletStep, r io.Reader, w io.Writer) error {
+		return bs.BuildMSI(ctx, r, w)
+	})
+}
+
+func (b *buildReleaseTasks) convertToZip(ctx *workflow.TaskContext, target *releasetargets.Target, binary string) (string, error) {
+	return b.runBuildStep(ctx, nil, "", binary, target.Name+".zip", func(_ *task.BuildletStep, r io.Reader, w io.Writer) error {
+		return task.ConvertTGZToZIP(r, w)
+	})
+}
+
+func (b *buildReleaseTasks) runTests(ctx *workflow.TaskContext, target *releasetargets.Target, buildlet, skipTests, binary string) (string, error) {
+	skipped := skipTests == "all"
+	skipTargets := strings.Fields(skipTests)
+	for _, skip := range skipTargets {
+		if target.Name == skip {
+			skipped = true
+		}
+	}
+	if skipped {
+		ctx.Printf("Skipping test")
+		return "", nil
+	}
+	return b.runBuildStep(ctx, target, buildlet, binary, "", func(bs *task.BuildletStep, r io.Reader, _ io.Writer) error {
+		return bs.TestTarget(ctx, r)
+	})
+}
+
+// runBuildStep is a convenience function that manages resources a build step might need.
+// If target and buildlet name are specified, a BuildletStep will be passed to f.
+// If inputName is specified, it will be opened and passed as a Reader to f.
+// If outputName is specified, a unique filename will be generated based off it, the file
+// will be opened and passed as a Writer to f, and its name will be returned as the result.
+func (b *buildReleaseTasks) runBuildStep(
+	ctx *workflow.TaskContext,
+	target *releasetargets.Target,
+	buildletName, inputName, outputName string,
+	f func(*task.BuildletStep, io.Reader, io.Writer) error,
+) (string, error) {
+	if (target == nil) != (buildletName == "") {
+		return "", fmt.Errorf("target and buildlet must be specified together")
+	}
+
+	var step *task.BuildletStep
+	if target != nil {
+		ctx.Printf("Creating buildlet %v.", buildletName)
+		client, err := b.createBuildlet(buildletName)
+		if err != nil {
+			return "", err
+		}
+		defer client.Close()
+		buildConfig, ok := dashboard.Builders[buildletName]
+		if !ok {
+			return "", fmt.Errorf("unknown builder: %v", buildConfig)
+		}
+		step = &task.BuildletStep{
+			Target:      target,
+			Buildlet:    client,
+			BuildConfig: buildConfig,
+			Watch:       true,
+		}
+		ctx.Printf("Buildlet ready.")
+	}
+
+	var in io.ReadCloser
+	var err error
+	if inputName != "" {
+		in, err = b.fs.Open(inputName)
+		if err != nil {
+			return "", err
+		}
+		defer in.Close()
+	}
+	var out io.WriteCloser
+	if outputName != "" {
+		out, err = b.fs.Create(outputName)
+		if err != nil {
+			return "", err
+		}
+		defer out.Close()
+	}
+	if err := f(step, in, out); err != nil {
+		return "", err
+	}
+	if step != nil {
+		if err := step.Buildlet.Close(); err != nil {
+			return "", err
+		}
+	}
+	// Don't check the error from in.Close: the steps may assert it down to
+	// Closer and close it themselves. (See https://pkg.go.dev/net/http#NewRequestWithContext.)
+	if out != nil {
+		if err := out.Close(); err != nil {
+			return "", err
+		}
+	}
+	return outputName, nil
+}
+
+func combineResults(ctx *workflow.TaskContext, artifacts, tests []string) (string, error) {
+	return strings.Join(artifacts, "\n") + strings.Join(tests, "\n"), nil
+}
+
+type WorkingFiles interface {
+	Create(string) (io.WriteCloser, error)
+	Open(string) (io.ReadCloser, error)
+}